OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
SqlToRelConverter.java
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one or more
3  * contributor license agreements. See the NOTICE file distributed with
4  * this work for additional information regarding copyright ownership.
5  * The ASF licenses this file to you under the Apache License, Version 2.0
6  * (the "License"); you may not use this file except in compliance with
7  * the License. You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 /* MAT 9 Jan 2021
19  * We have moved this code from calcite 1.25
20  * we keep our own copy as we have added some
21  * special handling of pushdown and correlation handling
22  *
23  * If we make changes from the original source please make
24  * sure you add good comments around the changed block
25  * as each time we move to a new calcite version we need to move
26  * our chnages to the new code
27  *
28  */
29 package org.apache.calcite.sql2rel;
30 
31 import static org.apache.calcite.sql.SqlUtil.stripAs;
32 
33 import com.google.common.base.Preconditions;
34 import com.google.common.collect.ImmutableList;
35 import com.google.common.collect.ImmutableList.Builder;
36 import com.google.common.collect.ImmutableMap;
37 import com.google.common.collect.ImmutableSet;
38 import com.google.common.collect.Iterables;
39 import com.google.common.collect.Sets;
41 
42 import org.apache.calcite.avatica.util.Spaces;
43 import org.apache.calcite.linq4j.Ord;
44 import org.apache.calcite.plan.Convention;
45 import org.apache.calcite.plan.RelOptCluster;
46 import org.apache.calcite.plan.RelOptPlanner;
47 import org.apache.calcite.plan.RelOptSamplingParameters;
48 import org.apache.calcite.plan.RelOptTable;
49 import org.apache.calcite.plan.RelOptUtil;
50 import org.apache.calcite.plan.RelTraitSet;
51 import org.apache.calcite.plan.ViewExpanders;
52 import org.apache.calcite.prepare.Prepare;
53 import org.apache.calcite.prepare.RelOptTableImpl;
54 import org.apache.calcite.rel.RelCollation;
55 import org.apache.calcite.rel.RelCollationTraitDef;
56 import org.apache.calcite.rel.RelCollations;
57 import org.apache.calcite.rel.RelFieldCollation;
58 import org.apache.calcite.rel.RelNode;
59 import org.apache.calcite.rel.RelRoot;
60 import org.apache.calcite.rel.RelShuttleImpl;
61 import org.apache.calcite.rel.SingleRel;
62 import org.apache.calcite.rel.core.AggregateCall;
63 import org.apache.calcite.rel.core.Collect;
64 import org.apache.calcite.rel.core.CorrelationId;
65 import org.apache.calcite.rel.core.Filter;
66 import org.apache.calcite.rel.core.Join;
67 import org.apache.calcite.rel.core.JoinInfo;
68 import org.apache.calcite.rel.core.JoinRelType;
69 import org.apache.calcite.rel.core.Project;
70 import org.apache.calcite.rel.core.RelFactories;
71 import org.apache.calcite.rel.core.Sample;
72 import org.apache.calcite.rel.core.Sort;
73 import org.apache.calcite.rel.hint.HintStrategyTable;
74 import org.apache.calcite.rel.hint.Hintable;
75 import org.apache.calcite.rel.hint.RelHint;
76 import org.apache.calcite.rel.logical.LogicalAggregate;
77 import org.apache.calcite.rel.logical.LogicalCorrelate;
78 import org.apache.calcite.rel.logical.LogicalFilter;
79 import org.apache.calcite.rel.logical.LogicalIntersect;
80 import org.apache.calcite.rel.logical.LogicalJoin;
81 import org.apache.calcite.rel.logical.LogicalMatch;
82 import org.apache.calcite.rel.logical.LogicalMinus;
83 import org.apache.calcite.rel.logical.LogicalProject;
84 import org.apache.calcite.rel.logical.LogicalSort;
85 import org.apache.calcite.rel.logical.LogicalTableFunctionScan;
86 import org.apache.calcite.rel.logical.LogicalTableModify;
87 import org.apache.calcite.rel.logical.LogicalTableScan;
88 import org.apache.calcite.rel.logical.LogicalUnion;
89 import org.apache.calcite.rel.logical.LogicalValues;
90 import org.apache.calcite.rel.metadata.RelColumnMapping;
91 import org.apache.calcite.rel.metadata.RelMetadataQuery;
92 import org.apache.calcite.rel.stream.Delta;
93 import org.apache.calcite.rel.stream.LogicalDelta;
94 import org.apache.calcite.rel.type.RelDataType;
95 import org.apache.calcite.rel.type.RelDataTypeFactory;
96 import org.apache.calcite.rel.type.RelDataTypeField;
97 import org.apache.calcite.rex.RexBuilder;
98 import org.apache.calcite.rex.RexCall;
99 import org.apache.calcite.rex.RexCallBinding;
100 import org.apache.calcite.rex.RexCorrelVariable;
101 import org.apache.calcite.rex.RexDynamicParam;
102 import org.apache.calcite.rex.RexFieldAccess;
103 import org.apache.calcite.rex.RexFieldCollation;
104 import org.apache.calcite.rex.RexInputRef;
105 import org.apache.calcite.rex.RexLiteral;
106 import org.apache.calcite.rex.RexNode;
107 import org.apache.calcite.rex.RexPatternFieldRef;
108 import org.apache.calcite.rex.RexRangeRef;
109 import org.apache.calcite.rex.RexShuttle;
110 import org.apache.calcite.rex.RexSubQuery;
111 import org.apache.calcite.rex.RexUtil;
112 import org.apache.calcite.rex.RexWindowBound;
113 import org.apache.calcite.rex.RexWindowBounds;
114 import org.apache.calcite.schema.ColumnStrategy;
115 import org.apache.calcite.schema.ModifiableTable;
116 import org.apache.calcite.schema.ModifiableView;
117 import org.apache.calcite.schema.Table;
118 import org.apache.calcite.schema.TranslatableTable;
119 import org.apache.calcite.schema.Wrapper;
120 import org.apache.calcite.sql.JoinConditionType;
122 import org.apache.calcite.sql.SqlAggFunction;
123 import org.apache.calcite.sql.SqlBasicCall;
124 import org.apache.calcite.sql.SqlCall;
125 import org.apache.calcite.sql.SqlCallBinding;
126 import org.apache.calcite.sql.SqlDataTypeSpec;
127 import org.apache.calcite.sql.SqlDelete;
128 import org.apache.calcite.sql.SqlDynamicParam;
129 import org.apache.calcite.sql.SqlExplainFormat;
130 import org.apache.calcite.sql.SqlExplainLevel;
131 import org.apache.calcite.sql.SqlFunction;
132 import org.apache.calcite.sql.SqlIdentifier;
133 import org.apache.calcite.sql.SqlInsert;
134 import org.apache.calcite.sql.SqlIntervalQualifier;
135 import org.apache.calcite.sql.SqlJoin;
136 import org.apache.calcite.sql.SqlKind;
137 import org.apache.calcite.sql.SqlLiteral;
138 import org.apache.calcite.sql.SqlMatchRecognize;
139 import org.apache.calcite.sql.SqlMerge;
140 import org.apache.calcite.sql.SqlNode;
141 import org.apache.calcite.sql.SqlNodeList;
142 import org.apache.calcite.sql.SqlNumericLiteral;
144 import org.apache.calcite.sql.SqlOperatorTable;
145 import org.apache.calcite.sql.SqlOrderBy;
146 import org.apache.calcite.sql.SqlSampleSpec;
147 import org.apache.calcite.sql.SqlSelect;
148 import org.apache.calcite.sql.SqlSelectKeyword;
149 import org.apache.calcite.sql.SqlSetOperator;
150 import org.apache.calcite.sql.SqlSnapshot;
151 import org.apache.calcite.sql.SqlUnnestOperator;
152 import org.apache.calcite.sql.SqlUpdate;
153 import org.apache.calcite.sql.SqlUtil;
154 import org.apache.calcite.sql.SqlValuesOperator;
155 import org.apache.calcite.sql.SqlWindow;
156 import org.apache.calcite.sql.SqlWith;
157 import org.apache.calcite.sql.SqlWithItem;
158 import org.apache.calcite.sql.fun.SqlCase;
159 import org.apache.calcite.sql.fun.SqlCountAggFunction;
160 import org.apache.calcite.sql.fun.SqlInOperator;
161 import org.apache.calcite.sql.fun.SqlQuantifyOperator;
162 import org.apache.calcite.sql.fun.SqlRowOperator;
163 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
164 import org.apache.calcite.sql.parser.SqlParserPos;
165 import org.apache.calcite.sql.type.SqlReturnTypeInference;
166 import org.apache.calcite.sql.type.SqlTypeName;
167 import org.apache.calcite.sql.type.SqlTypeUtil;
168 import org.apache.calcite.sql.type.TableFunctionReturnTypeInference;
169 import org.apache.calcite.sql.util.SqlBasicVisitor;
170 import org.apache.calcite.sql.util.SqlVisitor;
171 import org.apache.calcite.sql.validate.AggregatingSelectScope;
172 import org.apache.calcite.sql.validate.CollectNamespace;
173 import org.apache.calcite.sql.validate.DelegatingScope;
174 import org.apache.calcite.sql.validate.ListScope;
175 import org.apache.calcite.sql.validate.MatchRecognizeScope;
176 import org.apache.calcite.sql.validate.ParameterScope;
177 import org.apache.calcite.sql.validate.SelectScope;
178 import org.apache.calcite.sql.validate.SqlMonotonicity;
179 import org.apache.calcite.sql.validate.SqlNameMatcher;
180 import org.apache.calcite.sql.validate.SqlQualified;
181 import org.apache.calcite.sql.validate.SqlUserDefinedTableFunction;
182 import org.apache.calcite.sql.validate.SqlUserDefinedTableMacro;
183 import org.apache.calcite.sql.validate.SqlValidator;
185 import org.apache.calcite.sql.validate.SqlValidatorNamespace;
186 import org.apache.calcite.sql.validate.SqlValidatorScope;
187 import org.apache.calcite.sql.validate.SqlValidatorTable;
188 import org.apache.calcite.sql.validate.SqlValidatorUtil;
189 import org.apache.calcite.tools.RelBuilder;
190 import org.apache.calcite.tools.RelBuilderFactory;
191 import org.apache.calcite.util.ImmutableBitSet;
192 import org.apache.calcite.util.ImmutableIntList;
193 import org.apache.calcite.util.Litmus;
194 import org.apache.calcite.util.NlsString;
195 import org.apache.calcite.util.NumberUtil;
196 import org.apache.calcite.util.Pair;
197 import org.apache.calcite.util.Util;
198 import org.apache.calcite.util.trace.CalciteTrace;
199 import org.slf4j.Logger;
200 
202 import java.math.BigDecimal;
203 import java.util.AbstractList;
204 import java.util.ArrayDeque;
205 import java.util.ArrayList;
206 import java.util.BitSet;
207 import java.util.Collection;
208 import java.util.Collections;
209 import java.util.Deque;
210 import java.util.EnumSet;
211 import java.util.HashMap;
212 import java.util.HashSet;
213 import java.util.List;
214 import java.util.Map;
215 import java.util.Objects;
216 import java.util.Set;
217 import java.util.TreeSet;
218 import java.util.function.BiPredicate;
219 import java.util.function.Supplier;
220 import java.util.function.UnaryOperator;
221 import java.util.stream.Collectors;
222 
223 import javax.annotation.Nonnull;
233 public class SqlToRelConverter {
234  // ~ Static fields/initializers ---------------------------------------------
235 
236  protected static final Logger SQL2REL_LOGGER = CalciteTrace.getSqlToRelTracer();
237 
242  public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
243 
244  @Deprecated // to be removed before 2.0
246 
247  // ~ Instance fields --------------------------------------------------------
248 
249  protected final SqlValidator validator;
250  protected final RexBuilder rexBuilder;
251  protected final Prepare.CatalogReader catalogReader;
252  protected final RelOptCluster cluster;
253  private SubQueryConverter subQueryConverter;
254  protected final Map<RelNode, Integer> leaves = new HashMap<>();
255  private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
256  private final SqlOperatorTable opTab;
257  protected final RelDataTypeFactory typeFactory;
258  private final SqlNodeToRexConverter exprConverter;
259  private final HintStrategyTable hintStrategies;
260  private int explainParamCount;
261  public final SqlToRelConverter.Config config;
262  private final RelBuilder relBuilder;
263 
267  private final Map<CorrelationId, DeferredLookup> mapCorrelToDeferred = new HashMap<>();
268 
273  private final Deque<String> datasetStack = new ArrayDeque<>();
274 
280  private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs = new HashMap<>();
281 
282  public final RelOptTable.ViewExpander viewExpander;
283 
284  // ~ Constructors -----------------------------------------------------------
295  @Deprecated // to be removed before 2.0
296  public SqlToRelConverter(RelOptTable.ViewExpander viewExpander,
297  SqlValidator validator,
298  Prepare.CatalogReader catalogReader,
299  RelOptPlanner planner,
300  RexBuilder rexBuilder,
301  SqlRexConvertletTable convertletTable) {
302  this(viewExpander,
303  validator,
305  RelOptCluster.create(planner, rexBuilder),
306  convertletTable,
307  Config.DEFAULT);
308  }
309 
310  @Deprecated // to be removed before 2.0
311  public SqlToRelConverter(RelOptTable.ViewExpander viewExpander,
312  SqlValidator validator,
313  Prepare.CatalogReader catalogReader,
314  RelOptCluster cluster,
315  SqlRexConvertletTable convertletTable) {
316  this(viewExpander,
317  validator,
319  cluster,
320  convertletTable,
321  Config.DEFAULT);
322  }
323 
324  /* Creates a converter. */
325  public SqlToRelConverter(RelOptTable.ViewExpander viewExpander,
326  SqlValidator validator,
327  Prepare.CatalogReader catalogReader,
328  RelOptCluster cluster,
329  SqlRexConvertletTable convertletTable,
330  Config config) {
331  this.viewExpander = viewExpander;
332  this.opTab = (validator == null) ? SqlStdOperatorTable.instance()
333  : validator.getOperatorTable();
334  this.validator = validator;
335  this.catalogReader = catalogReader;
336  this.subQueryConverter = new NoOpSubQueryConverter();
337  this.rexBuilder = cluster.getRexBuilder();
338  this.typeFactory = rexBuilder.getTypeFactory();
339  this.exprConverter = new SqlNodeToRexConverterImpl(convertletTable);
340  this.explainParamCount = 0;
341  this.config = new ConfigBuilder().withConfig(config).build();
342  this.relBuilder = config.getRelBuilderFactory()
343  .create(cluster, null)
344  .transform(config.getRelBuilderConfigTransform());
345  this.hintStrategies = config.getHintStrategyTable();
346 
347  cluster.setHintStrategies(this.hintStrategies);
348  this.cluster = Objects.requireNonNull(cluster);
349  }
350 
351  // ~ Methods ----------------------------------------------------------------
352 
354  public RelOptCluster getCluster() {
355  return cluster;
356  }
357 
361  public RexBuilder getRexBuilder() {
362  return rexBuilder;
363  }
364 
371  public int getDynamicParamCount() {
372  return dynamicParamSqlNodes.size();
373  }
374 
381  public RelDataType getDynamicParamType(int index) {
382  SqlNode sqlNode = dynamicParamSqlNodes.get(index);
383  if (sqlNode == null) {
384  throw Util.needToImplement("dynamic param type inference");
385  }
386  return validator.getValidatedNodeType(sqlNode);
387  }
388 
396  public int getDynamicParamCountInExplain(boolean increment) {
397  int retVal = explainParamCount;
398  if (increment) {
400  }
401  return retVal;
402  }
403 
408  public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
410  }
411 
420  Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
421  mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
422  }
423 
430  public void setSubQueryConverter(SubQueryConverter converter) {
431  subQueryConverter = converter;
432  }
433 
440  assert config.isExplain();
441  this.explainParamCount = explainParamCount;
442  }
443 
444  private void checkConvertedType(SqlNode query, RelNode result) {
445  if (query.isA(SqlKind.DML)) {
446  return;
447  }
448  // Verify that conversion from SQL to relational algebra did
449  // not perturb any type information. (We can't do this if the
450  // SQL statement is something like an INSERT which has no
451  // validator type information associated with its result,
452  // hence the namespace check above.)
453  final List<RelDataTypeField> validatedFields =
454  validator.getValidatedNodeType(query).getFieldList();
455  final RelDataType validatedRowType =
456  validator.getTypeFactory().createStructType(Pair.right(validatedFields),
457  SqlValidatorUtil.uniquify(Pair.left(validatedFields),
458  catalogReader.nameMatcher().isCaseSensitive()));
459 
460  final List<RelDataTypeField> convertedFields =
461  result.getRowType().getFieldList().subList(0, validatedFields.size());
462  final RelDataType convertedRowType =
463  validator.getTypeFactory().createStructType(convertedFields);
464 
465  if (!RelOptUtil.equal("validated row type",
466  validatedRowType,
467  "converted row type",
468  convertedRowType,
469  Litmus.IGNORE)) {
470  throw new AssertionError("Conversion to relational algebra failed to "
471  + "preserve datatypes:\n"
472  + "validated type:\n" + validatedRowType.getFullTypeString()
473  + "\nconverted type:\n" + convertedRowType.getFullTypeString() + "\nrel:\n"
474  + RelOptUtil.toString(result));
475  }
476  }
477 
478  public RelNode flattenTypes(RelNode rootRel, boolean restructure) {
479  RelStructuredTypeFlattener typeFlattener = new RelStructuredTypeFlattener(
480  relBuilder, rexBuilder, createToRelContext(ImmutableList.of()), restructure);
481  return typeFlattener.rewrite(rootRel);
482  }
483 
492  public RelNode decorrelate(SqlNode query, RelNode rootRel) {
493  if (!config.isDecorrelationEnabled()) {
494  return rootRel;
495  }
496  final RelNode result = decorrelateQuery(rootRel);
497  if (result != rootRel) {
498  checkConvertedType(query, result);
499  }
500  return result;
501  }
502 
525  public RelNode trimUnusedFields(boolean ordered, RelNode rootRel) {
526  // Trim fields that are not used by their consumer.
527  if (isTrimUnusedFields()) {
528  final RelFieldTrimmer trimmer = newFieldTrimmer();
529  final List<RelCollation> collations =
530  rootRel.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE);
531  rootRel = trimmer.trim(rootRel);
532  if (!ordered && collations != null && !collations.isEmpty()
533  && !collations.equals(ImmutableList.of(RelCollations.EMPTY))) {
534  final RelTraitSet traitSet =
535  rootRel.getTraitSet().replace(RelCollationTraitDef.INSTANCE, collations);
536  rootRel = rootRel.copy(traitSet, rootRel.getInputs());
537  }
538  if (SQL2REL_LOGGER.isDebugEnabled()) {
539  SQL2REL_LOGGER.debug(RelOptUtil.dumpPlan("Plan after trimming unused fields",
540  rootRel,
541  SqlExplainFormat.TEXT,
542  SqlExplainLevel.EXPPLAN_ATTRIBUTES));
543  }
544  }
545  return rootRel;
546  }
547 
553  protected RelFieldTrimmer newFieldTrimmer() {
554  return new RelFieldTrimmer(validator, relBuilder);
555  }
556 
568  public RelRoot convertQuery(
569  SqlNode query, final boolean needsValidation, final boolean top) {
570  if (needsValidation) {
571  query = validator.validate(query);
572  }
573 
574  RelNode result = convertQueryRecursive(query, top, null).rel;
575  if (top) {
576  if (isStream(query)) {
577  result = new LogicalDelta(cluster, result.getTraitSet(), result);
578  }
579  }
580  RelCollation collation = RelCollations.EMPTY;
581  if (!query.isA(SqlKind.DML)) {
582  if (isOrdered(query)) {
583  collation = requiredCollation(result);
584  }
585  }
586  checkConvertedType(query, result);
587 
588  if (SQL2REL_LOGGER.isDebugEnabled()) {
589  SQL2REL_LOGGER.debug(RelOptUtil.dumpPlan("Plan after converting SqlNode to RelNode",
590  result,
591  SqlExplainFormat.TEXT,
592  SqlExplainLevel.EXPPLAN_ATTRIBUTES));
593  }
594 
595  final RelDataType validatedRowType = validator.getValidatedNodeType(query);
596  List<RelHint> hints = new ArrayList<>();
597  if (query.getKind() == SqlKind.SELECT) {
598  final SqlSelect select = (SqlSelect) query;
599  if (select.hasHints()) {
600  hints = SqlUtil.getRelHint(hintStrategies, select.getHints());
601  }
602  }
603  // propagate the hints.
604  result = RelOptUtil.propagateRelHints(result, false);
605  return RelRoot.of(result, validatedRowType, query.getKind())
606  .withCollation(collation)
607  .withHints(hints);
608  }
609 
610  private static boolean isStream(SqlNode query) {
611  return query instanceof SqlSelect
612  && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
613  }
614 
615  public static boolean isOrdered(SqlNode query) {
616  switch (query.getKind()) {
617  case SELECT:
618  return ((SqlSelect) query).getOrderList() != null
619  && ((SqlSelect) query).getOrderList().size() > 0;
620  case WITH:
621  return isOrdered(((SqlWith) query).body);
622  case ORDER_BY:
623  return ((SqlOrderBy) query).orderList.size() > 0;
624  default:
625  return false;
626  }
627  }
628 
629  private RelCollation requiredCollation(RelNode r) {
630  if (r instanceof Sort) {
631  return ((Sort) r).collation;
632  }
633  if (r instanceof Project) {
634  return requiredCollation(((Project) r).getInput());
635  }
636  if (r instanceof Delta) {
637  return requiredCollation(((Delta) r).getInput());
638  }
639  throw new AssertionError(r);
640  }
641 
645  public RelNode convertSelect(SqlSelect select, boolean top) {
646  final SqlValidatorScope selectScope = validator.getWhereScope(select);
647  final Blackboard bb = createBlackboard(selectScope, null, top);
648  convertSelectImpl(bb, select);
649  return bb.root;
650  }
651 
656  SqlValidatorScope scope, Map<String, RexNode> nameToNodeMap, boolean top) {
657  return new Blackboard(scope, nameToNodeMap, top);
658  }
659 
664  protected void convertSelectImpl(final Blackboard bb, SqlSelect select) {
665  convertFrom(bb, select.getFrom());
666  convertWhere(bb, select.getWhere());
667 
668  final List<SqlNode> orderExprList = new ArrayList<>();
669  final List<RelFieldCollation> collationList = new ArrayList<>();
670  gatherOrderExprs(bb, select, select.getOrderList(), orderExprList, collationList);
671  final RelCollation collation =
672  cluster.traitSet().canonize(RelCollations.of(collationList));
673 
674  if (validator.isAggregate(select)) {
675  convertAgg(bb, select, orderExprList);
676  } else {
677  convertSelectList(bb, select, orderExprList);
678  }
679 
680  if (select.isDistinct()) {
681  distinctify(bb, true);
682  }
683 
684  convertOrder(
685  select, bb, collation, orderExprList, select.getOffset(), select.getFetch());
686 
687  if (select.hasHints()) {
688  final List<RelHint> hints = SqlUtil.getRelHint(hintStrategies, select.getHints());
689  // Attach the hints to the first Hintable node we found from the root node.
690  bb.setRoot(bb.root.accept(new RelShuttleImpl() {
691  boolean attached = false;
692 
693  @Override
694  public RelNode visitChild(RelNode parent, int i, RelNode child) {
695  if (parent instanceof Hintable && !attached) {
696  attached = true;
697  return ((Hintable) parent).attachHints(hints);
698  } else {
699  return super.visitChild(parent, i, child);
700  }
701  }
702  }),
703  true);
704  } else {
705  bb.setRoot(bb.root, true);
706  }
707  }
708 
722  private void distinctify(Blackboard bb, boolean checkForDupExprs) {
723  // Look for duplicate expressions in the project.
724  // Say we have 'select x, y, x, z'.
725  // Then dups will be {[2, 0]}
726  // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]}
727  RelNode rel = bb.root;
728  if (checkForDupExprs && (rel instanceof LogicalProject)) {
729  LogicalProject project = (LogicalProject) rel;
730  final List<RexNode> projectExprs = project.getProjects();
731  final List<Integer> origins = new ArrayList<>();
732  int dupCount = 0;
733  for (int i = 0; i < projectExprs.size(); i++) {
734  int x = projectExprs.indexOf(projectExprs.get(i));
735  if (x >= 0 && x < i) {
736  origins.add(x);
737  ++dupCount;
738  } else {
739  origins.add(i);
740  }
741  }
742  if (dupCount == 0) {
743  distinctify(bb, false);
744  return;
745  }
746 
747  final Map<Integer, Integer> squished = new HashMap<>();
748  final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
749  final List<Pair<RexNode, String>> newProjects = new ArrayList<>();
750  for (int i = 0; i < fields.size(); i++) {
751  if (origins.get(i) == i) {
752  squished.put(i, newProjects.size());
753  newProjects.add(RexInputRef.of2(i, fields));
754  }
755  }
756  rel = LogicalProject.create(
757  rel, ImmutableList.of(), Pair.left(newProjects), Pair.right(newProjects));
758  bb.root = rel;
759  distinctify(bb, false);
760  rel = bb.root;
761 
762  // Create the expressions to reverse the mapping.
763  // Project($0, $1, $0, $2).
764  final List<Pair<RexNode, String>> undoProjects = new ArrayList<>();
765  for (int i = 0; i < fields.size(); i++) {
766  final int origin = origins.get(i);
767  RelDataTypeField field = fields.get(i);
768  undoProjects.add(Pair.of(
769  new RexInputRef(squished.get(origin), field.getType()), field.getName()));
770  }
771 
772  rel = LogicalProject.create(
773  rel, ImmutableList.of(), Pair.left(undoProjects), Pair.right(undoProjects));
774  bb.setRoot(rel, false);
775 
776  return;
777  }
778 
779  // Usual case: all of the expressions in the SELECT clause are
780  // different.
781  final ImmutableBitSet groupSet =
782  ImmutableBitSet.range(rel.getRowType().getFieldCount());
783  rel = createAggregate(bb, groupSet, ImmutableList.of(groupSet), ImmutableList.of());
784 
785  bb.setRoot(rel, false);
786  }
787 
804  protected void convertOrder(SqlSelect select,
805  Blackboard bb,
806  RelCollation collation,
807  List<SqlNode> orderExprList,
808  SqlNode offset,
809  SqlNode fetch) {
810  if (!bb.top || select.getOrderList() == null
811  || select.getOrderList().getList().isEmpty()) {
812  assert !bb.top || collation.getFieldCollations().isEmpty();
813  if ((offset == null
814  || (offset instanceof SqlLiteral
815  && ((SqlLiteral) offset)
816  .bigDecimalValue()
817  .equals(BigDecimal.ZERO)))
818  && fetch == null) {
819  return;
820  }
821  }
822 
823  // Create a sorter using the previously constructed collations.
824  bb.setRoot(LogicalSort.create(bb.root,
825  collation,
826  offset == null ? null : convertExpression(offset),
827  fetch == null ? null : convertExpression(fetch)),
828  false);
829 
830  // If extra expressions were added to the project list for sorting,
831  // add another project to remove them. But make the collation empty, because
832  // we can't represent the real collation.
833  //
834  // If it is the top node, use the real collation, but don't trim fields.
835  if (orderExprList.size() > 0 && !bb.top) {
836  final List<RexNode> exprs = new ArrayList<>();
837  final RelDataType rowType = bb.root.getRowType();
838  final int fieldCount = rowType.getFieldCount() - orderExprList.size();
839  for (int i = 0; i < fieldCount; i++) {
840  exprs.add(rexBuilder.makeInputRef(bb.root, i));
841  }
842  bb.setRoot(LogicalProject.create(bb.root,
843  ImmutableList.of(),
844  exprs,
845  rowType.getFieldNames().subList(0, fieldCount)),
846  false);
847  }
848  }
849 
855  private static boolean containsInOperator(SqlNode node) {
856  try {
857  SqlVisitor<Void> visitor = new SqlBasicVisitor<Void>() {
858  public Void visit(SqlCall call) {
859  if (call.getOperator() instanceof SqlInOperator) {
860  throw new Util.FoundOne(call);
861  }
862  return super.visit(call);
863  }
864  };
865  node.accept(visitor);
866  return false;
867  } catch (Util.FoundOne e) {
868  Util.swallow(e, null);
869  return true;
870  }
871  }
872 
880  private static SqlNode pushDownNotForIn(SqlValidatorScope scope, SqlNode sqlNode) {
881  if (!(sqlNode instanceof SqlCall) || !containsInOperator(sqlNode)) {
882  return sqlNode;
883  }
884  final SqlCall sqlCall = (SqlCall) sqlNode;
885  switch (sqlCall.getKind()) {
886  case AND:
887  case OR:
888  final List<SqlNode> operands = new ArrayList<>();
889  for (SqlNode operand : sqlCall.getOperandList()) {
890  operands.add(pushDownNotForIn(scope, operand));
891  }
892  final SqlCall newCall =
893  sqlCall.getOperator().createCall(sqlCall.getParserPosition(), operands);
894  return reg(scope, newCall);
895 
896  case NOT:
897  assert sqlCall.operand(0) instanceof SqlCall;
898  final SqlCall call = sqlCall.operand(0);
899  switch (sqlCall.operand(0).getKind()) {
900  case CASE:
901  final SqlCase caseNode = (SqlCase) call;
902  final SqlNodeList thenOperands = new SqlNodeList(SqlParserPos.ZERO);
903 
904  for (SqlNode thenOperand : caseNode.getThenOperands()) {
905  final SqlCall not =
906  SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, thenOperand);
907  thenOperands.add(pushDownNotForIn(scope, reg(scope, not)));
908  }
909  SqlNode elseOperand = caseNode.getElseOperand();
910  if (!SqlUtil.isNull(elseOperand)) {
911  // "not(unknown)" is "unknown", so no need to simplify
912  final SqlCall not =
913  SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, elseOperand);
914  elseOperand = pushDownNotForIn(scope, reg(scope, not));
915  }
916 
917  return reg(scope,
918  SqlStdOperatorTable.CASE.createCall(SqlParserPos.ZERO,
919  caseNode.getValueOperand(),
920  caseNode.getWhenOperands(),
921  thenOperands,
922  elseOperand));
923 
924  case AND:
925  final List<SqlNode> orOperands = new ArrayList<>();
926  for (SqlNode operand : call.getOperandList()) {
927  orOperands.add(pushDownNotForIn(scope,
928  reg(scope,
929  SqlStdOperatorTable.NOT.createCall(
930  SqlParserPos.ZERO, operand))));
931  }
932  return reg(scope,
933  SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands));
934 
935  case OR:
936  final List<SqlNode> andOperands = new ArrayList<>();
937  for (SqlNode operand : call.getOperandList()) {
938  andOperands.add(pushDownNotForIn(scope,
939  reg(scope,
940  SqlStdOperatorTable.NOT.createCall(
941  SqlParserPos.ZERO, operand))));
942  }
943  return reg(scope,
944  SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, andOperands));
945 
946  case NOT:
947  assert call.operandCount() == 1;
948  return pushDownNotForIn(scope, call.operand(0));
949 
950  case NOT_IN:
951  return reg(scope,
952  SqlStdOperatorTable.IN.createCall(
953  SqlParserPos.ZERO, call.getOperandList()));
954 
955  case IN:
956  return reg(scope,
957  SqlStdOperatorTable.NOT_IN.createCall(
958  SqlParserPos.ZERO, call.getOperandList()));
959  }
960  }
961  return sqlNode;
962  }
963 
968  private static SqlNode reg(SqlValidatorScope scope, SqlNode e) {
969  scope.getValidator().deriveType(scope, e);
970  return e;
971  }
972 
979  private void convertWhere(final Blackboard bb, final SqlNode where) {
980  if (where == null) {
981  return;
982  }
983  SqlNode newWhere = pushDownNotForIn(bb.scope, where);
984  replaceSubQueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
985  final RexNode convertedWhere = bb.convertExpression(newWhere);
986  final RexNode convertedWhere2 =
987  RexUtil.removeNullabilityCast(typeFactory, convertedWhere);
988 
989  // only allocate filter if the condition is not TRUE
990  if (convertedWhere2.isAlwaysTrue()) {
991  return;
992  }
993 
994  final RelFactories.FilterFactory filterFactory = RelFactories.DEFAULT_FILTER_FACTORY;
995  final RelNode filter =
996  filterFactory.createFilter(bb.root, convertedWhere2, ImmutableSet.of());
997  final RelNode r;
998  final CorrelationUse p = getCorrelationUse(bb, filter);
999  if (p != null) {
1000  assert p.r instanceof Filter;
1001  Filter f = (Filter) p.r;
1002  r = LogicalFilter.create(f.getInput(), f.getCondition(), ImmutableSet.of(p.id));
1003  } else {
1004  r = filter;
1005  }
1006 
1007  bb.setRoot(r, false);
1008  }
1009 
1010  private void replaceSubQueries(
1011  final Blackboard bb, final SqlNode expr, RelOptUtil.Logic logic) {
1012  findSubQueries(bb, expr, logic, false);
1013  for (SubQuery node : bb.subQueryList) {
1014  substituteSubQuery(bb, node);
1015  }
1016  }
1017 
1018  private void substituteSubQuery(Blackboard bb, SubQuery subQuery) {
1019  final RexNode expr = subQuery.expr;
1020  if (expr != null) {
1021  // Already done.
1022  return;
1023  }
1024 
1025  final SqlBasicCall call;
1026  final RelNode rel;
1027  final SqlNode query;
1028  final RelOptUtil.Exists converted;
1029 
1030  boolean isExpand = config.getExpandPredicate().test(bb.getTopNode(), subQuery.node);
1031 
1032  switch (subQuery.node.getKind()) {
1033  case CURSOR:
1034  convertCursor(bb, subQuery);
1035  return;
1036 
1037  case MULTISET_QUERY_CONSTRUCTOR:
1038  case MULTISET_VALUE_CONSTRUCTOR:
1039  case ARRAY_QUERY_CONSTRUCTOR:
1040  rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
1041  subQuery.expr = bb.register(rel, JoinRelType.INNER);
1042  return;
1043 
1044  case IN:
1045  case NOT_IN:
1046  case SOME:
1047  case ALL:
1048  call = (SqlBasicCall) subQuery.node;
1049  query = call.operand(1);
1050  if (!isExpand && !(query instanceof SqlNodeList)) {
1051  return;
1052  }
1053  final SqlNode leftKeyNode = call.operand(0);
1054 
1055  final List<RexNode> leftKeys;
1056  switch (leftKeyNode.getKind()) {
1057  case ROW:
1058  leftKeys = new ArrayList<>();
1059  for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
1060  leftKeys.add(bb.convertExpression(sqlExpr));
1061  }
1062  break;
1063  default:
1064  leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
1065  }
1066 
1067  if (query instanceof SqlNodeList) {
1068  SqlNodeList valueList = (SqlNodeList) query;
1069  if (!containsNullLiteral(valueList)
1070  && valueList.size() < config.getInSubQueryThreshold()) {
1071  // We're under the threshold, so convert to OR.
1072  subQuery.expr = convertInToOr(
1073  bb, leftKeys, valueList, (SqlInOperator) call.getOperator());
1074  return;
1075  }
1076 
1077  // Otherwise, let convertExists translate
1078  // values list into an inline table for the
1079  // reference to Q below.
1080  }
1081 
1082  // Project out the search columns from the left side
1083 
1084  // Q1:
1085  // "select from emp where emp.deptno in (select col1 from T)"
1086  //
1087  // is converted to
1088  //
1089  // "select from
1090  // emp inner join (select distinct col1 from T)) q
1091  // on emp.deptno = q.col1
1092  //
1093  // Q2:
1094  // "select from emp where emp.deptno not in (Q)"
1095  //
1096  // is converted to
1097  //
1098  // "select from
1099  // emp left outer join (select distinct col1, TRUE from T) q
1100  // on emp.deptno = q.col1
1101  // where emp.deptno <> null
1102  // and q.indicator <> TRUE"
1103  //
1104  // Note: Sub-query can be used as SqlUpdate#condition like below:
1105  //
1106  // UPDATE emp
1107  // SET empno = 1 WHERE emp.empno IN (
1108  // SELECT emp.empno FROM emp WHERE emp.empno = 2)
1109  //
1110  // In such case, when converting SqlUpdate#condition, bb.root is null
1111  // and it makes no sense to do the sub-query substitution.
1112  if (bb.root == null) {
1113  return;
1114  }
1115  final RelDataType targetRowType = SqlTypeUtil.promoteToRowType(
1116  typeFactory, validator.getValidatedNodeType(leftKeyNode), null);
1117  final boolean notIn = call.getOperator().kind == SqlKind.NOT_IN;
1118  converted = convertExists(
1119  query, RelOptUtil.SubQueryType.IN, subQuery.logic, notIn, targetRowType);
1120  if (converted.indicator) {
1121  // Generate
1122  // emp CROSS JOIN (SELECT COUNT(*) AS c,
1123  // COUNT(deptno) AS ck FROM dept)
1124  final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT);
1125  final RelNode seek = converted.r.getInput(0); // fragile
1126  final int keyCount = leftKeys.size();
1127  final List<Integer> args = ImmutableIntList.range(0, keyCount);
1128  LogicalAggregate aggregate = LogicalAggregate.create(seek,
1129  ImmutableList.of(),
1130  ImmutableBitSet.of(),
1131  null,
1132  ImmutableList.of(AggregateCall.create(SqlStdOperatorTable.COUNT,
1133  false,
1134  false,
1135  false,
1136  ImmutableList.of(),
1137  -1,
1138  RelCollations.EMPTY,
1139  longType,
1140  null),
1141  AggregateCall.create(SqlStdOperatorTable.COUNT,
1142  false,
1143  false,
1144  false,
1145  args,
1146  -1,
1147  RelCollations.EMPTY,
1148  longType,
1149  null)));
1150  LogicalJoin join = LogicalJoin.create(bb.root,
1151  aggregate,
1152  ImmutableList.of(),
1153  rexBuilder.makeLiteral(true),
1154  ImmutableSet.of(),
1155  JoinRelType.INNER);
1156  bb.setRoot(join, false);
1157  }
1158  final RexNode rex = bb.register(converted.r,
1159  converted.outerJoin ? JoinRelType.LEFT : JoinRelType.INNER,
1160  leftKeys);
1161 
1162  RelOptUtil.Logic logic = subQuery.logic;
1163  switch (logic) {
1164  case TRUE_FALSE_UNKNOWN:
1165  case UNKNOWN_AS_TRUE:
1166  if (!converted.indicator) {
1167  logic = RelOptUtil.Logic.TRUE_FALSE;
1168  }
1169  }
1170  subQuery.expr = translateIn(logic, bb.root, rex);
1171  if (notIn) {
1172  subQuery.expr = rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
1173  }
1174  return;
1175 
1176  case EXISTS:
1177  // "select from emp where exists (select a from T)"
1178  //
1179  // is converted to the following if the sub-query is correlated:
1180  //
1181  // "select from emp left outer join (select AGG_TRUE() as indicator
1182  // from T group by corr_var) q where q.indicator is true"
1183  //
1184  // If there is no correlation, the expression is replaced with a
1185  // boolean indicating whether the sub-query returned 0 or >= 1 row.
1186  call = (SqlBasicCall) subQuery.node;
1187  query = call.operand(0);
1188  if (!isExpand) {
1189  return;
1190  }
1191  final SqlValidatorScope seekScope = (query instanceof SqlSelect)
1192  ? validator.getSelectScope((SqlSelect) query)
1193  : null;
1194  final Blackboard seekBb = createBlackboard(seekScope, null, false);
1195  final RelNode seekRel = convertQueryOrInList(seekBb, query, null);
1196  // An EXIST sub-query whose inner child has at least 1 tuple
1197  // (e.g. an Aggregate with no grouping columns or non-empty Values
1198  // node) should be simplified to a Boolean constant expression.
1199  final RelMetadataQuery mq = seekRel.getCluster().getMetadataQuery();
1200  final Double minRowCount = mq.getMinRowCount(seekRel);
1201  if (minRowCount != null && minRowCount >= 1D) {
1202  subQuery.expr = rexBuilder.makeLiteral(true);
1203  return;
1204  }
1205  converted = RelOptUtil.createExistsPlan(seekRel,
1206  RelOptUtil.SubQueryType.EXISTS,
1207  subQuery.logic,
1208  true,
1209  relBuilder);
1210  assert !converted.indicator;
1211  if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, true)) {
1212  return;
1213  }
1214  subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
1215  return;
1216 
1217  case SCALAR_QUERY:
1218  // Convert the sub-query. If it's non-correlated, convert it
1219  // to a constant expression.
1220  if (!isExpand) {
1221  return;
1222  }
1223  call = (SqlBasicCall) subQuery.node;
1224  query = call.operand(0);
1225  converted = convertExists(
1226  query, RelOptUtil.SubQueryType.SCALAR, subQuery.logic, true, null);
1227  assert !converted.indicator;
1228  if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, false)) {
1229  return;
1230  }
1231  rel = convertToSingleValueSubq(query, converted.r);
1232  subQuery.expr = bb.register(rel, JoinRelType.LEFT);
1233  return;
1234 
1235  case SELECT:
1236  // This is used when converting multiset queries:
1237  //
1238  // select * from unnest(select multiset[deptno] from emps);
1239  //
1240  converted = convertExists(subQuery.node,
1241  RelOptUtil.SubQueryType.SCALAR,
1242  subQuery.logic,
1243  true,
1244  null);
1245  assert !converted.indicator;
1246  subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
1247 
1248  // This is used when converting window table functions:
1249  //
1250  // select * from table(tumble(table emps, descriptor(deptno), interval '3' DAY))
1251  //
1252  bb.cursors.add(converted.r);
1253  return;
1254 
1255  default:
1256  throw new AssertionError("unexpected kind of sub-query: " + subQuery.node);
1257  }
1258  }
1259 
1260  private RexNode translateIn(RelOptUtil.Logic logic, RelNode root, final RexNode rex) {
1261  switch (logic) {
1262  case TRUE:
1263  return rexBuilder.makeLiteral(true);
1264 
1265  case TRUE_FALSE:
1266  case UNKNOWN_AS_FALSE:
1267  assert rex instanceof RexRangeRef;
1268  final int fieldCount = rex.getType().getFieldCount();
1269  RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
1270  rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
1271 
1272  // Then append the IS NOT NULL(leftKeysForIn).
1273  //
1274  // RexRangeRef contains the following fields:
1275  // leftKeysForIn,
1276  // rightKeysForIn (the original sub-query select list),
1277  // nullIndicator
1278  //
1279  // The first two lists contain the same number of fields.
1280  final int k = (fieldCount - 1) / 2;
1281  for (int i = 0; i < k; i++) {
1282  rexNode = rexBuilder.makeCall(SqlStdOperatorTable.AND,
1283  rexNode,
1284  rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL,
1285  rexBuilder.makeFieldAccess(rex, i)));
1286  }
1287  return rexNode;
1288 
1289  case TRUE_FALSE_UNKNOWN:
1290  case UNKNOWN_AS_TRUE:
1291  // select e.deptno,
1292  // case
1293  // when ct.c = 0 then false
1294  // when dt.i is not null then true
1295  // when e.deptno is null then null
1296  // when ct.ck < ct.c then null
1297  // else false
1298  // end
1299  // from e
1300  // cross join (select count(*) as c, count(deptno) as ck from v) as ct
1301  // left join (select distinct deptno, true as i from v) as dt
1302  // on e.deptno = dt.deptno
1303  final Join join = (Join) root;
1304  final Project left = (Project) join.getLeft();
1305  final RelNode leftLeft = ((Join) left.getInput()).getLeft();
1306  final int leftLeftCount = leftLeft.getRowType().getFieldCount();
1307  final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT);
1308  final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
1309  final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
1310  final RexNode iRef =
1311  rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
1312 
1313  final RexLiteral zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
1314  final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
1315  final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
1316  final RexNode unknownLiteral = rexBuilder.makeNullLiteral(trueLiteral.getType());
1317 
1318  final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
1319  args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero),
1320  falseLiteral,
1321  rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
1322  trueLiteral);
1323  final JoinInfo joinInfo = join.analyzeCondition();
1324  for (int leftKey : joinInfo.leftKeys) {
1325  final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
1326  args.add(
1327  rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef), unknownLiteral);
1328  }
1329  args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef),
1330  unknownLiteral,
1331  falseLiteral);
1332 
1333  return rexBuilder.makeCall(SqlStdOperatorTable.CASE, args.build());
1334 
1335  default:
1336  throw new AssertionError(logic);
1337  }
1338  }
1339 
1340  private static boolean containsNullLiteral(SqlNodeList valueList) {
1341  for (SqlNode node : valueList.getList()) {
1342  if (node instanceof SqlLiteral) {
1343  SqlLiteral lit = (SqlLiteral) node;
1344  if (lit.getValue() == null) {
1345  return true;
1346  }
1347  }
1348  }
1349  return false;
1350  }
1351 
1363  SubQuery subQuery, Blackboard bb, RelNode converted, boolean isExists) {
1364  SqlCall call = (SqlBasicCall) subQuery.node;
1365  if (subQueryConverter.canConvertSubQuery()
1366  && isSubQueryNonCorrelated(converted, bb)) {
1367  // First check if the sub-query has already been converted
1368  // because it's a nested sub-query. If so, don't re-evaluate
1369  // it again.
1370  RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
1371  if (constExpr == null) {
1372  constExpr = subQueryConverter.convertSubQuery(
1373  call, this, isExists, config.isExplain());
1374  }
1375  if (constExpr != null) {
1376  subQuery.expr = constExpr;
1377  mapConvertedNonCorrSubqs.put(call, constExpr);
1378  return true;
1379  }
1380  }
1381  return false;
1382  }
1383 
1392  public RelNode convertToSingleValueSubq(SqlNode query, RelNode plan) {
1393  // Check whether query is guaranteed to produce a single value.
1394  if (query instanceof SqlSelect) {
1395  SqlSelect select = (SqlSelect) query;
1396  SqlNodeList selectList = select.getSelectList();
1397  SqlNodeList groupList = select.getGroup();
1398 
1399  if ((selectList.size() == 1) && ((groupList == null) || (groupList.size() == 0))) {
1400  SqlNode selectExpr = selectList.get(0);
1401  if (selectExpr instanceof SqlCall) {
1402  SqlCall selectExprCall = (SqlCall) selectExpr;
1403  if (Util.isSingleValue(selectExprCall)) {
1404  return plan;
1405  }
1406  }
1407 
1408  // If there is a limit with 0 or 1,
1409  // it is ensured to produce a single value
1410  if (select.getFetch() != null && select.getFetch() instanceof SqlNumericLiteral) {
1411  SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch();
1412  if (((BigDecimal) limitNum.getValue()).intValue() < 2) {
1413  return plan;
1414  }
1415  }
1416  }
1417  } else if (query instanceof SqlCall) {
1418  // If the query is (values ...),
1419  // it is necessary to look into the operands to determine
1420  // whether SingleValueAgg is necessary
1421  SqlCall exprCall = (SqlCall) query;
1422  if (exprCall.getOperator() instanceof SqlValuesOperator
1423  && Util.isSingleValue(exprCall)) {
1424  return plan;
1425  }
1426  }
1427 
1428  // If not, project SingleValueAgg
1429  return RelOptUtil.createSingleValueAggRel(cluster, plan);
1430  }
1431 
1440  private RexNode convertInToOr(final Blackboard bb,
1441  final List<RexNode> leftKeys,
1442  SqlNodeList valuesList,
1443  SqlInOperator op) {
1444  final List<RexNode> comparisons = new ArrayList<>();
1445  for (SqlNode rightVals : valuesList) {
1446  RexNode rexComparison;
1447  final SqlOperator comparisonOp;
1448  if (op instanceof SqlQuantifyOperator) {
1449  comparisonOp = RelOptUtil.op(
1450  ((SqlQuantifyOperator) op).comparisonKind, SqlStdOperatorTable.EQUALS);
1451  } else {
1452  comparisonOp = SqlStdOperatorTable.EQUALS;
1453  }
1454  if (leftKeys.size() == 1) {
1455  rexComparison = rexBuilder.makeCall(comparisonOp,
1456  leftKeys.get(0),
1457  ensureSqlType(
1458  leftKeys.get(0).getType(), bb.convertExpression(rightVals)));
1459  } else {
1460  assert rightVals instanceof SqlCall;
1461  final SqlBasicCall call = (SqlBasicCall) rightVals;
1462  assert (call.getOperator() instanceof SqlRowOperator)
1463  && call.operandCount() == leftKeys.size();
1464  rexComparison = RexUtil.composeConjunction(rexBuilder,
1465  Iterables.transform(Pair.zip(leftKeys, call.getOperandList()),
1466  pair
1467  -> rexBuilder.makeCall(comparisonOp,
1468  pair.left,
1469  ensureSqlType(pair.left.getType(),
1470  bb.convertExpression(pair.right)))));
1471  }
1472  comparisons.add(rexComparison);
1473  }
1474 
1475  switch (op.kind) {
1476  case ALL:
1477  return RexUtil.composeConjunction(rexBuilder, comparisons, true);
1478  case NOT_IN:
1479  return rexBuilder.makeCall(SqlStdOperatorTable.NOT,
1480  RexUtil.composeDisjunction(rexBuilder, comparisons, true));
1481  case IN:
1482  case SOME:
1483  return RexUtil.composeDisjunction(rexBuilder, comparisons, true);
1484  default:
1485  throw new AssertionError();
1486  }
1487  }
1488 
1494  private RexNode ensureSqlType(RelDataType type, RexNode node) {
1495  if (type.getSqlTypeName() == node.getType().getSqlTypeName()
1496  || (type.getSqlTypeName() == SqlTypeName.VARCHAR
1497  && node.getType().getSqlTypeName() == SqlTypeName.CHAR)) {
1498  return node;
1499  }
1500  return rexBuilder.ensureType(type, node, true);
1501  }
1502 
1512  @Deprecated // to be removed before 2.0
1513  protected int getInSubqueryThreshold() {
1514  return config.getInSubQueryThreshold();
1515  }
1516 
1534  private RelOptUtil.Exists convertExists(SqlNode seek,
1535  RelOptUtil.SubQueryType subQueryType,
1536  RelOptUtil.Logic logic,
1537  boolean notIn,
1538  RelDataType targetDataType) {
1539  final SqlValidatorScope seekScope = (seek instanceof SqlSelect)
1540  ? validator.getSelectScope((SqlSelect) seek)
1541  : null;
1542  final Blackboard seekBb = createBlackboard(seekScope, null, false);
1543  RelNode seekRel = convertQueryOrInList(seekBb, seek, targetDataType);
1544 
1545  return RelOptUtil.createExistsPlan(seekRel, subQueryType, logic, notIn, relBuilder);
1546  }
1547 
1548  private RelNode convertQueryOrInList(
1549  Blackboard bb, SqlNode seek, RelDataType targetRowType) {
1550  // NOTE: Once we start accepting single-row queries as row constructors,
1551  // there will be an ambiguity here for a case like X IN ((SELECT Y FROM
1552  // Z)). The SQL standard resolves the ambiguity by saying that a lone
1553  // select should be interpreted as a table expression, not a row
1554  // expression. The semantic difference is that a table expression can
1555  // return multiple rows.
1556  if (seek instanceof SqlNodeList) {
1557  return convertRowValues(
1558  bb, seek, ((SqlNodeList) seek).getList(), false, targetRowType);
1559  } else {
1560  return convertQueryRecursive(seek, false, null).project();
1561  }
1562  }
1563 
1564  private RelNode convertRowValues(Blackboard bb,
1565  SqlNode rowList,
1566  Collection<SqlNode> rows,
1567  boolean allowLiteralsOnly,
1568  RelDataType targetRowType) {
1569  // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
1570  // literals into a single LogicalValues; this gives the optimizer a smaller
1571  // input tree. For everything else (computed expressions, row
1572  // sub-queries), we union each row in as a projection on top of a
1573  // LogicalOneRow.
1574 
1575  final ImmutableList.Builder<ImmutableList<RexLiteral>> tupleList =
1576  ImmutableList.builder();
1577  final RelDataType rowType;
1578  if (targetRowType != null) {
1579  rowType = targetRowType;
1580  } else {
1581  rowType = SqlTypeUtil.promoteToRowType(
1582  typeFactory, validator.getValidatedNodeType(rowList), null);
1583  }
1584 
1585  final List<RelNode> unionInputs = new ArrayList<>();
1586  for (SqlNode node : rows) {
1587  SqlBasicCall call;
1588  if (isRowConstructor(node)) {
1589  call = (SqlBasicCall) node;
1590  ImmutableList.Builder<RexLiteral> tuple = ImmutableList.builder();
1591  for (Ord<SqlNode> operand : Ord.zip(call.operands)) {
1592  RexLiteral rexLiteral =
1593  convertLiteralInValuesList(operand.e, bb, rowType, operand.i);
1594  if ((rexLiteral == null) && allowLiteralsOnly) {
1595  return null;
1596  }
1597  if ((rexLiteral == null) || !config.isCreateValuesRel()) {
1598  // fallback to convertRowConstructor
1599  tuple = null;
1600  break;
1601  }
1602  tuple.add(rexLiteral);
1603  }
1604  if (tuple != null) {
1605  tupleList.add(tuple.build());
1606  continue;
1607  }
1608  } else {
1609  RexLiteral rexLiteral = convertLiteralInValuesList(node, bb, rowType, 0);
1610  if ((rexLiteral != null) && config.isCreateValuesRel()) {
1611  tupleList.add(ImmutableList.of(rexLiteral));
1612  continue;
1613  } else {
1614  if ((rexLiteral == null) && allowLiteralsOnly) {
1615  return null;
1616  }
1617  }
1618 
1619  // convert "1" to "row(1)"
1620  call = (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(SqlParserPos.ZERO, node);
1621  }
1622  unionInputs.add(convertRowConstructor(bb, call));
1623  }
1624  LogicalValues values = LogicalValues.create(cluster, rowType, tupleList.build());
1625  RelNode resultRel;
1626  if (unionInputs.isEmpty()) {
1627  resultRel = values;
1628  } else {
1629  if (!values.getTuples().isEmpty()) {
1630  unionInputs.add(values);
1631  }
1632  resultRel = LogicalUnion.create(unionInputs, true);
1633  }
1634  leaves.put(resultRel, resultRel.getRowType().getFieldCount());
1635  return resultRel;
1636  }
1637 
1638  private RexLiteral convertLiteralInValuesList(
1639  SqlNode sqlNode, Blackboard bb, RelDataType rowType, int iField) {
1640  if (!(sqlNode instanceof SqlLiteral)) {
1641  return null;
1642  }
1643  RelDataTypeField field = rowType.getFieldList().get(iField);
1644  RelDataType type = field.getType();
1645  if (type.isStruct()) {
1646  // null literals for weird stuff like UDT's need
1647  // special handling during type flattening, so
1648  // don't use LogicalValues for those
1649  return null;
1650  }
1651 
1652  RexNode literalExpr = exprConverter.convertLiteral(bb, (SqlLiteral) sqlNode);
1653 
1654  if (!(literalExpr instanceof RexLiteral)) {
1655  assert literalExpr.isA(SqlKind.CAST);
1656  RexNode child = ((RexCall) literalExpr).getOperands().get(0);
1657  assert RexLiteral.isNullLiteral(child);
1658 
1659  // NOTE jvs 22-Nov-2006: we preserve type info
1660  // in LogicalValues digest, so it's OK to lose it here
1661  return (RexLiteral) child;
1662  }
1663 
1664  RexLiteral literal = (RexLiteral) literalExpr;
1665 
1666  Comparable value = literal.getValue();
1667 
1668  if (SqlTypeUtil.isExactNumeric(type) && SqlTypeUtil.hasScale(type)) {
1669  BigDecimal roundedValue =
1670  NumberUtil.rescaleBigDecimal((BigDecimal) value, type.getScale());
1671  return rexBuilder.makeExactLiteral(roundedValue, type);
1672  }
1673 
1674  if ((value instanceof NlsString) && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
1675  // pad fixed character type
1676  NlsString unpadded = (NlsString) value;
1677  return rexBuilder.makeCharLiteral(
1678  new NlsString(Spaces.padRight(unpadded.getValue(), type.getPrecision()),
1679  unpadded.getCharsetName(),
1680  unpadded.getCollation()));
1681  }
1682  return literal;
1683  }
1684 
1685  private boolean isRowConstructor(SqlNode node) {
1686  if (!(node.getKind() == SqlKind.ROW)) {
1687  return false;
1688  }
1689  SqlCall call = (SqlCall) node;
1690  return call.getOperator().getName().equalsIgnoreCase("row");
1691  }
1692 
1709  private void findSubQueries(Blackboard bb,
1710  SqlNode node,
1711  RelOptUtil.Logic logic,
1712  boolean registerOnlyScalarSubQueries) {
1713  final SqlKind kind = node.getKind();
1714  switch (kind) {
1715  case EXISTS:
1716  case SELECT:
1717  case MULTISET_QUERY_CONSTRUCTOR:
1718  case MULTISET_VALUE_CONSTRUCTOR:
1719  case ARRAY_QUERY_CONSTRUCTOR:
1720  case CURSOR:
1721  case SCALAR_QUERY:
1722  if (!registerOnlyScalarSubQueries || (kind == SqlKind.SCALAR_QUERY)) {
1723  bb.registerSubQuery(node, RelOptUtil.Logic.TRUE_FALSE);
1724  }
1725  return;
1726  case IN:
1727  break;
1728  case NOT_IN:
1729  case NOT:
1730  logic = logic.negate();
1731  break;
1732  }
1733  if (node instanceof SqlCall) {
1734  switch (kind) {
1735  // Do no change logic for AND, IN and NOT IN expressions;
1736  // but do change logic for OR, NOT and others;
1737  // EXISTS was handled already.
1738  case AND:
1739  case IN:
1740  case NOT_IN:
1741  break;
1742  default:
1743  logic = RelOptUtil.Logic.TRUE_FALSE_UNKNOWN;
1744  break;
1745  }
1746  for (SqlNode operand : ((SqlCall) node).getOperandList()) {
1747  if (operand != null) {
1748  // In the case of an IN expression, locate scalar
1749  // sub-queries so we can convert them to constants
1750  findSubQueries(bb,
1751  operand,
1752  logic,
1753  kind == SqlKind.IN || kind == SqlKind.NOT_IN || kind == SqlKind.SOME
1754  || kind == SqlKind.ALL || registerOnlyScalarSubQueries);
1755  }
1756  }
1757  } else if (node instanceof SqlNodeList) {
1758  for (SqlNode child : (SqlNodeList) node) {
1759  findSubQueries(bb,
1760  child,
1761  logic,
1762  kind == SqlKind.IN || kind == SqlKind.NOT_IN || kind == SqlKind.SOME
1763  || kind == SqlKind.ALL || registerOnlyScalarSubQueries);
1764  }
1765  }
1766 
1767  // Now that we've located any scalar sub-queries inside the IN
1768  // expression, register the IN expression itself. We need to
1769  // register the scalar sub-queries first so they can be converted
1770  // before the IN expression is converted.
1771  switch (kind) {
1772  case IN:
1773  case NOT_IN:
1774  case SOME:
1775  case ALL:
1776  switch (logic) {
1777  case TRUE_FALSE_UNKNOWN:
1778  RelDataType type = validator.getValidatedNodeTypeIfKnown(node);
1779  if (type == null) {
1780  // The node might not be validated if we still don't know type of the node.
1781  // Therefore return directly.
1782  return;
1783  } else {
1784  break;
1785  }
1786  // fall through
1787  case UNKNOWN_AS_FALSE:
1788  logic = RelOptUtil.Logic.TRUE;
1789  }
1790  bb.registerSubQuery(node, logic);
1791  break;
1792  }
1793  }
1794 
1801  public RexNode convertExpression(SqlNode node) {
1802  Map<String, RelDataType> nameToTypeMap = Collections.emptyMap();
1803  final ParameterScope scope =
1804  new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
1805  final Blackboard bb = createBlackboard(scope, null, false);
1806  return bb.convertExpression(node);
1807  }
1808 
1820  public RexNode convertExpression(SqlNode node, Map<String, RexNode> nameToNodeMap) {
1821  final Map<String, RelDataType> nameToTypeMap = new HashMap<>();
1822  for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
1823  nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
1824  }
1825  final ParameterScope scope =
1826  new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
1827  final Blackboard bb = createBlackboard(scope, nameToNodeMap, false);
1828  return bb.convertExpression(node);
1829  }
1830 
1843  protected RexNode convertExtendedExpression(SqlNode node, Blackboard bb) {
1844  return null;
1845  }
1846 
1847  private RexNode convertOver(Blackboard bb, SqlNode node) {
1848  SqlCall call = (SqlCall) node;
1849  SqlCall aggCall = call.operand(0);
1850  boolean ignoreNulls = false;
1851  switch (aggCall.getKind()) {
1852  case IGNORE_NULLS:
1853  ignoreNulls = true;
1854  // fall through
1855  case RESPECT_NULLS:
1856  aggCall = aggCall.operand(0);
1857  }
1858 
1859  SqlNode windowOrRef = call.operand(1);
1860  final SqlWindow window = validator.resolveWindow(windowOrRef, bb.scope);
1861 
1862  SqlNode sqlLowerBound = window.getLowerBound();
1863  SqlNode sqlUpperBound = window.getUpperBound();
1864  boolean rows = window.isRows();
1865  SqlNodeList orderList = window.getOrderList();
1866 
1867  if (!aggCall.getOperator().allowsFraming()) {
1868  // If the operator does not allow framing, bracketing is implicitly
1869  // everything up to the current row.
1870  sqlLowerBound = SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO);
1871  sqlUpperBound = SqlWindow.createCurrentRow(SqlParserPos.ZERO);
1872  if (aggCall.getKind() == SqlKind.ROW_NUMBER) {
1873  // ROW_NUMBER() expects specific kind of framing.
1874  rows = true;
1875  }
1876  } else if (orderList.size() == 0) {
1877  // Without ORDER BY, there must be no bracketing.
1878  sqlLowerBound = SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO);
1879  sqlUpperBound = SqlWindow.createUnboundedFollowing(SqlParserPos.ZERO);
1880  } else if (sqlLowerBound == null && sqlUpperBound == null) {
1881  sqlLowerBound = SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO);
1882  sqlUpperBound = SqlWindow.createCurrentRow(SqlParserPos.ZERO);
1883  } else if (sqlUpperBound == null) {
1884  sqlUpperBound = SqlWindow.createCurrentRow(SqlParserPos.ZERO);
1885  } else if (sqlLowerBound == null) {
1886  sqlLowerBound = SqlWindow.createCurrentRow(SqlParserPos.ZERO);
1887  }
1888  final SqlNodeList partitionList = window.getPartitionList();
1889  final ImmutableList.Builder<RexNode> partitionKeys = ImmutableList.builder();
1890  for (SqlNode partition : partitionList) {
1891  partitionKeys.add(bb.convertExpression(partition));
1892  }
1893  final RexNode lowerBound = bb.convertExpression(sqlLowerBound);
1894  final RexNode upperBound = bb.convertExpression(sqlUpperBound);
1895  if (orderList.size() == 0 && !rows) {
1896  // A logical range requires an ORDER BY clause. Use the implicit
1897  // ordering of this relation. There must be one, otherwise it would
1898  // have failed validation.
1899  orderList = bb.scope.getOrderList();
1900  if (orderList == null) {
1901  throw new AssertionError("Relation should have sort key for implicit ORDER BY");
1902  }
1903  }
1904 
1905  final ImmutableList.Builder<RexFieldCollation> orderKeys = ImmutableList.builder();
1906  for (SqlNode order : orderList) {
1907  orderKeys.add(bb.convertSortExpression(order,
1908  RelFieldCollation.Direction.ASCENDING,
1909  RelFieldCollation.NullDirection.UNSPECIFIED));
1910  }
1911 
1912  try {
1913  Preconditions.checkArgument(bb.window == null, "already in window agg mode");
1914  bb.window = window;
1915  RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
1916  rexAgg = rexBuilder.ensureType(validator.getValidatedNodeType(call), rexAgg, false);
1917 
1918  // Walk over the tree and apply 'over' to all agg functions. This is
1919  // necessary because the returned expression is not necessarily a call
1920  // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x).
1921 
1922  final SqlLiteral q = aggCall.getFunctionQuantifier();
1923  final boolean isDistinct = q != null && q.getValue() == SqlSelectKeyword.DISTINCT;
1924 
1925  final RexShuttle visitor = new HistogramShuttle(partitionKeys.build(),
1926  orderKeys.build(),
1927  RexWindowBounds.create(sqlLowerBound, lowerBound),
1928  RexWindowBounds.create(sqlUpperBound, upperBound),
1929  rows,
1930  window.isAllowPartial(),
1931  isDistinct,
1932  ignoreNulls);
1933  return rexAgg.accept(visitor);
1934  } finally {
1935  bb.window = null;
1936  }
1937  }
1938 
1939  protected void convertFrom(Blackboard bb, SqlNode from) {
1940  convertFrom(bb, from, Collections.emptyList());
1941  }
1942 
1961  protected void convertFrom(Blackboard bb, SqlNode from, List<String> fieldNames) {
1962  if (from == null) {
1963  bb.setRoot(LogicalValues.createOneRow(cluster), false);
1964  return;
1965  }
1966 
1967  final SqlCall call;
1968  final SqlNode[] operands;
1969  switch (from.getKind()) {
1970  case MATCH_RECOGNIZE:
1971  convertMatchRecognize(bb, (SqlCall) from);
1972  return;
1973 
1974  case AS:
1975  call = (SqlCall) from;
1976  SqlNode firstOperand = call.operand(0);
1977  final List<String> fieldNameList = new ArrayList<>();
1978  if (call.operandCount() > 2) {
1979  for (SqlNode node : Util.skip(call.getOperandList(), 2)) {
1980  fieldNameList.add(((SqlIdentifier) node).getSimple());
1981  }
1982  }
1983  convertFrom(bb, firstOperand, fieldNameList);
1984  return;
1985 
1986  case WITH_ITEM:
1987  convertFrom(bb, ((SqlWithItem) from).query);
1988  return;
1989 
1990  case WITH:
1991  convertFrom(bb, ((SqlWith) from).body);
1992  return;
1993 
1994  case TABLESAMPLE:
1995  operands = ((SqlBasicCall) from).getOperands();
1996  SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]);
1997  if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
1998  String sampleName =
1999  ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec).getName();
2000  datasetStack.push(sampleName);
2001  convertFrom(bb, operands[0]);
2002  datasetStack.pop();
2003  } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
2004  SqlSampleSpec.SqlTableSampleSpec tableSampleSpec =
2005  (SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
2006  convertFrom(bb, operands[0]);
2007  RelOptSamplingParameters params =
2008  new RelOptSamplingParameters(tableSampleSpec.isBernoulli(),
2009  tableSampleSpec.getSamplePercentage(),
2010  tableSampleSpec.isRepeatable(),
2011  tableSampleSpec.getRepeatableSeed());
2012  bb.setRoot(new Sample(cluster, bb.root, params), false);
2013  } else {
2014  throw new AssertionError("unknown TABLESAMPLE type: " + sampleSpec);
2015  }
2016  return;
2017 
2018  case TABLE_REF:
2019  call = (SqlCall) from;
2020  convertIdentifier(bb, call.operand(0), null, call.operand(1));
2021  return;
2022 
2023  case IDENTIFIER:
2024  convertIdentifier(bb, (SqlIdentifier) from, null, null);
2025  return;
2026 
2027  case EXTEND:
2028  call = (SqlCall) from;
2029  final SqlNode operand0 = call.getOperandList().get(0);
2030  final SqlIdentifier id = operand0.getKind() == SqlKind.TABLE_REF
2031  ? ((SqlCall) operand0).operand(0)
2032  : (SqlIdentifier) operand0;
2033  SqlNodeList extendedColumns = (SqlNodeList) call.getOperandList().get(1);
2034  convertIdentifier(bb, id, extendedColumns, null);
2035  return;
2036 
2037  case SNAPSHOT:
2038  convertTemporalTable(bb, (SqlCall) from);
2039  return;
2040 
2041  case JOIN:
2042  final SqlJoin join = (SqlJoin) from;
2043  final SqlValidatorScope scope = validator.getJoinScope(from);
2044  final Blackboard fromBlackboard = createBlackboard(scope, null, false);
2045  SqlNode left = join.getLeft();
2046  SqlNode right = join.getRight();
2047  final boolean isNatural = join.isNatural();
2048  final JoinType joinType = join.getJoinType();
2049  final SqlValidatorScope leftScope = Util.first(
2050  validator.getJoinScope(left), ((DelegatingScope) bb.scope).getParent());
2051  final Blackboard leftBlackboard = createBlackboard(leftScope, null, false);
2052  final SqlValidatorScope rightScope = Util.first(
2053  validator.getJoinScope(right), ((DelegatingScope) bb.scope).getParent());
2054  final Blackboard rightBlackboard = createBlackboard(rightScope, null, false);
2055  convertFrom(leftBlackboard, left);
2056  RelNode leftRel = leftBlackboard.root;
2057  convertFrom(rightBlackboard, right);
2058  RelNode rightRel = rightBlackboard.root;
2059  JoinRelType convertedJoinType = convertJoinType(joinType);
2060  RexNode conditionExp;
2061  final SqlValidatorNamespace leftNamespace = validator.getNamespace(left);
2062  final SqlValidatorNamespace rightNamespace = validator.getNamespace(right);
2063  if (isNatural) {
2064  final RelDataType leftRowType = leftNamespace.getRowType();
2065  final RelDataType rightRowType = rightNamespace.getRowType();
2066  final List<String> columnList = SqlValidatorUtil.deriveNaturalJoinColumnList(
2067  catalogReader.nameMatcher(), leftRowType, rightRowType);
2068  conditionExp = convertUsing(leftNamespace, rightNamespace, columnList);
2069  } else {
2070  conditionExp = convertJoinCondition(fromBlackboard,
2071  leftNamespace,
2072  rightNamespace,
2073  join.getCondition(),
2074  join.getConditionType(),
2075  leftRel,
2076  rightRel);
2077  }
2078 
2079  final RelNode joinRel = createJoin(
2080  fromBlackboard, leftRel, rightRel, conditionExp, convertedJoinType);
2081  bb.setRoot(joinRel, false);
2082  return;
2083 
2084  case SELECT:
2085  case INTERSECT:
2086  case EXCEPT:
2087  case UNION:
2088  final RelNode rel = convertQueryRecursive(from, false, null).project();
2089  bb.setRoot(rel, true);
2090  return;
2091 
2092  case VALUES:
2093  convertValuesImpl(bb, (SqlCall) from, null);
2094  if (fieldNames.size() > 0) {
2095  bb.setRoot(relBuilder.push(bb.root).rename(fieldNames).build(), true);
2096  }
2097  return;
2098 
2099  case UNNEST:
2100  convertUnnest(bb, (SqlCall) from, fieldNames);
2101  return;
2102 
2103  case COLLECTION_TABLE:
2104  call = (SqlCall) from;
2105 
2106  // Dig out real call; TABLE() wrapper is just syntactic.
2107  assert call.getOperandList().size() == 1;
2108  final SqlCall call2 = call.operand(0);
2109  convertCollectionTable(bb, call2);
2110  return;
2111 
2112  default:
2113  throw new AssertionError("not a join operator " + from);
2114  }
2115  }
2116 
2117  private void convertUnnest(Blackboard bb, SqlCall call, List<String> fieldNames) {
2118  final List<SqlNode> nodes = call.getOperandList();
2119  final SqlUnnestOperator operator = (SqlUnnestOperator) call.getOperator();
2120  for (SqlNode node : nodes) {
2121  replaceSubQueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2122  }
2123  final List<RexNode> exprs = new ArrayList<>();
2124  for (Ord<SqlNode> node : Ord.zip(nodes)) {
2125  exprs.add(relBuilder.alias(
2126  bb.convertExpression(node.e), validator.deriveAlias(node.e, node.i)));
2127  }
2128  RelNode child = (null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster);
2129  RelNode uncollect;
2130  if (validator.config().sqlConformance().allowAliasUnnestItems()) {
2131  uncollect = relBuilder.push(child)
2132  .project(exprs)
2133  .uncollect(fieldNames, operator.withOrdinality)
2134  .build();
2135  } else {
2136  // REVIEW danny 2020-04-26: should we unify the normal field aliases and the
2137  // item aliases ?
2138  uncollect = relBuilder.push(child)
2139  .project(exprs)
2140  .uncollect(Collections.emptyList(), operator.withOrdinality)
2141  .rename(fieldNames)
2142  .build();
2143  }
2144  bb.setRoot(uncollect, true);
2145  }
2146 
2147  protected void convertMatchRecognize(Blackboard bb, SqlCall call) {
2148  final SqlMatchRecognize matchRecognize = (SqlMatchRecognize) call;
2149  final SqlValidatorNamespace ns = validator.getNamespace(matchRecognize);
2150  final SqlValidatorScope scope = validator.getMatchRecognizeScope(matchRecognize);
2151 
2152  final Blackboard matchBb = createBlackboard(scope, null, false);
2153  final RelDataType rowType = ns.getRowType();
2154  // convert inner query, could be a table name or a derived table
2155  SqlNode expr = matchRecognize.getTableRef();
2156  convertFrom(matchBb, expr);
2157  final RelNode input = matchBb.root;
2158 
2159  // PARTITION BY
2160  final SqlNodeList partitionList = matchRecognize.getPartitionList();
2161  final ImmutableBitSet.Builder partitionKeys = ImmutableBitSet.builder();
2162  for (SqlNode partition : partitionList) {
2163  RexNode e = matchBb.convertExpression(partition);
2164  partitionKeys.set(((RexInputRef) e).getIndex());
2165  }
2166 
2167  // ORDER BY
2168  final SqlNodeList orderList = matchRecognize.getOrderList();
2169  final List<RelFieldCollation> orderKeys = new ArrayList<>();
2170  for (SqlNode order : orderList) {
2171  final RelFieldCollation.Direction direction;
2172  switch (order.getKind()) {
2173  case DESCENDING:
2174  direction = RelFieldCollation.Direction.DESCENDING;
2175  order = ((SqlCall) order).operand(0);
2176  break;
2177  case NULLS_FIRST:
2178  case NULLS_LAST:
2179  throw new AssertionError();
2180  default:
2181  direction = RelFieldCollation.Direction.ASCENDING;
2182  break;
2183  }
2184  final RelFieldCollation.NullDirection nullDirection =
2185  validator.config().defaultNullCollation().last(desc(direction))
2186  ? RelFieldCollation.NullDirection.LAST
2187  : RelFieldCollation.NullDirection.FIRST;
2188  RexNode e = matchBb.convertExpression(order);
2189  orderKeys.add(new RelFieldCollation(
2190  ((RexInputRef) e).getIndex(), direction, nullDirection));
2191  }
2192  final RelCollation orders = cluster.traitSet().canonize(RelCollations.of(orderKeys));
2193 
2194  // convert pattern
2195  final Set<String> patternVarsSet = new HashSet<>();
2196  SqlNode pattern = matchRecognize.getPattern();
2197  final SqlBasicVisitor<RexNode> patternVarVisitor = new SqlBasicVisitor<RexNode>() {
2198  @Override
2199  public RexNode visit(SqlCall call) {
2200  List<SqlNode> operands = call.getOperandList();
2201  List<RexNode> newOperands = new ArrayList<>();
2202  for (SqlNode node : operands) {
2203  newOperands.add(node.accept(this));
2204  }
2205  return rexBuilder.makeCall(
2206  validator.getUnknownType(), call.getOperator(), newOperands);
2207  }
2208 
2209  @Override
2210  public RexNode visit(SqlIdentifier id) {
2211  assert id.isSimple();
2212  patternVarsSet.add(id.getSimple());
2213  return rexBuilder.makeLiteral(id.getSimple());
2214  }
2215 
2216  @Override
2217  public RexNode visit(SqlLiteral literal) {
2218  if (literal instanceof SqlNumericLiteral) {
2219  return rexBuilder.makeExactLiteral(BigDecimal.valueOf(literal.intValue(true)));
2220  } else {
2221  return rexBuilder.makeLiteral(literal.booleanValue());
2222  }
2223  }
2224  };
2225  final RexNode patternNode = pattern.accept(patternVarVisitor);
2226 
2227  SqlLiteral interval = matchRecognize.getInterval();
2228  RexNode intervalNode = null;
2229  if (interval != null) {
2230  intervalNode = matchBb.convertLiteral(interval);
2231  }
2232 
2233  // convert subset
2234  final SqlNodeList subsets = matchRecognize.getSubsetList();
2235  final Map<String, TreeSet<String>> subsetMap = new HashMap<>();
2236  for (SqlNode node : subsets) {
2237  List<SqlNode> operands = ((SqlCall) node).getOperandList();
2238  SqlIdentifier left = (SqlIdentifier) operands.get(0);
2239  patternVarsSet.add(left.getSimple());
2240  SqlNodeList rights = (SqlNodeList) operands.get(1);
2241  final TreeSet<String> list = new TreeSet<>();
2242  for (SqlNode right : rights) {
2243  assert right instanceof SqlIdentifier;
2244  list.add(((SqlIdentifier) right).getSimple());
2245  }
2246  subsetMap.put(left.getSimple(), list);
2247  }
2248 
2249  SqlNode afterMatch = matchRecognize.getAfter();
2250  if (afterMatch == null) {
2251  afterMatch =
2252  SqlMatchRecognize.AfterOption.SKIP_TO_NEXT_ROW.symbol(SqlParserPos.ZERO);
2253  }
2254 
2255  final RexNode after;
2256  if (afterMatch instanceof SqlCall) {
2257  List<SqlNode> operands = ((SqlCall) afterMatch).getOperandList();
2258  SqlOperator operator = ((SqlCall) afterMatch).getOperator();
2259  assert operands.size() == 1;
2260  SqlIdentifier id = (SqlIdentifier) operands.get(0);
2261  assert patternVarsSet.contains(id.getSimple())
2262  : id.getSimple()
2263  + " not defined in pattern";
2264  RexNode rex = rexBuilder.makeLiteral(id.getSimple());
2265  after = rexBuilder.makeCall(
2266  validator.getUnknownType(), operator, ImmutableList.of(rex));
2267  } else {
2268  after = matchBb.convertExpression(afterMatch);
2269  }
2270 
2271  matchBb.setPatternVarRef(true);
2272 
2273  // convert measures
2274  final ImmutableMap.Builder<String, RexNode> measureNodes = ImmutableMap.builder();
2275  for (SqlNode measure : matchRecognize.getMeasureList()) {
2276  List<SqlNode> operands = ((SqlCall) measure).getOperandList();
2277  String alias = ((SqlIdentifier) operands.get(1)).getSimple();
2278  RexNode rex = matchBb.convertExpression(operands.get(0));
2279  measureNodes.put(alias, rex);
2280  }
2281 
2282  // convert definitions
2283  final ImmutableMap.Builder<String, RexNode> definitionNodes = ImmutableMap.builder();
2284  for (SqlNode def : matchRecognize.getPatternDefList()) {
2285  replaceSubQueries(matchBb, def, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
2286  List<SqlNode> operands = ((SqlCall) def).getOperandList();
2287  String alias = ((SqlIdentifier) operands.get(1)).getSimple();
2288  RexNode rex = matchBb.convertExpression(operands.get(0));
2289  definitionNodes.put(alias, rex);
2290  }
2291 
2292  final SqlLiteral rowsPerMatch = matchRecognize.getRowsPerMatch();
2293  final boolean allRows = rowsPerMatch != null
2294  && rowsPerMatch.getValue() == SqlMatchRecognize.RowsPerMatchOption.ALL_ROWS;
2295 
2296  matchBb.setPatternVarRef(false);
2297 
2298  final RelFactories.MatchFactory factory = RelFactories.DEFAULT_MATCH_FACTORY;
2299  final RelNode rel = factory.createMatch(input,
2300  patternNode,
2301  rowType,
2302  matchRecognize.getStrictStart().booleanValue(),
2303  matchRecognize.getStrictEnd().booleanValue(),
2304  definitionNodes.build(),
2305  measureNodes.build(),
2306  after,
2307  subsetMap,
2308  allRows,
2309  partitionKeys.build(),
2310  orders,
2311  intervalNode);
2312  bb.setRoot(rel, false);
2313  }
2314 
2316  SqlIdentifier id,
2317  SqlNodeList extendedColumns,
2318  SqlNodeList tableHints) {
2319  final SqlValidatorNamespace fromNamespace = validator.getNamespace(id).resolve();
2320  if (fromNamespace.getNode() != null) {
2321  convertFrom(bb, fromNamespace.getNode());
2322  return;
2323  }
2324  final String datasetName = datasetStack.isEmpty() ? null : datasetStack.peek();
2325  final boolean[] usedDataset = {false};
2326  RelOptTable table = SqlValidatorUtil.getRelOptTable(
2327  fromNamespace, catalogReader, datasetName, usedDataset);
2328  if (extendedColumns != null && extendedColumns.size() > 0) {
2329  assert table != null;
2330  final SqlValidatorTable validatorTable = table.unwrap(SqlValidatorTable.class);
2331  final List<RelDataTypeField> extendedFields = SqlValidatorUtil.getExtendedColumns(
2332  validator, validatorTable, extendedColumns);
2333  table = table.extend(extendedFields);
2334  }
2335  final RelNode tableRel;
2336  // Review Danny 2020-01-13: hacky to construct a new table scan
2337  // in order to apply the hint strategies.
2338  final List<RelHint> hints =
2339  hintStrategies.apply(SqlUtil.getRelHint(hintStrategies, tableHints),
2340  LogicalTableScan.create(cluster, table, ImmutableList.of()));
2341  tableRel = toRel(table, hints);
2342  bb.setRoot(tableRel, true);
2343  if (usedDataset[0]) {
2344  bb.setDataset(datasetName);
2345  }
2346  }
2347 
2348  protected void convertCollectionTable(Blackboard bb, SqlCall call) {
2349  final SqlOperator operator = call.getOperator();
2350  if (operator == SqlStdOperatorTable.TABLESAMPLE) {
2351  final String sampleName =
2352  SqlLiteral.unchain(call.operand(0)).getValueAs(String.class);
2353  datasetStack.push(sampleName);
2354  SqlCall cursorCall = call.operand(1);
2355  SqlNode query = cursorCall.operand(0);
2356  RelNode converted = convertQuery(query, false, false).rel;
2357  bb.setRoot(converted, false);
2358  datasetStack.pop();
2359  return;
2360  }
2361  replaceSubQueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2362 
2363  // Expand table macro if possible. It's more efficient than
2364  // LogicalTableFunctionScan.
2365  final SqlCallBinding callBinding =
2366  new SqlCallBinding(bb.scope.getValidator(), bb.scope, call);
2367  if (operator instanceof SqlUserDefinedTableMacro) {
2368  final SqlUserDefinedTableMacro udf = (SqlUserDefinedTableMacro) operator;
2369  final TranslatableTable table = udf.getTable(callBinding);
2370  final RelDataType rowType = table.getRowType(typeFactory);
2371  RelOptTable relOptTable =
2372  RelOptTableImpl.create(null, rowType, table, udf.getNameAsId().names);
2373  RelNode converted = toRel(relOptTable, ImmutableList.of());
2374  bb.setRoot(converted, true);
2375  return;
2376  }
2377 
2378  Type elementType;
2379  if (operator instanceof SqlUserDefinedTableFunction) {
2380  SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator;
2381  elementType = udtf.getElementType(callBinding);
2382  } else {
2383  elementType = null;
2384  }
2385 
2386  RexNode rexCall = bb.convertExpression(call);
2387  final List<RelNode> inputs = bb.retrieveCursors();
2388  Set<RelColumnMapping> columnMappings = getColumnMappings(operator);
2389  LogicalTableFunctionScan callRel = LogicalTableFunctionScan.create(cluster,
2390  inputs,
2391  rexCall,
2392  elementType,
2393  validator.getValidatedNodeType(call),
2394  columnMappings);
2395  bb.setRoot(callRel, true);
2396  afterTableFunction(bb, call, callRel);
2397  }
2398 
2399  protected void afterTableFunction(SqlToRelConverter.Blackboard bb,
2400  SqlCall call,
2401  LogicalTableFunctionScan callRel) {}
2402 
2403  private void convertTemporalTable(Blackboard bb, SqlCall call) {
2404  final SqlSnapshot snapshot = (SqlSnapshot) call;
2405  final RexNode period = bb.convertExpression(snapshot.getPeriod());
2406 
2407  // convert inner query, could be a table name or a derived table
2408  SqlNode expr = snapshot.getTableRef();
2409  convertFrom(bb, expr);
2410 
2411  final RelNode snapshotRel = relBuilder.push(bb.root).snapshot(period).build();
2412 
2413  bb.setRoot(snapshotRel, false);
2414  }
2415 
2416  private Set<RelColumnMapping> getColumnMappings(SqlOperator op) {
2417  if (op instanceof ExtTableFunction) {
2418  ExtTableFunction tf = (ExtTableFunction) op;
2419  return tf.getColumnMappings();
2420  }
2421  SqlReturnTypeInference rti = op.getReturnTypeInference();
2422  if (rti == null) {
2423  return null;
2424  }
2425  if (rti instanceof TableFunctionReturnTypeInference) {
2426  TableFunctionReturnTypeInference tfrti = (TableFunctionReturnTypeInference) rti;
2427  return tfrti.getColumnMappings();
2428  } else {
2429  return null;
2430  }
2431  }
2432 
2438  private static class RexAccessShuttle extends RexShuttle {
2439  private final RexBuilder builder;
2440  private final RexCorrelVariable rexCorrel;
2441  private final BitSet varCols = new BitSet();
2442 
2443  RexAccessShuttle(RexBuilder builder, RexCorrelVariable rexCorrel) {
2444  this.builder = builder;
2445  this.rexCorrel = rexCorrel;
2446  }
2447 
2448  @Override
2449  public RexNode visitInputRef(RexInputRef input) {
2450  int i = input.getIndex() - rexCorrel.getType().getFieldCount();
2451  if (i < 0) {
2452  varCols.set(input.getIndex());
2453  return builder.makeFieldAccess(rexCorrel, input.getIndex());
2454  }
2455  return builder.makeInputRef(input.getType(), i);
2456  }
2457  }
2458 
2459  protected RelNode createJoin(Blackboard bb,
2460  RelNode leftRel,
2461  RelNode rightRel,
2462  RexNode joinCond,
2463  JoinRelType joinType) {
2464  assert joinCond != null;
2465 
2466  final CorrelationUse p = getCorrelationUse(bb, rightRel);
2467  if (p != null) {
2468  RelNode innerRel = p.r;
2469  ImmutableBitSet requiredCols = p.requiredColumns;
2470 
2471  if (!joinCond.isAlwaysTrue()) {
2472  final RelFactories.FilterFactory factory = RelFactories.DEFAULT_FILTER_FACTORY;
2473  final RexCorrelVariable rexCorrel =
2474  (RexCorrelVariable) rexBuilder.makeCorrel(leftRel.getRowType(), p.id);
2475  final RexAccessShuttle shuttle = new RexAccessShuttle(rexBuilder, rexCorrel);
2476 
2477  // Replace outer RexInputRef with RexFieldAccess,
2478  // and push lateral join predicate into inner child
2479  final RexNode newCond = joinCond.accept(shuttle);
2480  innerRel = factory.createFilter(p.r, newCond, ImmutableSet.of());
2481  requiredCols =
2482  ImmutableBitSet.fromBitSet(shuttle.varCols).union(p.requiredColumns);
2483  }
2484 
2485  return LogicalCorrelate.create(leftRel, innerRel, p.id, requiredCols, joinType);
2486  }
2487 
2488  // MAt 8 Jan 2021 original code
2489  // final RelNode node = relBuilder.push(leftRel).push(rightRel).join(joinType,
2490  // joinCond).build();
2491 
2492  // MAT 8 Jan 2021 this is HEAVY.AI code
2493  final Join originalJoin = (Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel,
2494  rightRel,
2495  ImmutableList.of(),
2496  joinCond,
2497  ImmutableSet.of(),
2498  joinType,
2499  false);
2500  RelNode node = originalJoin;
2501  // MAT 08 Jan 2021 end of HEAVY.AI Code
2502 
2503  // If join conditions are pushed down, update the leaves.
2504  if (node instanceof Project) {
2505  final Join newJoin = (Join) node.getInputs().get(0);
2506  if (leaves.containsKey(leftRel)) {
2507  leaves.put(newJoin.getLeft(), leaves.get(leftRel));
2508  }
2509  if (leaves.containsKey(rightRel)) {
2510  leaves.put(newJoin.getRight(), leaves.get(rightRel));
2511  }
2512  }
2513  return node;
2514  }
2515 
2516  private CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0) {
2517  final Set<CorrelationId> correlatedVariables = RelOptUtil.getVariablesUsed(r0);
2518  if (correlatedVariables.isEmpty()) {
2519  return null;
2520  }
2521  final ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder();
2522  final List<CorrelationId> correlNames = new ArrayList<>();
2523 
2524  // All correlations must refer the same namespace since correlation
2525  // produces exactly one correlation source.
2526  // The same source might be referenced by different variables since
2527  // DeferredLookups are not de-duplicated at create time.
2528  SqlValidatorNamespace prevNs = null;
2529 
2530  for (CorrelationId correlName : correlatedVariables) {
2531  DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
2532  RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName);
2533  String originalRelName = lookup.getOriginalRelName();
2534  String originalFieldName = fieldAccess.getField().getName();
2535 
2536  final SqlNameMatcher nameMatcher =
2537  bb.getValidator().getCatalogReader().nameMatcher();
2538  final SqlValidatorScope.ResolvedImpl resolved =
2539  new SqlValidatorScope.ResolvedImpl();
2540  lookup.bb.scope.resolve(
2541  ImmutableList.of(originalRelName), nameMatcher, false, resolved);
2542  assert resolved.count() == 1;
2543  final SqlValidatorScope.Resolve resolve = resolved.only();
2544  final SqlValidatorNamespace foundNs = resolve.namespace;
2545  final RelDataType rowType = resolve.rowType();
2546  final int childNamespaceIndex = resolve.path.steps().get(0).i;
2547  final SqlValidatorScope ancestorScope = resolve.scope;
2548  boolean correlInCurrentScope = bb.scope.isWithin(ancestorScope);
2549 
2550  if (!correlInCurrentScope) {
2551  continue;
2552  }
2553 
2554  if (prevNs == null) {
2555  prevNs = foundNs;
2556  } else {
2557  assert prevNs
2558  == foundNs : "All correlation variables should resolve"
2559  + " to the same namespace."
2560  + " Prev ns="
2561  + prevNs
2562  + ", new ns="
2563  + foundNs;
2564  }
2565 
2566  int namespaceOffset = 0;
2567  if (childNamespaceIndex > 0) {
2568  // If not the first child, need to figure out the width
2569  // of output types from all the preceding namespaces
2570  assert ancestorScope instanceof ListScope;
2571  List<SqlValidatorNamespace> children = ((ListScope) ancestorScope).getChildren();
2572 
2573  for (int i = 0; i < childNamespaceIndex; i++) {
2574  SqlValidatorNamespace child = children.get(i);
2575  namespaceOffset += child.getRowType().getFieldCount();
2576  }
2577  }
2578 
2579  RexFieldAccess topLevelFieldAccess = fieldAccess;
2580  while (topLevelFieldAccess.getReferenceExpr() instanceof RexFieldAccess) {
2581  topLevelFieldAccess = (RexFieldAccess) topLevelFieldAccess.getReferenceExpr();
2582  }
2583  final RelDataTypeField field = rowType.getFieldList().get(
2584  topLevelFieldAccess.getField().getIndex() - namespaceOffset);
2585  int pos = namespaceOffset + field.getIndex();
2586 
2587  assert field.getType() == topLevelFieldAccess.getField().getType();
2588 
2589  assert pos != -1;
2590 
2591  if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) {
2592  // bb.root is an aggregate and only projects group by
2593  // keys.
2594  Map<Integer, Integer> exprProjection =
2595  bb.mapRootRelToFieldProjection.get(bb.root);
2596 
2597  // sub-query can reference group by keys projected from
2598  // the root of the outer relation.
2599  if (exprProjection.containsKey(pos)) {
2600  pos = exprProjection.get(pos);
2601  } else {
2602  // correl not grouped
2603  throw new AssertionError("Identifier '" + originalRelName + "."
2604  + originalFieldName + "' is not a group expr");
2605  }
2606  }
2607 
2608  requiredColumns.set(pos);
2609  correlNames.add(correlName);
2610  }
2611 
2612  if (correlNames.isEmpty()) {
2613  // None of the correlating variables originated in this scope.
2614  return null;
2615  }
2616 
2617  RelNode r = r0;
2618  if (correlNames.size() > 1) {
2619  // The same table was referenced more than once.
2620  // So we deduplicate.
2621  r = DeduplicateCorrelateVariables.go(
2622  rexBuilder, correlNames.get(0), Util.skip(correlNames), r0);
2623  // Add new node to leaves.
2624  leaves.put(r, r.getRowType().getFieldCount());
2625  }
2626  return new CorrelationUse(correlNames.get(0), requiredColumns.build(), r);
2627  }
2628 
2639  private boolean isSubQueryNonCorrelated(RelNode subq, Blackboard bb) {
2640  Set<CorrelationId> correlatedVariables = RelOptUtil.getVariablesUsed(subq);
2641  for (CorrelationId correlName : correlatedVariables) {
2642  DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
2643  String originalRelName = lookup.getOriginalRelName();
2644 
2645  final SqlNameMatcher nameMatcher =
2646  lookup.bb.scope.getValidator().getCatalogReader().nameMatcher();
2647  final SqlValidatorScope.ResolvedImpl resolved =
2648  new SqlValidatorScope.ResolvedImpl();
2649  lookup.bb.scope.resolve(
2650  ImmutableList.of(originalRelName), nameMatcher, false, resolved);
2651 
2652  SqlValidatorScope ancestorScope = resolved.only().scope;
2653 
2654  // If the correlated reference is in a scope that's "above" the
2655  // sub-query, then this is a correlated sub-query.
2656  SqlValidatorScope parentScope = bb.scope;
2657  do {
2658  if (ancestorScope == parentScope) {
2659  return false;
2660  }
2661  if (parentScope instanceof DelegatingScope) {
2662  parentScope = ((DelegatingScope) parentScope).getParent();
2663  } else {
2664  break;
2665  }
2666  } while (parentScope != null);
2667  }
2668  return true;
2669  }
2670 
2676  protected List<RelDataTypeField> getSystemFields() {
2677  return Collections.emptyList();
2678  }
2679 
2680  private RexNode convertJoinCondition(Blackboard bb,
2681  SqlValidatorNamespace leftNamespace,
2682  SqlValidatorNamespace rightNamespace,
2683  SqlNode condition,
2684  JoinConditionType conditionType,
2685  RelNode leftRel,
2686  RelNode rightRel) {
2687  if (condition == null) {
2688  return rexBuilder.makeLiteral(true);
2689  }
2690  bb.setRoot(ImmutableList.of(leftRel, rightRel));
2691  replaceSubQueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
2692  switch (conditionType) {
2693  case ON:
2694  bb.setRoot(ImmutableList.of(leftRel, rightRel));
2695  return bb.convertExpression(condition);
2696  case USING:
2697  final SqlNodeList list = (SqlNodeList) condition;
2698  final List<String> nameList = new ArrayList<>();
2699  for (SqlNode columnName : list) {
2700  final SqlIdentifier id = (SqlIdentifier) columnName;
2701  String name = id.getSimple();
2702  nameList.add(name);
2703  }
2704  return convertUsing(leftNamespace, rightNamespace, nameList);
2705  default:
2706  throw Util.unexpected(conditionType);
2707  }
2708  }
2709 
2721  private RexNode convertUsing(SqlValidatorNamespace leftNamespace,
2722  SqlValidatorNamespace rightNamespace,
2723  List<String> nameList) {
2724  final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
2725  final List<RexNode> list = new ArrayList<>();
2726  for (String name : nameList) {
2727  List<RexNode> operands = new ArrayList<>();
2728  int offset = 0;
2729  for (SqlValidatorNamespace n : ImmutableList.of(leftNamespace, rightNamespace)) {
2730  final RelDataType rowType = n.getRowType();
2731  final RelDataTypeField field = nameMatcher.field(rowType, name);
2732  operands.add(rexBuilder.makeInputRef(field.getType(), offset + field.getIndex()));
2733  offset += rowType.getFieldList().size();
2734  }
2735  list.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, operands));
2736  }
2737  return RexUtil.composeConjunction(rexBuilder, list);
2738  }
2739 
2740  private static JoinRelType convertJoinType(JoinType joinType) {
2741  switch (joinType) {
2742  case COMMA:
2743  case INNER:
2744  case CROSS:
2745  return JoinRelType.INNER;
2746  case FULL:
2747  return JoinRelType.FULL;
2748  case LEFT:
2749  return JoinRelType.LEFT;
2750  case RIGHT:
2751  return JoinRelType.RIGHT;
2752  default:
2753  throw Util.unexpected(joinType);
2754  }
2755  }
2756 
2770  protected void convertAgg(
2771  Blackboard bb, SqlSelect select, List<SqlNode> orderExprList) {
2772  assert bb.root != null : "precondition: child != null";
2773  SqlNodeList groupList = select.getGroup();
2774  SqlNodeList selectList = select.getSelectList();
2775  SqlNode having = select.getHaving();
2776 
2777  final AggConverter aggConverter = new AggConverter(bb, select);
2778  createAggImpl(bb, aggConverter, selectList, groupList, having, orderExprList);
2779  }
2780 
2781  protected final void createAggImpl(Blackboard bb,
2782  final AggConverter aggConverter,
2783  SqlNodeList selectList,
2784  SqlNodeList groupList,
2785  SqlNode having,
2786  List<SqlNode> orderExprList) {
2787  // Find aggregate functions in SELECT and HAVING clause
2788  final AggregateFinder aggregateFinder = new AggregateFinder();
2789  selectList.accept(aggregateFinder);
2790  if (having != null) {
2791  having.accept(aggregateFinder);
2792  }
2793 
2794  // first replace the sub-queries inside the aggregates
2795  // because they will provide input rows to the aggregates.
2796  replaceSubQueries(bb, aggregateFinder.list, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2797 
2798  // also replace sub-queries inside filters in the aggregates
2800  bb, aggregateFinder.filterList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2801 
2802  // also replace sub-queries inside ordering spec in the aggregates
2803  replaceSubQueries(bb, aggregateFinder.orderList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2804 
2805  // If group-by clause is missing, pretend that it has zero elements.
2806  if (groupList == null) {
2807  groupList = SqlNodeList.EMPTY;
2808  }
2809 
2810  replaceSubQueries(bb, groupList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2811 
2812  // register the group exprs
2813 
2814  // build a map to remember the projections from the top scope to the
2815  // output of the current root.
2816  //
2817  // Calcite allows expressions, not just column references in
2818  // group by list. This is not SQL 2003 compliant, but hey.
2819 
2820  final AggregatingSelectScope scope = aggConverter.aggregatingSelectScope;
2821  final AggregatingSelectScope.Resolved r = scope.resolved.get();
2822  for (SqlNode groupExpr : r.groupExprList) {
2823  aggConverter.addGroupExpr(groupExpr);
2824  }
2825 
2826  final RexNode havingExpr;
2827  final List<Pair<RexNode, String>> projects = new ArrayList<>();
2828 
2829  try {
2830  Preconditions.checkArgument(bb.agg == null, "already in agg mode");
2831  bb.agg = aggConverter;
2832 
2833  // convert the select and having expressions, so that the
2834  // agg converter knows which aggregations are required
2835 
2836  selectList.accept(aggConverter);
2837  // Assert we don't have dangling items left in the stack
2838  assert !aggConverter.inOver;
2839  for (SqlNode expr : orderExprList) {
2840  expr.accept(aggConverter);
2841  assert !aggConverter.inOver;
2842  }
2843  if (having != null) {
2844  having.accept(aggConverter);
2845  assert !aggConverter.inOver;
2846  }
2847 
2848  // compute inputs to the aggregator
2849  List<Pair<RexNode, String>> preExprs = aggConverter.getPreExprs();
2850 
2851  if (preExprs.size() == 0) {
2852  // Special case for COUNT(*), where we can end up with no inputs
2853  // at all. The rest of the system doesn't like 0-tuples, so we
2854  // select a dummy constant here.
2855  final RexNode zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO);
2856  preExprs = ImmutableList.of(Pair.of(zero, (String) null));
2857  }
2858 
2859  final RelNode inputRel = bb.root;
2860 
2861  // Project the expressions required by agg and having.
2862  bb.setRoot(relBuilder.push(inputRel)
2863  .projectNamed(Pair.left(preExprs), Pair.right(preExprs), false)
2864  .build(),
2865  false);
2866  bb.mapRootRelToFieldProjection.put(bb.root, r.groupExprProjection);
2867 
2868  // REVIEW jvs 31-Oct-2007: doesn't the declaration of
2869  // monotonicity here assume sort-based aggregation at
2870  // the physical level?
2871 
2872  // Tell bb which of group columns are sorted.
2873  bb.columnMonotonicities.clear();
2874  for (SqlNode groupItem : groupList) {
2875  bb.columnMonotonicities.add(bb.scope.getMonotonicity(groupItem));
2876  }
2877 
2878  final RelNode relNode = aggConverter.containsGroupId()
2879  ? rewriteAggregateWithGroupId(bb, r, aggConverter)
2880  : createAggregate(bb, r.groupSet, r.groupSets, aggConverter.getAggCalls());
2881 
2882  bb.setRoot(relNode, false);
2883  bb.mapRootRelToFieldProjection.put(bb.root, r.groupExprProjection);
2884 
2885  // Replace sub-queries in having here and modify having to use
2886  // the replaced expressions
2887  if (having != null) {
2888  SqlNode newHaving = pushDownNotForIn(bb.scope, having);
2889  replaceSubQueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
2890  havingExpr = bb.convertExpression(newHaving);
2891  } else {
2892  havingExpr = relBuilder.literal(true);
2893  }
2894 
2895  // Now convert the other sub-queries in the select list.
2896  // This needs to be done separately from the sub-query inside
2897  // any aggregate in the select list, and after the aggregate rel
2898  // is allocated.
2899  replaceSubQueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2900 
2901  // Now sub-queries in the entire select list have been converted.
2902  // Convert the select expressions to get the final list to be
2903  // projected.
2904  int k = 0;
2905 
2906  // For select expressions, use the field names previously assigned
2907  // by the validator. If we derive afresh, we might generate names
2908  // like "EXPR$2" that don't match the names generated by the
2909  // validator. This is especially the case when there are system
2910  // fields; system fields appear in the relnode's rowtype but do not
2911  // (yet) appear in the validator type.
2912  final SelectScope selectScope = SqlValidatorUtil.getEnclosingSelectScope(bb.scope);
2913  assert selectScope != null;
2914  final SqlValidatorNamespace selectNamespace =
2915  validator.getNamespace(selectScope.getNode());
2916  final List<String> names = selectNamespace.getRowType().getFieldNames();
2917  int sysFieldCount = selectList.size() - names.size();
2918  for (SqlNode expr : selectList) {
2919  projects.add(Pair.of(bb.convertExpression(expr),
2920  k < sysFieldCount ? validator.deriveAlias(expr, k++)
2921  : names.get(k++ - sysFieldCount)));
2922  }
2923 
2924  for (SqlNode expr : orderExprList) {
2925  projects.add(
2926  Pair.of(bb.convertExpression(expr), validator.deriveAlias(expr, k++)));
2927  }
2928  } finally {
2929  bb.agg = null;
2930  }
2931 
2932  // implement HAVING (we have already checked that it is non-trivial)
2933  relBuilder.push(bb.root);
2934  if (havingExpr != null) {
2935  relBuilder.filter(havingExpr);
2936  }
2937 
2938  // implement the SELECT list
2939  relBuilder.project(Pair.left(projects), Pair.right(projects))
2940  .rename(Pair.right(projects));
2941  bb.setRoot(relBuilder.build(), false);
2942 
2943  // Tell bb which of group columns are sorted.
2944  bb.columnMonotonicities.clear();
2945  for (SqlNode selectItem : selectList) {
2946  bb.columnMonotonicities.add(bb.scope.getMonotonicity(selectItem));
2947  }
2948  }
2949 
2964  Blackboard bb, AggregatingSelectScope.Resolved r, AggConverter converter) {
2965  final List<AggregateCall> aggregateCalls = converter.getAggCalls();
2966  final ImmutableBitSet groupSet = r.groupSet;
2967  final Map<ImmutableBitSet, Integer> groupSetCount = r.groupSetCount;
2968 
2969  final List<String> fieldNamesIfNoRewrite =
2970  createAggregate(bb, groupSet, r.groupSets, aggregateCalls)
2971  .getRowType()
2972  .getFieldNames();
2973 
2974  // If n duplicates exist for a particular grouping, the {@code GROUP_ID()}
2975  // function produces values in the range 0 to n-1. For each value,
2976  // we need to figure out the corresponding group sets.
2977  //
2978  // For example, "... GROUPING SETS (a, a, b, c, c, c, c)"
2979  // (i) The max value of the GROUP_ID() function returns is 3
2980  // (ii) GROUPING SETS (a, b, c) produces value 0,
2981  // GROUPING SETS (a, c) produces value 1,
2982  // GROUPING SETS (c) produces value 2
2983  // GROUPING SETS (c) produces value 3
2984  final Map<Integer, Set<ImmutableBitSet>> groupIdToGroupSets = new HashMap<>();
2985  int maxGroupId = 0;
2986  for (Map.Entry<ImmutableBitSet, Integer> entry : groupSetCount.entrySet()) {
2987  int groupId = entry.getValue() - 1;
2988  if (groupId > maxGroupId) {
2989  maxGroupId = groupId;
2990  }
2991  for (int i = 0; i <= groupId; i++) {
2992  groupIdToGroupSets
2993  .computeIfAbsent(i, k -> Sets.newTreeSet(ImmutableBitSet.COMPARATOR))
2994  .add(entry.getKey());
2995  }
2996  }
2997 
2998  // AggregateCall list without GROUP_ID function
2999  final List<AggregateCall> aggregateCallsWithoutGroupId = new ArrayList<>();
3000  for (AggregateCall aggregateCall : aggregateCalls) {
3001  if (aggregateCall.getAggregation().kind != SqlKind.GROUP_ID) {
3002  aggregateCallsWithoutGroupId.add(aggregateCall);
3003  }
3004  }
3005  final List<RelNode> projects = new ArrayList<>();
3006  // For each group id value , we first construct an Aggregate without
3007  // GROUP_ID() function call, and then create a Project node on top of it.
3008  // The Project adds literal value for group id in right position.
3009  for (int groupId = 0; groupId <= maxGroupId; groupId++) {
3010  // Create the Aggregate node without GROUP_ID() call
3011  final ImmutableList<ImmutableBitSet> groupSets =
3012  ImmutableList.copyOf(groupIdToGroupSets.get(groupId));
3013  final RelNode aggregate =
3014  createAggregate(bb, groupSet, groupSets, aggregateCallsWithoutGroupId);
3015 
3016  // RexLiteral for each GROUP_ID, note the type should be BIGINT
3017  final RelDataType groupIdType = typeFactory.createSqlType(SqlTypeName.BIGINT);
3018  final RexNode groupIdLiteral =
3019  rexBuilder.makeExactLiteral(BigDecimal.valueOf(groupId), groupIdType);
3020 
3021  relBuilder.push(aggregate);
3022  final List<RexNode> selectList = new ArrayList<>();
3023  final int groupExprLength = r.groupExprList.size();
3024  // Project fields in group by expressions
3025  for (int i = 0; i < groupExprLength; i++) {
3026  selectList.add(relBuilder.field(i));
3027  }
3028  // Project fields in aggregate calls
3029  int groupIdCount = 0;
3030  for (int i = 0; i < aggregateCalls.size(); i++) {
3031  if (aggregateCalls.get(i).getAggregation().kind == SqlKind.GROUP_ID) {
3032  selectList.add(groupIdLiteral);
3033  groupIdCount++;
3034  } else {
3035  int ordinal = groupExprLength + i - groupIdCount;
3036  selectList.add(relBuilder.field(ordinal));
3037  }
3038  }
3039  final RelNode project =
3040  relBuilder.project(selectList, fieldNamesIfNoRewrite).build();
3041  projects.add(project);
3042  }
3043  // Skip to create Union when there is only one child, i.e., no duplicate group
3044  // set.
3045  if (projects.size() == 1) {
3046  return projects.get(0);
3047  }
3048  return LogicalUnion.create(projects, true);
3049  }
3050 
3070  protected RelNode createAggregate(Blackboard bb,
3071  ImmutableBitSet groupSet,
3072  ImmutableList<ImmutableBitSet> groupSets,
3073  List<AggregateCall> aggCalls) {
3074  return LogicalAggregate.create(
3075  bb.root, ImmutableList.of(), groupSet, groupSets, aggCalls);
3076  }
3077 
3078  public RexDynamicParam convertDynamicParam(final SqlDynamicParam dynamicParam) {
3079  // REVIEW jvs 8-Jan-2005: dynamic params may be encountered out of
3080  // order. Should probably cross-check with the count from the parser
3081  // at the end and make sure they all got filled in. Why doesn't List
3082  // have a resize() method?!? Make this a utility.
3083  while (dynamicParam.getIndex() >= dynamicParamSqlNodes.size()) {
3084  dynamicParamSqlNodes.add(null);
3085  }
3086 
3087  dynamicParamSqlNodes.set(dynamicParam.getIndex(), dynamicParam);
3088  return rexBuilder.makeDynamicParam(
3089  getDynamicParamType(dynamicParam.getIndex()), dynamicParam.getIndex());
3090  }
3091 
3106  protected void gatherOrderExprs(Blackboard bb,
3107  SqlSelect select,
3108  SqlNodeList orderList,
3109  List<SqlNode> extraOrderExprs,
3110  List<RelFieldCollation> collationList) {
3111  // TODO: add validation rules to SqlValidator also
3112  assert bb.root != null : "precondition: child != null";
3113  assert select != null;
3114  if (orderList == null) {
3115  return;
3116  }
3117 
3118  if (!bb.top) {
3119  SqlNode offset = select.getOffset();
3120  if ((offset == null
3121  || (offset instanceof SqlLiteral
3122  && ((SqlLiteral) offset)
3123  .bigDecimalValue()
3124  .equals(BigDecimal.ZERO)))
3125  && select.getFetch() == null) {
3126  return;
3127  }
3128  }
3129 
3130  for (SqlNode orderItem : orderList) {
3131  collationList.add(convertOrderItem(select,
3132  orderItem,
3133  extraOrderExprs,
3134  RelFieldCollation.Direction.ASCENDING,
3135  RelFieldCollation.NullDirection.UNSPECIFIED));
3136  }
3137  }
3138 
3139  protected RelFieldCollation convertOrderItem(SqlSelect select,
3140  SqlNode orderItem,
3141  List<SqlNode> extraExprs,
3142  RelFieldCollation.Direction direction,
3143  RelFieldCollation.NullDirection nullDirection) {
3144  assert select != null;
3145  // Handle DESC keyword, e.g. 'select a, b from t order by a desc'.
3146  switch (orderItem.getKind()) {
3147  case DESCENDING:
3148  return convertOrderItem(select,
3149  ((SqlCall) orderItem).operand(0),
3150  extraExprs,
3151  RelFieldCollation.Direction.DESCENDING,
3152  nullDirection);
3153  case NULLS_FIRST:
3154  return convertOrderItem(select,
3155  ((SqlCall) orderItem).operand(0),
3156  extraExprs,
3157  direction,
3158  RelFieldCollation.NullDirection.FIRST);
3159  case NULLS_LAST:
3160  return convertOrderItem(select,
3161  ((SqlCall) orderItem).operand(0),
3162  extraExprs,
3163  direction,
3164  RelFieldCollation.NullDirection.LAST);
3165  }
3166 
3167  SqlNode converted = validator.expandOrderExpr(select, orderItem);
3168 
3169  switch (nullDirection) {
3170  case UNSPECIFIED:
3171  nullDirection = validator.config().defaultNullCollation().last(desc(direction))
3172  ? RelFieldCollation.NullDirection.LAST
3173  : RelFieldCollation.NullDirection.FIRST;
3174  }
3175 
3176  // Scan the select list and order exprs for an identical expression.
3177  final SelectScope selectScope = validator.getRawSelectScope(select);
3178  int ordinal = -1;
3179  for (SqlNode selectItem : selectScope.getExpandedSelectList()) {
3180  ++ordinal;
3181  if (converted.equalsDeep(stripAs(selectItem), Litmus.IGNORE)) {
3182  return new RelFieldCollation(ordinal, direction, nullDirection);
3183  }
3184  }
3185 
3186  for (SqlNode extraExpr : extraExprs) {
3187  ++ordinal;
3188  if (converted.equalsDeep(extraExpr, Litmus.IGNORE)) {
3189  return new RelFieldCollation(ordinal, direction, nullDirection);
3190  }
3191  }
3192 
3193  // TODO: handle collation sequence
3194  // TODO: flag expressions as non-standard
3195 
3196  extraExprs.add(converted);
3197  return new RelFieldCollation(ordinal + 1, direction, nullDirection);
3198  }
3199 
3200  private static boolean desc(RelFieldCollation.Direction direction) {
3201  switch (direction) {
3202  case DESCENDING:
3203  case STRICTLY_DESCENDING:
3204  return true;
3205  default:
3206  return false;
3207  }
3208  }
3209 
3210  @Deprecated // to be removed before 2.0
3211  protected boolean enableDecorrelation() {
3212  // disable sub-query decorrelation when needed.
3213  // e.g. if outer joins are not supported.
3214  return config.isDecorrelationEnabled();
3215  }
3216 
3217  protected RelNode decorrelateQuery(RelNode rootRel) {
3218  return RelDecorrelator.decorrelateQuery(rootRel, relBuilder);
3219  }
3220 
3226  @Deprecated // to be removed before 2.0
3227  public boolean isTrimUnusedFields() {
3228  return config.isTrimUnusedFields();
3229  }
3230 
3240  protected RelRoot convertQueryRecursive(
3241  SqlNode query, boolean top, RelDataType targetRowType) {
3242  final SqlKind kind = query.getKind();
3243  switch (kind) {
3244  case SELECT:
3245  return RelRoot.of(convertSelect((SqlSelect) query, top), kind);
3246  case INSERT:
3247  return RelRoot.of(convertInsert((SqlInsert) query), kind);
3248  case DELETE:
3249  return RelRoot.of(convertDelete((SqlDelete) query), kind);
3250  case UPDATE:
3251  return RelRoot.of(convertUpdate((SqlUpdate) query), kind);
3252  case MERGE:
3253  return RelRoot.of(convertMerge((SqlMerge) query), kind);
3254  case UNION:
3255  case INTERSECT:
3256  case EXCEPT:
3257  return RelRoot.of(convertSetOp((SqlCall) query), kind);
3258  case WITH:
3259  return convertWith((SqlWith) query, top);
3260  case VALUES:
3261  return RelRoot.of(convertValues((SqlCall) query, targetRowType), kind);
3262  default:
3263  throw new AssertionError("not a query: " + query);
3264  }
3265  }
3266 
3274  protected RelNode convertSetOp(SqlCall call) {
3275  final RelNode left = convertQueryRecursive(call.operand(0), false, null).project();
3276  final RelNode right = convertQueryRecursive(call.operand(1), false, null).project();
3277  switch (call.getKind()) {
3278  case UNION:
3279  return LogicalUnion.create(ImmutableList.of(left, right), all(call));
3280 
3281  case INTERSECT:
3282  return LogicalIntersect.create(ImmutableList.of(left, right), all(call));
3283 
3284  case EXCEPT:
3285  return LogicalMinus.create(ImmutableList.of(left, right), all(call));
3286 
3287  default:
3288  throw Util.unexpected(call.getKind());
3289  }
3290  }
3291 
3292  private boolean all(SqlCall call) {
3293  return ((SqlSetOperator) call.getOperator()).isAll();
3294  }
3295 
3296  protected RelNode convertInsert(SqlInsert call) {
3297  RelOptTable targetTable = getTargetTable(call);
3298 
3299  final RelDataType targetRowType = validator.getValidatedNodeType(call);
3300  assert targetRowType != null;
3301  RelNode sourceRel =
3302  convertQueryRecursive(call.getSource(), true, targetRowType).project();
3303  RelNode massagedRel = convertColumnList(call, sourceRel);
3304 
3305  return createModify(targetTable, massagedRel);
3306  }
3307 
3309  private RelNode createModify(RelOptTable targetTable, RelNode source) {
3310  final ModifiableTable modifiableTable = targetTable.unwrap(ModifiableTable.class);
3311  if (modifiableTable != null && modifiableTable == targetTable.unwrap(Table.class)) {
3312  return modifiableTable.toModificationRel(cluster,
3313  targetTable,
3314  catalogReader,
3315  source,
3316  LogicalTableModify.Operation.INSERT,
3317  null,
3318  null,
3319  false);
3320  }
3321  final ModifiableView modifiableView = targetTable.unwrap(ModifiableView.class);
3322  if (modifiableView != null) {
3323  final Table delegateTable = modifiableView.getTable();
3324  final RelDataType delegateRowType = delegateTable.getRowType(typeFactory);
3325  final RelOptTable delegateRelOptTable = RelOptTableImpl.create(
3326  null, delegateRowType, delegateTable, modifiableView.getTablePath());
3327  final RelNode newSource =
3328  createSource(targetTable, source, modifiableView, delegateRowType);
3329  return createModify(delegateRelOptTable, newSource);
3330  }
3331  return LogicalTableModify.create(targetTable,
3332  catalogReader,
3333  source,
3334  LogicalTableModify.Operation.INSERT,
3335  null,
3336  null,
3337  false);
3338  }
3339 
3353  private RelNode createSource(RelOptTable targetTable,
3354  RelNode source,
3355  ModifiableView modifiableView,
3356  RelDataType delegateRowType) {
3357  final ImmutableIntList mapping = modifiableView.getColumnMapping();
3358  assert mapping.size() == targetTable.getRowType().getFieldCount();
3359 
3360  // For columns represented in the mapping, the expression is just a field
3361  // reference.
3362  final Map<Integer, RexNode> projectMap = new HashMap<>();
3363  final List<RexNode> filters = new ArrayList<>();
3364  for (int i = 0; i < mapping.size(); i++) {
3365  int target = mapping.get(i);
3366  if (target >= 0) {
3367  projectMap.put(target, RexInputRef.of(i, source.getRowType()));
3368  }
3369  }
3370 
3371  // For columns that are not in the mapping, and have a constraint of the
3372  // form "column = value", the expression is the literal "value".
3373  //
3374  // If a column has multiple constraints, the extra ones will become a
3375  // filter.
3376  final RexNode constraint = modifiableView.getConstraint(rexBuilder, delegateRowType);
3377  RelOptUtil.inferViewPredicates(projectMap, filters, constraint);
3378  final List<Pair<RexNode, String>> projects = new ArrayList<>();
3379  for (RelDataTypeField field : delegateRowType.getFieldList()) {
3380  RexNode node = projectMap.get(field.getIndex());
3381  if (node == null) {
3382  node = rexBuilder.makeNullLiteral(field.getType());
3383  }
3384  projects.add(Pair.of(
3385  rexBuilder.ensureType(field.getType(), node, false), field.getName()));
3386  }
3387 
3388  return relBuilder.push(source)
3389  .projectNamed(Pair.left(projects), Pair.right(projects), false)
3390  .filter(filters)
3391  .build();
3392  }
3393 
3394  private RelOptTable.ToRelContext createToRelContext(List<RelHint> hints) {
3395  return ViewExpanders.toRelContext(viewExpander, cluster, hints);
3396  }
3397 
3398  public RelNode toRel(final RelOptTable table, final List<RelHint> hints) {
3399  final RelNode scan = table.toRel(createToRelContext(hints));
3400 
3401  final InitializerExpressionFactory ief =
3402  Util.first(table.unwrap(InitializerExpressionFactory.class),
3403  NullInitializerExpressionFactory.INSTANCE);
3404 
3405  boolean hasVirtualFields = table.getRowType().getFieldList().stream().anyMatch(
3406  f -> ief.generationStrategy(table, f.getIndex()) == ColumnStrategy.VIRTUAL);
3407 
3408  if (hasVirtualFields) {
3409  final RexNode sourceRef = rexBuilder.makeRangeReference(scan);
3411  table, sourceRef, table.getRowType().getFieldNames());
3412  final List<RexNode> list = new ArrayList<>();
3413  for (RelDataTypeField f : table.getRowType().getFieldList()) {
3414  final ColumnStrategy strategy = ief.generationStrategy(table, f.getIndex());
3415  switch (strategy) {
3416  case VIRTUAL:
3417  list.add(ief.newColumnDefaultValue(table, f.getIndex(), bb));
3418  break;
3419  default:
3420  list.add(rexBuilder.makeInputRef(
3421  scan, RelOptTableImpl.realOrdinal(table, f.getIndex())));
3422  }
3423  }
3424  relBuilder.push(scan);
3425  relBuilder.project(list);
3426  final RelNode project = relBuilder.build();
3427  if (ief.postExpressionConversionHook() != null) {
3428  return ief.postExpressionConversionHook().apply(bb, project);
3429  } else {
3430  return project;
3431  }
3432  }
3433 
3434  return scan;
3435  }
3436 
3437  protected RelOptTable getTargetTable(SqlNode call) {
3438  final SqlValidatorNamespace targetNs = validator.getNamespace(call);
3439  if (targetNs.isWrapperFor(SqlValidatorImpl.DmlNamespace.class)) {
3440  final SqlValidatorImpl.DmlNamespace dmlNamespace =
3441  targetNs.unwrap(SqlValidatorImpl.DmlNamespace.class);
3442  return SqlValidatorUtil.getRelOptTable(dmlNamespace, catalogReader, null, null);
3443  }
3444  final SqlValidatorNamespace resolvedNamespace = targetNs.resolve();
3445  return SqlValidatorUtil.getRelOptTable(resolvedNamespace, catalogReader, null, null);
3446  }
3447 
3465  protected RelNode convertColumnList(final SqlInsert call, RelNode source) {
3466  RelDataType sourceRowType = source.getRowType();
3467  final RexNode sourceRef = rexBuilder.makeRangeReference(sourceRowType, 0, false);
3468  final List<String> targetColumnNames = new ArrayList<>();
3469  final List<RexNode> columnExprs = new ArrayList<>();
3470  collectInsertTargets(call, sourceRef, targetColumnNames, columnExprs);
3471 
3472  final RelOptTable targetTable = getTargetTable(call);
3473  final RelDataType targetRowType = RelOptTableImpl.realRowType(targetTable);
3474  final List<RelDataTypeField> targetFields = targetRowType.getFieldList();
3475  final List<RexNode> sourceExps =
3476  new ArrayList<>(Collections.nCopies(targetFields.size(), null));
3477  final List<String> fieldNames =
3478  new ArrayList<>(Collections.nCopies(targetFields.size(), null));
3479 
3480  final InitializerExpressionFactory initializerFactory =
3481  getInitializerFactory(validator.getNamespace(call).getTable());
3482 
3483  // Walk the name list and place the associated value in the
3484  // expression list according to the ordinal value returned from
3485  // the table construct, leaving nulls in the list for columns
3486  // that are not referenced.
3487  final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
3488  for (Pair<String, RexNode> p : Pair.zip(targetColumnNames, columnExprs)) {
3489  RelDataTypeField field = nameMatcher.field(targetRowType, p.left);
3490  assert field != null : "column " + p.left + " not found";
3491  sourceExps.set(field.getIndex(), p.right);
3492  }
3493 
3494  // Lazily create a blackboard that contains all non-generated columns.
3495  final Supplier<Blackboard> bb =
3496  () -> createInsertBlackboard(targetTable, sourceRef, targetColumnNames);
3497 
3498  // Walk the expression list and get default values for any columns
3499  // that were not supplied in the statement. Get field names too.
3500  for (int i = 0; i < targetFields.size(); ++i) {
3501  final RelDataTypeField field = targetFields.get(i);
3502  final String fieldName = field.getName();
3503  fieldNames.set(i, fieldName);
3504  if (sourceExps.get(i) == null || sourceExps.get(i).getKind() == SqlKind.DEFAULT) {
3505  sourceExps.set(
3506  i, initializerFactory.newColumnDefaultValue(targetTable, i, bb.get()));
3507 
3508  // bare nulls are dangerous in the wrong hands
3509  sourceExps.set(i, castNullLiteralIfNeeded(sourceExps.get(i), field.getType()));
3510  }
3511  }
3512 
3513  return relBuilder.push(source).projectNamed(sourceExps, fieldNames, false).build();
3514  }
3515 
3521  RelOptTable targetTable, RexNode sourceRef, List<String> targetColumnNames) {
3522  final Map<String, RexNode> nameToNodeMap = new HashMap<>();
3523  int j = 0;
3524 
3525  // Assign expressions for non-generated columns.
3526  final List<ColumnStrategy> strategies = targetTable.getColumnStrategies();
3527  final List<String> targetFields = targetTable.getRowType().getFieldNames();
3528  for (String targetColumnName : targetColumnNames) {
3529  final int i = targetFields.indexOf(targetColumnName);
3530  switch (strategies.get(i)) {
3531  case STORED:
3532  case VIRTUAL:
3533  break;
3534  default:
3535  nameToNodeMap.put(targetColumnName, rexBuilder.makeFieldAccess(sourceRef, j++));
3536  }
3537  }
3538  return createBlackboard(null, nameToNodeMap, false);
3539  }
3540 
3541  private InitializerExpressionFactory getInitializerFactory(
3542  SqlValidatorTable validatorTable) {
3543  // We might unwrap a null instead of a InitializerExpressionFactory.
3544  final Table table = unwrap(validatorTable, Table.class);
3545  if (table != null) {
3546  InitializerExpressionFactory f = unwrap(table, InitializerExpressionFactory.class);
3547  if (f != null) {
3548  return f;
3549  }
3550  }
3551  return NullInitializerExpressionFactory.INSTANCE;
3552  }
3553 
3554  private static <T> T unwrap(Object o, Class<T> clazz) {
3555  if (o instanceof Wrapper) {
3556  return ((Wrapper) o).unwrap(clazz);
3557  }
3558  return null;
3559  }
3560 
3561  private RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type) {
3562  if (!RexLiteral.isNullLiteral(node)) {
3563  return node;
3564  }
3565  return rexBuilder.makeCast(type, node);
3566  }
3567 
3578  protected void collectInsertTargets(SqlInsert call,
3579  final RexNode sourceRef,
3580  final List<String> targetColumnNames,
3581  List<RexNode> columnExprs) {
3582  final RelOptTable targetTable = getTargetTable(call);
3583  final RelDataType tableRowType = targetTable.getRowType();
3584  SqlNodeList targetColumnList = call.getTargetColumnList();
3585  if (targetColumnList == null) {
3586  if (validator.config().sqlConformance().isInsertSubsetColumnsAllowed()) {
3587  final RelDataType targetRowType =
3588  typeFactory.createStructType(tableRowType.getFieldList().subList(
3589  0, sourceRef.getType().getFieldCount()));
3590  targetColumnNames.addAll(targetRowType.getFieldNames());
3591  } else {
3592  targetColumnNames.addAll(tableRowType.getFieldNames());
3593  }
3594  } else {
3595  for (int i = 0; i < targetColumnList.size(); i++) {
3596  SqlIdentifier id = (SqlIdentifier) targetColumnList.get(i);
3597  RelDataTypeField field = SqlValidatorUtil.getTargetField(
3598  tableRowType, typeFactory, id, catalogReader, targetTable);
3599  assert field != null : "column " + id.toString() + " not found";
3600  targetColumnNames.add(field.getName());
3601  }
3602  }
3603 
3604  final Blackboard bb =
3605  createInsertBlackboard(targetTable, sourceRef, targetColumnNames);
3606 
3607  // Next, assign expressions for generated columns.
3608  final List<ColumnStrategy> strategies = targetTable.getColumnStrategies();
3609  for (String columnName : targetColumnNames) {
3610  final int i = tableRowType.getFieldNames().indexOf(columnName);
3611  final RexNode expr;
3612  switch (strategies.get(i)) {
3613  case STORED:
3614  final InitializerExpressionFactory f =
3615  Util.first(targetTable.unwrap(InitializerExpressionFactory.class),
3616  NullInitializerExpressionFactory.INSTANCE);
3617  expr = f.newColumnDefaultValue(targetTable, i, bb);
3618  break;
3619  case VIRTUAL:
3620  expr = null;
3621  break;
3622  default:
3623  expr = bb.nameToNodeMap.get(columnName);
3624  }
3625  columnExprs.add(expr);
3626  }
3627 
3628  // Remove virtual columns from the list.
3629  for (int i = 0; i < targetColumnNames.size(); i++) {
3630  if (columnExprs.get(i) == null) {
3631  columnExprs.remove(i);
3632  targetColumnNames.remove(i);
3633  --i;
3634  }
3635  }
3636  }
3637 
3638  private RelNode convertDelete(SqlDelete call) {
3639  RelOptTable targetTable = getTargetTable(call);
3640  RelNode sourceRel = convertSelect(call.getSourceSelect(), false);
3641  return LogicalTableModify.create(targetTable,
3642  catalogReader,
3643  sourceRel,
3644  LogicalTableModify.Operation.DELETE,
3645  null,
3646  null,
3647  false);
3648  }
3649 
3650  private RelNode convertUpdate(SqlUpdate call) {
3651  final SqlValidatorScope scope = validator.getWhereScope(call.getSourceSelect());
3652  Blackboard bb = createBlackboard(scope, null, false);
3653 
3654  replaceSubQueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
3655 
3656  RelOptTable targetTable = getTargetTable(call);
3657 
3658  // convert update column list from SqlIdentifier to String
3659  final List<String> targetColumnNameList = new ArrayList<>();
3660  final RelDataType targetRowType = targetTable.getRowType();
3661  for (SqlNode node : call.getTargetColumnList()) {
3662  SqlIdentifier id = (SqlIdentifier) node;
3663  RelDataTypeField field = SqlValidatorUtil.getTargetField(
3664  targetRowType, typeFactory, id, catalogReader, targetTable);
3665  assert field != null : "column " + id.toString() + " not found";
3666  targetColumnNameList.add(field.getName());
3667  }
3668 
3669  RelNode sourceRel = convertSelect(call.getSourceSelect(), false);
3670 
3671  bb.setRoot(sourceRel, false);
3672  Builder<RexNode> rexNodeSourceExpressionListBuilder = ImmutableList.builder();
3673  for (SqlNode n : call.getSourceExpressionList()) {
3674  RexNode rn = bb.convertExpression(n);
3675  rexNodeSourceExpressionListBuilder.add(rn);
3676  }
3677 
3678  return LogicalTableModify.create(targetTable,
3679  catalogReader,
3680  sourceRel,
3681  LogicalTableModify.Operation.UPDATE,
3682  targetColumnNameList,
3683  rexNodeSourceExpressionListBuilder.build(),
3684  false);
3685  }
3686 
3687  private RelNode convertMerge(SqlMerge call) {
3688  RelOptTable targetTable = getTargetTable(call);
3689 
3690  // convert update column list from SqlIdentifier to String
3691  final List<String> targetColumnNameList = new ArrayList<>();
3692  final RelDataType targetRowType = targetTable.getRowType();
3693  SqlUpdate updateCall = call.getUpdateCall();
3694  if (updateCall != null) {
3695  for (SqlNode targetColumn : updateCall.getTargetColumnList()) {
3696  SqlIdentifier id = (SqlIdentifier) targetColumn;
3697  RelDataTypeField field = SqlValidatorUtil.getTargetField(
3698  targetRowType, typeFactory, id, catalogReader, targetTable);
3699  assert field != null : "column " + id.toString() + " not found";
3700  targetColumnNameList.add(field.getName());
3701  }
3702  }
3703 
3704  // replace the projection of the source select with a
3705  // projection that contains the following:
3706  // 1) the expressions corresponding to the new insert row (if there is
3707  // an insert)
3708  // 2) all columns from the target table (if there is an update)
3709  // 3) the set expressions in the update call (if there is an update)
3710 
3711  // first, convert the merge's source select to construct the columns
3712  // from the target table and the set expressions in the update call
3713  RelNode mergeSourceRel = convertSelect(call.getSourceSelect(), false);
3714 
3715  // then, convert the insert statement so we can get the insert
3716  // values expressions
3717  SqlInsert insertCall = call.getInsertCall();
3718  int nLevel1Exprs = 0;
3719  List<RexNode> level1InsertExprs = null;
3720  List<RexNode> level2InsertExprs = null;
3721  if (insertCall != null) {
3722  RelNode insertRel = convertInsert(insertCall);
3723 
3724  // if there are 2 level of projections in the insert source, combine
3725  // them into a single project; level1 refers to the topmost project;
3726  // the level1 projection contains references to the level2
3727  // expressions, except in the case where no target expression was
3728  // provided, in which case, the expression is the default value for
3729  // the column; or if the expressions directly map to the source
3730  // table
3731  level1InsertExprs = ((LogicalProject) insertRel.getInput(0)).getProjects();
3732  if (insertRel.getInput(0).getInput(0) instanceof LogicalProject) {
3733  level2InsertExprs =
3734  ((LogicalProject) insertRel.getInput(0).getInput(0)).getProjects();
3735  }
3736  nLevel1Exprs = level1InsertExprs.size();
3737  }
3738 
3739  LogicalJoin join = (LogicalJoin) mergeSourceRel.getInput(0);
3740  int nSourceFields = join.getLeft().getRowType().getFieldCount();
3741  final List<RexNode> projects = new ArrayList<>();
3742  for (int level1Idx = 0; level1Idx < nLevel1Exprs; level1Idx++) {
3743  if ((level2InsertExprs != null)
3744  && (level1InsertExprs.get(level1Idx) instanceof RexInputRef)) {
3745  int level2Idx = ((RexInputRef) level1InsertExprs.get(level1Idx)).getIndex();
3746  projects.add(level2InsertExprs.get(level2Idx));
3747  } else {
3748  projects.add(level1InsertExprs.get(level1Idx));
3749  }
3750  }
3751  if (updateCall != null) {
3752  final LogicalProject project = (LogicalProject) mergeSourceRel;
3753  projects.addAll(Util.skip(project.getProjects(), nSourceFields));
3754  }
3755 
3756  relBuilder.push(join).project(projects);
3757 
3758  return LogicalTableModify.create(targetTable,
3759  catalogReader,
3760  relBuilder.build(),
3761  LogicalTableModify.Operation.MERGE,
3762  targetColumnNameList,
3763  null,
3764  false);
3765  }
3766 
3771  private RexNode convertIdentifier(Blackboard bb, SqlIdentifier identifier) {
3772  // first check for reserved identifiers like CURRENT_USER
3773  final SqlCall call = bb.getValidator().makeNullaryCall(identifier);
3774  if (call != null) {
3775  return bb.convertExpression(call);
3776  }
3777 
3778  String pv = null;
3779  if (bb.isPatternVarRef && identifier.names.size() > 1) {
3780  pv = identifier.names.get(0);
3781  }
3782 
3783  final SqlQualified qualified;
3784  if (bb.scope != null) {
3785  qualified = bb.scope.fullyQualify(identifier);
3786  } else {
3787  qualified = SqlQualified.create(null, 1, null, identifier);
3788  }
3789  final Pair<RexNode, Map<String, Integer>> e0 = bb.lookupExp(qualified);
3790  RexNode e = e0.left;
3791  for (String name : qualified.suffix()) {
3792  if (e == e0.left && e0.right != null) {
3793  int i = e0.right.get(name);
3794  e = rexBuilder.makeFieldAccess(e, i);
3795  } else {
3796  final boolean caseSensitive = true; // name already fully-qualified
3797  if (identifier.isStar() && bb.scope instanceof MatchRecognizeScope) {
3798  e = rexBuilder.makeFieldAccess(e, 0);
3799  } else {
3800  e = rexBuilder.makeFieldAccess(e, name, caseSensitive);
3801  }
3802  }
3803  }
3804  if (e instanceof RexInputRef) {
3805  // adjust the type to account for nulls introduced by outer joins
3806  e = adjustInputRef(bb, (RexInputRef) e);
3807  if (pv != null) {
3808  e = RexPatternFieldRef.of(pv, (RexInputRef) e);
3809  }
3810  }
3811 
3812  if (e0.left instanceof RexCorrelVariable) {
3813  assert e instanceof RexFieldAccess;
3814  final RexNode prev = bb.mapCorrelateToRex.put(
3815  ((RexCorrelVariable) e0.left).id, (RexFieldAccess) e);
3816  assert prev == null;
3817  }
3818  return e;
3819  }
3820 
3830  protected RexNode adjustInputRef(Blackboard bb, RexInputRef inputRef) {
3831  RelDataTypeField field = bb.getRootField(inputRef);
3832  if (field != null) {
3833  return rexBuilder.makeInputRef(field.getType(), inputRef.getIndex());
3834  }
3835  return inputRef;
3836  }
3837 
3845  private RelNode convertRowConstructor(Blackboard bb, SqlCall rowConstructor) {
3846  Preconditions.checkArgument(isRowConstructor(rowConstructor));
3847  final List<SqlNode> operands = rowConstructor.getOperandList();
3848  return convertMultisets(operands, bb);
3849  }
3850 
3851  private RelNode convertCursor(Blackboard bb, SubQuery subQuery) {
3852  final SqlCall cursorCall = (SqlCall) subQuery.node;
3853  assert cursorCall.operandCount() == 1;
3854  SqlNode query = cursorCall.operand(0);
3855  RelNode converted = convertQuery(query, false, false).rel;
3856  int iCursor = bb.cursors.size();
3857  bb.cursors.add(converted);
3858  subQuery.expr = new RexInputRef(iCursor, converted.getRowType());
3859  return converted;
3860  }
3861 
3862  private RelNode convertMultisets(final List<SqlNode> operands, Blackboard bb) {
3863  // NOTE: Wael 2/04/05: this implementation is not the most efficient in
3864  // terms of planning since it generates XOs that can be reduced.
3865  final List<Object> joinList = new ArrayList<>();
3866  List<SqlNode> lastList = new ArrayList<>();
3867  for (int i = 0; i < operands.size(); i++) {
3868  SqlNode operand = operands.get(i);
3869  if (!(operand instanceof SqlCall)) {
3870  lastList.add(operand);
3871  continue;
3872  }
3873 
3874  final SqlCall call = (SqlCall) operand;
3875  final RelNode input;
3876  switch (call.getKind()) {
3877  case MULTISET_VALUE_CONSTRUCTOR:
3878  case ARRAY_VALUE_CONSTRUCTOR:
3879  final SqlNodeList list =
3880  new SqlNodeList(call.getOperandList(), call.getParserPosition());
3881  CollectNamespace nss = (CollectNamespace) validator.getNamespace(call);
3882  Blackboard usedBb;
3883  if (null != nss) {
3884  usedBb = createBlackboard(nss.getScope(), null, false);
3885  } else {
3886  usedBb = createBlackboard(new ListScope(bb.scope) {
3887  public SqlNode getNode() {
3888  return call;
3889  }
3890  }, null, false);
3891  }
3892  RelDataType multisetType = validator.getValidatedNodeType(call);
3893  validator.setValidatedNodeType(list, multisetType.getComponentType());
3894  input = convertQueryOrInList(usedBb, list, null);
3895  break;
3896  case MULTISET_QUERY_CONSTRUCTOR:
3897  case ARRAY_QUERY_CONSTRUCTOR:
3898  final RelRoot root = convertQuery(call.operand(0), false, true);
3899  input = root.rel;
3900  break;
3901  default:
3902  lastList.add(operand);
3903  continue;
3904  }
3905 
3906  if (lastList.size() > 0) {
3907  joinList.add(lastList);
3908  }
3909  lastList = new ArrayList<>();
3910  Collect collect = new Collect(cluster,
3911  cluster.traitSetOf(Convention.NONE),
3912  input,
3913  validator.deriveAlias(call, i));
3914  joinList.add(collect);
3915  }
3916 
3917  if (joinList.size() == 0) {
3918  joinList.add(lastList);
3919  }
3920 
3921  for (int i = 0; i < joinList.size(); i++) {
3922  Object o = joinList.get(i);
3923  if (o instanceof List) {
3924  @SuppressWarnings("unchecked")
3925  List<SqlNode> projectList = (List<SqlNode>) o;
3926  final List<RexNode> selectList = new ArrayList<>();
3927  final List<String> fieldNameList = new ArrayList<>();
3928  for (int j = 0; j < projectList.size(); j++) {
3929  SqlNode operand = projectList.get(j);
3930  selectList.add(bb.convertExpression(operand));
3931 
3932  // REVIEW angel 5-June-2005: Use deriveAliasFromOrdinal
3933  // instead of deriveAlias to match field names from
3934  // SqlRowOperator. Otherwise, get error Type
3935  // 'RecordType(INTEGER EMPNO)' has no field 'EXPR$0' when
3936  // doing select * from unnest( select multiset[empno]
3937  // from sales.emps);
3938 
3939  fieldNameList.add(SqlUtil.deriveAliasFromOrdinal(j));
3940  }
3941 
3942  relBuilder.push(LogicalValues.createOneRow(cluster))
3943  .projectNamed(selectList, fieldNameList, true);
3944 
3945  joinList.set(i, relBuilder.build());
3946  }
3947  }
3948 
3949  RelNode ret = (RelNode) joinList.get(0);
3950  for (int i = 1; i < joinList.size(); i++) {
3951  RelNode relNode = (RelNode) joinList.get(i);
3952  ret = RelFactories.DEFAULT_JOIN_FACTORY.createJoin(ret,
3953  relNode,
3954  ImmutableList.of(),
3955  rexBuilder.makeLiteral(true),
3956  ImmutableSet.of(),
3957  JoinRelType.INNER,
3958  false);
3959  }
3960  return ret;
3961  }
3962 
3963  private void convertSelectList(
3964  Blackboard bb, SqlSelect select, List<SqlNode> orderList) {
3965  SqlNodeList selectList = select.getSelectList();
3966  selectList = validator.expandStar(selectList, select, false);
3967 
3968  replaceSubQueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
3969 
3970  List<String> fieldNames = new ArrayList<>();
3971  final List<RexNode> exprs = new ArrayList<>();
3972  final Collection<String> aliases = new TreeSet<>();
3973 
3974  // Project any system fields. (Must be done before regular select items,
3975  // because offsets may be affected.)
3976  final List<SqlMonotonicity> columnMonotonicityList = new ArrayList<>();
3977  extraSelectItems(bb, select, exprs, fieldNames, aliases, columnMonotonicityList);
3978 
3979  // Project select clause.
3980  int i = -1;
3981  for (SqlNode expr : selectList) {
3982  ++i;
3983  exprs.add(bb.convertExpression(expr));
3984  fieldNames.add(deriveAlias(expr, aliases, i));
3985  }
3986 
3987  // Project extra fields for sorting.
3988  for (SqlNode expr : orderList) {
3989  ++i;
3990  SqlNode expr2 = validator.expandOrderExpr(select, expr);
3991  exprs.add(bb.convertExpression(expr2));
3992  fieldNames.add(deriveAlias(expr, aliases, i));
3993  }
3994 
3995  fieldNames = SqlValidatorUtil.uniquify(
3996  fieldNames, catalogReader.nameMatcher().isCaseSensitive());
3997 
3998  relBuilder.push(bb.root).projectNamed(exprs, fieldNames, true);
3999  bb.setRoot(relBuilder.build(), false);
4000 
4001  assert bb.columnMonotonicities.isEmpty();
4002  bb.columnMonotonicities.addAll(columnMonotonicityList);
4003  for (SqlNode selectItem : selectList) {
4004  bb.columnMonotonicities.add(selectItem.getMonotonicity(bb.scope));
4005  }
4006  }
4007 
4021  protected void extraSelectItems(Blackboard bb,
4022  SqlSelect select,
4023  List<RexNode> exprList,
4024  List<String> nameList,
4025  Collection<String> aliasList,
4026  List<SqlMonotonicity> columnMonotonicityList) {}
4027 
4028  private String deriveAlias(
4029  final SqlNode node, Collection<String> aliases, final int ordinal) {
4030  String alias = validator.deriveAlias(node, ordinal);
4031  if ((alias == null) || aliases.contains(alias)) {
4032  String aliasBase = (alias == null) ? "EXPR$" : alias;
4033  for (int j = 0;; j++) {
4034  alias = aliasBase + j;
4035  if (!aliases.contains(alias)) {
4036  break;
4037  }
4038  }
4039  }
4040  aliases.add(alias);
4041  return alias;
4042  }
4043 
4047  public RelRoot convertWith(SqlWith with, boolean top) {
4048  return convertQuery(with.body, false, top);
4049  }
4050 
4054  public RelNode convertValues(SqlCall values, RelDataType targetRowType) {
4055  final SqlValidatorScope scope = validator.getOverScope(values);
4056  assert scope != null;
4057  final Blackboard bb = createBlackboard(scope, null, false);
4058  convertValuesImpl(bb, values, targetRowType);
4059  return bb.root;
4060  }
4061 
4070  private void convertValuesImpl(
4071  Blackboard bb, SqlCall values, RelDataType targetRowType) {
4072  // Attempt direct conversion to LogicalValues; if that fails, deal with
4073  // fancy stuff like sub-queries below.
4074  RelNode valuesRel =
4075  convertRowValues(bb, values, values.getOperandList(), true, targetRowType);
4076  if (valuesRel != null) {
4077  bb.setRoot(valuesRel, true);
4078  return;
4079  }
4080 
4081  final List<RelNode> unionRels = new ArrayList<>();
4082  for (SqlNode rowConstructor1 : values.getOperandList()) {
4083  SqlCall rowConstructor = (SqlCall) rowConstructor1;
4084  Blackboard tmpBb = createBlackboard(bb.scope, null, false);
4085  replaceSubQueries(tmpBb, rowConstructor, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
4086  final List<Pair<RexNode, String>> exps = new ArrayList<>();
4087  for (Ord<SqlNode> operand : Ord.zip(rowConstructor.getOperandList())) {
4088  exps.add(Pair.of(tmpBb.convertExpression(operand.e),
4089  validator.deriveAlias(operand.e, operand.i)));
4090  }
4091  RelNode in =
4092  (null == tmpBb.root) ? LogicalValues.createOneRow(cluster) : tmpBb.root;
4093  unionRels.add(
4094  relBuilder.push(in).project(Pair.left(exps), Pair.right(exps)).build());
4095  }
4096 
4097  if (unionRels.size() == 0) {
4098  throw new AssertionError("empty values clause");
4099  } else if (unionRels.size() == 1) {
4100  bb.setRoot(unionRels.get(0), true);
4101  } else {
4102  bb.setRoot(LogicalUnion.create(unionRels, true), true);
4103  }
4104  }
4105 
4106  // ~ Inner Classes ----------------------------------------------------------
4107 
4111  protected class Blackboard
4112  implements SqlRexContext, SqlVisitor<RexNode>, InitializerContext {
4116  public final SqlValidatorScope scope;
4117  private final Map<String, RexNode> nameToNodeMap;
4118  public RelNode root;
4119  private List<RelNode> inputs;
4120  private final Map<CorrelationId, RexFieldAccess> mapCorrelateToRex = new HashMap<>();
4121 
4122  private boolean isPatternVarRef = false;
4123 
4124  final List<RelNode> cursors = new ArrayList<>();
4125 
4130  private final List<SubQuery> subQueryList = new ArrayList<>();
4131 
4135  AggConverter agg;
4136 
4141  SqlWindow window;
4142 
4148  private final Map<RelNode, Map<Integer, Integer>> mapRootRelToFieldProjection =
4149  new HashMap<>();
4150 
4151  private final List<SqlMonotonicity> columnMonotonicities = new ArrayList<>();
4152 
4153  private final List<RelDataTypeField> systemFieldList = new ArrayList<>();
4154  final boolean top;
4155 
4156  private final InitializerExpressionFactory initializerExpressionFactory =
4157  new NullInitializerExpressionFactory();
4158 
4170  protected Blackboard(
4171  SqlValidatorScope scope, Map<String, RexNode> nameToNodeMap, boolean top) {
4172  this.scope = scope;
4173  this.nameToNodeMap = nameToNodeMap;
4174  this.top = top;
4175  }
4176 
4177  public SqlNode getTopNode() {
4178  try {
4179  if (null == scope) {
4180  return null;
4181  }
4182  return scope.getNode();
4183  } catch (Exception e) {
4184  return null;
4185  }
4186  }
4187 
4188  public void setPatternVarRef(boolean isVarRef) {
4189  this.isPatternVarRef = isVarRef;
4190  }
4191 
4192  public RexNode register(RelNode rel, JoinRelType joinType) {
4193  return register(rel, joinType, null);
4194  }
4195 
4205  public RexNode register(RelNode rel, JoinRelType joinType, List<RexNode> leftKeys) {
4206  assert joinType != null;
4207  if (root == null) {
4208  assert leftKeys == null;
4209  setRoot(rel, false);
4210  return rexBuilder.makeRangeReference(root.getRowType(), 0, false);
4211  }
4212 
4213  final RexNode joinCond;
4214  final int origLeftInputCount = root.getRowType().getFieldCount();
4215  if (leftKeys != null) {
4216  List<RexNode> newLeftInputExprs = new ArrayList<>();
4217  for (int i = 0; i < origLeftInputCount; i++) {
4218  newLeftInputExprs.add(rexBuilder.makeInputRef(root, i));
4219  }
4220 
4221  final List<Integer> leftJoinKeys = new ArrayList<>();
4222  for (RexNode leftKey : leftKeys) {
4223  int index = newLeftInputExprs.indexOf(leftKey);
4224  if (index < 0 || joinType == JoinRelType.LEFT) {
4225  index = newLeftInputExprs.size();
4226  newLeftInputExprs.add(leftKey);
4227  }
4228  leftJoinKeys.add(index);
4229  }
4230 
4231  RelNode newLeftInput = relBuilder.push(root).project(newLeftInputExprs).build();
4232 
4233  // maintain the group by mapping in the new LogicalProject
4234  if (mapRootRelToFieldProjection.containsKey(root)) {
4235  mapRootRelToFieldProjection.put(
4236  newLeftInput, mapRootRelToFieldProjection.get(root));
4237  }
4238 
4239  setRoot(newLeftInput, false);
4240 
4241  // right fields appear after the LHS fields.
4242  final int rightOffset = root.getRowType().getFieldCount()
4243  - newLeftInput.getRowType().getFieldCount();
4244  final List<Integer> rightKeys =
4245  Util.range(rightOffset, rightOffset + leftKeys.size());
4246 
4247  joinCond = RelOptUtil.createEquiJoinCondition(
4248  newLeftInput, leftJoinKeys, rel, rightKeys, rexBuilder);
4249  } else {
4250  joinCond = rexBuilder.makeLiteral(true);
4251  }
4252 
4253  int leftFieldCount = root.getRowType().getFieldCount();
4254  final RelNode join = createJoin(this, root, rel, joinCond, joinType);
4255 
4256  setRoot(join, false);
4257 
4258  if (leftKeys != null && joinType == JoinRelType.LEFT) {
4259  final int leftKeyCount = leftKeys.size();
4260  int rightFieldLength = rel.getRowType().getFieldCount();
4261  assert leftKeyCount == rightFieldLength - 1;
4262 
4263  final int rexRangeRefLength = leftKeyCount + rightFieldLength;
4264  RelDataType returnType = typeFactory.createStructType(
4265  new AbstractList<Map.Entry<String, RelDataType>>() {
4266  public Map.Entry<String, RelDataType> get(int index) {
4267  return join.getRowType().getFieldList().get(
4268  origLeftInputCount + index);
4269  }
4270 
4271  public int size() {
4272  return rexRangeRefLength;
4273  }
4274  });
4275 
4276  return rexBuilder.makeRangeReference(returnType, origLeftInputCount, false);
4277  } else {
4278  return rexBuilder.makeRangeReference(
4279  rel.getRowType(), leftFieldCount, joinType.generatesNullsOnRight());
4280  }
4281  }
4282 
4294  public void setRoot(RelNode root, boolean leaf) {
4295  setRoot(Collections.singletonList(root), root, root instanceof LogicalJoin);
4296  if (leaf) {
4297  leaves.put(root, root.getRowType().getFieldCount());
4298  }
4299  this.columnMonotonicities.clear();
4300  }
4301 
4302  private void setRoot(List<RelNode> inputs, RelNode root, boolean hasSystemFields) {
4303  this.inputs = inputs;
4304  this.root = root;
4305  this.systemFieldList.clear();
4306  if (hasSystemFields) {
4307  this.systemFieldList.addAll(getSystemFields());
4308  }
4309  }
4310 
4321  public void setDataset(String datasetName) {}
4322 
4323  void setRoot(List<RelNode> inputs) {
4324  setRoot(inputs, null, false);
4325  }
4326 
4333  Pair<RexNode, Map<String, Integer>> lookupExp(SqlQualified qualified) {
4334  if (nameToNodeMap != null && qualified.prefixLength == 1) {
4335  RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0));
4336  if (node == null) {
4337  throw new AssertionError("Unknown identifier '" + qualified.identifier
4338  + "' encountered while expanding expression");
4339  }
4340  return Pair.of(node, null);
4341  }
4342  final SqlNameMatcher nameMatcher =
4343  scope.getValidator().getCatalogReader().nameMatcher();
4344  final SqlValidatorScope.ResolvedImpl resolved =
4345  new SqlValidatorScope.ResolvedImpl();
4346  scope.resolve(qualified.prefix(), nameMatcher, false, resolved);
4347  if (!(resolved.count() == 1)) {
4348  return null;
4349  }
4350  final SqlValidatorScope.Resolve resolve = resolved.only();
4351  final RelDataType rowType = resolve.rowType();
4352 
4353  // Found in current query's from list. Find which from item.
4354  // We assume that the order of the from clause items has been
4355  // preserved.
4356  final SqlValidatorScope ancestorScope = resolve.scope;
4357  boolean isParent = ancestorScope != scope;
4358  if ((inputs != null) && !isParent) {
4359  final LookupContext rels =
4360  new LookupContext(this, inputs, systemFieldList.size());
4361  final RexNode node = lookup(resolve.path.steps().get(0).i, rels);
4362  if (node == null) {
4363  return null;
4364  } else {
4365  final Map<String, Integer> fieldOffsets = new HashMap<>();
4366  for (RelDataTypeField f : resolve.rowType().getFieldList()) {
4367  if (!fieldOffsets.containsKey(f.getName())) {
4368  fieldOffsets.put(f.getName(), f.getIndex());
4369  }
4370  }
4371  final Map<String, Integer> map = ImmutableMap.copyOf(fieldOffsets);
4372  return Pair.of(node, map);
4373  }
4374  } else {
4375  // We're referencing a relational expression which has not been
4376  // converted yet. This occurs when from items are correlated,
4377  // e.g. "select from emp as emp join emp.getDepts() as dept".
4378  // Create a temporary expression.
4379  DeferredLookup lookup =
4380  new DeferredLookup(this, qualified.identifier.names.get(0));
4381  final CorrelationId correlId = cluster.createCorrel();
4382  mapCorrelToDeferred.put(correlId, lookup);
4383  if (resolve.path.steps().get(0).i < 0) {
4384  return Pair.of(rexBuilder.makeCorrel(rowType, correlId), null);
4385  } else {
4386  final RelDataTypeFactory.Builder builder = typeFactory.builder();
4387  final ListScope ancestorScope1 = (ListScope) resolve.scope;
4388  final ImmutableMap.Builder<String, Integer> fields = ImmutableMap.builder();
4389  int i = 0;
4390  int offset = 0;
4391  for (SqlValidatorNamespace c : ancestorScope1.getChildren()) {
4392  builder.addAll(c.getRowType().getFieldList());
4393  if (i == resolve.path.steps().get(0).i) {
4394  for (RelDataTypeField field : c.getRowType().getFieldList()) {
4395  fields.put(field.getName(), field.getIndex() + offset);
4396  }
4397  }
4398  ++i;
4399  offset += c.getRowType().getFieldCount();
4400  }
4401  final RexNode c = rexBuilder.makeCorrel(builder.uniquify().build(), correlId);
4402  return Pair.of(c, fields.build());
4403  }
4404  }
4405  }
4406 
4411  RexNode lookup(int offset, LookupContext lookupContext) {
4412  Pair<RelNode, Integer> pair = lookupContext.findRel(offset);
4413  return rexBuilder.makeRangeReference(pair.left.getRowType(), pair.right, false);
4414  }
4415 
4416  RelDataTypeField getRootField(RexInputRef inputRef) {
4417  if (inputs == null) {
4418  return null;
4419  }
4420  int fieldOffset = inputRef.getIndex();
4421  for (RelNode input : inputs) {
4422  RelDataType rowType = input.getRowType();
4423  if (fieldOffset < rowType.getFieldCount()) {
4424  return rowType.getFieldList().get(fieldOffset);
4425  }
4426  fieldOffset -= rowType.getFieldCount();
4427  }
4428  return null;
4429  }
4430 
4431  public void flatten(List<RelNode> rels,
4432  int systemFieldCount,
4433  int[] start,
4434  List<Pair<RelNode, Integer>> relOffsetList) {
4435  for (RelNode rel : rels) {
4436  if (leaves.containsKey(rel)) {
4437  relOffsetList.add(Pair.of(rel, start[0]));
4438  start[0] += leaves.get(rel);
4439  } else if (rel instanceof LogicalMatch) {
4440  relOffsetList.add(Pair.of(rel, start[0]));
4441  start[0] += rel.getRowType().getFieldCount();
4442  } else {
4443  if (rel instanceof LogicalJoin || rel instanceof LogicalAggregate) {
4444  start[0] += systemFieldCount;
4445  }
4446  flatten(rel.getInputs(), systemFieldCount, start, relOffsetList);
4447  }
4448  }
4449  }
4450 
4451  void registerSubQuery(SqlNode node, RelOptUtil.Logic logic) {
4452  for (SubQuery subQuery : subQueryList) {
4453  // Compare the reference to make sure the matched node has
4454  // exact scope where it belongs.
4455  if (node == subQuery.node) {
4456  return;
4457  }
4458  }
4459  subQueryList.add(new SubQuery(node, logic));
4460  }
4461 
4462  SubQuery getSubQuery(SqlNode expr) {
4463  for (SubQuery subQuery : subQueryList) {
4464  // Compare the reference to make sure the matched node has
4465  // exact scope where it belongs.
4466  if (expr == subQuery.node) {
4467  return subQuery;
4468  }
4469  }
4470 
4471  return null;
4472  }
4473 
4474  ImmutableList<RelNode> retrieveCursors() {
4475  try {
4476  return ImmutableList.copyOf(cursors);
4477  } finally {
4478  cursors.clear();
4479  }
4480  }
4481 
4482  public RexNode convertExpression(SqlNode expr) {
4483  // If we're in aggregation mode and this is an expression in the
4484  // GROUP BY clause, return a reference to the field.
4485  if (agg != null) {
4486  final SqlNode expandedGroupExpr = validator.expand(expr, scope);
4487  final int ref = agg.lookupGroupExpr(expandedGroupExpr);
4488  if (ref >= 0) {
4489  return rexBuilder.makeInputRef(root, ref);
4490  }
4491  if (expr instanceof SqlCall) {
4492  final RexNode rex = agg.lookupAggregates((SqlCall) expr);
4493  if (rex != null) {
4494  return rex;
4495  }
4496  }
4497  }
4498 
4499  // Allow the derived class chance to override the standard
4500  // behavior for special kinds of expressions.
4501  RexNode rex = convertExtendedExpression(expr, this);
4502  if (rex != null) {
4503  return rex;
4504  }
4505 
4506  // Sub-queries and OVER expressions are not like ordinary
4507  // expressions.
4508  final SqlKind kind = expr.getKind();
4509  final SubQuery subQuery;
4510  boolean isExpand = config.getExpandPredicate().test(getTopNode(), expr);
4511  if (!isExpand) {
4512  final SqlCall call;
4513  final SqlNode query;
4514  final RelRoot root;
4515  switch (kind) {
4516  case IN:
4517  case NOT_IN:
4518  case SOME:
4519  case ALL:
4520  call = (SqlCall) expr;
4521  query = call.operand(1);
4522  if (!(query instanceof SqlNodeList)) {
4523  root = convertQueryRecursive(query, false, null);
4524  final SqlNode operand = call.operand(0);
4525  List<SqlNode> nodes;
4526  switch (operand.getKind()) {
4527  case ROW:
4528  nodes = ((SqlCall) operand).getOperandList();
4529  break;
4530  default:
4531  nodes = ImmutableList.of(operand);
4532  }
4533  final ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
4534  for (SqlNode node : nodes) {
4535  builder.add(convertExpression(node));
4536  }
4537  final ImmutableList<RexNode> list = builder.build();
4538  switch (kind) {
4539  case IN:
4540  return RexSubQuery.in(root.rel, list);
4541  case NOT_IN:
4542  return rexBuilder.makeCall(
4543  SqlStdOperatorTable.NOT, RexSubQuery.in(root.rel, list));
4544  case SOME:
4545  return RexSubQuery.some(
4546  root.rel, list, (SqlQuantifyOperator) call.getOperator());
4547  case ALL:
4548  return rexBuilder.makeCall(SqlStdOperatorTable.NOT,
4549  RexSubQuery.some(root.rel,
4550  list,
4551  negate((SqlQuantifyOperator) call.getOperator())));
4552  default:
4553  throw new AssertionError(kind);
4554  }
4555  }
4556  break;
4557 
4558  case EXISTS:
4559  call = (SqlCall) expr;
4560  query = Iterables.getOnlyElement(call.getOperandList());
4561  root = convertQueryRecursive(query, false, null);
4562  RelNode rel = root.rel;
4563  while (rel instanceof Project
4564  || rel instanceof Sort && ((Sort) rel).fetch == null
4565  && ((Sort) rel).offset == null) {
4566  rel = ((SingleRel) rel).getInput();
4567  }
4568  return RexSubQuery.exists(rel);
4569 
4570  case SCALAR_QUERY:
4571  call = (SqlCall) expr;
4572  query = Iterables.getOnlyElement(call.getOperandList());
4573  root = convertQueryRecursive(query, false, null);
4574  return RexSubQuery.scalar(root.rel);
4575  }
4576  }
4577 
4578  switch (kind) {
4579  case SOME:
4580  case ALL:
4581  if (isExpand) {
4582  throw new RuntimeException(kind + " is only supported if expand = false");
4583  }
4584  // fall through
4585  case CURSOR:
4586  case IN:
4587  case NOT_IN:
4588  subQuery = Objects.requireNonNull(getSubQuery(expr));
4589  rex = Objects.requireNonNull(subQuery.expr);
4590  return StandardConvertletTable.castToValidatedType(
4591  expr, rex, validator, rexBuilder);
4592 
4593  case SELECT:
4594  case EXISTS:
4595  case SCALAR_QUERY:
4596  subQuery = getSubQuery(expr);
4597  assert subQuery != null;
4598  rex = subQuery.expr;
4599  assert rex != null : "rex != null";
4600 
4601  if (((kind == SqlKind.SCALAR_QUERY) || (kind == SqlKind.EXISTS))
4602  && isConvertedSubq(rex)) {
4603  // scalar sub-query or EXISTS has been converted to a
4604  // constant
4605  return rex;
4606  }
4607 
4608  // The indicator column is the last field of the sub-query.
4609  RexNode fieldAccess =
4610  rexBuilder.makeFieldAccess(rex, rex.getType().getFieldCount() - 1);
4611 
4612  // The indicator column will be nullable if it comes from
4613  // the null-generating side of the join. For EXISTS, add an
4614  // "IS TRUE" check so that the result is "BOOLEAN NOT NULL".
4615  if (fieldAccess.getType().isNullable() && kind == SqlKind.EXISTS) {
4616  fieldAccess =
4617  rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, fieldAccess);
4618  }
4619  return fieldAccess;
4620 
4621  case OVER:
4622  return convertOver(this, expr);
4623 
4624  default:
4625  // fall through
4626  }
4627 
4628  // Apply standard conversions.
4629  rex = expr.accept(this);
4630  return Objects.requireNonNull(rex);
4631  }
4632 
4637  public RexFieldCollation convertSortExpression(SqlNode expr,
4638  RelFieldCollation.Direction direction,
4639  RelFieldCollation.NullDirection nullDirection) {
4640  switch (expr.getKind()) {
4641  case DESCENDING:
4642  return convertSortExpression(((SqlCall) expr).operand(0),
4643  RelFieldCollation.Direction.DESCENDING,
4644  nullDirection);
4645  case NULLS_LAST:
4646  return convertSortExpression(((SqlCall) expr).operand(0),
4647  direction,
4648  RelFieldCollation.NullDirection.LAST);
4649  case NULLS_FIRST:
4650  return convertSortExpression(((SqlCall) expr).operand(0),
4651  direction,
4652  RelFieldCollation.NullDirection.FIRST);
4653  default:
4654  final Set<SqlKind> flags = EnumSet.noneOf(SqlKind.class);
4655  switch (direction) {
4656  case DESCENDING:
4657  flags.add(SqlKind.DESCENDING);
4658  }
4659  switch (nullDirection) {
4660  case UNSPECIFIED:
4661  final RelFieldCollation.NullDirection nullDefaultDirection =
4662  validator.config().defaultNullCollation().last(desc(direction))
4663  ? RelFieldCollation.NullDirection.LAST
4664  : RelFieldCollation.NullDirection.FIRST;
4665  if (nullDefaultDirection != direction.defaultNullDirection()) {
4666  SqlKind nullDirectionSqlKind =
4667  validator.config().defaultNullCollation().last(desc(direction))
4668  ? SqlKind.NULLS_LAST
4669  : SqlKind.NULLS_FIRST;
4670  flags.add(nullDirectionSqlKind);
4671  }
4672  break;
4673  case FIRST:
4674  flags.add(SqlKind.NULLS_FIRST);
4675  break;
4676  case LAST:
4677  flags.add(SqlKind.NULLS_LAST);
4678  break;
4679  }
4680  return new RexFieldCollation(convertExpression(expr), flags);
4681  }
4682  }
4683 
4692  private boolean isConvertedSubq(RexNode rex) {
4693  if ((rex instanceof RexLiteral) || (rex instanceof RexDynamicParam)) {
4694  return true;
4695  }
4696  if (rex instanceof RexCall) {
4697  RexCall call = (RexCall) rex;
4698  if (call.getOperator() == SqlStdOperatorTable.CAST) {
4699  RexNode operand = call.getOperands().get(0);
4700  if (operand instanceof RexLiteral) {
4701  return true;
4702  }
4703  }
4704  }
4705  return false;
4706  }
4707 
4708  public int getGroupCount() {
4709  if (agg != null) {
4710  return agg.groupExprs.size();
4711  }
4712  if (window != null) {
4713  return window.isAlwaysNonEmpty() ? 1 : 0;
4714  }
4715  return -1;
4716  }
4717 
4718  public RexBuilder getRexBuilder() {
4719  return rexBuilder;
4720  }
4721 
4722  public SqlNode validateExpression(RelDataType rowType, SqlNode expr) {
4723  return SqlValidatorUtil
4724  .validateExprWithRowType(catalogReader.nameMatcher().isCaseSensitive(),
4725  opTab,
4726  typeFactory,
4727  rowType,
4728  expr)
4729  .left;
4730  }
4731 
4732  public RexRangeRef getSubQueryExpr(SqlCall call) {
4733  final SubQuery subQuery = getSubQuery(call);
4734  assert subQuery != null;
4735  return (RexRangeRef) subQuery.expr;
4736  }
4737 
4738  public RelDataTypeFactory getTypeFactory() {
4739  return typeFactory;
4740  }
4741 
4742  public InitializerExpressionFactory getInitializerExpressionFactory() {
4743  return initializerExpressionFactory;
4744  }
4745 
4746  public SqlValidator getValidator() {
4747  return validator;
4748  }
4749 
4750  public RexNode convertLiteral(SqlLiteral literal) {
4751  return exprConverter.convertLiteral(this, literal);
4752  }
4753 
4754  public RexNode convertInterval(SqlIntervalQualifier intervalQualifier) {
4755  return exprConverter.convertInterval(this, intervalQualifier);
4756  }
4757 
4758  public RexNode visit(SqlLiteral literal) {
4759  return exprConverter.convertLiteral(this, literal);
4760  }
4761 
4762  public RexNode visit(SqlCall call) {
4763  if (agg != null) {
4764  final SqlOperator op = call.getOperator();
4765  if (window == null
4766  && (op.isAggregator() || op.getKind() == SqlKind.FILTER
4767  || op.getKind() == SqlKind.WITHIN_GROUP)) {
4768  return agg.lookupAggregates(call);
4769  }
4770  }
4771  return exprConverter.convertCall(
4772  this, new SqlCallBinding(validator, scope, call).permutedCall());
4773  }
4774 
4775  public RexNode visit(SqlNodeList nodeList) {
4776  throw new UnsupportedOperationException();
4777  }
4778 
4779  public RexNode visit(SqlIdentifier id) {
4780  return convertIdentifier(this, id);
4781  }
4782 
4783  public RexNode visit(SqlDataTypeSpec type) {
4784  throw new UnsupportedOperationException();
4785  }
4786 
4787  public RexNode visit(SqlDynamicParam param) {
4788  return convertDynamicParam(param);
4789  }
4790 
4791  public RexNode visit(SqlIntervalQualifier intervalQualifier) {
4792  return convertInterval(intervalQualifier);
4793  }
4794 
4795  public List<SqlMonotonicity> getColumnMonotonicities() {
4796  return columnMonotonicities;
4797  }
4798  }
4799 
4800  private SqlQuantifyOperator negate(SqlQuantifyOperator operator) {
4801  assert operator.kind == SqlKind.ALL;
4802  return SqlStdOperatorTable.some(operator.comparisonKind.negateNullSafe());
4803  }
4804 
4806  private static class DeferredLookup {
4809 
4810  DeferredLookup(Blackboard bb, String originalRelName) {
4811  this.bb = bb;
4812  this.originalRelName = originalRelName;
4813  }
4814 
4815  public RexFieldAccess getFieldAccess(CorrelationId name) {
4816  return (RexFieldAccess) bb.mapCorrelateToRex.get(name);
4817  }
4818 
4819  public String getOriginalRelName() {
4820  return originalRelName;
4821  }
4822  }
4823 
4827  private class NoOpSubQueryConverter implements SubQueryConverter {
4828  public boolean canConvertSubQuery() {
4829  return false;
4830  }
4831 
4832  public RexNode convertSubQuery(SqlCall subQuery,
4833  SqlToRelConverter parentConverter,
4834  boolean isExists,
4835  boolean isExplain) {
4836  throw new IllegalArgumentException();
4837  }
4838  }
4839 
4859  protected class AggConverter implements SqlVisitor<Void> {
4860  private final Blackboard bb;
4861  public final AggregatingSelectScope aggregatingSelectScope;
4862 
4863  private final Map<String, String> nameMap = new HashMap<>();
4864 
4868  private final SqlNodeList groupExprs = new SqlNodeList(SqlParserPos.ZERO);
4869 
4873  private final Map<SqlNode, Ord<AuxiliaryConverter>> auxiliaryGroupExprs =
4874  new HashMap<>();
4875 
4883  private final List<Pair<RexNode, String>> convertedInputExprs = new ArrayList<>();
4884 
4891  private final List<AggregateCall> aggCalls = new ArrayList<>();
4892  private final Map<SqlNode, RexNode> aggMapping = new HashMap<>();
4893  private final Map<AggregateCall, RexNode> aggCallMapping = new HashMap<>();
4894 
4896  private boolean inOver = false;
4897 
4908  public AggConverter(Blackboard bb, SqlSelect select) {
4909  this.bb = bb;
4910  this.aggregatingSelectScope =
4911  (AggregatingSelectScope) bb.getValidator().getSelectScope(select);
4912 
4913  // Collect all expressions used in the select list so that aggregate
4914  // calls can be named correctly.
4915  final SqlNodeList selectList = select.getSelectList();
4916  for (int i = 0; i < selectList.size(); i++) {
4917  SqlNode selectItem = selectList.get(i);
4918  String name = null;
4919  if (SqlUtil.isCallTo(selectItem, SqlStdOperatorTable.AS)) {
4920  final SqlCall call = (SqlCall) selectItem;
4921  selectItem = call.operand(0);
4922  name = call.operand(1).toString();
4923  }
4924  if (name == null) {
4925  name = validator.deriveAlias(selectItem, i);
4926  }
4927  nameMap.put(selectItem.toString(), name);
4928  }
4929  }
4930 
4931  public int addGroupExpr(SqlNode expr) {
4932  int ref = lookupGroupExpr(expr);
4933  if (ref >= 0) {
4934  return ref;
4935  }
4936  final int index = groupExprs.size();
4937  groupExprs.add(expr);
4938  String name = nameMap.get(expr.toString());
4939  RexNode convExpr = bb.convertExpression(expr);
4940  addExpr(convExpr, name);
4941 
4942  if (expr instanceof SqlCall) {
4943  SqlCall call = (SqlCall) expr;
4944  for (Pair<SqlNode, AuxiliaryConverter> p :
4945  SqlStdOperatorTable.convertGroupToAuxiliaryCalls(call)) {
4946  addAuxiliaryGroupExpr(p.left, index, p.right);
4947  }
4948  }
4949 
4950  return index;
4951  }
4952 
4953  void addAuxiliaryGroupExpr(SqlNode node, int index, AuxiliaryConverter converter) {
4954  for (SqlNode node2 : auxiliaryGroupExprs.keySet()) {
4955  if (node2.equalsDeep(node, Litmus.IGNORE)) {
4956  return;
4957  }
4958  }
4959  auxiliaryGroupExprs.put(node, Ord.of(index, converter));
4960  }
4961 
4968  private void addExpr(RexNode expr, String name) {
4969  if ((name == null) && (expr instanceof RexInputRef)) {
4970  final int i = ((RexInputRef) expr).getIndex();
4971  name = bb.root.getRowType().getFieldList().get(i).getName();
4972  }
4973  if (Pair.right(convertedInputExprs).contains(name)) {
4974  // In case like 'SELECT ... GROUP BY x, y, x', don't add
4975  // name 'x' twice.
4976  name = null;
4977  }
4978  convertedInputExprs.add(Pair.of(expr, name));
4979  }
4980 
4981  public Void visit(SqlIdentifier id) {
4982  return null;
4983  }
4984 
4985  public Void visit(SqlNodeList nodeList) {
4986  for (int i = 0; i < nodeList.size(); i++) {
4987  nodeList.get(i).accept(this);
4988  }
4989  return null;
4990  }
4991 
4992  public Void visit(SqlLiteral lit) {
4993  return null;
4994  }
4995 
4996  public Void visit(SqlDataTypeSpec type) {
4997  return null;
4998  }
4999 
5000  public Void visit(SqlDynamicParam param) {
5001  return null;
5002  }
5003 
5004  public Void visit(SqlIntervalQualifier intervalQualifier) {
5005  return null;
5006  }
5007 
5008  public Void visit(SqlCall call) {
5009  switch (call.getKind()) {
5010  case FILTER:
5011  case WITHIN_GROUP:
5012  translateAgg(call);
5013  return null;
5014  case SELECT:
5015  // rchen 2006-10-17:
5016  // for now do not detect aggregates in sub-queries.
5017  return null;
5018  }
5019  final boolean prevInOver = inOver;
5020  // Ignore window aggregates and ranking functions (associated with OVER
5021  // operator). However, do not ignore nested window aggregates.
5022  if (call.getOperator().getKind() == SqlKind.OVER) {
5023  // Track aggregate nesting levels only within an OVER operator.
5024  List<SqlNode> operandList = call.getOperandList();
5025  assert operandList.size() == 2;
5026 
5027  // Ignore the top level window aggregates and ranking functions
5028  // positioned as the first operand of a OVER operator
5029  inOver = true;
5030  operandList.get(0).accept(this);
5031 
5032  // Normal translation for the second operand of a OVER operator
5033  inOver = false;
5034  operandList.get(1).accept(this);
5035  return null;
5036  }
5037 
5038  // Do not translate the top level window aggregate. Only do so for
5039  // nested aggregates, if present
5040  if (call.getOperator().isAggregator()) {
5041  if (inOver) {
5042  // Add the parent aggregate level before visiting its children
5043  inOver = false;
5044  } else {
5045  // We're beyond the one ignored level
5046  translateAgg(call);
5047  return null;
5048  }
5049  }
5050  for (SqlNode operand : call.getOperandList()) {
5051  // Operands are occasionally null, e.g. switched CASE arg 0.
5052  if (operand != null) {
5053  operand.accept(this);
5054  }
5055  }
5056  // Remove the parent aggregate level after visiting its children
5057  inOver = prevInOver;
5058  return null;
5059  }
5060 
5061  private void translateAgg(SqlCall call) {
5062  translateAgg(call, null, null, false, call);
5063  }
5064 
5065  private void translateAgg(SqlCall call,
5066  SqlNode filter,
5067  SqlNodeList orderList,
5068  boolean ignoreNulls,
5069  SqlCall outerCall) {
5070  assert bb.agg == this;
5071  assert outerCall != null;
5072  switch (call.getKind()) {
5073  case FILTER:
5074  assert filter == null;
5075  translateAgg(
5076  call.operand(0), call.operand(1), orderList, ignoreNulls, outerCall);
5077  return;
5078  case WITHIN_GROUP:
5079  assert orderList == null;
5080  translateAgg(call.operand(0), filter, call.operand(1), ignoreNulls, outerCall);
5081  return;
5082  case IGNORE_NULLS:
5083  ignoreNulls = true;
5084  // fall through
5085  case RESPECT_NULLS:
5086  translateAgg(call.operand(0), filter, orderList, ignoreNulls, outerCall);
5087  return;
5088  }
5089  final List<Integer> args = new ArrayList<>();
5090  int filterArg = -1;
5091  final List<RelDataType> argTypes = call.getOperator() instanceof SqlCountAggFunction
5092  ? new ArrayList<>(call.getOperandList().size())
5093  : null;
5094  try {
5095  // switch out of agg mode
5096  bb.agg = null;
5097  for (SqlNode operand : call.getOperandList()) {
5098  // special case for COUNT(*): delete the *
5099  if (operand instanceof SqlIdentifier) {
5100  SqlIdentifier id = (SqlIdentifier) operand;
5101  if (id.isStar()) {
5102  assert call.operandCount() == 1;
5103  assert args.isEmpty();
5104  break;
5105  }
5106  }
5107  RexNode convertedExpr = bb.convertExpression(operand);
5108  assert convertedExpr != null;
5109  if (argTypes != null) {
5110  argTypes.add(convertedExpr.getType());
5111  }
5112  args.add(lookupOrCreateGroupExpr(convertedExpr));
5113  }
5114 
5115  if (filter != null) {
5116  RexNode convertedExpr = bb.convertExpression(filter);
5117  assert convertedExpr != null;
5118  if (convertedExpr.getType().isNullable()) {
5119  convertedExpr =
5120  rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, convertedExpr);
5121  }
5122  filterArg = lookupOrCreateGroupExpr(convertedExpr);
5123  }
5124  } finally {
5125  // switch back into agg mode
5126  bb.agg = this;
5127  }
5128 
5129  SqlAggFunction aggFunction = (SqlAggFunction) call.getOperator();
5130  final RelDataType type = validator.deriveType(bb.scope, call);
5131  boolean distinct = false;
5132  SqlLiteral quantifier = call.getFunctionQuantifier();
5133  if ((null != quantifier) && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) {
5134  distinct = true;
5135  }
5136  boolean approximate = false;
5137  if (aggFunction == SqlStdOperatorTable.APPROX_COUNT_DISTINCT) {
5138  aggFunction = SqlStdOperatorTable.COUNT;
5139  distinct = true;
5140  approximate = true;
5141  }
5142  final RelCollation collation;
5143  if (orderList == null || orderList.size() == 0) {
5144  collation = RelCollations.EMPTY;
5145  } else {
5146  collation = RelCollations.of(
5147  orderList.getList()
5148  .stream()
5149  .map(order
5150  -> bb.convertSortExpression(order,
5151  RelFieldCollation.Direction.ASCENDING,
5152  RelFieldCollation.NullDirection.UNSPECIFIED))
5153  .map(fieldCollation
5154  -> new RelFieldCollation(
5155  lookupOrCreateGroupExpr(fieldCollation.left),
5156  fieldCollation.getDirection(),
5157  fieldCollation.getNullDirection()))
5158  .collect(Collectors.toList()));
5159  }
5160  final AggregateCall aggCall = AggregateCall.create(aggFunction,
5161  distinct,
5162  approximate,
5163  ignoreNulls,
5164  args,
5165  filterArg,
5166  collation,
5167  type,
5168  nameMap.get(outerCall.toString()));
5169  RexNode rex = rexBuilder.addAggCall(
5170  aggCall, groupExprs.size(), aggCalls, aggCallMapping, argTypes);
5171  aggMapping.put(outerCall, rex);
5172  }
5173 
5174  private int lookupOrCreateGroupExpr(RexNode expr) {
5175  int index = 0;
5176  for (RexNode convertedInputExpr : Pair.left(convertedInputExprs)) {
5177  if (expr.equals(convertedInputExpr)) {
5178  return index;
5179  }
5180  ++index;
5181  }
5182 
5183  // not found -- add it
5184  addExpr(expr, null);
5185  return index;
5186  }
5187 
5192  public int lookupGroupExpr(SqlNode expr) {
5193  for (int i = 0; i < groupExprs.size(); i++) {
5194  SqlNode groupExpr = groupExprs.get(i);
5195  if (expr.equalsDeep(groupExpr, Litmus.IGNORE)) {
5196  return i;
5197  }
5198  }
5199  return -1;
5200  }
5201 
5202  public RexNode lookupAggregates(SqlCall call) {
5203  // assert call.getOperator().isAggregator();
5204  assert bb.agg == this;
5205 
5206  for (Map.Entry<SqlNode, Ord<AuxiliaryConverter>> e :
5207  auxiliaryGroupExprs.entrySet()) {
5208  if (call.equalsDeep(e.getKey(), Litmus.IGNORE)) {
5209  AuxiliaryConverter converter = e.getValue().e;
5210  final int groupOrdinal = e.getValue().i;
5211  return converter.convert(rexBuilder,
5212  convertedInputExprs.get(groupOrdinal).left,
5213  rexBuilder.makeInputRef(bb.root, groupOrdinal));
5214  }
5215  }
5216 
5217  return aggMapping.get(call);
5218  }
5219 
5220  public List<Pair<RexNode, String>> getPreExprs() {
5221  return convertedInputExprs;
5222  }
5223 
5224  public List<AggregateCall> getAggCalls() {
5225  return aggCalls;
5226  }
5227 
5228  private boolean containsGroupId() {
5229  return aggCalls.stream().anyMatch(
5230  agg -> agg.getAggregation().kind == SqlKind.GROUP_ID);
5231  }
5232 
5233  public RelDataTypeFactory getTypeFactory() {
5234  return typeFactory;
5235  }
5236  }
5237 
5241  private static class LookupContext {
5242  private final List<Pair<RelNode, Integer>> relOffsetList = new ArrayList<>();
5243 
5251  LookupContext(Blackboard bb, List<RelNode> rels, int systemFieldCount) {
5252  bb.flatten(rels, systemFieldCount, new int[] {0}, relOffsetList);
5253  }
5254 
5267  Pair<RelNode, Integer> findRel(int offset) {
5268  return relOffsetList.get(offset);
5269  }
5270  }
5271 
5304  private class HistogramShuttle extends RexShuttle {
5309  static final boolean ENABLE_HISTOGRAM_AGG = false;
5310 
5311  private final List<RexNode> partitionKeys;
5312  private final ImmutableList<RexFieldCollation> orderKeys;
5313  private final RexWindowBound lowerBound;
5314  private final RexWindowBound upperBound;
5315  private final boolean rows;
5316  private final boolean allowPartial;
5317  private final boolean distinct;
5318  private final boolean ignoreNulls;
5319 
5320  HistogramShuttle(List<RexNode> partitionKeys,
5321  ImmutableList<RexFieldCollation> orderKeys,
5322  RexWindowBound lowerBound,
5323  RexWindowBound upperBound,
5324  boolean rows,
5325  boolean allowPartial,
5326  boolean distinct,
5327  boolean ignoreNulls) {
5328  this.partitionKeys = partitionKeys;
5329  this.orderKeys = orderKeys;
5330  this.lowerBound = lowerBound;
5331  this.upperBound = upperBound;
5332  this.rows = rows;
5333  this.allowPartial = allowPartial;
5334  this.distinct = distinct;
5335  this.ignoreNulls = ignoreNulls;
5336  }
5337 
5338  public RexNode visitCall(RexCall call) {
5339  final SqlOperator op = call.getOperator();
5340  if (!(op instanceof SqlAggFunction)) {
5341  return super.visitCall(call);
5342  }
5343  final SqlAggFunction aggOp = (SqlAggFunction) op;
5344  final RelDataType type = call.getType();
5345  List<RexNode> exprs = call.getOperands();
5346 
5347  SqlFunction histogramOp = !ENABLE_HISTOGRAM_AGG ? null : getHistogramOp(aggOp);
5348 
5349  if (histogramOp != null) {
5350  final RelDataType histogramType = computeHistogramType(type);
5351 
5352  // For DECIMAL, since it's already represented as a bigint we
5353  // want to do a reinterpretCast instead of a cast to avoid
5354  // losing any precision.
5355  boolean reinterpretCast = type.getSqlTypeName() == SqlTypeName.DECIMAL;
5356 
5357  // Replace original expression with CAST of not one
5358  // of the supported types
5359  if (histogramType != type) {
5360  exprs = new ArrayList<>(exprs);
5361  exprs.set(0,
5362  reinterpretCast ? rexBuilder.makeReinterpretCast(
5363  histogramType, exprs.get(0), rexBuilder.makeLiteral(false))
5364  : rexBuilder.makeCast(histogramType, exprs.get(0)));
5365  }
5366 
5367  RexCallBinding bind = new RexCallBinding(rexBuilder.getTypeFactory(),
5368  SqlStdOperatorTable.HISTOGRAM_AGG,
5369  exprs,
5370  ImmutableList.of());
5371 
5372  RexNode over = rexBuilder.makeOver(
5373  SqlStdOperatorTable.HISTOGRAM_AGG.inferReturnType(bind),
5374  SqlStdOperatorTable.HISTOGRAM_AGG,
5375  exprs,
5376  partitionKeys,
5377  orderKeys,
5378  lowerBound,
5379  upperBound,
5380  rows,
5381  allowPartial,
5382  false,
5383  distinct,
5384  ignoreNulls);
5385 
5386  RexNode histogramCall =
5387  rexBuilder.makeCall(histogramType, histogramOp, ImmutableList.of(over));
5388 
5389  // If needed, post Cast result back to original
5390  // type.
5391  if (histogramType != type) {
5392  if (reinterpretCast) {
5393  histogramCall = rexBuilder.makeReinterpretCast(
5394  type, histogramCall, rexBuilder.makeLiteral(false));
5395  } else {
5396  histogramCall = rexBuilder.makeCast(type, histogramCall);
5397  }
5398  }
5399 
5400  return histogramCall;
5401  } else {
5402  boolean needSum0 = aggOp == SqlStdOperatorTable.SUM && type.isNullable();
5403  SqlAggFunction aggOpToUse = needSum0 ? SqlStdOperatorTable.SUM0 : aggOp;
5404  return rexBuilder.makeOver(type,
5405  aggOpToUse,
5406  exprs,
5407  partitionKeys,
5408  orderKeys,
5409  lowerBound,
5410  upperBound,
5411  rows,
5412  allowPartial,
5413  needSum0,
5414  distinct,
5415  ignoreNulls);
5416  }
5417  }
5418 
5430  SqlFunction getHistogramOp(SqlAggFunction aggFunction) {
5431  if (aggFunction == SqlStdOperatorTable.MIN) {
5432  return SqlStdOperatorTable.HISTOGRAM_MIN;
5433  } else if (aggFunction == SqlStdOperatorTable.MAX) {
5434  return SqlStdOperatorTable.HISTOGRAM_MAX;
5435  } else if (aggFunction == SqlStdOperatorTable.FIRST_VALUE) {
5436  return SqlStdOperatorTable.HISTOGRAM_FIRST_VALUE;
5437  } else if (aggFunction == SqlStdOperatorTable.LAST_VALUE) {
5438  return SqlStdOperatorTable.HISTOGRAM_LAST_VALUE;
5439  } else {
5440  return null;
5441  }
5442  }
5443 
5448  private RelDataType computeHistogramType(RelDataType type) {
5449  if (SqlTypeUtil.isExactNumeric(type)
5450  && type.getSqlTypeName() != SqlTypeName.BIGINT) {
5451  return typeFactory.createSqlType(SqlTypeName.BIGINT);
5452  } else if (SqlTypeUtil.isApproximateNumeric(type)
5453  && type.getSqlTypeName() != SqlTypeName.DOUBLE) {
5454  return typeFactory.createSqlType(SqlTypeName.DOUBLE);
5455  } else {
5456  return type;
5457  }
5458  }
5459  }
5460 
5464  private static class SubQuery {
5465  final SqlNode node;
5466  final RelOptUtil.Logic logic;
5467  RexNode expr;
5468 
5469  private SubQuery(SqlNode node, RelOptUtil.Logic logic) {
5470  this.node = node;
5471  this.logic = logic;
5472  }
5473  }
5474 
5478  private static class AggregateFinder extends SqlBasicVisitor<Void> {
5479  final SqlNodeList list = new SqlNodeList(SqlParserPos.ZERO);
5480  final SqlNodeList filterList = new SqlNodeList(SqlParserPos.ZERO);
5481  final SqlNodeList orderList = new SqlNodeList(SqlParserPos.ZERO);
5482 
5483  @Override
5484  public Void visit(SqlCall call) {
5485  // ignore window aggregates and ranking functions (associated with OVER
5486  // operator)
5487  if (call.getOperator().getKind() == SqlKind.OVER) {
5488  return null;
5489  }
5490 
5491  if (call.getOperator().getKind() == SqlKind.FILTER) {
5492  // the WHERE in a FILTER must be tracked too so we can call replaceSubQueries on
5493  // it.
5494  // see https://issues.apache.org/jira/browse/CALCITE-1910
5495  final SqlNode aggCall = call.getOperandList().get(0);
5496  final SqlNode whereCall = call.getOperandList().get(1);
5497  list.add(aggCall);
5498  filterList.add(whereCall);
5499  return null;
5500  }
5501 
5502  if (call.getOperator().getKind() == SqlKind.WITHIN_GROUP) {
5503  // the WHERE in a WITHIN_GROUP must be tracked too so we can call
5504  // replaceSubQueries on it.
5505  // see https://issues.apache.org/jira/browse/CALCITE-1910
5506  final SqlNode aggCall = call.getOperandList().get(0);
5507  final SqlNodeList orderList = (SqlNodeList) call.getOperandList().get(1);
5508  list.add(aggCall);
5509  orderList.getList().forEach(this.orderList::add);
5510  return null;
5511  }
5512 
5513  if (call.getOperator().isAggregator()) {
5514  list.add(call);
5515  return null;
5516  }
5517 
5518  // Don't traverse into sub-queries, even if they contain aggregate
5519  // functions.
5520  if (call instanceof SqlSelect) {
5521  return null;
5522  }
5523 
5524  return call.getOperator().acceptCall(this, call);
5525  }
5526  }
5527 
5531  private static class CorrelationUse {
5532  private final CorrelationId id;
5533  private final ImmutableBitSet requiredColumns;
5535  private final RelNode r;
5536 
5537  CorrelationUse(CorrelationId id, ImmutableBitSet requiredColumns, RelNode r) {
5538  this.id = id;
5539  this.requiredColumns = requiredColumns;
5540  this.r = r;
5541  }
5542  }
5543 
5545  public static ConfigBuilder configBuilder() {
5546  return new ConfigBuilder();
5547  }
5548 
5556  public interface Config {
5558  Config DEFAULT = configBuilder().build();
5559 
5564  boolean isDecorrelationEnabled();
5565 
5570  boolean isTrimUnusedFields();
5571 
5577  boolean isCreateValuesRel();
5578 
5583  boolean isExplain();
5584 
5589  boolean isExpand();
5590 
5600  int getInSubQueryThreshold();
5601 
5606  RelBuilderFactory getRelBuilderFactory();
5607 
5612  UnaryOperator<RelBuilder.Config> getRelBuilderConfigTransform();
5613 
5618  HintStrategyTable getHintStrategyTable();
5619 
5626  BiPredicate<SqlNode, SqlNode> getExpandPredicate();
5627  }
5628 
5630  public static class ConfigBuilder {
5631  private boolean decorrelationEnabled = true;
5632  private boolean trimUnusedFields = false;
5633  private boolean createValuesRel = true;
5634  private boolean explain;
5635  private boolean expand = true;
5636  private int inSubQueryThreshold = DEFAULT_IN_SUB_QUERY_THRESHOLD;
5637  private UnaryOperator<RelBuilder.Config> relBuilderConfigTransform =
5638  c -> c.withPushJoinCondition(true);
5639  private RelBuilderFactory relBuilderFactory = RelFactories.LOGICAL_BUILDER;
5640  private HintStrategyTable hintStrategyTable = HintStrategyTable.EMPTY;
5641  // MAT 08 Jan 2021 HEAVY.AI code
5642  // pushdown join condition incurs an extra projection overhead, so remove it (21 Apr
5643  // 2021)
5644  private BiPredicate<SqlNode, SqlNode> expandPredicate;
5645  // MAT 08 Jan 2021 HEAVY.AI code ends
5646 
5647  private ConfigBuilder() {}
5648 
5650  public ConfigBuilder withConfig(Config config) {
5651  this.decorrelationEnabled = config.isDecorrelationEnabled();
5652  this.trimUnusedFields = config.isTrimUnusedFields();
5653  this.createValuesRel = config.isCreateValuesRel();
5654  this.explain = config.isExplain();
5655  this.expand = config.isExpand();
5656  this.inSubQueryThreshold = config.getInSubQueryThreshold();
5657  this.relBuilderConfigTransform = config.getRelBuilderConfigTransform();
5658  this.relBuilderFactory = config.getRelBuilderFactory();
5659  this.hintStrategyTable = config.getHintStrategyTable();
5660 
5661  // MAT 08 Jan 2021 HEAVY.AI code
5662  if (!(config.getExpandPredicate() instanceof ConfigImpl.DefaultExpandPredicate)) {
5663  this.expandPredicate = config.getExpandPredicate();
5664  }
5665  // MAT 08 Jan 2021 HEAVY.AI code ends
5666  return this;
5667  }
5668 
5669  public ConfigBuilder withDecorrelationEnabled(boolean enabled) {
5670  this.decorrelationEnabled = enabled;
5671  return this;
5672  }
5673 
5674  public ConfigBuilder withTrimUnusedFields(boolean trimUnusedFields) {
5675  this.trimUnusedFields = trimUnusedFields;
5676  return this;
5677  }
5678 
5679  public ConfigBuilder withCreateValuesRel(boolean createValuesRel) {
5680  this.createValuesRel = createValuesRel;
5681  return this;
5682  }
5683 
5684  public ConfigBuilder withExplain(boolean explain) {
5685  this.explain = explain;
5686  return this;
5687  }
5688 
5689  public ConfigBuilder withExpand(boolean expand) {
5690  this.expand = expand;
5691  return this;
5692  }
5693 
5694  public ConfigBuilder withExpandPredicate(BiPredicate<SqlNode, SqlNode> predicate) {
5695  this.expandPredicate = predicate;
5696  return this;
5697  }
5698 
5700  public ConfigBuilder withPushJoinCondition(boolean pushJoinCondition) {
5701  return withRelBuilderConfigTransform(Util.andThen(relBuilderConfigTransform,
5702  c -> c.withPushJoinCondition(pushJoinCondition)));
5703  }
5704 
5705  @Deprecated // to be removed before 2.0
5706  public ConfigBuilder withInSubqueryThreshold(int inSubQueryThreshold) {
5707  return withInSubQueryThreshold(inSubQueryThreshold);
5708  }
5709 
5710  public ConfigBuilder withInSubQueryThreshold(int inSubQueryThreshold) {
5711  this.inSubQueryThreshold = inSubQueryThreshold;
5712  return this;
5713  }
5714 
5715  public ConfigBuilder withRelBuilderConfigTransform(
5716  UnaryOperator<RelBuilder.Config> relBuilderConfigTransform) {
5717  this.relBuilderConfigTransform = relBuilderConfigTransform;
5718  return this;
5719  }
5720 
5721  public ConfigBuilder withRelBuilderFactory(RelBuilderFactory relBuilderFactory) {
5722  this.relBuilderFactory = relBuilderFactory;
5723  return this;
5724  }
5725 
5726  public ConfigBuilder withHintStrategyTable(HintStrategyTable hintStrategyTable) {
5727  this.hintStrategyTable = hintStrategyTable;
5728  return this;
5729  }
5730 
5732  public Config build() {
5733  return new ConfigImpl(decorrelationEnabled,
5734  trimUnusedFields,
5735  createValuesRel,
5736  explain,
5737  expand,
5738  expandPredicate,
5739  inSubQueryThreshold,
5740  relBuilderConfigTransform,
5741  relBuilderFactory,
5742  hintStrategyTable);
5743  }
5744  }
5745 
5750  private static class ConfigImpl implements Config {
5751  private final boolean decorrelationEnabled;
5752  private final boolean trimUnusedFields;
5753  private final boolean createValuesRel;
5754  private final boolean explain;
5755  private final boolean expand;
5756  private final int inSubQueryThreshold;
5757  private final UnaryOperator<RelBuilder.Config> relBuilderConfigTransform;
5758  private final RelBuilderFactory relBuilderFactory;
5759  private final HintStrategyTable hintStrategyTable;
5760 
5761  private final BiPredicate<SqlNode, SqlNode> expandPredicate;
5762 
5763  // MAT 08 Jab 2021 HEAVY.AI code
5764  private class DefaultExpandPredicate implements BiPredicate<SqlNode, SqlNode> {
5765  @Override
5766  public boolean test(SqlNode t, SqlNode u) {
5767  return expand;
5768  }
5769  }
5770 
5771  // MAT 08 Jan 2021 HEAVY.AI code ends
5772 
5773  private ConfigImpl(boolean decorrelationEnabled,
5774  boolean trimUnusedFields,
5775  boolean createValuesRel,
5776  boolean explain,
5777  boolean expand,
5778  // MAT 08 Jan 2021 HEAVY.AI code
5779  BiPredicate<SqlNode, SqlNode> expandPredicate,
5780  // MAT 08 Jan 2021 HEAVY.AI code ends
5781  int inSubQueryThreshold,
5782  UnaryOperator<RelBuilder.Config> relBuilderConfigTransform,
5783  RelBuilderFactory relBuilderFactory,
5784  HintStrategyTable hintStrategyTable) {
5785  this.decorrelationEnabled = decorrelationEnabled;
5786  this.trimUnusedFields = trimUnusedFields;
5787  this.createValuesRel = createValuesRel;
5788  // MAT 08 Jan 2021 HEAVY.AI code
5789  this.explain = explain;
5790  this.expand = expand;
5791  // MAT 08 Jan 2021 HEAVY.AI code ends
5792  this.inSubQueryThreshold = inSubQueryThreshold;
5793  this.relBuilderConfigTransform = relBuilderConfigTransform;
5794  this.relBuilderFactory = relBuilderFactory;
5795  this.hintStrategyTable = hintStrategyTable;
5796  // MAT 08 Jan 2021 HEAVY.AI code
5797  if (null == expandPredicate) {
5798  expandPredicate = new DefaultExpandPredicate();
5799  }
5800  this.expandPredicate = expandPredicate;
5801 
5802  // MAT 08 Jan 2021 HEAVY.AI code ends
5803  }
5804 
5805  // TODO MAT 08 Jan 2021 add our added types to this equals
5806  @Override
5807  public boolean equals(Object obj) {
5808  return this == obj
5809  || obj instanceof ConfigImpl
5810  && decorrelationEnabled == ((ConfigImpl) obj).decorrelationEnabled
5811  && trimUnusedFields == ((ConfigImpl) obj).trimUnusedFields
5812  && createValuesRel == ((ConfigImpl) obj).createValuesRel
5813  && explain == ((ConfigImpl) obj).explain
5814  && expand == ((ConfigImpl) obj).expand
5815  && inSubQueryThreshold == ((ConfigImpl) obj).inSubQueryThreshold
5816  && relBuilderFactory == ((ConfigImpl) obj).relBuilderFactory
5817  && hintStrategyTable == ((ConfigImpl) obj).hintStrategyTable;
5818  }
5819 
5820  @Override
5821  public int hashCode() {
5822  return Objects.hash(decorrelationEnabled,
5823  trimUnusedFields,
5824  createValuesRel,
5825  explain,
5826  expand,
5827  inSubQueryThreshold,
5828  relBuilderFactory,
5829  hintStrategyTable);
5830  }
5831 
5832  public boolean isDecorrelationEnabled() {
5833  return decorrelationEnabled;
5834  }
5835 
5836  public boolean isTrimUnusedFields() {
5837  return trimUnusedFields;
5838  }
5839 
5840  public boolean isCreateValuesRel() {
5841  return createValuesRel;
5842  }
5843 
5844  public boolean isExplain() {
5845  return explain;
5846  }
5847 
5848  public boolean isExpand() {
5849  return expand;
5850  }
5851 
5852  public int getInSubQueryThreshold() {
5853  return inSubQueryThreshold;
5854  }
5855 
5856  public UnaryOperator<RelBuilder.Config> getRelBuilderConfigTransform() {
5857  return relBuilderConfigTransform;
5858  }
5859 
5860  public RelBuilderFactory getRelBuilderFactory() {
5861  return relBuilderFactory;
5862  }
5863 
5864  public HintStrategyTable getHintStrategyTable() {
5865  return hintStrategyTable;
5866  }
5867 
5868  // MAT 08 Jan 2021 HEAVY.AI code
5869  public BiPredicate<SqlNode, SqlNode> getExpandPredicate() {
5870  return expandPredicate;
5871  }
5872  // MAT 08 Jan 2021 HEAVY.AI code ends
5873  }
5874 }
RelRoot convertWith(SqlWith with, boolean top)
RelOptTable.ToRelContext createToRelContext(List< RelHint > hints)
SqlQuantifyOperator negate(SqlQuantifyOperator operator)
JoinType
Definition: sqldefs.h:165
void setDynamicParamCountInExplain(int explainParamCount)
static SqlNode pushDownNotForIn(SqlValidatorScope scope, SqlNode sqlNode)
void extraSelectItems(Blackboard bb, SqlSelect select, List< RexNode > exprList, List< String > nameList, Collection< String > aliasList, List< SqlMonotonicity > columnMonotonicityList)
void substituteSubQuery(Blackboard bb, SubQuery subQuery)
void convertCollectionTable(Blackboard bb, SqlCall call)
RexNode convertSubQuery(SqlCall subQuery, SqlToRelConverter parentConverter, boolean isExists, boolean isExplain)
void checkConvertedType(SqlNode query, RelNode result)
RexNode translateIn(RelOptUtil.Logic logic, RelNode root, final RexNode rex)
RelNode convertCursor(Blackboard bb, SubQuery subQuery)
RelNode convertQueryOrInList(Blackboard bb, SqlNode seek, RelDataType targetRowType)
std::string join(T const &container, std::string const &delim)
RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type)
RexNode ensureSqlType(RelDataType type, RexNode node)
tuple root
Definition: setup.in.py:14
void afterTableFunction(SqlToRelConverter.Blackboard bb, SqlCall call, LogicalTableFunctionScan callRel)
RelNode toRel(final RelOptTable table, final List< RelHint > hints)
std::pair< FILE *, std::string > create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:57
SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable)
void convertValuesImpl(Blackboard bb, SqlCall values, RelDataType targetRowType)
void convertFrom(Blackboard bb, SqlNode from)
InitializerExpressionFactory getInitializerFactory(SqlValidatorTable validatorTable)
String deriveAlias(final SqlNode node, Collection< String > aliases, final int ordinal)
constexpr double f
Definition: Utm.h:31
RelNode decorrelate(SqlNode query, RelNode rootRel)
void setRoot(List< RelNode > inputs, RelNode root, boolean hasSystemFields)
RexDynamicParam convertDynamicParam(final SqlDynamicParam dynamicParam)
RelNode convertRowValues(Blackboard bb, SqlNode rowList, Collection< SqlNode > rows, boolean allowLiteralsOnly, RelDataType targetRowType)
RexNode convertInToOr(final Blackboard bb, final List< RexNode > leftKeys, SqlNodeList valuesList, SqlInOperator op)
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31
final Map< CorrelationId, DeferredLookup > mapCorrelToDeferred
RelOptUtil.Exists convertExists(SqlNode seek, RelOptUtil.SubQueryType subQueryType, RelOptUtil.Logic logic, boolean notIn, RelDataType targetDataType)
RelNode createSource(RelOptTable targetTable, RelNode source, ModifiableView modifiableView, RelDataType delegateRowType)
RelNode convertRowConstructor(Blackboard bb, SqlCall rowConstructor)
RelRoot convertQueryRecursive(SqlNode query, boolean top, RelDataType targetRowType)
SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptPlanner planner, RexBuilder rexBuilder, SqlRexConvertletTable convertletTable)
final List< SqlDynamicParam > dynamicParamSqlNodes
final Map< SqlNode, RexNode > mapConvertedNonCorrSubqs
RexLiteral convertLiteralInValuesList(SqlNode sqlNode, Blackboard bb, RelDataType rowType, int iField)
void convertUnnest(Blackboard bb, SqlCall call, List< String > fieldNames)
void distinctify(Blackboard bb, boolean checkForDupExprs)
RelNode createModify(RelOptTable targetTable, RelNode source)
RelNode trimUnusedFields(boolean ordered, RelNode rootRel)
RexNode convertUsing(SqlValidatorNamespace leftNamespace, SqlValidatorNamespace rightNamespace, List< String > nameList)
void convertSelectImpl(final Blackboard bb, SqlSelect select)
std::string toString(const ExecutorDeviceType &device_type)
void addConvertedNonCorrSubqs(Map< SqlNode, RexNode > alreadyConvertedNonCorrSubqs)
RexNode convertJoinCondition(Blackboard bb, SqlValidatorNamespace leftNamespace, SqlValidatorNamespace rightNamespace, SqlNode condition, JoinConditionType conditionType, RelNode leftRel, RelNode rightRel)
void collectInsertTargets(SqlInsert call, final RexNode sourceRef, final List< String > targetColumnNames, List< RexNode > columnExprs)
void replaceSubQueries(final Blackboard bb, final SqlNode expr, RelOptUtil.Logic logic)
static< T > T unwrap(Object o, Class< T > clazz)
size_t indexOf(std::vector< T > &vec, T val)
SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable, Config config)
RelNode flattenTypes(RelNode rootRel, boolean restructure)
RelNode createAggregate(Blackboard bb, ImmutableBitSet groupSet, ImmutableList< ImmutableBitSet > groupSets, List< AggregateCall > aggCalls)
CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0)
RelNode convertValues(SqlCall values, RelDataType targetRowType)
void convertAgg(Blackboard bb, SqlSelect select, List< SqlNode > orderExprList)
RexNode convertOver(Blackboard bb, SqlNode node)
final void createAggImpl(Blackboard bb, final AggConverter aggConverter, SqlNodeList selectList, SqlNodeList groupList, SqlNode having, List< SqlNode > orderExprList)
Blackboard createInsertBlackboard(RelOptTable targetTable, RexNode sourceRef, List< String > targetColumnNames)
void convertIdentifier(Blackboard bb, SqlIdentifier id, SqlNodeList extendedColumns, SqlNodeList tableHints)
RelNode convertMultisets(final List< SqlNode > operands, Blackboard bb)
static boolean containsNullLiteral(SqlNodeList valueList)
void gatherOrderExprs(Blackboard bb, SqlSelect select, SqlNodeList orderList, List< SqlNode > extraOrderExprs, List< RelFieldCollation > collationList)
RexNode convertExtendedExpression(SqlNode node, Blackboard bb)
void convertMatchRecognize(Blackboard bb, SqlCall call)
void convertWhere(final Blackboard bb, final SqlNode where)
RexNode convertExpression(SqlNode node, Map< String, RexNode > nameToNodeMap)
Blackboard(SqlValidatorScope scope, Map< String, RexNode > nameToNodeMap, boolean top)
RelNode rewriteAggregateWithGroupId(Blackboard bb, AggregatingSelectScope.Resolved r, AggConverter converter)
static SqlNode reg(SqlValidatorScope scope, SqlNode e)
boolean convertNonCorrelatedSubQuery(SubQuery subQuery, Blackboard bb, RelNode converted, boolean isExists)
void findSubQueries(Blackboard bb, SqlNode node, RelOptUtil.Logic logic, boolean registerOnlyScalarSubQueries)
Pair< RexNode, Map< String, Integer > > lookupExp(SqlQualified qualified)
boolean isSubQueryNonCorrelated(RelNode subq, Blackboard bb)
RexNode convertIdentifier(Blackboard bb, SqlIdentifier identifier)
RelNode createJoin(Blackboard bb, RelNode leftRel, RelNode rightRel, RexNode joinCond, JoinRelType joinType)
RexAccessShuttle(RexBuilder builder, RexCorrelVariable rexCorrel)
RelNode convertColumnList(final SqlInsert call, RelNode source)
void convertFrom(Blackboard bb, SqlNode from, List< String > fieldNames)
void convertTemporalTable(Blackboard bb, SqlCall call)
static boolean desc(RelFieldCollation.Direction direction)
string name
Definition: setup.in.py:72
constexpr double n
Definition: Utm.h:38
Set< RelColumnMapping > getColumnMappings(SqlOperator op)
Blackboard createBlackboard(SqlValidatorScope scope, Map< String, RexNode > nameToNodeMap, boolean top)
RelRoot convertQuery(SqlNode query, final boolean needsValidation, final boolean top)
RelFieldCollation convertOrderItem(SqlSelect select, SqlNode orderItem, List< SqlNode > extraExprs, RelFieldCollation.Direction direction, RelFieldCollation.NullDirection nullDirection)
final Map< RelNode, Map< Integer, Integer > > mapRootRelToFieldProjection
RelNode convertToSingleValueSubq(SqlNode query, RelNode plan)
static JoinRelType convertJoinType(JoinType joinType)
void setSubQueryConverter(SubQueryConverter converter)
RexNode adjustInputRef(Blackboard bb, RexInputRef inputRef)
void convertOrder(SqlSelect select, Blackboard bb, RelCollation collation, List< SqlNode > orderExprList, SqlNode offset, SqlNode fetch)
void convertSelectList(Blackboard bb, SqlSelect select, List< SqlNode > orderList)
RelNode convertSelect(SqlSelect select, boolean top)