OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
SqlToRelConverter.java
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one or more
3  * contributor license agreements. See the NOTICE file distributed with
4  * this work for additional information regarding copyright ownership.
5  * The ASF licenses this file to you under the Apache License, Version 2.0
6  * (the "License"); you may not use this file except in compliance with
7  * the License. You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 /* MAT 9 Jan 2021
19  * We have moved this code from calcite 1.25
20  * we keep our own copy as we have added some
21  * special handling of pushdown and correlation handling
22  *
23  * If we make changes from the original source please make
24  * sure you add good comments around the changed block
25  * as each time we move to a new calcite version we need to move
26  * our chnages to the new code
27  *
28  */
29 package org.apache.calcite.sql2rel;
30 
31 import static org.apache.calcite.sql.SqlUtil.stripAs;
32 
33 import com.google.common.base.Preconditions;
34 import com.google.common.collect.ImmutableList;
35 import com.google.common.collect.ImmutableList.Builder;
36 import com.google.common.collect.ImmutableMap;
37 import com.google.common.collect.ImmutableSet;
38 import com.google.common.collect.Iterables;
39 import com.google.common.collect.Sets;
41 
42 import org.apache.calcite.avatica.util.Spaces;
43 import org.apache.calcite.linq4j.Ord;
44 import org.apache.calcite.plan.Convention;
45 import org.apache.calcite.plan.RelOptCluster;
46 import org.apache.calcite.plan.RelOptPlanner;
47 import org.apache.calcite.plan.RelOptSamplingParameters;
48 import org.apache.calcite.plan.RelOptTable;
49 import org.apache.calcite.plan.RelOptUtil;
50 import org.apache.calcite.plan.RelTraitSet;
51 import org.apache.calcite.plan.ViewExpanders;
52 import org.apache.calcite.prepare.Prepare;
53 import org.apache.calcite.prepare.RelOptTableImpl;
54 import org.apache.calcite.rel.RelCollation;
55 import org.apache.calcite.rel.RelCollationTraitDef;
56 import org.apache.calcite.rel.RelCollations;
57 import org.apache.calcite.rel.RelFieldCollation;
58 import org.apache.calcite.rel.RelNode;
59 import org.apache.calcite.rel.RelRoot;
60 import org.apache.calcite.rel.RelShuttleImpl;
61 import org.apache.calcite.rel.SingleRel;
62 import org.apache.calcite.rel.core.AggregateCall;
63 import org.apache.calcite.rel.core.Collect;
64 import org.apache.calcite.rel.core.CorrelationId;
65 import org.apache.calcite.rel.core.Filter;
66 import org.apache.calcite.rel.core.Join;
67 import org.apache.calcite.rel.core.JoinInfo;
68 import org.apache.calcite.rel.core.JoinRelType;
69 import org.apache.calcite.rel.core.Project;
70 import org.apache.calcite.rel.core.RelFactories;
71 import org.apache.calcite.rel.core.Sample;
72 import org.apache.calcite.rel.core.Sort;
73 import org.apache.calcite.rel.hint.HintStrategyTable;
74 import org.apache.calcite.rel.hint.Hintable;
75 import org.apache.calcite.rel.hint.RelHint;
76 import org.apache.calcite.rel.logical.LogicalAggregate;
77 import org.apache.calcite.rel.logical.LogicalCorrelate;
78 import org.apache.calcite.rel.logical.LogicalFilter;
79 import org.apache.calcite.rel.logical.LogicalIntersect;
80 import org.apache.calcite.rel.logical.LogicalJoin;
81 import org.apache.calcite.rel.logical.LogicalMatch;
82 import org.apache.calcite.rel.logical.LogicalMinus;
83 import org.apache.calcite.rel.logical.LogicalProject;
84 import org.apache.calcite.rel.logical.LogicalSort;
85 import org.apache.calcite.rel.logical.LogicalTableFunctionScan;
86 import org.apache.calcite.rel.logical.LogicalTableModify;
87 import org.apache.calcite.rel.logical.LogicalTableScan;
88 import org.apache.calcite.rel.logical.LogicalUnion;
89 import org.apache.calcite.rel.logical.LogicalValues;
90 import org.apache.calcite.rel.metadata.RelColumnMapping;
91 import org.apache.calcite.rel.metadata.RelMetadataQuery;
92 import org.apache.calcite.rel.stream.Delta;
93 import org.apache.calcite.rel.stream.LogicalDelta;
94 import org.apache.calcite.rel.type.RelDataType;
95 import org.apache.calcite.rel.type.RelDataTypeFactory;
96 import org.apache.calcite.rel.type.RelDataTypeField;
97 import org.apache.calcite.rex.RexBuilder;
98 import org.apache.calcite.rex.RexCall;
99 import org.apache.calcite.rex.RexCallBinding;
100 import org.apache.calcite.rex.RexCorrelVariable;
101 import org.apache.calcite.rex.RexDynamicParam;
102 import org.apache.calcite.rex.RexFieldAccess;
103 import org.apache.calcite.rex.RexFieldCollation;
104 import org.apache.calcite.rex.RexInputRef;
105 import org.apache.calcite.rex.RexLiteral;
106 import org.apache.calcite.rex.RexNode;
107 import org.apache.calcite.rex.RexPatternFieldRef;
108 import org.apache.calcite.rex.RexRangeRef;
109 import org.apache.calcite.rex.RexShuttle;
110 import org.apache.calcite.rex.RexSubQuery;
111 import org.apache.calcite.rex.RexUtil;
112 import org.apache.calcite.rex.RexWindowBound;
113 import org.apache.calcite.rex.RexWindowBounds;
114 import org.apache.calcite.schema.ColumnStrategy;
115 import org.apache.calcite.schema.ModifiableTable;
116 import org.apache.calcite.schema.ModifiableView;
117 import org.apache.calcite.schema.Table;
118 import org.apache.calcite.schema.TranslatableTable;
119 import org.apache.calcite.schema.Wrapper;
120 import org.apache.calcite.sql.JoinConditionType;
122 import org.apache.calcite.sql.SqlAggFunction;
123 import org.apache.calcite.sql.SqlBasicCall;
124 import org.apache.calcite.sql.SqlCall;
125 import org.apache.calcite.sql.SqlCallBinding;
126 import org.apache.calcite.sql.SqlDataTypeSpec;
127 import org.apache.calcite.sql.SqlDelete;
128 import org.apache.calcite.sql.SqlDynamicParam;
129 import org.apache.calcite.sql.SqlExplainFormat;
130 import org.apache.calcite.sql.SqlExplainLevel;
131 import org.apache.calcite.sql.SqlFunction;
132 import org.apache.calcite.sql.SqlIdentifier;
133 import org.apache.calcite.sql.SqlInsert;
134 import org.apache.calcite.sql.SqlIntervalQualifier;
135 import org.apache.calcite.sql.SqlJoin;
136 import org.apache.calcite.sql.SqlKind;
137 import org.apache.calcite.sql.SqlLiteral;
138 import org.apache.calcite.sql.SqlMatchRecognize;
139 import org.apache.calcite.sql.SqlMerge;
140 import org.apache.calcite.sql.SqlNode;
141 import org.apache.calcite.sql.SqlNodeList;
142 import org.apache.calcite.sql.SqlNumericLiteral;
144 import org.apache.calcite.sql.SqlOperatorTable;
145 import org.apache.calcite.sql.SqlOrderBy;
146 import org.apache.calcite.sql.SqlSampleSpec;
147 import org.apache.calcite.sql.SqlSelect;
148 import org.apache.calcite.sql.SqlSelectKeyword;
149 import org.apache.calcite.sql.SqlSetOperator;
150 import org.apache.calcite.sql.SqlSnapshot;
151 import org.apache.calcite.sql.SqlUnnestOperator;
152 import org.apache.calcite.sql.SqlUpdate;
153 import org.apache.calcite.sql.SqlUtil;
154 import org.apache.calcite.sql.SqlValuesOperator;
155 import org.apache.calcite.sql.SqlWindow;
156 import org.apache.calcite.sql.SqlWith;
157 import org.apache.calcite.sql.SqlWithItem;
158 import org.apache.calcite.sql.fun.SqlCase;
159 import org.apache.calcite.sql.fun.SqlCountAggFunction;
160 import org.apache.calcite.sql.fun.SqlInOperator;
161 import org.apache.calcite.sql.fun.SqlQuantifyOperator;
162 import org.apache.calcite.sql.fun.SqlRowOperator;
163 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
164 import org.apache.calcite.sql.parser.SqlParserPos;
165 import org.apache.calcite.sql.type.SqlReturnTypeInference;
166 import org.apache.calcite.sql.type.SqlTypeName;
167 import org.apache.calcite.sql.type.SqlTypeUtil;
168 import org.apache.calcite.sql.type.TableFunctionReturnTypeInference;
169 import org.apache.calcite.sql.util.SqlBasicVisitor;
170 import org.apache.calcite.sql.util.SqlVisitor;
171 import org.apache.calcite.sql.validate.AggregatingSelectScope;
172 import org.apache.calcite.sql.validate.CollectNamespace;
173 import org.apache.calcite.sql.validate.DelegatingScope;
174 import org.apache.calcite.sql.validate.ListScope;
175 import org.apache.calcite.sql.validate.MatchRecognizeScope;
176 import org.apache.calcite.sql.validate.ParameterScope;
177 import org.apache.calcite.sql.validate.SelectScope;
178 import org.apache.calcite.sql.validate.SqlMonotonicity;
179 import org.apache.calcite.sql.validate.SqlNameMatcher;
180 import org.apache.calcite.sql.validate.SqlQualified;
181 import org.apache.calcite.sql.validate.SqlUserDefinedTableFunction;
182 import org.apache.calcite.sql.validate.SqlUserDefinedTableMacro;
183 import org.apache.calcite.sql.validate.SqlValidator;
185 import org.apache.calcite.sql.validate.SqlValidatorNamespace;
186 import org.apache.calcite.sql.validate.SqlValidatorScope;
187 import org.apache.calcite.sql.validate.SqlValidatorTable;
188 import org.apache.calcite.sql.validate.SqlValidatorUtil;
189 import org.apache.calcite.tools.RelBuilder;
190 import org.apache.calcite.tools.RelBuilderFactory;
191 import org.apache.calcite.util.ImmutableBitSet;
192 import org.apache.calcite.util.ImmutableIntList;
193 import org.apache.calcite.util.Litmus;
194 import org.apache.calcite.util.NlsString;
195 import org.apache.calcite.util.NumberUtil;
196 import org.apache.calcite.util.Pair;
197 import org.apache.calcite.util.Util;
198 import org.apache.calcite.util.trace.CalciteTrace;
199 import org.slf4j.Logger;
200 
202 import java.math.BigDecimal;
203 import java.util.AbstractList;
204 import java.util.ArrayDeque;
205 import java.util.ArrayList;
206 import java.util.BitSet;
207 import java.util.Collection;
208 import java.util.Collections;
209 import java.util.Deque;
210 import java.util.EnumSet;
211 import java.util.HashMap;
212 import java.util.HashSet;
213 import java.util.List;
214 import java.util.Map;
215 import java.util.Objects;
216 import java.util.Set;
217 import java.util.TreeSet;
218 import java.util.function.BiPredicate;
219 import java.util.function.Supplier;
220 import java.util.function.UnaryOperator;
221 import java.util.stream.Collectors;
222 
223 import javax.annotation.Nonnull;
233 public class SqlToRelConverter {
234  // ~ Static fields/initializers ---------------------------------------------
235 
236  protected static final Logger SQL2REL_LOGGER = CalciteTrace.getSqlToRelTracer();
237 
242  public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
243 
244  @Deprecated // to be removed before 2.0
246 
247  // ~ Instance fields --------------------------------------------------------
248 
249  protected final SqlValidator validator;
250  protected final RexBuilder rexBuilder;
251  protected final Prepare.CatalogReader catalogReader;
252  protected final RelOptCluster cluster;
253  private SubQueryConverter subQueryConverter;
254  protected final Map<RelNode, Integer> leaves = new HashMap<>();
255  private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
256  private final SqlOperatorTable opTab;
257  protected final RelDataTypeFactory typeFactory;
258  private final SqlNodeToRexConverter exprConverter;
259  private final HintStrategyTable hintStrategies;
260  private int explainParamCount;
261  public final SqlToRelConverter.Config config;
262  private final RelBuilder relBuilder;
263 
267  private final Map<CorrelationId, DeferredLookup> mapCorrelToDeferred = new HashMap<>();
268 
273  private final Deque<String> datasetStack = new ArrayDeque<>();
274 
280  private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs = new HashMap<>();
281 
282  public final RelOptTable.ViewExpander viewExpander;
283 
284  // ~ Constructors -----------------------------------------------------------
295  @Deprecated // to be removed before 2.0
296  public SqlToRelConverter(RelOptTable.ViewExpander viewExpander,
297  SqlValidator validator,
298  Prepare.CatalogReader catalogReader,
299  RelOptPlanner planner,
300  RexBuilder rexBuilder,
301  SqlRexConvertletTable convertletTable) {
302  this(viewExpander,
303  validator,
305  RelOptCluster.create(planner, rexBuilder),
306  convertletTable,
307  Config.DEFAULT);
308  }
309 
310  @Deprecated // to be removed before 2.0
311  public SqlToRelConverter(RelOptTable.ViewExpander viewExpander,
312  SqlValidator validator,
313  Prepare.CatalogReader catalogReader,
314  RelOptCluster cluster,
315  SqlRexConvertletTable convertletTable) {
316  this(viewExpander,
317  validator,
319  cluster,
320  convertletTable,
321  Config.DEFAULT);
322  }
323 
324  /* Creates a converter. */
325  public SqlToRelConverter(RelOptTable.ViewExpander viewExpander,
326  SqlValidator validator,
327  Prepare.CatalogReader catalogReader,
328  RelOptCluster cluster,
329  SqlRexConvertletTable convertletTable,
330  Config config) {
331  this.viewExpander = viewExpander;
332  this.opTab = (validator == null) ? SqlStdOperatorTable.instance()
333  : validator.getOperatorTable();
334  this.validator = validator;
335  this.catalogReader = catalogReader;
336  this.subQueryConverter = new NoOpSubQueryConverter();
337  this.rexBuilder = cluster.getRexBuilder();
338  this.typeFactory = rexBuilder.getTypeFactory();
339  this.exprConverter = new SqlNodeToRexConverterImpl(convertletTable);
340  this.explainParamCount = 0;
341  this.config = new ConfigBuilder().withConfig(config).build();
342  this.relBuilder = config.getRelBuilderFactory()
343  .create(cluster, null)
344  .transform(config.getRelBuilderConfigTransform());
345  this.hintStrategies = config.getHintStrategyTable();
346 
347  cluster.setHintStrategies(this.hintStrategies);
348  this.cluster = Objects.requireNonNull(cluster);
349  }
350 
351  // ~ Methods ----------------------------------------------------------------
352 
354  public RelOptCluster getCluster() {
355  return cluster;
356  }
357 
361  public RexBuilder getRexBuilder() {
362  return rexBuilder;
363  }
364 
371  public int getDynamicParamCount() {
372  return dynamicParamSqlNodes.size();
373  }
374 
381  public RelDataType getDynamicParamType(int index) {
382  SqlNode sqlNode = dynamicParamSqlNodes.get(index);
383  if (sqlNode == null) {
384  throw Util.needToImplement("dynamic param type inference");
385  }
386  return validator.getValidatedNodeType(sqlNode);
387  }
388 
396  public int getDynamicParamCountInExplain(boolean increment) {
397  int retVal = explainParamCount;
398  if (increment) {
400  }
401  return retVal;
402  }
403 
408  public Map<SqlNode, RexNode> getMapConvertedNonCorrSubqs() {
410  }
411 
420  Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
421  mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
422  }
423 
430  public void setSubQueryConverter(SubQueryConverter converter) {
431  subQueryConverter = converter;
432  }
433 
440  assert config.isExplain();
441  this.explainParamCount = explainParamCount;
442  }
443 
444  private void checkConvertedType(SqlNode query, RelNode result) {
445  if (query.isA(SqlKind.DML)) {
446  return;
447  }
448  // Verify that conversion from SQL to relational algebra did
449  // not perturb any type information. (We can't do this if the
450  // SQL statement is something like an INSERT which has no
451  // validator type information associated with its result,
452  // hence the namespace check above.)
453  final List<RelDataTypeField> validatedFields =
454  validator.getValidatedNodeType(query).getFieldList();
455  final RelDataType validatedRowType =
456  validator.getTypeFactory().createStructType(Pair.right(validatedFields),
457  SqlValidatorUtil.uniquify(Pair.left(validatedFields),
458  catalogReader.nameMatcher().isCaseSensitive()));
459 
460  final List<RelDataTypeField> convertedFields =
461  result.getRowType().getFieldList().subList(0, validatedFields.size());
462  final RelDataType convertedRowType =
463  validator.getTypeFactory().createStructType(convertedFields);
464 
465  if (!RelOptUtil.equal("validated row type",
466  validatedRowType,
467  "converted row type",
468  convertedRowType,
469  Litmus.IGNORE)) {
470  throw new AssertionError("Conversion to relational algebra failed to "
471  + "preserve datatypes:\n"
472  + "validated type:\n" + validatedRowType.getFullTypeString()
473  + "\nconverted type:\n" + convertedRowType.getFullTypeString() + "\nrel:\n"
474  + RelOptUtil.toString(result));
475  }
476  }
477 
478  public RelNode flattenTypes(RelNode rootRel, boolean restructure) {
479  RelStructuredTypeFlattener typeFlattener = new RelStructuredTypeFlattener(
480  relBuilder, rexBuilder, createToRelContext(ImmutableList.of()), restructure);
481  return typeFlattener.rewrite(rootRel);
482  }
483 
492  public RelNode decorrelate(SqlNode query, RelNode rootRel) {
493  if (!config.isDecorrelationEnabled()) {
494  return rootRel;
495  }
496  final RelNode result = decorrelateQuery(rootRel);
497  if (result != rootRel) {
498  checkConvertedType(query, result);
499  }
500  return result;
501  }
502 
525  public RelNode trimUnusedFields(boolean ordered, RelNode rootRel) {
526  // Trim fields that are not used by their consumer.
527  if (isTrimUnusedFields()) {
528  final RelFieldTrimmer trimmer = newFieldTrimmer();
529  final List<RelCollation> collations =
530  rootRel.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE);
531  rootRel = trimmer.trim(rootRel);
532  if (!ordered && collations != null && !collations.isEmpty()
533  && !collations.equals(ImmutableList.of(RelCollations.EMPTY))) {
534  final RelTraitSet traitSet =
535  rootRel.getTraitSet().replace(RelCollationTraitDef.INSTANCE, collations);
536  rootRel = rootRel.copy(traitSet, rootRel.getInputs());
537  }
538  if (SQL2REL_LOGGER.isDebugEnabled()) {
539  SQL2REL_LOGGER.debug(RelOptUtil.dumpPlan("Plan after trimming unused fields",
540  rootRel,
541  SqlExplainFormat.TEXT,
542  SqlExplainLevel.EXPPLAN_ATTRIBUTES));
543  }
544  }
545  return rootRel;
546  }
547 
553  protected RelFieldTrimmer newFieldTrimmer() {
554  return new RelFieldTrimmer(validator, relBuilder);
555  }
556 
568  public RelRoot convertQuery(
569  SqlNode query, final boolean needsValidation, final boolean top) {
570  if (needsValidation) {
571  query = validator.validate(query);
572  }
573 
574  RelNode result = convertQueryRecursive(query, top, null).rel;
575  if (top) {
576  if (isStream(query)) {
577  result = new LogicalDelta(cluster, result.getTraitSet(), result);
578  }
579  }
580  RelCollation collation = RelCollations.EMPTY;
581  if (!query.isA(SqlKind.DML)) {
582  if (isOrdered(query)) {
583  collation = requiredCollation(result);
584  }
585  }
586  checkConvertedType(query, result);
587 
588  if (SQL2REL_LOGGER.isDebugEnabled()) {
589  SQL2REL_LOGGER.debug(RelOptUtil.dumpPlan("Plan after converting SqlNode to RelNode",
590  result,
591  SqlExplainFormat.TEXT,
592  SqlExplainLevel.EXPPLAN_ATTRIBUTES));
593  }
594 
595  final RelDataType validatedRowType = validator.getValidatedNodeType(query);
596  List<RelHint> hints = new ArrayList<>();
597  if (query.getKind() == SqlKind.SELECT) {
598  final SqlSelect select = (SqlSelect) query;
599  if (select.hasHints()) {
600  hints = SqlUtil.getRelHint(hintStrategies, select.getHints());
601  }
602  }
603  // propagate the hints.
604  result = RelOptUtil.propagateRelHints(result, false);
605  return RelRoot.of(result, validatedRowType, query.getKind())
606  .withCollation(collation)
607  .withHints(hints);
608  }
609 
610  private static boolean isStream(SqlNode query) {
611  return query instanceof SqlSelect
612  && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
613  }
614 
615  public static boolean isOrdered(SqlNode query) {
616  switch (query.getKind()) {
617  case SELECT:
618  return ((SqlSelect) query).getOrderList() != null
619  && ((SqlSelect) query).getOrderList().size() > 0;
620  case WITH:
621  return isOrdered(((SqlWith) query).body);
622  case ORDER_BY:
623  return ((SqlOrderBy) query).orderList.size() > 0;
624  default:
625  return false;
626  }
627  }
628 
629  private RelCollation requiredCollation(RelNode r) {
630  if (r instanceof Sort) {
631  return ((Sort) r).collation;
632  }
633  if (r instanceof Project) {
634  return requiredCollation(((Project) r).getInput());
635  }
636  if (r instanceof Delta) {
637  return requiredCollation(((Delta) r).getInput());
638  }
639  throw new AssertionError(r);
640  }
641 
645  public RelNode convertSelect(SqlSelect select, boolean top) {
646  final SqlValidatorScope selectScope = validator.getWhereScope(select);
647  final Blackboard bb = createBlackboard(selectScope, null, top);
648  convertSelectImpl(bb, select);
649  return bb.root;
650  }
651 
656  SqlValidatorScope scope, Map<String, RexNode> nameToNodeMap, boolean top) {
657  return new Blackboard(scope, nameToNodeMap, top);
658  }
659 
664  protected void convertSelectImpl(final Blackboard bb, SqlSelect select) {
665  convertFrom(bb, select.getFrom());
666  convertWhere(bb, select.getWhere());
667 
668  final List<SqlNode> orderExprList = new ArrayList<>();
669  final List<RelFieldCollation> collationList = new ArrayList<>();
670  gatherOrderExprs(bb, select, select.getOrderList(), orderExprList, collationList);
671  final RelCollation collation =
672  cluster.traitSet().canonize(RelCollations.of(collationList));
673 
674  if (validator.isAggregate(select)) {
675  convertAgg(bb, select, orderExprList);
676  } else {
677  convertSelectList(bb, select, orderExprList);
678  }
679 
680  if (select.isDistinct()) {
681  distinctify(bb, true);
682  }
683 
684  convertOrder(
685  select, bb, collation, orderExprList, select.getOffset(), select.getFetch());
686 
687  if (select.hasHints()) {
688  final List<RelHint> hints = SqlUtil.getRelHint(hintStrategies, select.getHints());
689  // Attach the hints to the first Hintable node we found from the root node.
690  bb.setRoot(bb.root.accept(new RelShuttleImpl() {
691  boolean attached = false;
692 
693  @Override
694  public RelNode visitChild(RelNode parent, int i, RelNode child) {
695  if (parent instanceof Hintable && !attached) {
696  attached = true;
697  return ((Hintable) parent).attachHints(hints);
698  } else {
699  return super.visitChild(parent, i, child);
700  }
701  }
702  }),
703  true);
704  } else {
705  bb.setRoot(bb.root, true);
706  }
707  }
708 
722  private void distinctify(Blackboard bb, boolean checkForDupExprs) {
723  // Look for duplicate expressions in the project.
724  // Say we have 'select x, y, x, z'.
725  // Then dups will be {[2, 0]}
726  // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]}
727  RelNode rel = bb.root;
728  if (checkForDupExprs && (rel instanceof LogicalProject)) {
729  LogicalProject project = (LogicalProject) rel;
730  final List<RexNode> projectExprs = project.getProjects();
731  final List<Integer> origins = new ArrayList<>();
732  int dupCount = 0;
733  for (int i = 0; i < projectExprs.size(); i++) {
734  int x = projectExprs.indexOf(projectExprs.get(i));
735  if (x >= 0 && x < i) {
736  origins.add(x);
737  ++dupCount;
738  } else {
739  origins.add(i);
740  }
741  }
742  if (dupCount == 0) {
743  distinctify(bb, false);
744  return;
745  }
746 
747  final Map<Integer, Integer> squished = new HashMap<>();
748  final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
749  final List<Pair<RexNode, String>> newProjects = new ArrayList<>();
750  for (int i = 0; i < fields.size(); i++) {
751  if (origins.get(i) == i) {
752  squished.put(i, newProjects.size());
753  newProjects.add(RexInputRef.of2(i, fields));
754  }
755  }
756  rel = LogicalProject.create(
757  rel, ImmutableList.of(), Pair.left(newProjects), Pair.right(newProjects));
758  bb.root = rel;
759  distinctify(bb, false);
760  rel = bb.root;
761 
762  // Create the expressions to reverse the mapping.
763  // Project($0, $1, $0, $2).
764  final List<Pair<RexNode, String>> undoProjects = new ArrayList<>();
765  for (int i = 0; i < fields.size(); i++) {
766  final int origin = origins.get(i);
767  RelDataTypeField field = fields.get(i);
768  undoProjects.add(Pair.of(
769  new RexInputRef(squished.get(origin), field.getType()), field.getName()));
770  }
771 
772  rel = LogicalProject.create(
773  rel, ImmutableList.of(), Pair.left(undoProjects), Pair.right(undoProjects));
774  bb.setRoot(rel, false);
775 
776  return;
777  }
778 
779  // Usual case: all of the expressions in the SELECT clause are
780  // different.
781  final ImmutableBitSet groupSet =
782  ImmutableBitSet.range(rel.getRowType().getFieldCount());
783  rel = createAggregate(bb, groupSet, ImmutableList.of(groupSet), ImmutableList.of());
784 
785  bb.setRoot(rel, false);
786  }
787 
804  protected void convertOrder(SqlSelect select,
805  Blackboard bb,
806  RelCollation collation,
807  List<SqlNode> orderExprList,
808  SqlNode offset,
809  SqlNode fetch) {
810  if (!bb.top || select.getOrderList() == null
811  || select.getOrderList().getList().isEmpty()) {
812  assert !bb.top || collation.getFieldCollations().isEmpty();
813  if ((offset == null
814  || (offset instanceof SqlLiteral
815  && ((SqlLiteral) offset)
816  .bigDecimalValue()
817  .equals(BigDecimal.ZERO)))
818  && fetch == null) {
819  return;
820  }
821  }
822 
823  // Create a sorter using the previously constructed collations.
824  bb.setRoot(LogicalSort.create(bb.root,
825  collation,
826  offset == null ? null : convertExpression(offset),
827  fetch == null ? null : convertExpression(fetch)),
828  false);
829 
830  // If extra expressions were added to the project list for sorting,
831  // add another project to remove them. But make the collation empty, because
832  // we can't represent the real collation.
833  //
834  // If it is the top node, use the real collation, but don't trim fields.
835  if (orderExprList.size() > 0 && !bb.top) {
836  final List<RexNode> exprs = new ArrayList<>();
837  final RelDataType rowType = bb.root.getRowType();
838  final int fieldCount = rowType.getFieldCount() - orderExprList.size();
839  for (int i = 0; i < fieldCount; i++) {
840  exprs.add(rexBuilder.makeInputRef(bb.root, i));
841  }
842  bb.setRoot(LogicalProject.create(bb.root,
843  ImmutableList.of(),
844  exprs,
845  rowType.getFieldNames().subList(0, fieldCount)),
846  false);
847  }
848  }
849 
855  private static boolean containsInOperator(SqlNode node) {
856  try {
857  SqlVisitor<Void> visitor = new SqlBasicVisitor<Void>() {
858  public Void visit(SqlCall call) {
859  if (call.getOperator() instanceof SqlInOperator) {
860  throw new Util.FoundOne(call);
861  }
862  return super.visit(call);
863  }
864  };
865  node.accept(visitor);
866  return false;
867  } catch (Util.FoundOne e) {
868  Util.swallow(e, null);
869  return true;
870  }
871  }
872 
880  private static SqlNode pushDownNotForIn(SqlValidatorScope scope, SqlNode sqlNode) {
881  if (!(sqlNode instanceof SqlCall) || !containsInOperator(sqlNode)) {
882  return sqlNode;
883  }
884  final SqlCall sqlCall = (SqlCall) sqlNode;
885  switch (sqlCall.getKind()) {
886  case AND:
887  case OR:
888  final List<SqlNode> operands = new ArrayList<>();
889  for (SqlNode operand : sqlCall.getOperandList()) {
890  operands.add(pushDownNotForIn(scope, operand));
891  }
892  final SqlCall newCall =
893  sqlCall.getOperator().createCall(sqlCall.getParserPosition(), operands);
894  return reg(scope, newCall);
895 
896  case NOT:
897  assert sqlCall.operand(0) instanceof SqlCall;
898  final SqlCall call = sqlCall.operand(0);
899  switch (sqlCall.operand(0).getKind()) {
900  case CASE:
901  final SqlCase caseNode = (SqlCase) call;
902  final SqlNodeList thenOperands = new SqlNodeList(SqlParserPos.ZERO);
903 
904  for (SqlNode thenOperand : caseNode.getThenOperands()) {
905  final SqlCall not =
906  SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, thenOperand);
907  thenOperands.add(pushDownNotForIn(scope, reg(scope, not)));
908  }
909  SqlNode elseOperand = caseNode.getElseOperand();
910  if (!SqlUtil.isNull(elseOperand)) {
911  // "not(unknown)" is "unknown", so no need to simplify
912  final SqlCall not =
913  SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, elseOperand);
914  elseOperand = pushDownNotForIn(scope, reg(scope, not));
915  }
916 
917  return reg(scope,
918  SqlStdOperatorTable.CASE.createCall(SqlParserPos.ZERO,
919  caseNode.getValueOperand(),
920  caseNode.getWhenOperands(),
921  thenOperands,
922  elseOperand));
923 
924  case AND:
925  final List<SqlNode> orOperands = new ArrayList<>();
926  for (SqlNode operand : call.getOperandList()) {
927  orOperands.add(pushDownNotForIn(scope,
928  reg(scope,
929  SqlStdOperatorTable.NOT.createCall(
930  SqlParserPos.ZERO, operand))));
931  }
932  return reg(scope,
933  SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands));
934 
935  case OR:
936  final List<SqlNode> andOperands = new ArrayList<>();
937  for (SqlNode operand : call.getOperandList()) {
938  andOperands.add(pushDownNotForIn(scope,
939  reg(scope,
940  SqlStdOperatorTable.NOT.createCall(
941  SqlParserPos.ZERO, operand))));
942  }
943  return reg(scope,
944  SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, andOperands));
945 
946  case NOT:
947  assert call.operandCount() == 1;
948  return pushDownNotForIn(scope, call.operand(0));
949 
950  case NOT_IN:
951  return reg(scope,
952  SqlStdOperatorTable.IN.createCall(
953  SqlParserPos.ZERO, call.getOperandList()));
954 
955  case IN:
956  return reg(scope,
957  SqlStdOperatorTable.NOT_IN.createCall(
958  SqlParserPos.ZERO, call.getOperandList()));
959  }
960  }
961  return sqlNode;
962  }
963 
968  private static SqlNode reg(SqlValidatorScope scope, SqlNode e) {
969  scope.getValidator().deriveType(scope, e);
970  return e;
971  }
972 
979  private void convertWhere(final Blackboard bb, final SqlNode where) {
980  if (where == null) {
981  return;
982  }
983  SqlNode newWhere = pushDownNotForIn(bb.scope, where);
984  replaceSubQueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
985  final RexNode convertedWhere = bb.convertExpression(newWhere);
986  final RexNode convertedWhere2 =
987  RexUtil.removeNullabilityCast(typeFactory, convertedWhere);
988 
989  // only allocate filter if the condition is not TRUE
990  if (convertedWhere2.isAlwaysTrue()) {
991  return;
992  }
993 
994  final RelFactories.FilterFactory filterFactory = RelFactories.DEFAULT_FILTER_FACTORY;
995  final RelNode filter =
996  filterFactory.createFilter(bb.root, convertedWhere2, ImmutableSet.of());
997  final RelNode r;
998  final CorrelationUse p = getCorrelationUse(bb, filter);
999  if (p != null) {
1000  assert p.r instanceof Filter;
1001  Filter f = (Filter) p.r;
1002  r = LogicalFilter.create(f.getInput(), f.getCondition(), ImmutableSet.of(p.id));
1003  } else {
1004  r = filter;
1005  }
1006 
1007  bb.setRoot(r, false);
1008  }
1009 
1010  private void replaceSubQueries(
1011  final Blackboard bb, final SqlNode expr, RelOptUtil.Logic logic) {
1012  findSubQueries(bb, expr, logic, false);
1013  for (SubQuery node : bb.subQueryList) {
1014  substituteSubQuery(bb, node);
1015  }
1016  }
1017 
1018  private void substituteSubQuery(Blackboard bb, SubQuery subQuery) {
1019  final RexNode expr = subQuery.expr;
1020  if (expr != null) {
1021  // Already done.
1022  return;
1023  }
1024 
1025  final SqlBasicCall call;
1026  final RelNode rel;
1027  final SqlNode query;
1028  final RelOptUtil.Exists converted;
1029 
1030  boolean isExpand = config.getExpandPredicate().test(bb.getTopNode(), subQuery.node);
1031 
1032  switch (subQuery.node.getKind()) {
1033  case CURSOR:
1034  convertCursor(bb, subQuery);
1035  return;
1036 
1037  case MULTISET_QUERY_CONSTRUCTOR:
1038  case MULTISET_VALUE_CONSTRUCTOR:
1039  case ARRAY_QUERY_CONSTRUCTOR:
1040  rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
1041  subQuery.expr = bb.register(rel, JoinRelType.INNER);
1042  return;
1043 
1044  case IN:
1045  case NOT_IN:
1046  case SOME:
1047  case ALL:
1048  call = (SqlBasicCall) subQuery.node;
1049  query = call.operand(1);
1050  if (!isExpand && !(query instanceof SqlNodeList)) {
1051  return;
1052  }
1053  final SqlNode leftKeyNode = call.operand(0);
1054 
1055  final List<RexNode> leftKeys;
1056  switch (leftKeyNode.getKind()) {
1057  case ROW:
1058  leftKeys = new ArrayList<>();
1059  for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
1060  leftKeys.add(bb.convertExpression(sqlExpr));
1061  }
1062  break;
1063  default:
1064  leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
1065  }
1066 
1067  if (query instanceof SqlNodeList) {
1068  SqlNodeList valueList = (SqlNodeList) query;
1069  if (!containsNullLiteral(valueList)
1070  && valueList.size() < config.getInSubQueryThreshold()) {
1071  // We're under the threshold, so convert to OR.
1072  subQuery.expr = convertInToOr(
1073  bb, leftKeys, valueList, (SqlInOperator) call.getOperator());
1074  return;
1075  }
1076 
1077  // Otherwise, let convertExists translate
1078  // values list into an inline table for the
1079  // reference to Q below.
1080  }
1081 
1082  // Project out the search columns from the left side
1083 
1084  // Q1:
1085  // "select from emp where emp.deptno in (select col1 from T)"
1086  //
1087  // is converted to
1088  //
1089  // "select from
1090  // emp inner join (select distinct col1 from T)) q
1091  // on emp.deptno = q.col1
1092  //
1093  // Q2:
1094  // "select from emp where emp.deptno not in (Q)"
1095  //
1096  // is converted to
1097  //
1098  // "select from
1099  // emp left outer join (select distinct col1, TRUE from T) q
1100  // on emp.deptno = q.col1
1101  // where emp.deptno <> null
1102  // and q.indicator <> TRUE"
1103  //
1104  // Note: Sub-query can be used as SqlUpdate#condition like below:
1105  //
1106  // UPDATE emp
1107  // SET empno = 1 WHERE emp.empno IN (
1108  // SELECT emp.empno FROM emp WHERE emp.empno = 2)
1109  //
1110  // In such case, when converting SqlUpdate#condition, bb.root is null
1111  // and it makes no sense to do the sub-query substitution.
1112  if (bb.root == null) {
1113  return;
1114  }
1115  final RelDataType targetRowType = SqlTypeUtil.promoteToRowType(
1116  typeFactory, validator.getValidatedNodeType(leftKeyNode), null);
1117  final boolean notIn = call.getOperator().kind == SqlKind.NOT_IN;
1118  converted = convertExists(
1119  query, RelOptUtil.SubQueryType.IN, subQuery.logic, notIn, targetRowType);
1120  if (converted.indicator) {
1121  // Generate
1122  // emp CROSS JOIN (SELECT COUNT(*) AS c,
1123  // COUNT(deptno) AS ck FROM dept)
1124  final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT);
1125  final RelNode seek = converted.r.getInput(0); // fragile
1126  final int keyCount = leftKeys.size();
1127  final List<Integer> args = ImmutableIntList.range(0, keyCount);
1128  LogicalAggregate aggregate = LogicalAggregate.create(seek,
1129  ImmutableList.of(),
1130  ImmutableBitSet.of(),
1131  null,
1132  ImmutableList.of(AggregateCall.create(SqlStdOperatorTable.COUNT,
1133  false,
1134  false,
1135  false,
1136  ImmutableList.of(),
1137  -1,
1138  RelCollations.EMPTY,
1139  longType,
1140  null),
1141  AggregateCall.create(SqlStdOperatorTable.COUNT,
1142  false,
1143  false,
1144  false,
1145  args,
1146  -1,
1147  RelCollations.EMPTY,
1148  longType,
1149  null)));
1150  LogicalJoin join = LogicalJoin.create(bb.root,
1151  aggregate,
1152  ImmutableList.of(),
1153  rexBuilder.makeLiteral(true),
1154  ImmutableSet.of(),
1155  JoinRelType.INNER);
1156  bb.setRoot(join, false);
1157  }
1158  final RexNode rex = bb.register(converted.r,
1159  converted.outerJoin ? JoinRelType.LEFT : JoinRelType.INNER,
1160  leftKeys);
1161 
1162  RelOptUtil.Logic logic = subQuery.logic;
1163  switch (logic) {
1164  case TRUE_FALSE_UNKNOWN:
1165  case UNKNOWN_AS_TRUE:
1166  if (!converted.indicator) {
1167  logic = RelOptUtil.Logic.TRUE_FALSE;
1168  }
1169  }
1170  subQuery.expr = translateIn(logic, bb.root, rex);
1171  if (notIn) {
1172  subQuery.expr = rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
1173  }
1174  return;
1175 
1176  case EXISTS:
1177  // "select from emp where exists (select a from T)"
1178  //
1179  // is converted to the following if the sub-query is correlated:
1180  //
1181  // "select from emp left outer join (select AGG_TRUE() as indicator
1182  // from T group by corr_var) q where q.indicator is true"
1183  //
1184  // If there is no correlation, the expression is replaced with a
1185  // boolean indicating whether the sub-query returned 0 or >= 1 row.
1186  call = (SqlBasicCall) subQuery.node;
1187  query = call.operand(0);
1188  if (!isExpand) {
1189  return;
1190  }
1191  final SqlValidatorScope seekScope = (query instanceof SqlSelect)
1192  ? validator.getSelectScope((SqlSelect) query)
1193  : null;
1194  final Blackboard seekBb = createBlackboard(seekScope, null, false);
1195  final RelNode seekRel = convertQueryOrInList(seekBb, query, null);
1196  // An EXIST sub-query whose inner child has at least 1 tuple
1197  // (e.g. an Aggregate with no grouping columns or non-empty Values
1198  // node) should be simplified to a Boolean constant expression.
1199  final RelMetadataQuery mq = seekRel.getCluster().getMetadataQuery();
1200  final Double minRowCount = mq.getMinRowCount(seekRel);
1201  if (minRowCount != null && minRowCount >= 1D) {
1202  subQuery.expr = rexBuilder.makeLiteral(true);
1203  return;
1204  }
1205  converted = RelOptUtil.createExistsPlan(seekRel,
1206  RelOptUtil.SubQueryType.EXISTS,
1207  subQuery.logic,
1208  true,
1209  relBuilder);
1210  assert !converted.indicator;
1211  if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, true)) {
1212  return;
1213  }
1214  subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
1215  return;
1216 
1217  case SCALAR_QUERY:
1218  // Convert the sub-query. If it's non-correlated, convert it
1219  // to a constant expression.
1220  if (!isExpand) {
1221  return;
1222  }
1223  call = (SqlBasicCall) subQuery.node;
1224  query = call.operand(0);
1225  converted = convertExists(
1226  query, RelOptUtil.SubQueryType.SCALAR, subQuery.logic, true, null);
1227  assert !converted.indicator;
1228  if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, false)) {
1229  return;
1230  }
1231  rel = convertToSingleValueSubq(query, converted.r);
1232  subQuery.expr = bb.register(rel, JoinRelType.LEFT);
1233  return;
1234 
1235  case SELECT:
1236  // This is used when converting multiset queries:
1237  //
1238  // select * from unnest(select multiset[deptno] from emps);
1239  //
1240  converted = convertExists(subQuery.node,
1241  RelOptUtil.SubQueryType.SCALAR,
1242  subQuery.logic,
1243  true,
1244  null);
1245  assert !converted.indicator;
1246  subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
1247 
1248  // This is used when converting window table functions:
1249  //
1250  // select * from table(tumble(table emps, descriptor(deptno), interval '3' DAY))
1251  //
1252  bb.cursors.add(converted.r);
1253  return;
1254 
1255  default:
1256  throw new AssertionError("unexpected kind of sub-query: " + subQuery.node);
1257  }
1258  }
1259 
1260  private RexNode translateIn(RelOptUtil.Logic logic, RelNode root, final RexNode rex) {
1261  switch (logic) {
1262  case TRUE:
1263  return rexBuilder.makeLiteral(true);
1264 
1265  case TRUE_FALSE:
1266  case UNKNOWN_AS_FALSE:
1267  assert rex instanceof RexRangeRef;
1268  final int fieldCount = rex.getType().getFieldCount();
1269  RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
1270  rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
1271 
1272  // Then append the IS NOT NULL(leftKeysForIn).
1273  //
1274  // RexRangeRef contains the following fields:
1275  // leftKeysForIn,
1276  // rightKeysForIn (the original sub-query select list),
1277  // nullIndicator
1278  //
1279  // The first two lists contain the same number of fields.
1280  final int k = (fieldCount - 1) / 2;
1281  for (int i = 0; i < k; i++) {
1282  rexNode = rexBuilder.makeCall(SqlStdOperatorTable.AND,
1283  rexNode,
1284  rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL,
1285  rexBuilder.makeFieldAccess(rex, i)));
1286  }
1287  return rexNode;
1288 
1289  case TRUE_FALSE_UNKNOWN:
1290  case UNKNOWN_AS_TRUE:
1291  // select e.deptno,
1292  // case
1293  // when ct.c = 0 then false
1294  // when dt.i is not null then true
1295  // when e.deptno is null then null
1296  // when ct.ck < ct.c then null
1297  // else false
1298  // end
1299  // from e
1300  // cross join (select count(*) as c, count(deptno) as ck from v) as ct
1301  // left join (select distinct deptno, true as i from v) as dt
1302  // on e.deptno = dt.deptno
1303  final Join join = (Join) root;
1304  final Project left = (Project) join.getLeft();
1305  final RelNode leftLeft = ((Join) left.getInput()).getLeft();
1306  final int leftLeftCount = leftLeft.getRowType().getFieldCount();
1307  final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT);
1308  final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
1309  final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
1310  final RexNode iRef =
1311  rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
1312 
1313  final RexLiteral zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
1314  final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
1315  final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
1316  final RexNode unknownLiteral = rexBuilder.makeNullLiteral(trueLiteral.getType());
1317 
1318  final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
1319  args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero),
1320  falseLiteral,
1321  rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
1322  trueLiteral);
1323  final JoinInfo joinInfo = join.analyzeCondition();
1324  for (int leftKey : joinInfo.leftKeys) {
1325  final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
1326  args.add(
1327  rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef), unknownLiteral);
1328  }
1329  args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef),
1330  unknownLiteral,
1331  falseLiteral);
1332 
1333  return rexBuilder.makeCall(SqlStdOperatorTable.CASE, args.build());
1334 
1335  default:
1336  throw new AssertionError(logic);
1337  }
1338  }
1339 
1340  private static boolean containsNullLiteral(SqlNodeList valueList) {
1341  for (SqlNode node : valueList.getList()) {
1342  if (node instanceof SqlLiteral) {
1343  SqlLiteral lit = (SqlLiteral) node;
1344  if (lit.getValue() == null) {
1345  return true;
1346  }
1347  }
1348  }
1349  return false;
1350  }
1351 
1363  SubQuery subQuery, Blackboard bb, RelNode converted, boolean isExists) {
1364  SqlCall call = (SqlBasicCall) subQuery.node;
1365  if (subQueryConverter.canConvertSubQuery()
1366  && isSubQueryNonCorrelated(converted, bb)) {
1367  // First check if the sub-query has already been converted
1368  // because it's a nested sub-query. If so, don't re-evaluate
1369  // it again.
1370  RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
1371  if (constExpr == null) {
1372  constExpr = subQueryConverter.convertSubQuery(
1373  call, this, isExists, config.isExplain());
1374  }
1375  if (constExpr != null) {
1376  subQuery.expr = constExpr;
1377  mapConvertedNonCorrSubqs.put(call, constExpr);
1378  return true;
1379  }
1380  }
1381  return false;
1382  }
1383 
1392  public RelNode convertToSingleValueSubq(SqlNode query, RelNode plan) {
1393  // Check whether query is guaranteed to produce a single value.
1394  if (query instanceof SqlSelect) {
1395  SqlSelect select = (SqlSelect) query;
1396  SqlNodeList selectList = select.getSelectList();
1397  SqlNodeList groupList = select.getGroup();
1398 
1399  if ((selectList.size() == 1) && ((groupList == null) || (groupList.size() == 0))) {
1400  SqlNode selectExpr = selectList.get(0);
1401  if (selectExpr instanceof SqlCall) {
1402  SqlCall selectExprCall = (SqlCall) selectExpr;
1403  if (Util.isSingleValue(selectExprCall)) {
1404  return plan;
1405  }
1406  }
1407 
1408  // If there is a limit with 0 or 1,
1409  // it is ensured to produce a single value
1410  if (select.getFetch() != null && select.getFetch() instanceof SqlNumericLiteral) {
1411  SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch();
1412  if (((BigDecimal) limitNum.getValue()).intValue() < 2) {
1413  return plan;
1414  }
1415  }
1416  }
1417  } else if (query instanceof SqlCall) {
1418  // If the query is (values ...),
1419  // it is necessary to look into the operands to determine
1420  // whether SingleValueAgg is necessary
1421  SqlCall exprCall = (SqlCall) query;
1422  if (exprCall.getOperator() instanceof SqlValuesOperator
1423  && Util.isSingleValue(exprCall)) {
1424  return plan;
1425  }
1426  }
1427 
1428  // If not, project SingleValueAgg
1429  return RelOptUtil.createSingleValueAggRel(cluster, plan);
1430  }
1431 
1440  private RexNode convertInToOr(final Blackboard bb,
1441  final List<RexNode> leftKeys,
1442  SqlNodeList valuesList,
1443  SqlInOperator op) {
1444  final List<RexNode> comparisons = new ArrayList<>();
1445  for (SqlNode rightVals : valuesList) {
1446  RexNode rexComparison;
1447  final SqlOperator comparisonOp;
1448  if (op instanceof SqlQuantifyOperator) {
1449  comparisonOp = RelOptUtil.op(
1450  ((SqlQuantifyOperator) op).comparisonKind, SqlStdOperatorTable.EQUALS);
1451  } else {
1452  comparisonOp = SqlStdOperatorTable.EQUALS;
1453  }
1454  if (leftKeys.size() == 1) {
1455  rexComparison = rexBuilder.makeCall(comparisonOp,
1456  leftKeys.get(0),
1457  ensureSqlType(
1458  leftKeys.get(0).getType(), bb.convertExpression(rightVals)));
1459  } else {
1460  assert rightVals instanceof SqlCall;
1461  final SqlBasicCall call = (SqlBasicCall) rightVals;
1462  assert (call.getOperator() instanceof SqlRowOperator)
1463  && call.operandCount() == leftKeys.size();
1464  rexComparison = RexUtil.composeConjunction(rexBuilder,
1465  Iterables.transform(Pair.zip(leftKeys, call.getOperandList()),
1466  pair
1467  -> rexBuilder.makeCall(comparisonOp,
1468  pair.left,
1469  ensureSqlType(pair.left.getType(),
1470  bb.convertExpression(pair.right)))));
1471  }
1472  comparisons.add(rexComparison);
1473  }
1474 
1475  switch (op.kind) {
1476  case ALL:
1477  return RexUtil.composeConjunction(rexBuilder, comparisons, true);
1478  case NOT_IN:
1479  return rexBuilder.makeCall(SqlStdOperatorTable.NOT,
1480  RexUtil.composeDisjunction(rexBuilder, comparisons, true));
1481  case IN:
1482  case SOME:
1483  return RexUtil.composeDisjunction(rexBuilder, comparisons, true);
1484  default:
1485  throw new AssertionError();
1486  }
1487  }
1488 
1494  private RexNode ensureSqlType(RelDataType type, RexNode node) {
1495  if (type.getSqlTypeName() == node.getType().getSqlTypeName()
1496  || (type.getSqlTypeName() == SqlTypeName.VARCHAR
1497  && node.getType().getSqlTypeName() == SqlTypeName.CHAR)) {
1498  return node;
1499  }
1500  return rexBuilder.ensureType(type, node, true);
1501  }
1502 
1512  @Deprecated // to be removed before 2.0
1513  protected int getInSubqueryThreshold() {
1514  return config.getInSubQueryThreshold();
1515  }
1516 
1534  private RelOptUtil.Exists convertExists(SqlNode seek,
1535  RelOptUtil.SubQueryType subQueryType,
1536  RelOptUtil.Logic logic,
1537  boolean notIn,
1538  RelDataType targetDataType) {
1539  final SqlValidatorScope seekScope = (seek instanceof SqlSelect)
1540  ? validator.getSelectScope((SqlSelect) seek)
1541  : null;
1542  final Blackboard seekBb = createBlackboard(seekScope, null, false);
1543  RelNode seekRel = convertQueryOrInList(seekBb, seek, targetDataType);
1544 
1545  return RelOptUtil.createExistsPlan(seekRel, subQueryType, logic, notIn, relBuilder);
1546  }
1547 
1548  private RelNode convertQueryOrInList(
1549  Blackboard bb, SqlNode seek, RelDataType targetRowType) {
1550  // NOTE: Once we start accepting single-row queries as row constructors,
1551  // there will be an ambiguity here for a case like X IN ((SELECT Y FROM
1552  // Z)). The SQL standard resolves the ambiguity by saying that a lone
1553  // select should be interpreted as a table expression, not a row
1554  // expression. The semantic difference is that a table expression can
1555  // return multiple rows.
1556  if (seek instanceof SqlNodeList) {
1557  return convertRowValues(
1558  bb, seek, ((SqlNodeList) seek).getList(), false, targetRowType);
1559  } else {
1560  return convertQueryRecursive(seek, false, null).project();
1561  }
1562  }
1563 
1564  private RelNode convertRowValues(Blackboard bb,
1565  SqlNode rowList,
1566  Collection<SqlNode> rows,
1567  boolean allowLiteralsOnly,
1568  RelDataType targetRowType) {
1569  // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
1570  // literals into a single LogicalValues; this gives the optimizer a smaller
1571  // input tree. For everything else (computed expressions, row
1572  // sub-queries), we union each row in as a projection on top of a
1573  // LogicalOneRow.
1574 
1575  final ImmutableList.Builder<ImmutableList<RexLiteral>> tupleList =
1576  ImmutableList.builder();
1577  final RelDataType rowType;
1578  if (targetRowType != null) {
1579  rowType = targetRowType;
1580  } else {
1581  rowType = SqlTypeUtil.promoteToRowType(
1582  typeFactory, validator.getValidatedNodeType(rowList), null);
1583  }
1584 
1585  final List<RelNode> unionInputs = new ArrayList<>();
1586  for (SqlNode node : rows) {
1587  SqlBasicCall call;
1588  if (isRowConstructor(node)) {
1589  call = (SqlBasicCall) node;
1590  ImmutableList.Builder<RexLiteral> tuple = ImmutableList.builder();
1591  for (Ord<SqlNode> operand : Ord.zip(call.operands)) {
1592  RexLiteral rexLiteral =
1593  convertLiteralInValuesList(operand.e, bb, rowType, operand.i);
1594  if ((rexLiteral == null) && allowLiteralsOnly) {
1595  return null;
1596  }
1597  if ((rexLiteral == null) || !config.isCreateValuesRel()) {
1598  // fallback to convertRowConstructor
1599  tuple = null;
1600  break;
1601  }
1602  tuple.add(rexLiteral);
1603  }
1604  if (tuple != null) {
1605  tupleList.add(tuple.build());
1606  continue;
1607  }
1608  } else {
1609  RexLiteral rexLiteral = convertLiteralInValuesList(node, bb, rowType, 0);
1610  if ((rexLiteral != null) && config.isCreateValuesRel()) {
1611  tupleList.add(ImmutableList.of(rexLiteral));
1612  continue;
1613  } else {
1614  if ((rexLiteral == null) && allowLiteralsOnly) {
1615  return null;
1616  }
1617  }
1618 
1619  // convert "1" to "row(1)"
1620  call = (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(SqlParserPos.ZERO, node);
1621  }
1622  unionInputs.add(convertRowConstructor(bb, call));
1623  }
1624  LogicalValues values = LogicalValues.create(cluster, rowType, tupleList.build());
1625  RelNode resultRel;
1626  if (unionInputs.isEmpty()) {
1627  resultRel = values;
1628  } else {
1629  if (!values.getTuples().isEmpty()) {
1630  unionInputs.add(values);
1631  }
1632  resultRel = LogicalUnion.create(unionInputs, true);
1633  }
1634  leaves.put(resultRel, resultRel.getRowType().getFieldCount());
1635  return resultRel;
1636  }
1637 
1638  private RexLiteral convertLiteralInValuesList(
1639  SqlNode sqlNode, Blackboard bb, RelDataType rowType, int iField) {
1640  if (!(sqlNode instanceof SqlLiteral)) {
1641  return null;
1642  }
1643  RelDataTypeField field = rowType.getFieldList().get(iField);
1644  RelDataType type = field.getType();
1645  if (type.isStruct()) {
1646  // null literals for weird stuff like UDT's need
1647  // special handling during type flattening, so
1648  // don't use LogicalValues for those
1649  return null;
1650  }
1651 
1652  RexNode literalExpr = exprConverter.convertLiteral(bb, (SqlLiteral) sqlNode);
1653 
1654  if (!(literalExpr instanceof RexLiteral)) {
1655  assert literalExpr.isA(SqlKind.CAST);
1656  RexNode child = ((RexCall) literalExpr).getOperands().get(0);
1657  assert RexLiteral.isNullLiteral(child);
1658 
1659  // NOTE jvs 22-Nov-2006: we preserve type info
1660  // in LogicalValues digest, so it's OK to lose it here
1661  return (RexLiteral) child;
1662  }
1663 
1664  RexLiteral literal = (RexLiteral) literalExpr;
1665 
1666  Comparable value = literal.getValue();
1667 
1668  if (SqlTypeUtil.isExactNumeric(type) && SqlTypeUtil.hasScale(type)) {
1669  BigDecimal roundedValue =
1670  NumberUtil.rescaleBigDecimal((BigDecimal) value, type.getScale());
1671  return rexBuilder.makeExactLiteral(roundedValue, type);
1672  }
1673 
1674  if ((value instanceof NlsString) && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
1675  // pad fixed character type
1676  NlsString unpadded = (NlsString) value;
1677  return rexBuilder.makeCharLiteral(
1678  new NlsString(Spaces.padRight(unpadded.getValue(), type.getPrecision()),
1679  unpadded.getCharsetName(),
1680  unpadded.getCollation()));
1681  }
1682  return literal;
1683  }
1684 
1685  private boolean isRowConstructor(SqlNode node) {
1686  if (!(node.getKind() == SqlKind.ROW)) {
1687  return false;
1688  }
1689  SqlCall call = (SqlCall) node;
1690  return call.getOperator().getName().equalsIgnoreCase("row");
1691  }
1692 
1709  private void findSubQueries(Blackboard bb,
1710  SqlNode node,
1711  RelOptUtil.Logic logic,
1712  boolean registerOnlyScalarSubQueries) {
1713  final SqlKind kind = node.getKind();
1714  switch (kind) {
1715  case EXISTS:
1716  case SELECT:
1717  case MULTISET_QUERY_CONSTRUCTOR:
1718  case MULTISET_VALUE_CONSTRUCTOR:
1719  case ARRAY_QUERY_CONSTRUCTOR:
1720  case CURSOR:
1721  case SCALAR_QUERY:
1722  if (!registerOnlyScalarSubQueries || (kind == SqlKind.SCALAR_QUERY)) {
1723  bb.registerSubQuery(node, RelOptUtil.Logic.TRUE_FALSE);
1724  }
1725  return;
1726  case IN:
1727  break;
1728  case NOT_IN:
1729  case NOT:
1730  logic = logic.negate();
1731  break;
1732  }
1733  if (node instanceof SqlCall) {
1734  switch (kind) {
1735  // Do no change logic for AND, IN and NOT IN expressions;
1736  // but do change logic for OR, NOT and others;
1737  // EXISTS was handled already.
1738  case AND:
1739  case IN:
1740  case NOT_IN:
1741  break;
1742  default:
1743  logic = RelOptUtil.Logic.TRUE_FALSE_UNKNOWN;
1744  break;
1745  }
1746  for (SqlNode operand : ((SqlCall) node).getOperandList()) {
1747  if (operand != null) {
1748  // In the case of an IN expression, locate scalar
1749  // sub-queries so we can convert them to constants
1750  findSubQueries(bb,
1751  operand,
1752  logic,
1753  kind == SqlKind.IN || kind == SqlKind.NOT_IN || kind == SqlKind.SOME
1754  || kind == SqlKind.ALL || registerOnlyScalarSubQueries);
1755  }
1756  }
1757  } else if (node instanceof SqlNodeList) {
1758  for (SqlNode child : (SqlNodeList) node) {
1759  findSubQueries(bb,
1760  child,
1761  logic,
1762  kind == SqlKind.IN || kind == SqlKind.NOT_IN || kind == SqlKind.SOME
1763  || kind == SqlKind.ALL || registerOnlyScalarSubQueries);
1764  }
1765  }
1766 
1767  // Now that we've located any scalar sub-queries inside the IN
1768  // expression, register the IN expression itself. We need to
1769  // register the scalar sub-queries first so they can be converted
1770  // before the IN expression is converted.
1771  switch (kind) {
1772  case IN:
1773  case NOT_IN:
1774  case SOME:
1775  case ALL:
1776  switch (logic) {
1777  case TRUE_FALSE_UNKNOWN:
1778  RelDataType type = validator.getValidatedNodeTypeIfKnown(node);
1779  if (type == null) {
1780  // The node might not be validated if we still don't know type of the node.
1781  // Therefore return directly.
1782  return;
1783  } else {
1784  break;
1785  }
1786  // fall through
1787  case UNKNOWN_AS_FALSE:
1788  logic = RelOptUtil.Logic.TRUE;
1789  }
1790  bb.registerSubQuery(node, logic);
1791  break;
1792  }
1793  }
1794 
1801  public RexNode convertExpression(SqlNode node) {
1802  Map<String, RelDataType> nameToTypeMap = Collections.emptyMap();
1803  final ParameterScope scope =
1804  new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
1805  final Blackboard bb = createBlackboard(scope, null, false);
1806  return bb.convertExpression(node);
1807  }
1808 
1820  public RexNode convertExpression(SqlNode node, Map<String, RexNode> nameToNodeMap) {
1821  final Map<String, RelDataType> nameToTypeMap = new HashMap<>();
1822  for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
1823  nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
1824  }
1825  final ParameterScope scope =
1826  new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
1827  final Blackboard bb = createBlackboard(scope, nameToNodeMap, false);
1828  return bb.convertExpression(node);
1829  }
1830 
1843  protected RexNode convertExtendedExpression(SqlNode node, Blackboard bb) {
1844  return null;
1845  }
1846 
1847  private RexNode convertOver(Blackboard bb, SqlNode node) {
1848  SqlCall call = (SqlCall) node;
1849  SqlCall aggCall = call.operand(0);
1850  boolean ignoreNulls = false;
1851  switch (aggCall.getKind()) {
1852  case IGNORE_NULLS:
1853  ignoreNulls = true;
1854  // fall through
1855  case RESPECT_NULLS:
1856  aggCall = aggCall.operand(0);
1857  }
1858 
1859  SqlNode windowOrRef = call.operand(1);
1860  final SqlWindow window = validator.resolveWindow(windowOrRef, bb.scope);
1861 
1862  SqlNode sqlLowerBound = window.getLowerBound();
1863  SqlNode sqlUpperBound = window.getUpperBound();
1864  boolean rows = window.isRows();
1865  SqlNodeList orderList = window.getOrderList();
1866 
1867  if (!aggCall.getOperator().allowsFraming()) {
1868  // If the operator does not allow framing, bracketing is implicitly
1869  // everything up to the current row.
1870  sqlLowerBound = SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO);
1871  sqlUpperBound = SqlWindow.createCurrentRow(SqlParserPos.ZERO);
1872  if (aggCall.getKind() == SqlKind.ROW_NUMBER) {
1873  // ROW_NUMBER() expects specific kind of framing.
1874  rows = true;
1875  }
1876  } else if (orderList.size() == 0) {
1877  // Without ORDER BY, there must be no bracketing.
1878  sqlLowerBound = SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO);
1879  sqlUpperBound = SqlWindow.createUnboundedFollowing(SqlParserPos.ZERO);
1880  } else if (sqlLowerBound == null && sqlUpperBound == null) {
1881  sqlLowerBound = SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO);
1882  sqlUpperBound = SqlWindow.createCurrentRow(SqlParserPos.ZERO);
1883  } else if (sqlUpperBound == null) {
1884  sqlUpperBound = SqlWindow.createCurrentRow(SqlParserPos.ZERO);
1885  } else if (sqlLowerBound == null) {
1886  sqlLowerBound = SqlWindow.createCurrentRow(SqlParserPos.ZERO);
1887  }
1888  final SqlNodeList partitionList = window.getPartitionList();
1889  final ImmutableList.Builder<RexNode> partitionKeys = ImmutableList.builder();
1890  for (SqlNode partition : partitionList) {
1891  partitionKeys.add(bb.convertExpression(partition));
1892  }
1893  final RexNode lowerBound = bb.convertExpression(sqlLowerBound);
1894  final RexNode upperBound = bb.convertExpression(sqlUpperBound);
1895  if (orderList.size() == 0 && !rows) {
1896  // A logical range requires an ORDER BY clause. Use the implicit
1897  // ordering of this relation. There must be one, otherwise it would
1898  // have failed validation.
1899  orderList = bb.scope.getOrderList();
1900  if (orderList == null) {
1901  throw new AssertionError("Relation should have sort key for implicit ORDER BY");
1902  }
1903  }
1904 
1905  final ImmutableList.Builder<RexFieldCollation> orderKeys = ImmutableList.builder();
1906  for (SqlNode order : orderList) {
1907  orderKeys.add(bb.convertSortExpression(order,
1908  RelFieldCollation.Direction.ASCENDING,
1909  RelFieldCollation.NullDirection.UNSPECIFIED));
1910  }
1911 
1912  try {
1913  Preconditions.checkArgument(bb.window == null, "already in window agg mode");
1914  bb.window = window;
1915  RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
1916  rexAgg = rexBuilder.ensureType(validator.getValidatedNodeType(call), rexAgg, false);
1917 
1918  // Walk over the tree and apply 'over' to all agg functions. This is
1919  // necessary because the returned expression is not necessarily a call
1920  // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x).
1921 
1922  final SqlLiteral q = aggCall.getFunctionQuantifier();
1923  final boolean isDistinct = q != null && q.getValue() == SqlSelectKeyword.DISTINCT;
1924 
1925  final RexShuttle visitor = new HistogramShuttle(partitionKeys.build(),
1926  orderKeys.build(),
1927  RexWindowBounds.create(sqlLowerBound, lowerBound),
1928  RexWindowBounds.create(sqlUpperBound, upperBound),
1929  rows,
1930  window.isAllowPartial(),
1931  isDistinct,
1932  ignoreNulls);
1933  return rexAgg.accept(visitor);
1934  } finally {
1935  bb.window = null;
1936  }
1937  }
1938 
1939  protected void convertFrom(Blackboard bb, SqlNode from) {
1940  convertFrom(bb, from, Collections.emptyList());
1941  }
1942 
1961  protected void convertFrom(Blackboard bb, SqlNode from, List<String> fieldNames) {
1962  if (from == null) {
1963  bb.setRoot(LogicalValues.createOneRow(cluster), false);
1964  return;
1965  }
1966 
1967  final SqlCall call;
1968  final SqlNode[] operands;
1969  switch (from.getKind()) {
1970  case MATCH_RECOGNIZE:
1971  convertMatchRecognize(bb, (SqlCall) from);
1972  return;
1973 
1974  case AS:
1975  call = (SqlCall) from;
1976  SqlNode firstOperand = call.operand(0);
1977  final List<String> fieldNameList = new ArrayList<>();
1978  if (call.operandCount() > 2) {
1979  for (SqlNode node : Util.skip(call.getOperandList(), 2)) {
1980  fieldNameList.add(((SqlIdentifier) node).getSimple());
1981  }
1982  }
1983  convertFrom(bb, firstOperand, fieldNameList);
1984  return;
1985 
1986  case WITH_ITEM:
1987  convertFrom(bb, ((SqlWithItem) from).query);
1988  return;
1989 
1990  case WITH:
1991  convertFrom(bb, ((SqlWith) from).body);
1992  return;
1993 
1994  case TABLESAMPLE:
1995  operands = ((SqlBasicCall) from).getOperands();
1996  SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]);
1997  if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
1998  String sampleName =
1999  ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec).getName();
2000  datasetStack.push(sampleName);
2001  convertFrom(bb, operands[0]);
2002  datasetStack.pop();
2003  } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
2004  SqlSampleSpec.SqlTableSampleSpec tableSampleSpec =
2005  (SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
2006  convertFrom(bb, operands[0]);
2007  RelOptSamplingParameters params =
2008  new RelOptSamplingParameters(tableSampleSpec.isBernoulli(),
2009  tableSampleSpec.getSamplePercentage(),
2010  tableSampleSpec.isRepeatable(),
2011  tableSampleSpec.getRepeatableSeed());
2012  bb.setRoot(new Sample(cluster, bb.root, params), false);
2013  } else {
2014  throw new AssertionError("unknown TABLESAMPLE type: " + sampleSpec);
2015  }
2016  return;
2017 
2018  case TABLE_REF:
2019  call = (SqlCall) from;
2020  convertIdentifier(bb, call.operand(0), null, call.operand(1));
2021  return;
2022 
2023  case IDENTIFIER:
2024  convertIdentifier(bb, (SqlIdentifier) from, null, null);
2025  return;
2026 
2027  case EXTEND:
2028  call = (SqlCall) from;
2029  final SqlNode operand0 = call.getOperandList().get(0);
2030  final SqlIdentifier id = operand0.getKind() == SqlKind.TABLE_REF
2031  ? ((SqlCall) operand0).operand(0)
2032  : (SqlIdentifier) operand0;
2033  SqlNodeList extendedColumns = (SqlNodeList) call.getOperandList().get(1);
2034  convertIdentifier(bb, id, extendedColumns, null);
2035  return;
2036 
2037  case SNAPSHOT:
2038  convertTemporalTable(bb, (SqlCall) from);
2039  return;
2040 
2041  case JOIN:
2042  final SqlJoin join = (SqlJoin) from;
2043  final SqlValidatorScope scope = validator.getJoinScope(from);
2044  final Blackboard fromBlackboard = createBlackboard(scope, null, false);
2045  SqlNode left = join.getLeft();
2046  SqlNode right = join.getRight();
2047  final boolean isNatural = join.isNatural();
2048  final JoinType joinType = join.getJoinType();
2049  final SqlValidatorScope leftScope = Util.first(
2050  validator.getJoinScope(left), ((DelegatingScope) bb.scope).getParent());
2051  final Blackboard leftBlackboard = createBlackboard(leftScope, null, false);
2052  final SqlValidatorScope rightScope = Util.first(
2053  validator.getJoinScope(right), ((DelegatingScope) bb.scope).getParent());
2054  final Blackboard rightBlackboard = createBlackboard(rightScope, null, false);
2055  convertFrom(leftBlackboard, left);
2056  RelNode leftRel = leftBlackboard.root;
2057  convertFrom(rightBlackboard, right);
2058  RelNode rightRel = rightBlackboard.root;
2059  JoinRelType convertedJoinType = convertJoinType(joinType);
2060  RexNode conditionExp;
2061  final SqlValidatorNamespace leftNamespace = validator.getNamespace(left);
2062  final SqlValidatorNamespace rightNamespace = validator.getNamespace(right);
2063  if (isNatural) {
2064  final RelDataType leftRowType = leftNamespace.getRowType();
2065  final RelDataType rightRowType = rightNamespace.getRowType();
2066  final List<String> columnList = SqlValidatorUtil.deriveNaturalJoinColumnList(
2067  catalogReader.nameMatcher(), leftRowType, rightRowType);
2068  conditionExp = convertUsing(leftNamespace, rightNamespace, columnList);
2069  } else {
2070  conditionExp = convertJoinCondition(fromBlackboard,
2071  leftNamespace,
2072  rightNamespace,
2073  join.getCondition(),
2074  join.getConditionType(),
2075  leftRel,
2076  rightRel);
2077  }
2078 
2079  final RelNode joinRel = createJoin(
2080  fromBlackboard, leftRel, rightRel, conditionExp, convertedJoinType);
2081  bb.setRoot(joinRel, false);
2082  return;
2083 
2084  case SELECT:
2085  case INTERSECT:
2086  case EXCEPT:
2087  case UNION:
2088  final RelNode rel = convertQueryRecursive(from, false, null).project();
2089  bb.setRoot(rel, true);
2090  return;
2091 
2092  case VALUES:
2093  convertValuesImpl(bb, (SqlCall) from, null);
2094  if (fieldNames.size() > 0) {
2095  bb.setRoot(relBuilder.push(bb.root).rename(fieldNames).build(), true);
2096  }
2097  return;
2098 
2099  case UNNEST:
2100  convertUnnest(bb, (SqlCall) from, fieldNames);
2101  return;
2102 
2103  case COLLECTION_TABLE:
2104  call = (SqlCall) from;
2105 
2106  // Dig out real call; TABLE() wrapper is just syntactic.
2107  assert call.getOperandList().size() == 1;
2108  final SqlCall call2 = call.operand(0);
2109  convertCollectionTable(bb, call2);
2110  return;
2111 
2112  default:
2113  throw new AssertionError("not a join operator " + from);
2114  }
2115  }
2116 
2117  private void convertUnnest(Blackboard bb, SqlCall call, List<String> fieldNames) {
2118  final List<SqlNode> nodes = call.getOperandList();
2119  final SqlUnnestOperator operator = (SqlUnnestOperator) call.getOperator();
2120  for (SqlNode node : nodes) {
2121  replaceSubQueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2122  }
2123  final List<RexNode> exprs = new ArrayList<>();
2124  for (Ord<SqlNode> node : Ord.zip(nodes)) {
2125  exprs.add(relBuilder.alias(
2126  bb.convertExpression(node.e), validator.deriveAlias(node.e, node.i)));
2127  }
2128  RelNode child = (null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster);
2129  RelNode uncollect;
2130  if (validator.config().sqlConformance().allowAliasUnnestItems()) {
2131  uncollect = relBuilder.push(child)
2132  .project(exprs)
2133  .uncollect(fieldNames, operator.withOrdinality)
2134  .build();
2135  } else {
2136  // REVIEW danny 2020-04-26: should we unify the normal field aliases and the
2137  // item aliases ?
2138  uncollect = relBuilder.push(child)
2139  .project(exprs)
2140  .uncollect(Collections.emptyList(), operator.withOrdinality)
2141  .rename(fieldNames)
2142  .build();
2143  }
2144  bb.setRoot(uncollect, true);
2145  }
2146 
2147  protected void convertMatchRecognize(Blackboard bb, SqlCall call) {
2148  final SqlMatchRecognize matchRecognize = (SqlMatchRecognize) call;
2149  final SqlValidatorNamespace ns = validator.getNamespace(matchRecognize);
2150  final SqlValidatorScope scope = validator.getMatchRecognizeScope(matchRecognize);
2151 
2152  final Blackboard matchBb = createBlackboard(scope, null, false);
2153  final RelDataType rowType = ns.getRowType();
2154  // convert inner query, could be a table name or a derived table
2155  SqlNode expr = matchRecognize.getTableRef();
2156  convertFrom(matchBb, expr);
2157  final RelNode input = matchBb.root;
2158 
2159  // PARTITION BY
2160  final SqlNodeList partitionList = matchRecognize.getPartitionList();
2161  final ImmutableBitSet.Builder partitionKeys = ImmutableBitSet.builder();
2162  for (SqlNode partition : partitionList) {
2163  RexNode e = matchBb.convertExpression(partition);
2164  partitionKeys.set(((RexInputRef) e).getIndex());
2165  }
2166 
2167  // ORDER BY
2168  final SqlNodeList orderList = matchRecognize.getOrderList();
2169  final List<RelFieldCollation> orderKeys = new ArrayList<>();
2170  for (SqlNode order : orderList) {
2171  final RelFieldCollation.Direction direction;
2172  switch (order.getKind()) {
2173  case DESCENDING:
2174  direction = RelFieldCollation.Direction.DESCENDING;
2175  order = ((SqlCall) order).operand(0);
2176  break;
2177  case NULLS_FIRST:
2178  case NULLS_LAST:
2179  throw new AssertionError();
2180  default:
2181  direction = RelFieldCollation.Direction.ASCENDING;
2182  break;
2183  }
2184  final RelFieldCollation.NullDirection nullDirection =
2185  validator.config().defaultNullCollation().last(desc(direction))
2186  ? RelFieldCollation.NullDirection.LAST
2187  : RelFieldCollation.NullDirection.FIRST;
2188  RexNode e = matchBb.convertExpression(order);
2189  orderKeys.add(new RelFieldCollation(
2190  ((RexInputRef) e).getIndex(), direction, nullDirection));
2191  }
2192  final RelCollation orders = cluster.traitSet().canonize(RelCollations.of(orderKeys));
2193 
2194  // convert pattern
2195  final Set<String> patternVarsSet = new HashSet<>();
2196  SqlNode pattern = matchRecognize.getPattern();
2197  final SqlBasicVisitor<RexNode> patternVarVisitor = new SqlBasicVisitor<RexNode>() {
2198  @Override
2199  public RexNode visit(SqlCall call) {
2200  List<SqlNode> operands = call.getOperandList();
2201  List<RexNode> newOperands = new ArrayList<>();
2202  for (SqlNode node : operands) {
2203  newOperands.add(node.accept(this));
2204  }
2205  return rexBuilder.makeCall(
2206  validator.getUnknownType(), call.getOperator(), newOperands);
2207  }
2208 
2209  @Override
2210  public RexNode visit(SqlIdentifier id) {
2211  assert id.isSimple();
2212  patternVarsSet.add(id.getSimple());
2213  return rexBuilder.makeLiteral(id.getSimple());
2214  }
2215 
2216  @Override
2217  public RexNode visit(SqlLiteral literal) {
2218  if (literal instanceof SqlNumericLiteral) {
2219  return rexBuilder.makeExactLiteral(BigDecimal.valueOf(literal.intValue(true)));
2220  } else {
2221  return rexBuilder.makeLiteral(literal.booleanValue());
2222  }
2223  }
2224  };
2225  final RexNode patternNode = pattern.accept(patternVarVisitor);
2226 
2227  SqlLiteral interval = matchRecognize.getInterval();
2228  RexNode intervalNode = null;
2229  if (interval != null) {
2230  intervalNode = matchBb.convertLiteral(interval);
2231  }
2232 
2233  // convert subset
2234  final SqlNodeList subsets = matchRecognize.getSubsetList();
2235  final Map<String, TreeSet<String>> subsetMap = new HashMap<>();
2236  for (SqlNode node : subsets) {
2237  List<SqlNode> operands = ((SqlCall) node).getOperandList();
2238  SqlIdentifier left = (SqlIdentifier) operands.get(0);
2239  patternVarsSet.add(left.getSimple());
2240  SqlNodeList rights = (SqlNodeList) operands.get(1);
2241  final TreeSet<String> list = new TreeSet<>();
2242  for (SqlNode right : rights) {
2243  assert right instanceof SqlIdentifier;
2244  list.add(((SqlIdentifier) right).getSimple());
2245  }
2246  subsetMap.put(left.getSimple(), list);
2247  }
2248 
2249  SqlNode afterMatch = matchRecognize.getAfter();
2250  if (afterMatch == null) {
2251  afterMatch =
2252  SqlMatchRecognize.AfterOption.SKIP_TO_NEXT_ROW.symbol(SqlParserPos.ZERO);
2253  }
2254 
2255  final RexNode after;
2256  if (afterMatch instanceof SqlCall) {
2257  List<SqlNode> operands = ((SqlCall) afterMatch).getOperandList();
2258  SqlOperator operator = ((SqlCall) afterMatch).getOperator();
2259  assert operands.size() == 1;
2260  SqlIdentifier id = (SqlIdentifier) operands.get(0);
2261  assert patternVarsSet.contains(id.getSimple())
2262  : id.getSimple()
2263  + " not defined in pattern";
2264  RexNode rex = rexBuilder.makeLiteral(id.getSimple());
2265  after = rexBuilder.makeCall(
2266  validator.getUnknownType(), operator, ImmutableList.of(rex));
2267  } else {
2268  after = matchBb.convertExpression(afterMatch);
2269  }
2270 
2271  matchBb.setPatternVarRef(true);
2272 
2273  // convert measures
2274  final ImmutableMap.Builder<String, RexNode> measureNodes = ImmutableMap.builder();
2275  for (SqlNode measure : matchRecognize.getMeasureList()) {
2276  List<SqlNode> operands = ((SqlCall) measure).getOperandList();
2277  String alias = ((SqlIdentifier) operands.get(1)).getSimple();
2278  RexNode rex = matchBb.convertExpression(operands.get(0));
2279  measureNodes.put(alias, rex);
2280  }
2281 
2282  // convert definitions
2283  final ImmutableMap.Builder<String, RexNode> definitionNodes = ImmutableMap.builder();
2284  for (SqlNode def : matchRecognize.getPatternDefList()) {
2285  replaceSubQueries(matchBb, def, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
2286  List<SqlNode> operands = ((SqlCall) def).getOperandList();
2287  String alias = ((SqlIdentifier) operands.get(1)).getSimple();
2288  RexNode rex = matchBb.convertExpression(operands.get(0));
2289  definitionNodes.put(alias, rex);
2290  }
2291 
2292  final SqlLiteral rowsPerMatch = matchRecognize.getRowsPerMatch();
2293  final boolean allRows = rowsPerMatch != null
2294  && rowsPerMatch.getValue() == SqlMatchRecognize.RowsPerMatchOption.ALL_ROWS;
2295 
2296  matchBb.setPatternVarRef(false);
2297 
2298  final RelFactories.MatchFactory factory = RelFactories.DEFAULT_MATCH_FACTORY;
2299  final RelNode rel = factory.createMatch(input,
2300  patternNode,
2301  rowType,
2302  matchRecognize.getStrictStart().booleanValue(),
2303  matchRecognize.getStrictEnd().booleanValue(),
2304  definitionNodes.build(),
2305  measureNodes.build(),
2306  after,
2307  subsetMap,
2308  allRows,
2309  partitionKeys.build(),
2310  orders,
2311  intervalNode);
2312  bb.setRoot(rel, false);
2313  }
2314 
2316  SqlIdentifier id,
2317  SqlNodeList extendedColumns,
2318  SqlNodeList tableHints) {
2319  final SqlValidatorNamespace fromNamespace = validator.getNamespace(id).resolve();
2320  if (fromNamespace.getNode() != null) {
2321  convertFrom(bb, fromNamespace.getNode());
2322  return;
2323  }
2324  final String datasetName = datasetStack.isEmpty() ? null : datasetStack.peek();
2325  final boolean[] usedDataset = {false};
2326  RelOptTable table = SqlValidatorUtil.getRelOptTable(
2327  fromNamespace, catalogReader, datasetName, usedDataset);
2328  if (extendedColumns != null && extendedColumns.size() > 0) {
2329  assert table != null;
2330  final SqlValidatorTable validatorTable = table.unwrap(SqlValidatorTable.class);
2331  final List<RelDataTypeField> extendedFields = SqlValidatorUtil.getExtendedColumns(
2332  validator, validatorTable, extendedColumns);
2333  table = table.extend(extendedFields);
2334  }
2335  final RelNode tableRel;
2336  // Review Danny 2020-01-13: hacky to construct a new table scan
2337  // in order to apply the hint strategies.
2338  final List<RelHint> hints =
2339  hintStrategies.apply(SqlUtil.getRelHint(hintStrategies, tableHints),
2340  LogicalTableScan.create(cluster, table, ImmutableList.of()));
2341  tableRel = toRel(table, hints);
2342  bb.setRoot(tableRel, true);
2343  if (usedDataset[0]) {
2344  bb.setDataset(datasetName);
2345  }
2346  }
2347 
2348  protected void convertCollectionTable(Blackboard bb, SqlCall call) {
2349  final SqlOperator operator = call.getOperator();
2350  if (operator == SqlStdOperatorTable.TABLESAMPLE) {
2351  final String sampleName =
2352  SqlLiteral.unchain(call.operand(0)).getValueAs(String.class);
2353  datasetStack.push(sampleName);
2354  SqlCall cursorCall = call.operand(1);
2355  SqlNode query = cursorCall.operand(0);
2356  RelNode converted = convertQuery(query, false, false).rel;
2357  bb.setRoot(converted, false);
2358  datasetStack.pop();
2359  return;
2360  }
2361  replaceSubQueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2362 
2363  // Expand table macro if possible. It's more efficient than
2364  // LogicalTableFunctionScan.
2365  final SqlCallBinding callBinding =
2366  new SqlCallBinding(bb.scope.getValidator(), bb.scope, call);
2367  if (operator instanceof SqlUserDefinedTableMacro) {
2368  final SqlUserDefinedTableMacro udf = (SqlUserDefinedTableMacro) operator;
2369  final TranslatableTable table = udf.getTable(callBinding);
2370  final RelDataType rowType = table.getRowType(typeFactory);
2371  RelOptTable relOptTable =
2372  RelOptTableImpl.create(null, rowType, table, udf.getNameAsId().names);
2373  RelNode converted = toRel(relOptTable, ImmutableList.of());
2374  bb.setRoot(converted, true);
2375  return;
2376  }
2377 
2378  Type elementType;
2379  if (operator instanceof SqlUserDefinedTableFunction) {
2380  SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator;
2381  elementType = udtf.getElementType(callBinding);
2382  } else {
2383  elementType = null;
2384  }
2385 
2386  RexNode rexCall = bb.convertExpression(call);
2387  final List<RelNode> inputs = bb.retrieveCursors();
2388  Set<RelColumnMapping> columnMappings = getColumnMappings(operator);
2389  LogicalTableFunctionScan callRel = LogicalTableFunctionScan.create(cluster,
2390  inputs,
2391  rexCall,
2392  elementType,
2393  validator.getValidatedNodeType(call),
2394  columnMappings);
2395  bb.setRoot(callRel, true);
2396  afterTableFunction(bb, call, callRel);
2397  }
2398 
2399  protected void afterTableFunction(SqlToRelConverter.Blackboard bb,
2400  SqlCall call,
2401  LogicalTableFunctionScan callRel) {}
2402 
2403  private void convertTemporalTable(Blackboard bb, SqlCall call) {
2404  final SqlSnapshot snapshot = (SqlSnapshot) call;
2405  final RexNode period = bb.convertExpression(snapshot.getPeriod());
2406 
2407  // convert inner query, could be a table name or a derived table
2408  SqlNode expr = snapshot.getTableRef();
2409  convertFrom(bb, expr);
2410 
2411  final RelNode snapshotRel = relBuilder.push(bb.root).snapshot(period).build();
2412 
2413  bb.setRoot(snapshotRel, false);
2414  }
2415 
2416  private Set<RelColumnMapping> getColumnMappings(SqlOperator op) {
2417  if (op instanceof ExtTableFunction) {
2418  ExtTableFunction tf = (ExtTableFunction) op;
2419  return tf.getColumnMappings();
2420  }
2421  SqlReturnTypeInference rti = op.getReturnTypeInference();
2422  if (rti == null) {
2423  return null;
2424  }
2425  if (rti instanceof TableFunctionReturnTypeInference) {
2426  TableFunctionReturnTypeInference tfrti = (TableFunctionReturnTypeInference) rti;
2427  return tfrti.getColumnMappings();
2428  } else {
2429  return null;
2430  }
2431  }
2432 
2438  private static class RexAccessShuttle extends RexShuttle {
2439  private final RexBuilder builder;
2440  private final RexCorrelVariable rexCorrel;
2441  private final BitSet varCols = new BitSet();
2442 
2443  RexAccessShuttle(RexBuilder builder, RexCorrelVariable rexCorrel) {
2444  this.builder = builder;
2445  this.rexCorrel = rexCorrel;
2446  }
2447 
2448  @Override
2449  public RexNode visitInputRef(RexInputRef input) {
2450  int i = input.getIndex() - rexCorrel.getType().getFieldCount();
2451  if (i < 0) {
2452  varCols.set(input.getIndex());
2453  return builder.makeFieldAccess(rexCorrel, input.getIndex());
2454  }
2455  return builder.makeInputRef(input.getType(), i);
2456  }
2457  }
2458 
2459  protected RelNode createJoin(Blackboard bb,
2460  RelNode leftRel,
2461  RelNode rightRel,
2462  RexNode joinCond,
2463  JoinRelType joinType) {
2464  assert joinCond != null;
2465 
2466  final CorrelationUse p = getCorrelationUse(bb, rightRel);
2467  if (p != null) {
2468  RelNode innerRel = p.r;
2469  ImmutableBitSet requiredCols = p.requiredColumns;
2470 
2471  if (!joinCond.isAlwaysTrue()) {
2472  final RelFactories.FilterFactory factory = RelFactories.DEFAULT_FILTER_FACTORY;
2473  final RexCorrelVariable rexCorrel =
2474  (RexCorrelVariable) rexBuilder.makeCorrel(leftRel.getRowType(), p.id);
2475  final RexAccessShuttle shuttle = new RexAccessShuttle(rexBuilder, rexCorrel);
2476 
2477  // Replace outer RexInputRef with RexFieldAccess,
2478  // and push lateral join predicate into inner child
2479  final RexNode newCond = joinCond.accept(shuttle);
2480  innerRel = factory.createFilter(p.r, newCond, ImmutableSet.of());
2481  requiredCols =
2482  ImmutableBitSet.fromBitSet(shuttle.varCols).union(p.requiredColumns);
2483  }
2484 
2485  return LogicalCorrelate.create(leftRel, innerRel, p.id, requiredCols, joinType);
2486  }
2487 
2488  // MAt 8 Jan 2021 original code
2489  // final RelNode node = relBuilder.push(leftRel).push(rightRel).join(joinType,
2490  // joinCond).build();
2491 
2492  // MAT 8 Jan 2021 this is HEAVY.AI code
2493  final Join originalJoin = (Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel,
2494  rightRel,
2495  ImmutableList.of(),
2496  joinCond,
2497  ImmutableSet.of(),
2498  joinType,
2499  false);
2500  RelNode node = originalJoin;
2501  // MAT 08 Jan 2021 end of HEAVY.AI Code
2502 
2503  // If join conditions are pushed down, update the leaves.
2504  if (node instanceof Project) {
2505  final Join newJoin = (Join) node.getInputs().get(0);
2506  if (leaves.containsKey(leftRel)) {
2507  leaves.put(newJoin.getLeft(), leaves.get(leftRel));
2508  }
2509  if (leaves.containsKey(rightRel)) {
2510  leaves.put(newJoin.getRight(), leaves.get(rightRel));
2511  }
2512  }
2513  return node;
2514  }
2515 
2516  private CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0) {
2517  final Set<CorrelationId> correlatedVariables = RelOptUtil.getVariablesUsed(r0);
2518  if (correlatedVariables.isEmpty()) {
2519  return null;
2520  }
2521  final ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder();
2522  final List<CorrelationId> correlNames = new ArrayList<>();
2523 
2524  // All correlations must refer the same namespace since correlation
2525  // produces exactly one correlation source.
2526  // The same source might be referenced by different variables since
2527  // DeferredLookups are not de-duplicated at create time.
2528  SqlValidatorNamespace prevNs = null;
2529 
2530  for (CorrelationId correlName : correlatedVariables) {
2531  DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
2532  RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName);
2533  String originalRelName = lookup.getOriginalRelName();
2534  String originalFieldName = fieldAccess.getField().getName();
2535 
2536  final SqlNameMatcher nameMatcher =
2537  bb.getValidator().getCatalogReader().nameMatcher();
2538  final SqlValidatorScope.ResolvedImpl resolved =
2539  new SqlValidatorScope.ResolvedImpl();
2540  lookup.bb.scope.resolve(
2541  ImmutableList.of(originalRelName), nameMatcher, false, resolved);
2542  assert resolved.count() == 1;
2543  final SqlValidatorScope.Resolve resolve = resolved.only();
2544  final SqlValidatorNamespace foundNs = resolve.namespace;
2545  final RelDataType rowType = resolve.rowType();
2546  final int childNamespaceIndex = resolve.path.steps().get(0).i;
2547  final SqlValidatorScope ancestorScope = resolve.scope;
2548  boolean correlInCurrentScope = bb.scope.isWithin(ancestorScope);
2549 
2550  if (!correlInCurrentScope) {
2551  continue;
2552  }
2553 
2554  if (prevNs == null) {
2555  prevNs = foundNs;
2556  } else {
2557  assert prevNs
2558  == foundNs : "All correlation variables should resolve"
2559  + " to the same namespace."
2560  + " Prev ns="
2561  + prevNs
2562  + ", new ns="
2563  + foundNs;
2564  }
2565 
2566  int namespaceOffset = 0;
2567  if (childNamespaceIndex > 0) {
2568  // If not the first child, need to figure out the width
2569  // of output types from all the preceding namespaces
2570  assert ancestorScope instanceof ListScope;
2571  List<SqlValidatorNamespace> children = ((ListScope) ancestorScope).getChildren();
2572 
2573  for (int i = 0; i < childNamespaceIndex; i++) {
2574  SqlValidatorNamespace child = children.get(i);
2575  namespaceOffset += child.getRowType().getFieldCount();
2576  }
2577  }
2578 
2579  RexFieldAccess topLevelFieldAccess = fieldAccess;
2580  while (topLevelFieldAccess.getReferenceExpr() instanceof RexFieldAccess) {
2581  topLevelFieldAccess = (RexFieldAccess) topLevelFieldAccess.getReferenceExpr();
2582  }
2583  final RelDataTypeField field = rowType.getFieldList().get(
2584  topLevelFieldAccess.getField().getIndex() - namespaceOffset);
2585  int pos = namespaceOffset + field.getIndex();
2586 
2587  assert field.getType() == topLevelFieldAccess.getField().getType();
2588 
2589  assert pos != -1;
2590 
2591  if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) {
2592  // bb.root is an aggregate and only projects group by
2593  // keys.
2594  Map<Integer, Integer> exprProjection =
2595  bb.mapRootRelToFieldProjection.get(bb.root);
2596 
2597  // sub-query can reference group by keys projected from
2598  // the root of the outer relation.
2599  if (exprProjection.containsKey(pos)) {
2600  pos = exprProjection.get(pos);
2601  } else {
2602  // correl not grouped
2603  throw new AssertionError("Identifier '" + originalRelName + "."
2604  + originalFieldName + "' is not a group expr");
2605  }
2606  }
2607 
2608  requiredColumns.set(pos);
2609  correlNames.add(correlName);
2610  }
2611 
2612  if (correlNames.isEmpty()) {
2613  // None of the correlating variables originated in this scope.
2614  return null;
2615  }
2616 
2617  RelNode r = r0;
2618  if (correlNames.size() > 1) {
2619  // The same table was referenced more than once.
2620  // So we deduplicate.
2621  r = DeduplicateCorrelateVariables.go(
2622  rexBuilder, correlNames.get(0), Util.skip(correlNames), r0);
2623  // Add new node to leaves.
2624  leaves.put(r, r.getRowType().getFieldCount());
2625  }
2626  return new CorrelationUse(correlNames.get(0), requiredColumns.build(), r);
2627  }
2628 
2639  private boolean isSubQueryNonCorrelated(RelNode subq, Blackboard bb) {
2640  Set<CorrelationId> correlatedVariables = RelOptUtil.getVariablesUsed(subq);
2641  for (CorrelationId correlName : correlatedVariables) {
2642  DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
2643  String originalRelName = lookup.getOriginalRelName();
2644 
2645  final SqlNameMatcher nameMatcher =
2646  lookup.bb.scope.getValidator().getCatalogReader().nameMatcher();
2647  final SqlValidatorScope.ResolvedImpl resolved =
2648  new SqlValidatorScope.ResolvedImpl();
2649  lookup.bb.scope.resolve(
2650  ImmutableList.of(originalRelName), nameMatcher, false, resolved);
2651 
2652  SqlValidatorScope ancestorScope = resolved.only().scope;
2653 
2654  // If the correlated reference is in a scope that's "above" the
2655  // sub-query, then this is a correlated sub-query.
2656  SqlValidatorScope parentScope = bb.scope;
2657  do {
2658  if (ancestorScope == parentScope) {
2659  return false;
2660  }
2661  if (parentScope instanceof DelegatingScope) {
2662  parentScope = ((DelegatingScope) parentScope).getParent();
2663  } else {
2664  break;
2665  }
2666  } while (parentScope != null);
2667  }
2668  return true;
2669  }
2670 
2676  protected List<RelDataTypeField> getSystemFields() {
2677  return Collections.emptyList();
2678  }
2679 
2680  private RexNode convertJoinCondition(Blackboard bb,
2681  SqlValidatorNamespace leftNamespace,
2682  SqlValidatorNamespace rightNamespace,
2683  SqlNode condition,
2684  JoinConditionType conditionType,
2685  RelNode leftRel,
2686  RelNode rightRel) {
2687  if (condition == null) {
2688  return rexBuilder.makeLiteral(true);
2689  }
2690  bb.setRoot(ImmutableList.of(leftRel, rightRel));
2691  replaceSubQueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
2692  switch (conditionType) {
2693  case ON:
2694  bb.setRoot(ImmutableList.of(leftRel, rightRel));
2695  return bb.convertExpression(condition);
2696  case USING:
2697  final SqlNodeList list = (SqlNodeList) condition;
2698  final List<String> nameList = new ArrayList<>();
2699  for (SqlNode columnName : list) {
2700  final SqlIdentifier id = (SqlIdentifier) columnName;
2701  String name = id.getSimple();
2702  nameList.add(name);
2703  }
2704  return convertUsing(leftNamespace, rightNamespace, nameList);
2705  default:
2706  throw Util.unexpected(conditionType);
2707  }
2708  }
2709 
2721  private RexNode convertUsing(SqlValidatorNamespace leftNamespace,
2722  SqlValidatorNamespace rightNamespace,
2723  List<String> nameList) {
2724  final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
2725  final List<RexNode> list = new ArrayList<>();
2726  for (String name : nameList) {
2727  List<RexNode> operands = new ArrayList<>();
2728  int offset = 0;
2729  for (SqlValidatorNamespace n : ImmutableList.of(leftNamespace, rightNamespace)) {
2730  final RelDataType rowType = n.getRowType();
2731  final RelDataTypeField field = nameMatcher.field(rowType, name);
2732  operands.add(rexBuilder.makeInputRef(field.getType(), offset + field.getIndex()));
2733  offset += rowType.getFieldList().size();
2734  }
2735  list.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, operands));
2736  }
2737  return RexUtil.composeConjunction(rexBuilder, list);
2738  }
2739 
2740  private static JoinRelType convertJoinType(JoinType joinType) {
2741  switch (joinType) {
2742  case COMMA:
2743  case INNER:
2744  case CROSS:
2745  return JoinRelType.INNER;
2746  case FULL:
2747  return JoinRelType.FULL;
2748  case LEFT:
2749  return JoinRelType.LEFT;
2750  case RIGHT:
2751  return JoinRelType.RIGHT;
2752  default:
2753  throw Util.unexpected(joinType);
2754  }
2755  }
2756 
2770  protected void convertAgg(
2771  Blackboard bb, SqlSelect select, List<SqlNode> orderExprList) {
2772  assert bb.root != null : "precondition: child != null";
2773  SqlNodeList groupList = select.getGroup();
2774  SqlNodeList selectList = select.getSelectList();
2775  SqlNode having = select.getHaving();
2776 
2777  final AggConverter aggConverter = new AggConverter(bb, select);
2778  createAggImpl(bb, aggConverter, selectList, groupList, having, orderExprList);
2779  }
2780 
2781  protected final void createAggImpl(Blackboard bb,
2782  final AggConverter aggConverter,
2783  SqlNodeList selectList,
2784  SqlNodeList groupList,
2785  SqlNode having,
2786  List<SqlNode> orderExprList) {
2787  // Find aggregate functions in SELECT and HAVING clause
2788  final AggregateFinder aggregateFinder = new AggregateFinder();
2789  selectList.accept(aggregateFinder);
2790  if (having != null) {
2791  having.accept(aggregateFinder);
2792  }
2793 
2794  // first replace the sub-queries inside the aggregates
2795  // because they will provide input rows to the aggregates.
2796  replaceSubQueries(bb, aggregateFinder.list, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2797 
2798  // also replace sub-queries inside filters in the aggregates
2800  bb, aggregateFinder.filterList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2801 
2802  // also replace sub-queries inside ordering spec in the aggregates
2803  replaceSubQueries(bb, aggregateFinder.orderList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2804 
2805  // If group-by clause is missing, pretend that it has zero elements.
2806  if (groupList == null) {
2807  groupList = SqlNodeList.EMPTY;
2808  }
2809 
2810  replaceSubQueries(bb, groupList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2811 
2812  // register the group exprs
2813 
2814  // build a map to remember the projections from the top scope to the
2815  // output of the current root.
2816  //
2817  // Calcite allows expressions, not just column references in
2818  // group by list. This is not SQL 2003 compliant, but hey.
2819 
2820  final AggregatingSelectScope scope = aggConverter.aggregatingSelectScope;
2821  final AggregatingSelectScope.Resolved r = scope.resolved.get();
2822  for (SqlNode groupExpr : r.groupExprList) {
2823  aggConverter.addGroupExpr(groupExpr);
2824  }
2825 
2826  final RexNode havingExpr;
2827  final List<Pair<RexNode, String>> projects = new ArrayList<>();
2828 
2829  try {
2830  Preconditions.checkArgument(bb.agg == null, "already in agg mode");
2831  bb.agg = aggConverter;
2832 
2833  // convert the select and having expressions, so that the
2834  // agg converter knows which aggregations are required
2835 
2836  selectList.accept(aggConverter);
2837  // Assert we don't have dangling items left in the stack
2838  assert !aggConverter.inOver;
2839  for (SqlNode expr : orderExprList) {
2840  expr.accept(aggConverter);
2841  assert !aggConverter.inOver;
2842  }
2843  if (having != null) {
2844  having.accept(aggConverter);
2845  assert !aggConverter.inOver;
2846  }
2847 
2848  // compute inputs to the aggregator
2849  List<Pair<RexNode, String>> preExprs = aggConverter.getPreExprs();
2850 
2851  if (preExprs.size() == 0) {
2852  // Special case for COUNT(*), where we can end up with no inputs
2853  // at all. The rest of the system doesn't like 0-tuples, so we
2854  // select a dummy constant here.
2855  final RexNode zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO);
2856  preExprs = ImmutableList.of(Pair.of(zero, (String) null));
2857  }
2858 
2859  final RelNode inputRel = bb.root;
2860 
2861  // Project the expressions required by agg and having.
2862  bb.setRoot(relBuilder.push(inputRel)
2863  .projectNamed(Pair.left(preExprs), Pair.right(preExprs), false)
2864  .build(),
2865  false);
2866  bb.mapRootRelToFieldProjection.put(bb.root, r.groupExprProjection);
2867 
2868  // REVIEW jvs 31-Oct-2007: doesn't the declaration of
2869  // monotonicity here assume sort-based aggregation at
2870  // the physical level?
2871 
2872  // Tell bb which of group columns are sorted.
2873  bb.columnMonotonicities.clear();
2874  for (SqlNode groupItem : groupList) {
2875  bb.columnMonotonicities.add(bb.scope.getMonotonicity(groupItem));
2876  }
2877 
2878  final RelNode relNode = aggConverter.containsGroupId()
2879  ? rewriteAggregateWithGroupId(bb, r, aggConverter)
2880  : createAggregate(bb, r.groupSet, r.groupSets, aggConverter.getAggCalls());
2881 
2882  bb.setRoot(relNode, false);
2883  bb.mapRootRelToFieldProjection.put(bb.root, r.groupExprProjection);
2884 
2885  // Replace sub-queries in having here and modify having to use
2886  // the replaced expressions
2887  if (having != null) {
2888  SqlNode newHaving = pushDownNotForIn(bb.scope, having);
2889  replaceSubQueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
2890  havingExpr = bb.convertExpression(newHaving);
2891  } else {
2892  havingExpr = relBuilder.literal(true);
2893  }
2894 
2895  // Now convert the other sub-queries in the select list.
2896  // This needs to be done separately from the sub-query inside
2897  // any aggregate in the select list, and after the aggregate rel
2898  // is allocated.
2899  replaceSubQueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
2900 
2901  // Now sub-queries in the entire select list have been converted.
2902  // Convert the select expressions to get the final list to be
2903  // projected.
2904  int k = 0;
2905 
2906  // For select expressions, use the field names previously assigned
2907  // by the validator. If we derive afresh, we might generate names
2908  // like "EXPR$2" that don't match the names generated by the
2909  // validator. This is especially the case when there are system
2910  // fields; system fields appear in the relnode's rowtype but do not
2911  // (yet) appear in the validator type.
2912  final SelectScope selectScope = SqlValidatorUtil.getEnclosingSelectScope(bb.scope);
2913  assert selectScope != null;
2914  final SqlValidatorNamespace selectNamespace =
2915  validator.getNamespace(selectScope.getNode());
2916  final List<String> names = selectNamespace.getRowType().getFieldNames();
2917  int sysFieldCount = selectList.size() - names.size();
2918  for (SqlNode expr : selectList) {
2919  projects.add(Pair.of(bb.convertExpression(expr),
2920  k < sysFieldCount ? validator.deriveAlias(expr, k++)
2921  : names.get(k++ - sysFieldCount)));
2922  }
2923 
2924  for (SqlNode expr : orderExprList) {
2925  projects.add(
2926  Pair.of(bb.convertExpression(expr), validator.deriveAlias(expr, k++)));
2927  }
2928  } finally {
2929  bb.agg = null;
2930  }
2931 
2932  // implement HAVING (we have already checked that it is non-trivial)
2933  relBuilder.push(bb.root);
2934  if (havingExpr != null) {
2935  relBuilder.filter(havingExpr);
2936  }
2937 
2938  // implement the SELECT list
2939  relBuilder.project(Pair.left(projects), Pair.right(projects))
2940  .rename(Pair.right(projects));
2941  bb.setRoot(relBuilder.build(), false);
2942 
2943  // Tell bb which of group columns are sorted.
2944  bb.columnMonotonicities.clear();
2945  for (SqlNode selectItem : selectList) {
2946  bb.columnMonotonicities.add(bb.scope.getMonotonicity(selectItem));
2947  }
2948  }
2949 
2964  Blackboard bb, AggregatingSelectScope.Resolved r, AggConverter converter) {
2965  final List<AggregateCall> aggregateCalls = converter.getAggCalls();
2966  final ImmutableBitSet groupSet = r.groupSet;
2967  final Map<ImmutableBitSet, Integer> groupSetCount = r.groupSetCount;
2968 
2969  final List<String> fieldNamesIfNoRewrite =
2970  createAggregate(bb, groupSet, r.groupSets, aggregateCalls)
2971  .getRowType()
2972  .getFieldNames();
2973 
2974  // If n duplicates exist for a particular grouping, the {@code GROUP_ID()}
2975  // function produces values in the range 0 to n-1. For each value,
2976  // we need to figure out the corresponding group sets.
2977  //
2978  // For example, "... GROUPING SETS (a, a, b, c, c, c, c)"
2979  // (i) The max value of the GROUP_ID() function returns is 3
2980  // (ii) GROUPING SETS (a, b, c) produces value 0,
2981  // GROUPING SETS (a, c) produces value 1,
2982  // GROUPING SETS (c) produces value 2
2983  // GROUPING SETS (c) produces value 3
2984  final Map<Integer, Set<ImmutableBitSet>> groupIdToGroupSets = new HashMap<>();
2985  int maxGroupId = 0;
2986  for (Map.Entry<ImmutableBitSet, Integer> entry : groupSetCount.entrySet()) {
2987  int groupId = entry.getValue() - 1;
2988  if (groupId > maxGroupId) {
2989  maxGroupId = groupId;
2990  }
2991  for (int i = 0; i <= groupId; i++) {
2992  groupIdToGroupSets
2993  .computeIfAbsent(i, k -> Sets.newTreeSet(ImmutableBitSet.COMPARATOR))
2994  .add(entry.getKey());
2995  }
2996  }
2997 
2998  // AggregateCall list without GROUP_ID function
2999  final List<AggregateCall> aggregateCallsWithoutGroupId = new ArrayList<>();
3000  for (AggregateCall aggregateCall : aggregateCalls) {
3001  if (aggregateCall.getAggregation().kind != SqlKind.GROUP_ID) {
3002  aggregateCallsWithoutGroupId.add(aggregateCall);
3003  }
3004  }
3005  final List<RelNode> projects = new ArrayList<>();
3006  // For each group id value , we first construct an Aggregate without
3007  // GROUP_ID() function call, and then create a Project node on top of it.
3008  // The Project adds literal value for group id in right position.
3009  for (int groupId = 0; groupId <= maxGroupId; groupId++) {
3010  // Create the Aggregate node without GROUP_ID() call
3011  final ImmutableList<ImmutableBitSet> groupSets =
3012  ImmutableList.copyOf(groupIdToGroupSets.get(groupId));
3013  final RelNode aggregate =
3014  createAggregate(bb, groupSet, groupSets, aggregateCallsWithoutGroupId);
3015 
3016  // RexLiteral for each GROUP_ID, note the type should be BIGINT
3017  final RelDataType groupIdType = typeFactory.createSqlType(SqlTypeName.BIGINT);
3018  final RexNode groupIdLiteral =
3019  rexBuilder.makeExactLiteral(BigDecimal.valueOf(groupId), groupIdType);
3020 
3021  relBuilder.push(aggregate);
3022  final List<RexNode> selectList = new ArrayList<>();
3023  final int groupExprLength = r.groupExprList.size();
3024  // Project fields in group by expressions
3025  for (int i = 0; i < groupExprLength; i++) {
3026  selectList.add(relBuilder.field(i));
3027  }
3028  // Project fields in aggregate calls
3029  int groupIdCount = 0;
3030  for (int i = 0; i < aggregateCalls.size(); i++) {
3031  if (aggregateCalls.get(i).getAggregation().kind == SqlKind.GROUP_ID) {
3032  selectList.add(groupIdLiteral);
3033  groupIdCount++;
3034  } else {
3035  int ordinal = groupExprLength + i - groupIdCount;
3036  selectList.add(relBuilder.field(ordinal));
3037  }
3038  }
3039  final RelNode project =
3040  relBuilder.project(selectList, fieldNamesIfNoRewrite).build();
3041  projects.add(project);
3042  }
3043  // Skip to create Union when there is only one child, i.e., no duplicate group
3044  // set.
3045  if (projects.size() == 1) {
3046  return projects.get(0);
3047  }
3048  return LogicalUnion.create(projects, true);
3049  }
3050 
3070  protected RelNode createAggregate(Blackboard bb,
3071  ImmutableBitSet groupSet,
3072  ImmutableList<ImmutableBitSet> groupSets,
3073  List<AggregateCall> aggCalls) {
3074  return LogicalAggregate.create(
3075  bb.root, ImmutableList.of(), groupSet, groupSets, aggCalls);
3076  }
3077 
3078  public RexDynamicParam convertDynamicParam(final SqlDynamicParam dynamicParam) {
3079  // REVIEW jvs 8-Jan-2005: dynamic params may be encountered out of
3080  // order. Should probably cross-check with the count from the parser
3081  // at the end and make sure they all got filled in. Why doesn't List
3082  // have a resize() method?!? Make this a utility.
3083  while (dynamicParam.getIndex() >= dynamicParamSqlNodes.size()) {
3084  dynamicParamSqlNodes.add(null);
3085  }
3086 
3087  dynamicParamSqlNodes.set(dynamicParam.getIndex(), dynamicParam);
3088  return rexBuilder.makeDynamicParam(
3089  getDynamicParamType(dynamicParam.getIndex()), dynamicParam.getIndex());
3090  }
3091 
3106  protected void gatherOrderExprs(Blackboard bb,
3107  SqlSelect select,
3108  SqlNodeList orderList,
3109  List<SqlNode> extraOrderExprs,
3110  List<RelFieldCollation> collationList) {
3111  // TODO: add validation rules to SqlValidator also
3112  assert bb.root != null : "precondition: child != null";
3113  assert select != null;
3114  if (orderList == null) {
3115  return;
3116  }
3117 
3118  if (!bb.top) {
3119  SqlNode offset = select.getOffset();
3120  if ((offset == null
3121  || (offset instanceof SqlLiteral
3122  && ((SqlLiteral) offset)
3123  .bigDecimalValue()
3124  .equals(BigDecimal.ZERO)))
3125  && select.getFetch() == null) {
3126  return;
3127  }
3128  }
3129 
3130  for (SqlNode orderItem : orderList) {
3131  collationList.add(convertOrderItem(select,
3132  orderItem,
3133  extraOrderExprs,
3134  RelFieldCollation.Direction.ASCENDING,
3135  RelFieldCollation.NullDirection.UNSPECIFIED));
3136  }
3137  }
3138 
3139  protected RelFieldCollation convertOrderItem(SqlSelect select,
3140  SqlNode orderItem,
3141  List<SqlNode> extraExprs,
3142  RelFieldCollation.Direction direction,
3143  RelFieldCollation.NullDirection nullDirection) {
3144  assert select != null;
3145  // Handle DESC keyword, e.g. 'select a, b from t order by a desc'.
3146  switch (orderItem.getKind()) {
3147  case DESCENDING:
3148  return convertOrderItem(select,
3149  ((SqlCall) orderItem).operand(0),
3150  extraExprs,
3151  RelFieldCollation.Direction.DESCENDING,
3152  nullDirection);
3153  case NULLS_FIRST:
3154  return convertOrderItem(select,
3155  ((SqlCall) orderItem).operand(0),
3156  extraExprs,
3157  direction,
3158  RelFieldCollation.NullDirection.FIRST);
3159  case NULLS_LAST:
3160  return convertOrderItem(select,
3161  ((SqlCall) orderItem).operand(0),
3162  extraExprs,
3163  direction,
3164  RelFieldCollation.NullDirection.LAST);
3165  }
3166 
3167  SqlNode converted = validator.expandOrderExpr(select, orderItem);
3168 
3169  switch (nullDirection) {
3170  case UNSPECIFIED:
3171  nullDirection = validator.config().defaultNullCollation().last(desc(direction))
3172  ? RelFieldCollation.NullDirection.LAST
3173  : RelFieldCollation.NullDirection.FIRST;
3174  }
3175 
3176  // Scan the select list and order exprs for an identical expression.
3177  final SelectScope selectScope = validator.getRawSelectScope(select);
3178  int ordinal = -1;
3179  for (SqlNode selectItem : selectScope.getExpandedSelectList()) {
3180  ++ordinal;
3181  if (converted.equalsDeep(stripAs(selectItem), Litmus.IGNORE)) {
3182  return new RelFieldCollation(ordinal, direction, nullDirection);
3183  }
3184  }
3185 
3186  for (SqlNode extraExpr : extraExprs) {
3187  ++ordinal;
3188  if (converted.equalsDeep(extraExpr, Litmus.IGNORE)) {
3189  return new RelFieldCollation(ordinal, direction, nullDirection);
3190  }
3191  }
3192 
3193  // TODO: handle collation sequence
3194  // TODO: flag expressions as non-standard
3195 
3196  extraExprs.add(converted);
3197  return new RelFieldCollation(ordinal + 1, direction, nullDirection);
3198  }
3199 
3200  private static boolean desc(RelFieldCollation.Direction direction) {
3201  switch (direction) {
3202  case DESCENDING:
3203  case STRICTLY_DESCENDING:
3204  return true;
3205  default:
3206  return false;
3207  }
3208  }
3209 
3210  @Deprecated // to be removed before 2.0
3211  protected boolean enableDecorrelation() {
3212  // disable sub-query decorrelation when needed.
3213  // e.g. if outer joins are not supported.
3214  return config.isDecorrelationEnabled();
3215  }
3216 
3217  protected RelNode decorrelateQuery(RelNode rootRel) {
3218  return RelDecorrelator.decorrelateQuery(rootRel, relBuilder);
3219  }
3220 
3226  @Deprecated // to be removed before 2.0
3227  public boolean isTrimUnusedFields() {
3228  return config.isTrimUnusedFields();
3229  }
3230 
3240  protected RelRoot convertQueryRecursive(
3241  SqlNode query, boolean top, RelDataType targetRowType) {
3242  final SqlKind kind = query.getKind();
3243  switch (kind) {
3244  case SELECT:
3245  return RelRoot.of(convertSelect((SqlSelect) query, top), kind);
3246  case INSERT:
3247  return RelRoot.of(convertInsert((SqlInsert) query), kind);
3248  case DELETE:
3249  return RelRoot.of(convertDelete((SqlDelete) query), kind);
3250  case UPDATE:
3251  return RelRoot.of(convertUpdate((SqlUpdate) query), kind);
3252  case MERGE:
3253  return RelRoot.of(convertMerge((SqlMerge) query), kind);
3254  case UNION:
3255  case INTERSECT:
3256  case EXCEPT:
3257  return RelRoot.of(convertSetOp((SqlCall) query), kind);
3258  case WITH:
3259  return convertWith((SqlWith) query, top);
3260  case VALUES:
3261  return RelRoot.of(convertValues((SqlCall) query, targetRowType), kind);
3262  default:
3263  throw new AssertionError("not a query: " + query);
3264  }
3265  }
3266 
3274  protected RelNode convertSetOp(SqlCall call) {
3275  final RelNode left = convertQueryRecursive(call.operand(0), false, null).project();
3276  final RelNode right = convertQueryRecursive(call.operand(1), false, null).project();
3277  switch (call.getKind()) {
3278  case UNION:
3279  return LogicalUnion.create(ImmutableList.of(left, right), all(call));
3280 
3281  case INTERSECT:
3282  return LogicalIntersect.create(ImmutableList.of(left, right), all(call));
3283 
3284  case EXCEPT:
3285  return LogicalMinus.create(ImmutableList.of(left, right), all(call));
3286 
3287  default:
3288  throw Util.unexpected(call.getKind());
3289  }
3290  }
3291 
3292  private boolean all(SqlCall call) {
3293  return ((SqlSetOperator) call.getOperator()).isAll();
3294  }
3295 
3296  protected RelNode convertInsert(SqlInsert call) {
3297  RelOptTable targetTable = getTargetTable(call);
3298 
3299  final RelDataType targetRowType = validator.getValidatedNodeType(call);
3300  assert targetRowType != null;
3301  RelNode sourceRel =
3302  convertQueryRecursive(call.getSource(), true, targetRowType).project();
3303  RelNode massagedRel = convertColumnList(call, sourceRel);
3304 
3305  return createModify(targetTable, massagedRel);
3306  }
3307 
3309  private RelNode createModify(RelOptTable targetTable, RelNode source) {
3310  final ModifiableTable modifiableTable = targetTable.unwrap(ModifiableTable.class);
3311  if (modifiableTable != null && modifiableTable == targetTable.unwrap(Table.class)) {
3312  return modifiableTable.toModificationRel(cluster,
3313  targetTable,
3314  catalogReader,
3315  source,
3316  LogicalTableModify.Operation.INSERT,
3317  null,
3318  null,
3319  false);
3320  }
3321  final ModifiableView modifiableView = targetTable.unwrap(ModifiableView.class);
3322  if (modifiableView != null) {
3323  final Table delegateTable = modifiableView.getTable();
3324  final RelDataType delegateRowType = delegateTable.getRowType(typeFactory);
3325  final RelOptTable delegateRelOptTable = RelOptTableImpl.create(
3326  null, delegateRowType, delegateTable, modifiableView.getTablePath());
3327  final RelNode newSource =
3328  createSource(targetTable, source, modifiableView, delegateRowType);
3329  return createModify(delegateRelOptTable, newSource);
3330  }
3331  return LogicalTableModify.create(targetTable,
3332  catalogReader,
3333  source,
3334  LogicalTableModify.Operation.INSERT,
3335  null,
3336  null,
3337  false);
3338  }
3339 
3353  private RelNode createSource(RelOptTable targetTable,
3354  RelNode source,
3355  ModifiableView modifiableView,
3356  RelDataType delegateRowType) {
3357  final ImmutableIntList mapping = modifiableView.getColumnMapping();
3358  assert mapping.size() == targetTable.getRowType().getFieldCount();
3359 
3360  // For columns represented in the mapping, the expression is just a field
3361  // reference.
3362  final Map<Integer, RexNode> projectMap = new HashMap<>();
3363  final List<RexNode> filters = new ArrayList<>();
3364  for (int i = 0; i < mapping.size(); i++) {
3365  int target = mapping.get(i);
3366  if (target >= 0) {
3367  projectMap.put(target, RexInputRef.of(i, source.getRowType()));
3368  }
3369  }
3370 
3371  // For columns that are not in the mapping, and have a constraint of the
3372  // form "column = value", the expression is the literal "value".
3373  //
3374  // If a column has multiple constraints, the extra ones will become a
3375  // filter.
3376  final RexNode constraint = modifiableView.getConstraint(rexBuilder, delegateRowType);
3377  RelOptUtil.inferViewPredicates(projectMap, filters, constraint);
3378  final List<Pair<RexNode, String>> projects = new ArrayList<>();
3379  for (RelDataTypeField field : delegateRowType.getFieldList()) {
3380  RexNode node = projectMap.get(field.getIndex());
3381  if (node == null) {
3382  node = rexBuilder.makeNullLiteral(field.getType());
3383  }
3384  projects.add(Pair.of(
3385  rexBuilder.ensureType(field.getType(), node, false), field.getName()));
3386  }
3387 
3388  return relBuilder.push(source)
3389  .projectNamed(Pair.left(projects), Pair.right(projects), false)
3390  .filter(filters)
3391  .build();
3392  }
3393 
3394  private RelOptTable.ToRelContext createToRelContext(List<RelHint> hints) {
3395  return ViewExpanders.toRelContext(viewExpander, cluster, hints);
3396  }
3397 
3398  public RelNode toRel(final RelOptTable table, final List<RelHint> hints) {
3399  final RelNode scan = table.toRel(createToRelContext(hints));
3400 
3401  final InitializerExpressionFactory ief =
3402  Util.first(table.unwrap(InitializerExpressionFactory.class),
3403  NullInitializerExpressionFactory.INSTANCE);
3404 
3405  boolean hasVirtualFields = table.getRowType().getFieldList().stream().anyMatch(
3406  f -> ief.generationStrategy(table, f.getIndex()) == ColumnStrategy.VIRTUAL);
3407 
3408  if (hasVirtualFields) {
3409  final RexNode sourceRef = rexBuilder.makeRangeReference(scan);
3411  table, sourceRef, table.getRowType().getFieldNames());
3412  final List<RexNode> list = new ArrayList<>();
3413  for (RelDataTypeField f : table.getRowType().getFieldList()) {
3414  final ColumnStrategy strategy = ief.generationStrategy(table, f.getIndex());
3415  switch (strategy) {
3416  case VIRTUAL:
3417  list.add(ief.newColumnDefaultValue(table, f.getIndex(), bb));
3418  break;
3419  default:
3420  list.add(rexBuilder.makeInputRef(
3421  scan, RelOptTableImpl.realOrdinal(table, f.getIndex())));
3422  }
3423  }
3424  relBuilder.push(scan);
3425  relBuilder.project(list);
3426  final RelNode project = relBuilder.build();
3427  if (ief.postExpressionConversionHook() != null) {
3428  return ief.postExpressionConversionHook().apply(bb, project);
3429  } else {
3430  return project;
3431  }
3432  }
3433 
3434  return scan;
3435  }
3436 
3437  protected RelOptTable getTargetTable(SqlNode call) {
3438  final SqlValidatorNamespace targetNs = validator.getNamespace(call);
3439  if (targetNs.isWrapperFor(SqlValidatorImpl.DmlNamespace.class)) {
3440  final SqlValidatorImpl.DmlNamespace dmlNamespace =
3441  targetNs.unwrap(SqlValidatorImpl.DmlNamespace.class);
3442  return SqlValidatorUtil.getRelOptTable(dmlNamespace, catalogReader, null, null);
3443  }
3444  final SqlValidatorNamespace resolvedNamespace = targetNs.resolve();
3445  return SqlValidatorUtil.getRelOptTable(resolvedNamespace, catalogReader, null, null);
3446  }
3447 
3465  protected RelNode convertColumnList(final SqlInsert call, RelNode source) {
3466  RelDataType sourceRowType = source.getRowType();
3467  final RexNode sourceRef = rexBuilder.makeRangeReference(sourceRowType, 0, false);
3468  final List<String> targetColumnNames = new ArrayList<>();
3469  final List<RexNode> columnExprs = new ArrayList<>();
3470  collectInsertTargets(call, sourceRef, targetColumnNames, columnExprs);
3471 
3472  final RelOptTable targetTable = getTargetTable(call);
3473  final RelDataType targetRowType = RelOptTableImpl.realRowType(targetTable);
3474  final List<RelDataTypeField> targetFields = targetRowType.getFieldList();
3475  final List<RexNode> sourceExps =
3476  new ArrayList<>(Collections.nCopies(targetFields.size(), null));
3477  final List<String> fieldNames =
3478  new ArrayList<>(Collections.nCopies(targetFields.size(), null));
3479 
3480  final InitializerExpressionFactory initializerFactory =
3481  getInitializerFactory(validator.getNamespace(call).getTable());
3482 
3483  // Walk the name list and place the associated value in the
3484  // expression list according to the ordinal value returned from
3485  // the table construct, leaving nulls in the list for columns
3486  // that are not referenced.
3487  final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
3488  for (Pair<String, RexNode> p : Pair.zip(targetColumnNames, columnExprs)) {
3489  RelDataTypeField field = nameMatcher.field(targetRowType, p.left);
3490  assert field != null : "column " + p.left + " not found";
3491  sourceExps.set(field.getIndex(), p.right);
3492  }
3493 
3494  // Lazily create a blackboard that contains all non-generated columns.
3495  final Supplier<Blackboard> bb =
3496  () -> createInsertBlackboard(targetTable, sourceRef, targetColumnNames);
3497 
3498  // Walk the expression list and get default values for any columns
3499  // that were not supplied in the statement. Get field names too.
3500  for (int i = 0; i < targetFields.size(); ++i) {
3501  final RelDataTypeField field = targetFields.get(i);
3502  final String fieldName = field.getName();
3503  fieldNames.set(i, fieldName);
3504  if (sourceExps.get(i) == null || sourceExps.get(i).getKind() == SqlKind.DEFAULT) {
3505  sourceExps.set(
3506  i, initializerFactory.newColumnDefaultValue(targetTable, i, bb.get()));
3507 
3508  // bare nulls are dangerous in the wrong hands
3509  sourceExps.set(i, castNullLiteralIfNeeded(sourceExps.get(i), field.getType()));
3510  }
3511  }
3512 
3513  return relBuilder.push(source).projectNamed(sourceExps, fieldNames, false).build();
3514  }
3515 
3521  RelOptTable targetTable, RexNode sourceRef, List<String> targetColumnNames) {
3522  final Map<String, RexNode> nameToNodeMap = new HashMap<>();
3523  int j = 0;
3524 
3525  // Assign expressions for non-generated columns.
3526  final List<ColumnStrategy> strategies = targetTable.getColumnStrategies();
3527  final List<String> targetFields = targetTable.getRowType().getFieldNames();
3528  for (String targetColumnName : targetColumnNames) {
3529  final int i = targetFields.indexOf(targetColumnName);
3530  switch (strategies.get(i)) {
3531  case STORED:
3532  case VIRTUAL:
3533  break;
3534  default:
3535  nameToNodeMap.put(targetColumnName, rexBuilder.makeFieldAccess(sourceRef, j++));
3536  }
3537  }
3538  return createBlackboard(null, nameToNodeMap, false);
3539  }
3540 
3541  private InitializerExpressionFactory getInitializerFactory(
3542  SqlValidatorTable validatorTable) {
3543  // We might unwrap a null instead of a InitializerExpressionFactory.
3544  final Table table = unwrap(validatorTable, Table.class);
3545  if (table != null) {
3546  InitializerExpressionFactory f = unwrap(table, InitializerExpressionFactory.class);
3547  if (f != null) {
3548  return f;
3549  }
3550  }
3551  return NullInitializerExpressionFactory.INSTANCE;
3552  }
3553 
3554  private static <T> T unwrap(Object o, Class<T> clazz) {
3555  if (o instanceof Wrapper) {
3556  return ((Wrapper) o).unwrap(clazz);
3557  }
3558  return null;
3559  }
3560 
3561  private RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type) {
3562  if (!RexLiteral.isNullLiteral(node)) {
3563  return node;
3564  }
3565  return rexBuilder.makeCast(type, node);
3566  }
3567 
3578  protected void collectInsertTargets(SqlInsert call,
3579  final RexNode sourceRef,
3580  final List<String> targetColumnNames,
3581  List<RexNode> columnExprs) {
3582  final RelOptTable targetTable = getTargetTable(call);
3583  final RelDataType tableRowType = targetTable.getRowType();
3584  SqlNodeList targetColumnList = call.getTargetColumnList();
3585  if (targetColumnList == null) {
3586  if (validator.config().sqlConformance().isInsertSubsetColumnsAllowed()) {
3587  final RelDataType targetRowType =
3588  typeFactory.createStructType(tableRowType.getFieldList().subList(
3589  0, sourceRef.getType().getFieldCount()));
3590  targetColumnNames.addAll(targetRowType.getFieldNames());
3591  } else {
3592  targetColumnNames.addAll(tableRowType.getFieldNames());
3593  }
3594  } else {
3595  for (int i = 0; i < targetColumnList.size(); i++) {
3596  SqlIdentifier id = (SqlIdentifier) targetColumnList.get(i);
3597  RelDataTypeField field = SqlValidatorUtil.getTargetField(
3598  tableRowType, typeFactory, id, catalogReader, targetTable);
3599  assert field != null : "column " + id.toString() + " not found";
3600  targetColumnNames.add(field.getName());
3601  }
3602  }
3603 
3604  final Blackboard bb =
3605  createInsertBlackboard(targetTable, sourceRef, targetColumnNames);
3606 
3607  // Next, assign expressions for generated columns.
3608  final List<ColumnStrategy> strategies = targetTable.getColumnStrategies();
3609  for (String columnName : targetColumnNames) {
3610  final int i = tableRowType.getFieldNames().indexOf(columnName);
3611  final RexNode expr;
3612  switch (strategies.get(i)) {
3613  case STORED:
3614  final InitializerExpressionFactory f =
3615  Util.first(targetTable.unwrap(InitializerExpressionFactory.class),
3616  NullInitializerExpressionFactory.INSTANCE);
3617  expr = f.newColumnDefaultValue(targetTable, i, bb);
3618  break;
3619  case VIRTUAL:
3620  expr = null;
3621  break;
3622  default:
3623  expr = bb.nameToNodeMap.get(columnName);
3624  }
3625  columnExprs.add(expr);
3626  }
3627 
3628  // Remove virtual columns from the list.
3629  for (int i = 0; i < targetColumnNames.size(); i++) {
3630  if (columnExprs.get(i) == null) {
3631  columnExprs.remove(i);
3632  targetColumnNames.remove(i);
3633  --i;
3634  }
3635  }
3636  }
3637 
3638  private RelNode convertDelete(SqlDelete call) {
3639  RelOptTable targetTable = getTargetTable(call);
3640  RelNode sourceRel = convertSelect(call.getSourceSelect(), false);
3641  return LogicalTableModify.create(targetTable,
3642  catalogReader,
3643  sourceRel,
3644  LogicalTableModify.Operation.DELETE,
3645  null,
3646  null,
3647  false);
3648  }
3649 
3650  private RelNode convertUpdate(SqlUpdate call) {
3651  final SqlValidatorScope scope = validator.getWhereScope(call.getSourceSelect());
3652  Blackboard bb = createBlackboard(scope, null, false);
3653 
3654  replaceSubQueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
3655 
3656  RelOptTable targetTable = getTargetTable(call);
3657 
3658  // convert update column list from SqlIdentifier to String
3659  final List<String> targetColumnNameList = new ArrayList<>();
3660  final RelDataType targetRowType = targetTable.getRowType();
3661  for (SqlNode node : call.getTargetColumnList()) {
3662  SqlIdentifier id = (SqlIdentifier) node;
3663  RelDataTypeField field = SqlValidatorUtil.getTargetField(
3664  targetRowType, typeFactory, id, catalogReader, targetTable);
3665  assert field != null : "column " + id.toString() + " not found";
3666  targetColumnNameList.add(field.getName());
3667  }
3668 
3669  RelNode sourceRel = convertSelect(call.getSourceSelect(), false);
3670 
3671  bb.setRoot(sourceRel, false);
3672  Builder<RexNode> rexNodeSourceExpressionListBuilder = ImmutableList.builder();
3673  for (SqlNode n : call.getSourceExpressionList()) {
3674  RexNode rn = bb.convertExpression(n);
3675  rexNodeSourceExpressionListBuilder.add(rn);
3676  }
3677 
3678  return LogicalTableModify.create(targetTable,
3679  catalogReader,
3680  sourceRel,
3681  LogicalTableModify.Operation.UPDATE,
3682  targetColumnNameList,
3683  rexNodeSourceExpressionListBuilder.build(),
3684  false);
3685  }
3686 
3687  private RelNode convertMerge(SqlMerge call) {
3688  RelOptTable targetTable = getTargetTable(call);
3689 
3690  // convert update column list from SqlIdentifier to String
3691  final List<String> targetColumnNameList = new ArrayList<>();
3692  final RelDataType targetRowType = targetTable.getRowType();
3693  SqlUpdate updateCall = call.getUpdateCall();
3694  if (updateCall != null) {
3695  for (SqlNode targetColumn : updateCall.getTargetColumnList()) {
3696  SqlIdentifier id = (SqlIdentifier) targetColumn;
3697  RelDataTypeField field = SqlValidatorUtil.getTargetField(
3698  targetRowType, typeFactory, id, catalogReader, targetTable);
3699  assert field != null : "column " + id.toString() + " not found";
3700  targetColumnNameList.add(field.getName());
3701  }
3702  }
3703 
3704  // replace the projection of the source select with a
3705  // projection that contains the following:
3706  // 1) the expressions corresponding to the new insert row (if there is
3707  // an insert)
3708  // 2) all columns from the target table (if there is an update)
3709  // 3) the set expressions in the update call (if there is an update)
3710 
3711  // first, convert the merge's source select to construct the columns
3712  // from the target table and the set expressions in the update call
3713  RelNode mergeSourceRel = convertSelect(call.getSourceSelect(), false);
3714 
3715  // then, convert the insert statement so we can get the insert
3716  // values expressions
3717  SqlInsert insertCall = call.getInsertCall();
3718  int nLevel1Exprs = 0;
3719  List<RexNode> level1InsertExprs = null;
3720  List<RexNode> level2InsertExprs = null;
3721  if (insertCall != null) {
3722  RelNode insertRel = convertInsert(insertCall);
3723 
3724  // if there are 2 level of projections in the insert source, combine
3725  // them into a single project; level1 refers to the topmost project;
3726  // the level1 projection contains references to the level2
3727  // expressions, except in the case where no target expression was
3728  // provided, in which case, the expression is the default value for
3729  // the column; or if the expressions directly map to the source
3730  // table
3731  level1InsertExprs = ((LogicalProject) insertRel.getInput(0)).getProjects();
3732  if (insertRel.getInput(0).getInput(0) instanceof LogicalProject) {
3733  level2InsertExprs =
3734  ((LogicalProject) insertRel.getInput(0).getInput(0)).getProjects();
3735  }
3736  nLevel1Exprs = level1InsertExprs.size();
3737  }
3738 
3739  LogicalJoin join = (LogicalJoin) mergeSourceRel.getInput(0);
3740  int nSourceFields = join.getLeft().getRowType().getFieldCount();
3741  final List<RexNode> projects = new ArrayList<>();
3742  for (int level1Idx = 0; level1Idx < nLevel1Exprs; level1Idx++) {
3743  if ((level2InsertExprs != null)
3744  && (level1InsertExprs.get(level1Idx) instanceof RexInputRef)) {
3745  int level2Idx = ((RexInputRef) level1InsertExprs.get(level1Idx)).getIndex();
3746  projects.add(level2InsertExprs.get(level2Idx));
3747  } else {
3748  projects.add(level1InsertExprs.get(level1Idx));
3749  }
3750  }
3751  if (updateCall != null) {
3752  final LogicalProject project = (LogicalProject) mergeSourceRel;
3753  projects.addAll(Util.skip(project.getProjects(), nSourceFields));
3754  }
3755 
3756  relBuilder.push(join).project(projects);
3757 
3758  return LogicalTableModify.create(targetTable,
3759  catalogReader,
3760  relBuilder.build(),
3761  LogicalTableModify.Operation.MERGE,
3762  targetColumnNameList,
3763  null,
3764  false);
3765  }
3766 
3771  private RexNode convertIdentifier(Blackboard bb, SqlIdentifier identifier) {
3772  // first check for reserved identifiers like CURRENT_USER
3773  final SqlCall call = bb.getValidator().makeNullaryCall(identifier);
3774  if (call != null) {
3775  return bb.convertExpression(call);
3776  }
3777 
3778  String pv = null;
3779  if (bb.isPatternVarRef && identifier.names.size() > 1) {
3780  pv = identifier.names.get(0);
3781  }
3782 
3783  final SqlQualified qualified;
3784  if (bb.scope != null) {
3785  qualified = bb.scope.fullyQualify(identifier);
3786  } else {
3787  qualified = SqlQualified.create(null, 1, null, identifier);
3788  }
3789  final Pair<RexNode, Map<String, Integer>> e0 = bb.lookupExp(qualified);
3790  RexNode e = e0.left;
3791  for (String name : qualified.suffix()) {
3792  if (e == e0.left && e0.right != null) {
3793  int i = e0.right.get(name);
3794  e = rexBuilder.makeFieldAccess(e, i);
3795  } else {
3796  final boolean caseSensitive = true; // name already fully-qualified
3797  if (identifier.isStar() && bb.scope instanceof MatchRecognizeScope) {
3798  e = rexBuilder.makeFieldAccess(e, 0);
3799  } else {
3800  e = rexBuilder.makeFieldAccess(e, name, caseSensitive);
3801  }
3802  }
3803  }
3804  if (e instanceof RexInputRef) {
3805  // adjust the type to account for nulls introduced by outer joins
3806  e = adjustInputRef(bb, (RexInputRef) e);
3807  if (pv != null) {
3808  e = RexPatternFieldRef.of(pv, (RexInputRef) e);
3809  }
3810  }
3811 
3812  if (e0.left instanceof RexCorrelVariable) {
3813  assert e instanceof RexFieldAccess;
3814  final RexNode prev = bb.mapCorrelateToRex.put(
3815  ((RexCorrelVariable) e0.left).id, (RexFieldAccess) e);
3816  assert prev == null;
3817  }
3818  return e;
3819  }
3820 
3830  protected RexNode adjustInputRef(Blackboard bb, RexInputRef inputRef) {
3831  RelDataTypeField field = bb.getRootField(inputRef);
3832  if (field != null) {
3833  return rexBuilder.makeInputRef(field.getType(), inputRef.getIndex());
3834  }
3835  return inputRef;
3836  }
3837 
3845  private RelNode convertRowConstructor(Blackboard bb, SqlCall rowConstructor) {
3846  Preconditions.checkArgument(isRowConstructor(rowConstructor));
3847  final List<SqlNode> operands = rowConstructor.getOperandList();
3848  return convertMultisets(operands, bb);
3849  }
3850 
3851  private RelNode convertCursor(Blackboard bb, SubQuery subQuery) {
3852  final SqlCall cursorCall = (SqlCall) subQuery.node;
3853  assert cursorCall.operandCount() == 1;
3854  SqlNode query = cursorCall.operand(0);
3855  RelNode converted = convertQuery(query, false, false).rel;
3856  int iCursor = bb.cursors.size();
3857  bb.cursors.add(converted);
3858  subQuery.expr = new RexInputRef(iCursor, converted.getRowType());
3859  return converted;
3860  }
3861 
3862  private RelNode convertMultisets(final List<SqlNode> operands, Blackboard bb) {
3863  // NOTE: Wael 2/04/05: this implementation is not the most efficient in
3864  // terms of planning since it generates XOs that can be reduced.
3865  final List<Object> joinList = new ArrayList<>();
3866  List<SqlNode> lastList = new ArrayList<>();
3867  for (int i = 0; i < operands.size(); i++) {
3868  SqlNode operand = operands.get(i);
3869  if (!(operand instanceof SqlCall)) {
3870  lastList.add(operand);
3871  continue;
3872  }
3873 
3874  final SqlCall call = (SqlCall) operand;
3875  final RelNode input;
3876  switch (call.getKind()) {
3877  case MULTISET_VALUE_CONSTRUCTOR:
3878  case ARRAY_VALUE_CONSTRUCTOR:
3879  final SqlNodeList list =
3880  new SqlNodeList(call.getOperandList(), call.getParserPosition());
3881  CollectNamespace nss = (CollectNamespace) validator.getNamespace(call);
3882  Blackboard usedBb;
3883  if (null != nss) {
3884  usedBb = createBlackboard(nss.getScope(), null, false);
3885  } else {
3886  usedBb = createBlackboard(new ListScope(bb.scope) {
3887  public SqlNode getNode() {
3888  return call;
3889  }
3890  }, null, false);
3891  }
3892  RelDataType multisetType = validator.getValidatedNodeType(call);
3893  validator.setValidatedNodeType(list, multisetType.getComponentType());
3894  input = convertQueryOrInList(usedBb, list, null);
3895  break;
3896  case MULTISET_QUERY_CONSTRUCTOR:
3897  case ARRAY_QUERY_CONSTRUCTOR:
3898  final RelRoot root = convertQuery(call.operand(0), false, true);
3899  input = root.rel;
3900  break;
3901  default:
3902  lastList.add(operand);
3903  continue;
3904  }
3905 
3906  if (lastList.size() > 0) {
3907  joinList.add(lastList);
3908  }
3909  lastList = new ArrayList<>();
3910  Collect collect = new Collect(cluster,
3911  cluster.traitSetOf(Convention.NONE),
3912  input,
3913  validator.deriveAlias(call, i));
3914  joinList.add(collect);
3915  }
3916 
3917  if (joinList.size() == 0) {
3918  joinList.add(lastList);
3919  }
3920 
3921  for (int i = 0; i < joinList.size(); i++) {
3922  Object o = joinList.get(i);
3923  if (o instanceof List) {
3924  @SuppressWarnings("unchecked")
3925  List<SqlNode> projectList = (List<SqlNode>) o;
3926  final List<RexNode> selectList = new ArrayList<>();
3927  final List<String> fieldNameList = new ArrayList<>();
3928  for (int j = 0; j < projectList.size(); j++) {
3929  SqlNode operand = projectList.get(j);
3930  selectList.add(bb.convertExpression(operand));
3931 
3932  // REVIEW angel 5-June-2005: Use deriveAliasFromOrdinal
3933  // instead of deriveAlias to match field names from
3934  // SqlRowOperator. Otherwise, get error Type
3935  // 'RecordType(INTEGER EMPNO)' has no field 'EXPR$0' when
3936  // doing select * from unnest( select multiset[empno]
3937  // from sales.emps);
3938 
3939  fieldNameList.add(SqlUtil.deriveAliasFromOrdinal(j));
3940  }
3941 
3942  relBuilder.push(LogicalValues.createOneRow(cluster))
3943  .projectNamed(selectList, fieldNameList, true);
3944 
3945  joinList.set(i, relBuilder.build());
3946  }
3947  }
3948 
3949  RelNode ret = (RelNode) joinList.get(0);
3950  for (int i = 1; i < joinList.size(); i++) {
3951  RelNode relNode = (RelNode) joinList.get(i);
3952  ret = RelFactories.DEFAULT_JOIN_FACTORY.createJoin(ret,
3953  relNode,
3954  ImmutableList.of(),
3955  rexBuilder.makeLiteral(true),
3956  ImmutableSet.of(),
3957  JoinRelType.INNER,
3958  false);
3959  }
3960  return ret;
3961  }
3962 
3963  private void convertSelectList(
3964  Blackboard bb, SqlSelect select, List<SqlNode> orderList) {
3965  SqlNodeList selectList = select.getSelectList();
3966  selectList = validator.expandStar(selectList, select, false);
3967 
3968  replaceSubQueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
3969 
3970  List<String> fieldNames = new ArrayList<>();
3971  final List<RexNode> exprs = new ArrayList<>();
3972  final Collection<String> aliases = new TreeSet<>();
3973 
3974  // Project any system fields. (Must be done before regular select items,
3975  // because offsets may be affected.)
3976  final List<SqlMonotonicity> columnMonotonicityList = new ArrayList<>();
3977  extraSelectItems(bb, select, exprs, fieldNames, aliases, columnMonotonicityList);
3978 
3979  // Project select clause.
3980  int i = -1;
3981  for (SqlNode expr : selectList) {
3982  ++i;
3983  exprs.add(bb.convertExpression(expr));
3984  fieldNames.add(deriveAlias(expr, aliases, i));
3985  }
3986 
3987  // Project extra fields for sorting.
3988  for (SqlNode expr : orderList) {
3989  ++i;
3990  SqlNode expr2 = validator.expandOrderExpr(select, expr);
3991  exprs.add(bb.convertExpression(expr2));
3992  fieldNames.add(deriveAlias(expr, aliases, i));
3993  }
3994 
3995  fieldNames = SqlValidatorUtil.uniquify(
3996  fieldNames, catalogReader.nameMatcher().isCaseSensitive());
3997 
3998  relBuilder.push(bb.root).projectNamed(exprs, fieldNames, true);
3999  bb.setRoot(relBuilder.build(), false);
4000 
4001  assert bb.columnMonotonicities.isEmpty();
4002  bb.columnMonotonicities.addAll(columnMonotonicityList);
4003  for (SqlNode selectItem : selectList) {
4004  bb.columnMonotonicities.add(selectItem.getMonotonicity(bb.scope));
4005  }
4006  }
4007 
4021  protected void extraSelectItems(Blackboard bb,
4022  SqlSelect select,
4023  List<RexNode> exprList,
4024  List<String> nameList,
4025  Collection<String> aliasList,
4026  List<SqlMonotonicity> columnMonotonicityList) {}
4027 
4028  private String deriveAlias(
4029  final SqlNode node, Collection<String> aliases, final int ordinal) {
4030  String alias = validator.deriveAlias(node, ordinal);
4031  if ((alias == null) || aliases.contains(alias)) {
4032  String aliasBase = (alias == null) ? "EXPR$" : alias;
4033  for (int j = 0;; j++) {
4034  alias = aliasBase + j;
4035  if (!aliases.contains(alias)) {
4036  break;
4037  }
4038  }
4039  }
4040  aliases.add(alias);
4041  return alias;
4042  }
4043 
4047  public RelRoot convertWith(SqlWith with, boolean top) {
4048  return convertQuery(with.body, false, top);
4049  }
4050 
4054  public RelNode convertValues(SqlCall values, RelDataType targetRowType) {
4055  final SqlValidatorScope scope = validator.getOverScope(values);
4056  assert scope != null;
4057  final Blackboard bb = createBlackboard(scope, null, false);
4058  convertValuesImpl(bb, values, targetRowType);
4059  return bb.root;
4060  }
4061 
4070  private void convertValuesImpl(
4071  Blackboard bb, SqlCall values, RelDataType targetRowType) {
4072  // Attempt direct conversion to LogicalValues; if that fails, deal with
4073  // fancy stuff like sub-queries below.
4074  RelNode valuesRel =
4075  convertRowValues(bb, values, values.getOperandList(), true, targetRowType);
4076  if (valuesRel != null) {
4077  bb.setRoot(valuesRel, true);
4078  return;
4079  }
4080 
4081  final List<RelNode> unionRels = new ArrayList<>();
4082  for (SqlNode rowConstructor1 : values.getOperandList()) {
4083  SqlCall rowConstructor = (SqlCall) rowConstructor1;
4084  Blackboard tmpBb = createBlackboard(bb.scope, null, false);
4085  replaceSubQueries(tmpBb, rowConstructor, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
4086  final List<Pair<RexNode, String>> exps = new ArrayList<>();
4087  for (Ord<SqlNode> operand : Ord.zip(rowConstructor.getOperandList())) {
4088  exps.add(Pair.of(tmpBb.convertExpression(operand.e),
4089  validator.deriveAlias(operand.e, operand.i)));
4090  }
4091  RelNode in =
4092  (null == tmpBb.root) ? LogicalValues.createOneRow(cluster) : tmpBb.root;
4093  unionRels.add(
4094  relBuilder.push(in).project(Pair.left(exps), Pair.right(exps)).build());
4095  }
4096 
4097  if (unionRels.size() == 0) {
4098  throw new AssertionError("empty values clause");
4099  } else if (unionRels.size() == 1) {
4100  bb.setRoot(unionRels.get(0), true);
4101  } else {
4102  bb.setRoot(LogicalUnion.create(unionRels, true), true);
4103  }
4104  }
4105 
4106  // ~ Inner Classes ----------------------------------------------------------
4107 
4111  protected class Blackboard
4112  implements SqlRexContext, SqlVisitor<RexNode>, InitializerContext {
4116  public final SqlValidatorScope scope;
4117  private final Map<String, RexNode> nameToNodeMap;
4118  public RelNode root;
4119  private List<RelNode> inputs;
4120  private final Map<CorrelationId, RexFieldAccess> mapCorrelateToRex = new HashMap<>();
4121 
4122  private boolean isPatternVarRef = false;
4123 
4124  final List<RelNode> cursors = new ArrayList<>();
4125 
4130  private final List<SubQuery> subQueryList = new ArrayList<>();
4131 
4135  AggConverter agg;
4136 
4141  SqlWindow window;
4142 
4148  private final Map<RelNode, Map<Integer, Integer>> mapRootRelToFieldProjection =
4149  new HashMap<>();
4150 
4151  private final List<SqlMonotonicity> columnMonotonicities = new ArrayList<>();
4152 
4153  private final List<RelDataTypeField> systemFieldList = new ArrayList<>();
4154  final boolean top;
4155 
4156  private final InitializerExpressionFactory initializerExpressionFactory =
4157  new NullInitializerExpressionFactory();
4158 
4170  protected Blackboard(
4171  SqlValidatorScope scope, Map<String, RexNode> nameToNodeMap, boolean top) {
4172  this.scope = scope;
4173  this.nameToNodeMap = nameToNodeMap;
4174  this.top = top;
4175  }
4176 
4177  public SqlNode getTopNode() {
4178  try {
4179  if (null == scope) {
4180  return null;
4181  }
4182  return scope.getNode();
4183  } catch (Exception e) {
4184  return null;
4185  }
4186  }
4187 
4188  public void setPatternVarRef(boolean isVarRef) {
4189  this.isPatternVarRef = isVarRef;
4190  }
4191 
4192  public RexNode register(RelNode rel, JoinRelType joinType) {
4193  return register(rel, joinType, null);
4194  }
4195 
4205  public RexNode register(RelNode rel, JoinRelType joinType, List<RexNode> leftKeys) {
4206  assert joinType != null;
4207  if (root == null) {
4208  assert leftKeys == null;
4209  setRoot(rel, false);
4210  return rexBuilder.makeRangeReference(root.getRowType(), 0, false);
4211  }
4212 
4213  final RexNode joinCond;
4214  final int origLeftInputCount = root.getRowType().getFieldCount();
4215  if (leftKeys != null) {
4216  List<RexNode> newLeftInputExprs = new ArrayList<>();
4217  for (int i = 0; i < origLeftInputCount; i++) {
4218  newLeftInputExprs.add(rexBuilder.makeInputRef(root, i));
4219  }
4220 
4221  final List<Integer> leftJoinKeys = new ArrayList<>();
4222  for (RexNode leftKey : leftKeys) {
4223  int index = newLeftInputExprs.indexOf(leftKey);
4224  if (index < 0 || joinType == JoinRelType.LEFT) {
4225  index = newLeftInputExprs.size();
4226  newLeftInputExprs.add(leftKey);
4227  }
4228  leftJoinKeys.add(index);
4229  }
4230 
4231  RelNode newLeftInput = relBuilder.push(root).project(newLeftInputExprs).build();
4232 
4233  // maintain the group by mapping in the new LogicalProject
4234  if (mapRootRelToFieldProjection.containsKey(root)) {
4235  mapRootRelToFieldProjection.put(
4236  newLeftInput, mapRootRelToFieldProjection.get(root));
4237  }
4238 
4239  setRoot(newLeftInput, false);
4240 
4241  // right fields appear after the LHS fields.
4242  final int rightOffset = root.getRowType().getFieldCount()
4243  - newLeftInput.getRowType().getFieldCount();
4244  final List<Integer> rightKeys =
4245  Util.range(rightOffset, rightOffset + leftKeys.size());
4246 
4247  joinCond = RelOptUtil.createEquiJoinCondition(
4248  newLeftInput, leftJoinKeys, rel, rightKeys, rexBuilder);
4249  } else {
4250  joinCond = rexBuilder.makeLiteral(true);
4251  }
4252 
4253  int leftFieldCount = root.getRowType().getFieldCount();
4254  final RelNode join = createJoin(this, root, rel, joinCond, joinType);
4255 
4256  setRoot(join, false);
4257 
4258  if (leftKeys != null && joinType == JoinRelType.LEFT) {
4259  final int leftKeyCount = leftKeys.size();
4260  int rightFieldLength = rel.getRowType().getFieldCount();
4261  assert leftKeyCount == rightFieldLength - 1;
4262 
4263  final int rexRangeRefLength = leftKeyCount + rightFieldLength;
4264  RelDataType returnType = typeFactory.createStructType(
4265  new AbstractList<Map.Entry<String, RelDataType>>() {
4266  public Map.Entry<String, RelDataType> get(int index) {
4267  return join.getRowType().getFieldList().get(
4268  origLeftInputCount + index);
4269  }
4270 
4271  public int size() {
4272  return rexRangeRefLength;
4273  }
4274  });
4275 
4276  return rexBuilder.makeRangeReference(returnType, origLeftInputCount, false);
4277  } else {
4278  return rexBuilder.makeRangeReference(
4279  rel.getRowType(), leftFieldCount, joinType.generatesNullsOnRight());
4280  }
4281  }
4282 
4294  public void setRoot(RelNode root, boolean leaf) {
4295  setRoot(Collections.singletonList(root), root, root instanceof LogicalJoin);
4296  if (leaf) {
4297  leaves.put(root, root.getRowType().getFieldCount());
4298  }
4299  this.columnMonotonicities.clear();
4300  }
4301 
4302  private void setRoot(List<RelNode> inputs, RelNode root, boolean hasSystemFields) {
4303  this.inputs = inputs;
4304  this.root = root;
4305  this.systemFieldList.clear();
4306  if (hasSystemFields) {
4307  this.systemFieldList.addAll(getSystemFields());
4308  }
4309  }
4310 
4321  public void setDataset(String datasetName) {}
4322 
4323  void setRoot(List<RelNode> inputs) {
4324  setRoot(inputs, null, false);
4325  }
4326 
4333  Pair<RexNode, Map<String, Integer>> lookupExp(SqlQualified qualified) {
4334  if (nameToNodeMap != null && qualified.prefixLength == 1) {
4335  RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0));
4336  if (node == null) {
4337  throw new AssertionError("Unknown identifier '" + qualified.identifier
4338  + "' encountered while expanding expression");
4339  }
4340  return Pair.of(node, null);
4341  }
4342  final SqlNameMatcher nameMatcher =
4343  scope.getValidator().getCatalogReader().nameMatcher();
4344  final SqlValidatorScope.ResolvedImpl resolved =
4345  new SqlValidatorScope.ResolvedImpl();
4346  scope.resolve(qualified.prefix(), nameMatcher, false, resolved);
4347  if (!(resolved.count() == 1)) {
4348  return null;
4349  }
4350  final SqlValidatorScope.Resolve resolve = resolved.only();
4351  final RelDataType rowType = resolve.rowType();
4352 
4353  // Found in current query's from list. Find which from item.
4354  // We assume that the order of the from clause items has been
4355  // preserved.
4356  final SqlValidatorScope ancestorScope = resolve.scope;
4357  boolean isParent = ancestorScope != scope;
4358  if ((inputs != null) && !isParent) {
4359  final LookupContext rels =
4360  new LookupContext(this, inputs, systemFieldList.size());
4361  final RexNode node = lookup(resolve.path.steps().get(0).i, rels);
4362  if (node == null) {
4363  return null;
4364  } else {
4365  final Map<String, Integer> fieldOffsets = new HashMap<>();
4366  for (RelDataTypeField f : resolve.rowType().getFieldList()) {
4367  if (!fieldOffsets.containsKey(f.getName())) {
4368  fieldOffsets.put(f.getName(), f.getIndex());
4369  }
4370  }
4371  final Map<String, Integer> map = ImmutableMap.copyOf(fieldOffsets);
4372  return Pair.of(node, map);
4373  }
4374  } else {
4375  // We're referencing a relational expression which has not been
4376  // converted yet. This occurs when from items are correlated,
4377  // e.g. "select from emp as emp join emp.getDepts() as dept".
4378  // Create a temporary expression.
4379  DeferredLookup lookup =
4380  new DeferredLookup(this, qualified.identifier.names.get(0));
4381  final CorrelationId correlId = cluster.createCorrel();
4382  mapCorrelToDeferred.put(correlId, lookup);
4383  if (resolve.path.steps().get(0).i < 0) {
4384  return Pair.of(rexBuilder.makeCorrel(rowType, correlId), null);
4385  } else {
4386  final RelDataTypeFactory.Builder builder = typeFactory.builder();
4387  final ListScope ancestorScope1 = (ListScope) resolve.scope;
4388  final ImmutableMap.Builder<String, Integer> fields = ImmutableMap.builder();
4389  int i = 0;
4390  int offset = 0;
4391  for (SqlValidatorNamespace c : ancestorScope1.getChildren()) {
4392  builder.addAll(c.getRowType().getFieldList());
4393  if (i == resolve.path.steps().get(0).i) {
4394  for (RelDataTypeField field : c.getRowType().getFieldList()) {
4395  fields.put(field.getName(), field.getIndex() + offset);
4396  }
4397  }
4398  ++i;
4399  offset += c.getRowType().getFieldCount();
4400  }
4401  final RexNode c = rexBuilder.makeCorrel(builder.uniquify().build(), correlId);
4402  return Pair.of(c, fields.build());
4403  }
4404  }
4405  }
4406 
4411  RexNode lookup(int offset, LookupContext lookupContext) {
4412  Pair<RelNode, Integer> pair = lookupContext.findRel(offset);
4413  return rexBuilder.makeRangeReference(pair.left.getRowType(), pair.right, false);
4414  }
4415 
4416  RelDataTypeField getRootField(RexInputRef inputRef) {
4417  if (inputs == null) {
4418  return null;
4419  }
4420  int fieldOffset = inputRef.getIndex();
4421  for (RelNode input : inputs) {
4422  RelDataType rowType = input.getRowType();
4423  if (fieldOffset < rowType.getFieldCount()) {
4424  return rowType.getFieldList().get(fieldOffset);
4425  }
4426  fieldOffset -= rowType.getFieldCount();
4427  }
4428  return null;
4429  }
4430 
4431  public void flatten(List<RelNode> rels,
4432  int systemFieldCount,
4433  int[] start,
4434  List<Pair<RelNode, Integer>> relOffsetList) {
4435  for (RelNode rel : rels) {
4436  if (leaves.containsKey(rel)) {
4437  relOffsetList.add(Pair.of(rel, start[0]));
4438  start[0] += leaves.get(rel);
4439  } else if (rel instanceof LogicalMatch) {
4440  relOffsetList.add(Pair.of(rel, start[0]));
4441  start[0] += rel.getRowType().getFieldCount();
4442  } else {
4443  if (rel instanceof LogicalJoin || rel instanceof LogicalAggregate) {
4444  start[0] += systemFieldCount;
4445  }
4446  flatten(rel.getInputs(), systemFieldCount, start, relOffsetList);
4447  }
4448  }
4449  }
4450 
4451  void registerSubQuery(SqlNode node, RelOptUtil.Logic logic) {
4452  for (SubQuery subQuery : subQueryList) {
4453  // Compare the reference to make sure the matched node has
4454  // exact scope where it belongs.
4455  if (node == subQuery.node) {
4456  return;
4457  }
4458  }
4459  subQueryList.add(new SubQuery(node, logic));
4460  }
4461 
4462  SubQuery getSubQuery(SqlNode expr) {
4463  for (SubQuery subQuery : subQueryList) {
4464  // Compare the reference to make sure the matched node has
4465  // exact scope where it belongs.
4466  if (expr == subQuery.node) {
4467  return subQuery;
4468  }
4469  }
4470 
4471  return null;
4472  }
4473 
4474  ImmutableList<RelNode> retrieveCursors() {
4475  try {
4476  return ImmutableList.copyOf(cursors);
4477  } finally {
4478  cursors.clear();
4479  }
4480  }
4481 
4482  public RexNode convertExpression(SqlNode expr) {
4483  // If we're in aggregation mode and this is an expression in the
4484  // GROUP BY clause, return a reference to the field.
4485  if (agg != null) {
4486  final SqlNode expandedGroupExpr = validator.expand(expr, scope);
4487  final int ref = agg.lookupGroupExpr(expandedGroupExpr);
4488  if (ref >= 0) {
4489  return rexBuilder.makeInputRef(root, ref);
4490  }
4491  if (expr instanceof SqlCall) {
4492  final RexNode rex = agg.lookupAggregates((SqlCall) expr);
4493  if (rex != null) {
4494  return rex;
4495  }
4496  }
4497  }
4498 
4499  // Allow the derived class chance to override the standard
4500  // behavior for special kinds of expressions.
4501  RexNode rex = convertExtendedExpression(expr, this);
4502  if (rex != null) {
4503  return rex;
4504  }
4505 
4506  // Sub-queries and OVER expressions are not like ordinary
4507  // expressions.
4508  final SqlKind kind = expr.getKind();
4509  final SubQuery subQuery;
4510  boolean isExpand = config.getExpandPredicate().test(getTopNode(), expr);
4511  if (!isExpand) {
4512  final SqlCall call;
4513  final SqlNode query;
4514  final RelRoot root;
4515  switch (kind) {
4516  case IN:
4517  case NOT_IN:
4518  case SOME:
4519  case ALL:
4520  call = (SqlCall) expr;
4521  query = call.operand(1);
4522  if (!(query instanceof SqlNodeList)) {
4523  root = convertQueryRecursive(query, false, null);
4524  final SqlNode operand = call.operand(0);
4525  List<SqlNode> nodes;
4526  switch (operand.getKind()) {
4527  case ROW:
4528  nodes = ((SqlCall) operand).getOperandList();
4529  break;
4530  default:
4531  nodes = ImmutableList.of(operand);
4532  }
4533  final ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
4534  for (SqlNode node : nodes) {
4535  builder.add(convertExpression(node));
4536  }
4537  final ImmutableList<RexNode> list = builder.build();
4538  switch (kind) {
4539  case IN:
4540  return RexSubQuery.in(root.rel, list);
4541  case NOT_IN:
4542  return rexBuilder.makeCall(
4543  SqlStdOperatorTable.NOT, RexSubQuery.in(root.rel, list));
4544  case SOME:
4545  return RexSubQuery.some(
4546  root.rel, list, (SqlQuantifyOperator) call.getOperator());
4547  case ALL:
4548  return rexBuilder.makeCall(SqlStdOperatorTable.NOT,
4549  RexSubQuery.some(root.rel,
4550  list,
4551  negate((SqlQuantifyOperator) call.getOperator())));
4552  default:
4553  throw new AssertionError(kind);
4554  }
4555  }
4556  break;
4557 
4558  case EXISTS:
4559  call = (SqlCall) expr;
4560  query = Iterables.getOnlyElement(call.getOperandList());
4561  root = convertQueryRecursive(query, false, null);
4562  RelNode rel = root.rel;
4563  while (rel instanceof Project
4564  || rel instanceof Sort && ((Sort) rel).fetch == null
4565  && ((Sort) rel).offset == null) {
4566  rel = ((SingleRel) rel).getInput();
4567  }
4568  return RexSubQuery.exists(rel);
4569 
4570  case SCALAR_QUERY:
4571  call = (SqlCall) expr;
4572  query = Iterables.getOnlyElement(call.getOperandList());
4573  root = convertQueryRecursive(query, false, null);
4574  return RexSubQuery.scalar(root.rel);
4575  }
4576  }
4577 
4578  switch (kind) {
4579  case SOME:
4580  case ALL:
4581  if (isExpand) {
4582  throw new RuntimeException(kind + " is only supported if expand = false");
4583  }
4584  // fall through
4585  case CURSOR:
4586  case IN:
4587  case NOT_IN:
4588  subQuery = Objects.requireNonNull(getSubQuery(expr));
4589  rex = Objects.requireNonNull(subQuery.expr);
4590  return StandardConvertletTable.castToValidatedType(
4591  expr, rex, validator, rexBuilder);
4592 
4593  case SELECT:
4594  case EXISTS:
4595  case SCALAR_QUERY:
4596  subQuery = getSubQuery(expr);
4597  assert subQuery != null;
4598  rex = subQuery.expr;
4599  assert rex != null : "rex != null";
4600 
4601  if (((kind == SqlKind.SCALAR_QUERY) || (kind == SqlKind.EXISTS))
4602  && isConvertedSubq(rex)) {
4603  // scalar sub-query or EXISTS has been converted to a
4604  // constant
4605  return rex;
4606  }
4607 
4608  // The indicator column is the last field of the sub-query.
4609  RexNode fieldAccess =
4610  rexBuilder.makeFieldAccess(rex, rex.getType().getFieldCount() - 1);
4611 
4612  // The indicator column will be nullable if it comes from
4613  // the null-generating side of the join. For EXISTS, add an
4614  // "IS TRUE" check so that the result is "BOOLEAN NOT NULL".
4615  if (fieldAccess.getType().isNullable() && kind == SqlKind.EXISTS) {
4616  fieldAccess =
4617  rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, fieldAccess);
4618  }
4619  return fieldAccess;
4620 
4621  case OVER:
4622  return convertOver(this, expr);
4623 
4624  default:
4625  // fall through
4626  }
4627 
4628  // Apply standard conversions.
4629  rex = expr.accept(this);
4630  return Objects.requireNonNull(rex);
4631  }
4632 
4637  public RexFieldCollation convertSortExpression(SqlNode expr,
4638  RelFieldCollation.Direction direction,
4639  RelFieldCollation.NullDirection nullDirection) {
4640  switch (expr.getKind()) {
4641  case DESCENDING:
4642  return convertSortExpression(((SqlCall) expr).operand(0),
4643  RelFieldCollation.Direction.DESCENDING,
4644  nullDirection);
4645  case NULLS_LAST:
4646  return convertSortExpression(((SqlCall) expr).operand(0),
4647  direction,
4648  RelFieldCollation.NullDirection.LAST);
4649  case NULLS_FIRST:
4650  return convertSortExpression(((SqlCall) expr).operand(0),
4651  direction,
4652  RelFieldCollation.NullDirection.FIRST);
4653  default:
4654  final Set<SqlKind> flags = EnumSet.noneOf(SqlKind.class);
4655  switch (direction) {
4656  case DESCENDING:
4657  flags.add(SqlKind.DESCENDING);
4658  }
4659  switch (nullDirection) {
4660  case UNSPECIFIED:
4661  final RelFieldCollation.NullDirection nullDefaultDirection =
4662  validator.config().defaultNullCollation().last(desc(direction))
4663  ? RelFieldCollation.NullDirection.LAST
4664  : RelFieldCollation.NullDirection.FIRST;
4665  if (nullDefaultDirection != direction.defaultNullDirection()) {
4666  SqlKind nullDirectionSqlKind =
4667  validator.config().defaultNullCollation().last(desc(direction))
4668  ? SqlKind.NULLS_LAST
4669  : SqlKind.NULLS_FIRST;
4670  flags.add(nullDirectionSqlKind);
4671  }
4672  break;
4673  case FIRST:
4674  flags.add(SqlKind.NULLS_FIRST);
4675  break;
4676  case LAST:
4677  flags.add(SqlKind.NULLS_LAST);
4678  break;
4679  }
4680  return new RexFieldCollation(convertExpression(expr), flags);
4681  }
4682  }
4683 
4692  private boolean isConvertedSubq(RexNode rex) {
4693  if ((rex instanceof RexLiteral) || (rex instanceof RexDynamicParam)) {
4694  return true;
4695  }
4696  if (rex instanceof RexCall) {
4697  RexCall call = (RexCall) rex;
4698  if (call.getOperator() == SqlStdOperatorTable.CAST) {
4699  RexNode operand = call.getOperands().get(0);
4700  if (operand instanceof RexLiteral) {
4701  return true;
4702  }
4703  }
4704  }
4705  return false;
4706  }
4707 
4708  public int getGroupCount() {
4709  if (agg != null) {
4710  return agg.groupExprs.size();
4711  }
4712  if (window != null) {
4713  return window.isAlwaysNonEmpty() ? 1 : 0;
4714  }
4715  return -1;
4716  }
4717 
4718  public RexBuilder getRexBuilder() {
4719  return rexBuilder;
4720  }
4721 
4722  public SqlNode validateExpression(RelDataType rowType, SqlNode expr) {
4723  return SqlValidatorUtil
4724  .validateExprWithRowType(catalogReader.nameMatcher().isCaseSensitive(),
4725  opTab,
4726  typeFactory,
4727  rowType,
4728  expr)
4729  .left;
4730  }
4731 
4732  public RexRangeRef getSubQueryExpr(SqlCall call) {
4733  final SubQuery subQuery = getSubQuery(call);
4734  assert subQuery != null;
4735  return (RexRangeRef) subQuery.expr;
4736  }
4737 
4738  public RelDataTypeFactory getTypeFactory() {
4739  return typeFactory;
4740  }
4741 
4742  public InitializerExpressionFactory getInitializerExpressionFactory() {
4743  return initializerExpressionFactory;
4744  }
4745 
4746  public SqlValidator getValidator() {
4747  return validator;
4748  }
4749 
4750  public RexNode convertLiteral(SqlLiteral literal) {
4751  return exprConverter.convertLiteral(this, literal);
4752  }
4753 
4754  public RexNode convertInterval(SqlIntervalQualifier intervalQualifier) {
4755  return exprConverter.convertInterval(this, intervalQualifier);
4756  }
4757 
4758  public RexNode visit(SqlLiteral literal) {
4759  return exprConverter.convertLiteral(this, literal);
4760  }
4761 
4762  public RexNode visit(SqlCall call) {
4763  if (agg != null) {
4764  final SqlOperator op = call.getOperator();
4765  if (window == null
4766  && (op.isAggregator() || op.getKind() == SqlKind.FILTER
4767  || op.getKind() == SqlKind.WITHIN_GROUP)) {
4768  return agg.lookupAggregates(call);
4769  }
4770  }
4771  SqlCall permutedCall =
4772  new SqlCallBinding(validator, scope, call).permutedCall(); // HEAVY.AI new
4773  if (permutedCall.getOperator() instanceof ExtTableFunction) { // HEAVY.AI new
4774  ExtTableFunction tf =
4775  (ExtTableFunction) permutedCall.getOperator(); // HEAVY.AI new
4776  if (tf.supportsDefaultArguments() // HEAVY.AI new
4777  && permutedCall.getOperandList().stream().anyMatch( // HEAVY.AI new
4778  op -> op.getKind() == SqlKind.DEFAULT)) { // HEAVY.AI new
4779  permutedCall = tf.rewriteCallWithDefaultArguments(permutedCall); // HEAVY.AI new
4780  } // HEAVY.AI new
4781  } // HEAVY.AI new
4782  return exprConverter.convertCall(this, permutedCall); // HEAVY.AI new
4783  }
4784 
4785  public RexNode visit(SqlNodeList nodeList) {
4786  throw new UnsupportedOperationException();
4787  }
4788 
4789  public RexNode visit(SqlIdentifier id) {
4790  return convertIdentifier(this, id);
4791  }
4792 
4793  public RexNode visit(SqlDataTypeSpec type) {
4794  throw new UnsupportedOperationException();
4795  }
4796 
4797  public RexNode visit(SqlDynamicParam param) {
4798  return convertDynamicParam(param);
4799  }
4800 
4801  public RexNode visit(SqlIntervalQualifier intervalQualifier) {
4802  return convertInterval(intervalQualifier);
4803  }
4804 
4805  public List<SqlMonotonicity> getColumnMonotonicities() {
4806  return columnMonotonicities;
4807  }
4808  }
4809 
4810  private SqlQuantifyOperator negate(SqlQuantifyOperator operator) {
4811  assert operator.kind == SqlKind.ALL;
4812  return SqlStdOperatorTable.some(operator.comparisonKind.negateNullSafe());
4813  }
4814 
4816  private static class DeferredLookup {
4819 
4820  DeferredLookup(Blackboard bb, String originalRelName) {
4821  this.bb = bb;
4822  this.originalRelName = originalRelName;
4823  }
4824 
4825  public RexFieldAccess getFieldAccess(CorrelationId name) {
4826  return (RexFieldAccess) bb.mapCorrelateToRex.get(name);
4827  }
4828 
4829  public String getOriginalRelName() {
4830  return originalRelName;
4831  }
4832  }
4833 
4837  private class NoOpSubQueryConverter implements SubQueryConverter {
4838  public boolean canConvertSubQuery() {
4839  return false;
4840  }
4841 
4842  public RexNode convertSubQuery(SqlCall subQuery,
4843  SqlToRelConverter parentConverter,
4844  boolean isExists,
4845  boolean isExplain) {
4846  throw new IllegalArgumentException();
4847  }
4848  }
4849 
4869  protected class AggConverter implements SqlVisitor<Void> {
4870  private final Blackboard bb;
4871  public final AggregatingSelectScope aggregatingSelectScope;
4872 
4873  private final Map<String, String> nameMap = new HashMap<>();
4874 
4878  private final SqlNodeList groupExprs = new SqlNodeList(SqlParserPos.ZERO);
4879 
4883  private final Map<SqlNode, Ord<AuxiliaryConverter>> auxiliaryGroupExprs =
4884  new HashMap<>();
4885 
4893  private final List<Pair<RexNode, String>> convertedInputExprs = new ArrayList<>();
4894 
4901  private final List<AggregateCall> aggCalls = new ArrayList<>();
4902  private final Map<SqlNode, RexNode> aggMapping = new HashMap<>();
4903  private final Map<AggregateCall, RexNode> aggCallMapping = new HashMap<>();
4904 
4906  private boolean inOver = false;
4907 
4918  public AggConverter(Blackboard bb, SqlSelect select) {
4919  this.bb = bb;
4920  this.aggregatingSelectScope =
4921  (AggregatingSelectScope) bb.getValidator().getSelectScope(select);
4922 
4923  // Collect all expressions used in the select list so that aggregate
4924  // calls can be named correctly.
4925  final SqlNodeList selectList = select.getSelectList();
4926  for (int i = 0; i < selectList.size(); i++) {
4927  SqlNode selectItem = selectList.get(i);
4928  String name = null;
4929  if (SqlUtil.isCallTo(selectItem, SqlStdOperatorTable.AS)) {
4930  final SqlCall call = (SqlCall) selectItem;
4931  selectItem = call.operand(0);
4932  name = call.operand(1).toString();
4933  }
4934  if (name == null) {
4935  name = validator.deriveAlias(selectItem, i);
4936  }
4937  nameMap.put(selectItem.toString(), name);
4938  }
4939  }
4940 
4941  public int addGroupExpr(SqlNode expr) {
4942  int ref = lookupGroupExpr(expr);
4943  if (ref >= 0) {
4944  return ref;
4945  }
4946  final int index = groupExprs.size();
4947  groupExprs.add(expr);
4948  String name = nameMap.get(expr.toString());
4949  RexNode convExpr = bb.convertExpression(expr);
4950  addExpr(convExpr, name);
4951 
4952  if (expr instanceof SqlCall) {
4953  SqlCall call = (SqlCall) expr;
4954  for (Pair<SqlNode, AuxiliaryConverter> p :
4955  SqlStdOperatorTable.convertGroupToAuxiliaryCalls(call)) {
4956  addAuxiliaryGroupExpr(p.left, index, p.right);
4957  }
4958  }
4959 
4960  return index;
4961  }
4962 
4963  void addAuxiliaryGroupExpr(SqlNode node, int index, AuxiliaryConverter converter) {
4964  for (SqlNode node2 : auxiliaryGroupExprs.keySet()) {
4965  if (node2.equalsDeep(node, Litmus.IGNORE)) {
4966  return;
4967  }
4968  }
4969  auxiliaryGroupExprs.put(node, Ord.of(index, converter));
4970  }
4971 
4978  private void addExpr(RexNode expr, String name) {
4979  if ((name == null) && (expr instanceof RexInputRef)) {
4980  final int i = ((RexInputRef) expr).getIndex();
4981  name = bb.root.getRowType().getFieldList().get(i).getName();
4982  }
4983  if (Pair.right(convertedInputExprs).contains(name)) {
4984  // In case like 'SELECT ... GROUP BY x, y, x', don't add
4985  // name 'x' twice.
4986  name = null;
4987  }
4988  convertedInputExprs.add(Pair.of(expr, name));
4989  }
4990 
4991  public Void visit(SqlIdentifier id) {
4992  return null;
4993  }
4994 
4995  public Void visit(SqlNodeList nodeList) {
4996  for (int i = 0; i < nodeList.size(); i++) {
4997  nodeList.get(i).accept(this);
4998  }
4999  return null;
5000  }
5001 
5002  public Void visit(SqlLiteral lit) {
5003  return null;
5004  }
5005 
5006  public Void visit(SqlDataTypeSpec type) {
5007  return null;
5008  }
5009 
5010  public Void visit(SqlDynamicParam param) {
5011  return null;
5012  }
5013 
5014  public Void visit(SqlIntervalQualifier intervalQualifier) {
5015  return null;
5016  }
5017 
5018  public Void visit(SqlCall call) {
5019  switch (call.getKind()) {
5020  case FILTER:
5021  case WITHIN_GROUP:
5022  translateAgg(call);
5023  return null;
5024  case SELECT:
5025  // rchen 2006-10-17:
5026  // for now do not detect aggregates in sub-queries.
5027  return null;
5028  }
5029  final boolean prevInOver = inOver;
5030  // Ignore window aggregates and ranking functions (associated with OVER
5031  // operator). However, do not ignore nested window aggregates.
5032  if (call.getOperator().getKind() == SqlKind.OVER) {
5033  // Track aggregate nesting levels only within an OVER operator.
5034  List<SqlNode> operandList = call.getOperandList();
5035  assert operandList.size() == 2;
5036 
5037  // Ignore the top level window aggregates and ranking functions
5038  // positioned as the first operand of a OVER operator
5039  inOver = true;
5040  operandList.get(0).accept(this);
5041 
5042  // Normal translation for the second operand of a OVER operator
5043  inOver = false;
5044  operandList.get(1).accept(this);
5045  return null;
5046  }
5047 
5048  // Do not translate the top level window aggregate. Only do so for
5049  // nested aggregates, if present
5050  if (call.getOperator().isAggregator()) {
5051  if (inOver) {
5052  // Add the parent aggregate level before visiting its children
5053  inOver = false;
5054  } else {
5055  // We're beyond the one ignored level
5056  translateAgg(call);
5057  return null;
5058  }
5059  }
5060  for (SqlNode operand : call.getOperandList()) {
5061  // Operands are occasionally null, e.g. switched CASE arg 0.
5062  if (operand != null) {
5063  operand.accept(this);
5064  }
5065  }
5066  // Remove the parent aggregate level after visiting its children
5067  inOver = prevInOver;
5068  return null;
5069  }
5070 
5071  private void translateAgg(SqlCall call) {
5072  translateAgg(call, null, null, false, call);
5073  }
5074 
5075  private void translateAgg(SqlCall call,
5076  SqlNode filter,
5077  SqlNodeList orderList,
5078  boolean ignoreNulls,
5079  SqlCall outerCall) {
5080  assert bb.agg == this;
5081  assert outerCall != null;
5082  switch (call.getKind()) {
5083  case FILTER:
5084  assert filter == null;
5085  translateAgg(
5086  call.operand(0), call.operand(1), orderList, ignoreNulls, outerCall);
5087  return;
5088  case WITHIN_GROUP:
5089  assert orderList == null;
5090  translateAgg(call.operand(0), filter, call.operand(1), ignoreNulls, outerCall);
5091  return;
5092  case IGNORE_NULLS:
5093  ignoreNulls = true;
5094  // fall through
5095  case RESPECT_NULLS:
5096  translateAgg(call.operand(0), filter, orderList, ignoreNulls, outerCall);
5097  return;
5098  }
5099  final List<Integer> args = new ArrayList<>();
5100  int filterArg = -1;
5101  final List<RelDataType> argTypes = call.getOperator() instanceof SqlCountAggFunction
5102  ? new ArrayList<>(call.getOperandList().size())
5103  : null;
5104  try {
5105  // switch out of agg mode
5106  bb.agg = null;
5107  for (SqlNode operand : call.getOperandList()) {
5108  // special case for COUNT(*): delete the *
5109  if (operand instanceof SqlIdentifier) {
5110  SqlIdentifier id = (SqlIdentifier) operand;
5111  if (id.isStar()) {
5112  assert call.operandCount() == 1;
5113  assert args.isEmpty();
5114  break;
5115  }
5116  }
5117  RexNode convertedExpr = bb.convertExpression(operand);
5118  assert convertedExpr != null;
5119  if (argTypes != null) {
5120  argTypes.add(convertedExpr.getType());
5121  }
5122  args.add(lookupOrCreateGroupExpr(convertedExpr));
5123  }
5124 
5125  if (filter != null) {
5126  RexNode convertedExpr = bb.convertExpression(filter);
5127  assert convertedExpr != null;
5128  if (convertedExpr.getType().isNullable()) {
5129  convertedExpr =
5130  rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, convertedExpr);
5131  }
5132  filterArg = lookupOrCreateGroupExpr(convertedExpr);
5133  }
5134  } finally {
5135  // switch back into agg mode
5136  bb.agg = this;
5137  }
5138 
5139  SqlAggFunction aggFunction = (SqlAggFunction) call.getOperator();
5140  final RelDataType type = validator.deriveType(bb.scope, call);
5141  boolean distinct = false;
5142  SqlLiteral quantifier = call.getFunctionQuantifier();
5143  if ((null != quantifier) && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) {
5144  distinct = true;
5145  }
5146  boolean approximate = false;
5147  if (aggFunction == SqlStdOperatorTable.APPROX_COUNT_DISTINCT) {
5148  aggFunction = SqlStdOperatorTable.COUNT;
5149  distinct = true;
5150  approximate = true;
5151  }
5152  final RelCollation collation;
5153  if (orderList == null || orderList.size() == 0) {
5154  collation = RelCollations.EMPTY;
5155  } else {
5156  collation = RelCollations.of(
5157  orderList.getList()
5158  .stream()
5159  .map(order
5160  -> bb.convertSortExpression(order,
5161  RelFieldCollation.Direction.ASCENDING,
5162  RelFieldCollation.NullDirection.UNSPECIFIED))
5163  .map(fieldCollation
5164  -> new RelFieldCollation(
5165  lookupOrCreateGroupExpr(fieldCollation.left),
5166  fieldCollation.getDirection(),
5167  fieldCollation.getNullDirection()))
5168  .collect(Collectors.toList()));
5169  }
5170  final AggregateCall aggCall = AggregateCall.create(aggFunction,
5171  distinct,
5172  approximate,
5173  ignoreNulls,
5174  args,
5175  filterArg,
5176  collation,
5177  type,
5178  nameMap.get(outerCall.toString()));
5179  RexNode rex = rexBuilder.addAggCall(
5180  aggCall, groupExprs.size(), aggCalls, aggCallMapping, argTypes);
5181  aggMapping.put(outerCall, rex);
5182  }
5183 
5184  private int lookupOrCreateGroupExpr(RexNode expr) {
5185  int index = 0;
5186  for (RexNode convertedInputExpr : Pair.left(convertedInputExprs)) {
5187  if (expr.equals(convertedInputExpr)) {
5188  return index;
5189  }
5190  ++index;
5191  }
5192 
5193  // not found -- add it
5194  addExpr(expr, null);
5195  return index;
5196  }
5197 
5202  public int lookupGroupExpr(SqlNode expr) {
5203  for (int i = 0; i < groupExprs.size(); i++) {
5204  SqlNode groupExpr = groupExprs.get(i);
5205  if (expr.equalsDeep(groupExpr, Litmus.IGNORE)) {
5206  return i;
5207  }
5208  }
5209  return -1;
5210  }
5211 
5212  public RexNode lookupAggregates(SqlCall call) {
5213  // assert call.getOperator().isAggregator();
5214  assert bb.agg == this;
5215 
5216  for (Map.Entry<SqlNode, Ord<AuxiliaryConverter>> e :
5217  auxiliaryGroupExprs.entrySet()) {
5218  if (call.equalsDeep(e.getKey(), Litmus.IGNORE)) {
5219  AuxiliaryConverter converter = e.getValue().e;
5220  final int groupOrdinal = e.getValue().i;
5221  return converter.convert(rexBuilder,
5222  convertedInputExprs.get(groupOrdinal).left,
5223  rexBuilder.makeInputRef(bb.root, groupOrdinal));
5224  }
5225  }
5226 
5227  return aggMapping.get(call);
5228  }
5229 
5230  public List<Pair<RexNode, String>> getPreExprs() {
5231  return convertedInputExprs;
5232  }
5233 
5234  public List<AggregateCall> getAggCalls() {
5235  return aggCalls;
5236  }
5237 
5238  private boolean containsGroupId() {
5239  return aggCalls.stream().anyMatch(
5240  agg -> agg.getAggregation().kind == SqlKind.GROUP_ID);
5241  }
5242 
5243  public RelDataTypeFactory getTypeFactory() {
5244  return typeFactory;
5245  }
5246  }
5247 
5251  private static class LookupContext {
5252  private final List<Pair<RelNode, Integer>> relOffsetList = new ArrayList<>();
5253 
5261  LookupContext(Blackboard bb, List<RelNode> rels, int systemFieldCount) {
5262  bb.flatten(rels, systemFieldCount, new int[] {0}, relOffsetList);
5263  }
5264 
5277  Pair<RelNode, Integer> findRel(int offset) {
5278  return relOffsetList.get(offset);
5279  }
5280  }
5281 
5314  private class HistogramShuttle extends RexShuttle {
5319  static final boolean ENABLE_HISTOGRAM_AGG = false;
5320 
5321  private final List<RexNode> partitionKeys;
5322  private final ImmutableList<RexFieldCollation> orderKeys;
5323  private final RexWindowBound lowerBound;
5324  private final RexWindowBound upperBound;
5325  private final boolean rows;
5326  private final boolean allowPartial;
5327  private final boolean distinct;
5328  private final boolean ignoreNulls;
5329 
5330  HistogramShuttle(List<RexNode> partitionKeys,
5331  ImmutableList<RexFieldCollation> orderKeys,
5332  RexWindowBound lowerBound,
5333  RexWindowBound upperBound,
5334  boolean rows,
5335  boolean allowPartial,
5336  boolean distinct,
5337  boolean ignoreNulls) {
5338  this.partitionKeys = partitionKeys;
5339  this.orderKeys = orderKeys;
5340  this.lowerBound = lowerBound;
5341  this.upperBound = upperBound;
5342  this.rows = rows;
5343  this.allowPartial = allowPartial;
5344  this.distinct = distinct;
5345  this.ignoreNulls = ignoreNulls;
5346  }
5347 
5348  public RexNode visitCall(RexCall call) {
5349  final SqlOperator op = call.getOperator();
5350  if (!(op instanceof SqlAggFunction)) {
5351  return super.visitCall(call);
5352  }
5353  final SqlAggFunction aggOp = (SqlAggFunction) op;
5354  final RelDataType type = call.getType();
5355  List<RexNode> exprs = call.getOperands();
5356 
5357  SqlFunction histogramOp = !ENABLE_HISTOGRAM_AGG ? null : getHistogramOp(aggOp);
5358 
5359  if (histogramOp != null) {
5360  final RelDataType histogramType = computeHistogramType(type);
5361 
5362  // For DECIMAL, since it's already represented as a bigint we
5363  // want to do a reinterpretCast instead of a cast to avoid
5364  // losing any precision.
5365  boolean reinterpretCast = type.getSqlTypeName() == SqlTypeName.DECIMAL;
5366 
5367  // Replace original expression with CAST of not one
5368  // of the supported types
5369  if (histogramType != type) {
5370  exprs = new ArrayList<>(exprs);
5371  exprs.set(0,
5372  reinterpretCast ? rexBuilder.makeReinterpretCast(
5373  histogramType, exprs.get(0), rexBuilder.makeLiteral(false))
5374  : rexBuilder.makeCast(histogramType, exprs.get(0)));
5375  }
5376 
5377  RexCallBinding bind = new RexCallBinding(rexBuilder.getTypeFactory(),
5378  SqlStdOperatorTable.HISTOGRAM_AGG,
5379  exprs,
5380  ImmutableList.of());
5381 
5382  RexNode over = rexBuilder.makeOver(
5383  SqlStdOperatorTable.HISTOGRAM_AGG.inferReturnType(bind),
5384  SqlStdOperatorTable.HISTOGRAM_AGG,
5385  exprs,
5386  partitionKeys,
5387  orderKeys,
5388  lowerBound,
5389  upperBound,
5390  rows,
5391  allowPartial,
5392  false,
5393  distinct,
5394  ignoreNulls);
5395 
5396  RexNode histogramCall =
5397  rexBuilder.makeCall(histogramType, histogramOp, ImmutableList.of(over));
5398 
5399  // If needed, post Cast result back to original
5400  // type.
5401  if (histogramType != type) {
5402  if (reinterpretCast) {
5403  histogramCall = rexBuilder.makeReinterpretCast(
5404  type, histogramCall, rexBuilder.makeLiteral(false));
5405  } else {
5406  histogramCall = rexBuilder.makeCast(type, histogramCall);
5407  }
5408  }
5409 
5410  return histogramCall;
5411  } else {
5412  boolean needSum0 = aggOp == SqlStdOperatorTable.SUM && type.isNullable();
5413  SqlAggFunction aggOpToUse = needSum0 ? SqlStdOperatorTable.SUM0 : aggOp;
5414  return rexBuilder.makeOver(type,
5415  aggOpToUse,
5416  exprs,
5417  partitionKeys,
5418  orderKeys,
5419  lowerBound,
5420  upperBound,
5421  rows,
5422  allowPartial,
5423  needSum0,
5424  distinct,
5425  ignoreNulls);
5426  }
5427  }
5428 
5440  SqlFunction getHistogramOp(SqlAggFunction aggFunction) {
5441  if (aggFunction == SqlStdOperatorTable.MIN) {
5442  return SqlStdOperatorTable.HISTOGRAM_MIN;
5443  } else if (aggFunction == SqlStdOperatorTable.MAX) {
5444  return SqlStdOperatorTable.HISTOGRAM_MAX;
5445  } else if (aggFunction == SqlStdOperatorTable.FIRST_VALUE) {
5446  return SqlStdOperatorTable.HISTOGRAM_FIRST_VALUE;
5447  } else if (aggFunction == SqlStdOperatorTable.LAST_VALUE) {
5448  return SqlStdOperatorTable.HISTOGRAM_LAST_VALUE;
5449  } else {
5450  return null;
5451  }
5452  }
5453 
5458  private RelDataType computeHistogramType(RelDataType type) {
5459  if (SqlTypeUtil.isExactNumeric(type)
5460  && type.getSqlTypeName() != SqlTypeName.BIGINT) {
5461  return typeFactory.createSqlType(SqlTypeName.BIGINT);
5462  } else if (SqlTypeUtil.isApproximateNumeric(type)
5463  && type.getSqlTypeName() != SqlTypeName.DOUBLE) {
5464  return typeFactory.createSqlType(SqlTypeName.DOUBLE);
5465  } else {
5466  return type;
5467  }
5468  }
5469  }
5470 
5474  private static class SubQuery {
5475  final SqlNode node;
5476  final RelOptUtil.Logic logic;
5477  RexNode expr;
5478 
5479  private SubQuery(SqlNode node, RelOptUtil.Logic logic) {
5480  this.node = node;
5481  this.logic = logic;
5482  }
5483  }
5484 
5488  private static class AggregateFinder extends SqlBasicVisitor<Void> {
5489  final SqlNodeList list = new SqlNodeList(SqlParserPos.ZERO);
5490  final SqlNodeList filterList = new SqlNodeList(SqlParserPos.ZERO);
5491  final SqlNodeList orderList = new SqlNodeList(SqlParserPos.ZERO);
5492 
5493  @Override
5494  public Void visit(SqlCall call) {
5495  // ignore window aggregates and ranking functions (associated with OVER
5496  // operator)
5497  if (call.getOperator().getKind() == SqlKind.OVER) {
5498  return null;
5499  }
5500 
5501  if (call.getOperator().getKind() == SqlKind.FILTER) {
5502  // the WHERE in a FILTER must be tracked too so we can call replaceSubQueries on
5503  // it.
5504  // see https://issues.apache.org/jira/browse/CALCITE-1910
5505  final SqlNode aggCall = call.getOperandList().get(0);
5506  final SqlNode whereCall = call.getOperandList().get(1);
5507  list.add(aggCall);
5508  filterList.add(whereCall);
5509  return null;
5510  }
5511 
5512  if (call.getOperator().getKind() == SqlKind.WITHIN_GROUP) {
5513  // the WHERE in a WITHIN_GROUP must be tracked too so we can call
5514  // replaceSubQueries on it.
5515  // see https://issues.apache.org/jira/browse/CALCITE-1910
5516  final SqlNode aggCall = call.getOperandList().get(0);
5517  final SqlNodeList orderList = (SqlNodeList) call.getOperandList().get(1);
5518  list.add(aggCall);
5519  orderList.getList().forEach(this.orderList::add);
5520  return null;
5521  }
5522 
5523  if (call.getOperator().isAggregator()) {
5524  list.add(call);
5525  return null;
5526  }
5527 
5528  // Don't traverse into sub-queries, even if they contain aggregate
5529  // functions.
5530  if (call instanceof SqlSelect) {
5531  return null;
5532  }
5533 
5534  return call.getOperator().acceptCall(this, call);
5535  }
5536  }
5537 
5541  private static class CorrelationUse {
5542  private final CorrelationId id;
5543  private final ImmutableBitSet requiredColumns;
5545  private final RelNode r;
5546 
5547  CorrelationUse(CorrelationId id, ImmutableBitSet requiredColumns, RelNode r) {
5548  this.id = id;
5549  this.requiredColumns = requiredColumns;
5550  this.r = r;
5551  }
5552  }
5553 
5555  public static ConfigBuilder configBuilder() {
5556  return new ConfigBuilder();
5557  }
5558 
5566  public interface Config {
5568  Config DEFAULT = configBuilder().build();
5569 
5574  boolean isDecorrelationEnabled();
5575 
5580  boolean isTrimUnusedFields();
5581 
5587  boolean isCreateValuesRel();
5588 
5593  boolean isExplain();
5594 
5599  boolean isExpand();
5600 
5610  int getInSubQueryThreshold();
5611 
5616  RelBuilderFactory getRelBuilderFactory();
5617 
5622  UnaryOperator<RelBuilder.Config> getRelBuilderConfigTransform();
5623 
5628  HintStrategyTable getHintStrategyTable();
5629 
5636  BiPredicate<SqlNode, SqlNode> getExpandPredicate();
5637  }
5638 
5640  public static class ConfigBuilder {
5641  private boolean decorrelationEnabled = true;
5642  private boolean trimUnusedFields = false;
5643  private boolean createValuesRel = true;
5644  private boolean explain;
5645  private boolean expand = true;
5646  private int inSubQueryThreshold = DEFAULT_IN_SUB_QUERY_THRESHOLD;
5647  private UnaryOperator<RelBuilder.Config> relBuilderConfigTransform =
5648  c -> c.withPushJoinCondition(true);
5649  private RelBuilderFactory relBuilderFactory = RelFactories.LOGICAL_BUILDER;
5650  private HintStrategyTable hintStrategyTable = HintStrategyTable.EMPTY;
5651  // MAT 08 Jan 2021 HEAVY.AI code
5652  // pushdown join condition incurs an extra projection overhead, so remove it (21 Apr
5653  // 2021)
5654  private BiPredicate<SqlNode, SqlNode> expandPredicate;
5655  // MAT 08 Jan 2021 HEAVY.AI code ends
5656 
5657  private ConfigBuilder() {}
5658 
5660  public ConfigBuilder withConfig(Config config) {
5661  this.decorrelationEnabled = config.isDecorrelationEnabled();
5662  this.trimUnusedFields = config.isTrimUnusedFields();
5663  this.createValuesRel = config.isCreateValuesRel();
5664  this.explain = config.isExplain();
5665  this.expand = config.isExpand();
5666  this.inSubQueryThreshold = config.getInSubQueryThreshold();
5667  this.relBuilderConfigTransform = config.getRelBuilderConfigTransform();
5668  this.relBuilderFactory = config.getRelBuilderFactory();
5669  this.hintStrategyTable = config.getHintStrategyTable();
5670 
5671  // MAT 08 Jan 2021 HEAVY.AI code
5672  if (!(config.getExpandPredicate() instanceof ConfigImpl.DefaultExpandPredicate)) {
5673  this.expandPredicate = config.getExpandPredicate();
5674  }
5675  // MAT 08 Jan 2021 HEAVY.AI code ends
5676  return this;
5677  }
5678 
5679  public ConfigBuilder withDecorrelationEnabled(boolean enabled) {
5680  this.decorrelationEnabled = enabled;
5681  return this;
5682  }
5683 
5684  public ConfigBuilder withTrimUnusedFields(boolean trimUnusedFields) {
5685  this.trimUnusedFields = trimUnusedFields;
5686  return this;
5687  }
5688 
5689  public ConfigBuilder withCreateValuesRel(boolean createValuesRel) {
5690  this.createValuesRel = createValuesRel;
5691  return this;
5692  }
5693 
5694  public ConfigBuilder withExplain(boolean explain) {
5695  this.explain = explain;
5696  return this;
5697  }
5698 
5699  public ConfigBuilder withExpand(boolean expand) {
5700  this.expand = expand;
5701  return this;
5702  }
5703 
5704  public ConfigBuilder withExpandPredicate(BiPredicate<SqlNode, SqlNode> predicate) {
5705  this.expandPredicate = predicate;
5706  return this;
5707  }
5708 
5710  public ConfigBuilder withPushJoinCondition(boolean pushJoinCondition) {
5711  return withRelBuilderConfigTransform(Util.andThen(relBuilderConfigTransform,
5712  c -> c.withPushJoinCondition(pushJoinCondition)));
5713  }
5714 
5715  @Deprecated // to be removed before 2.0
5716  public ConfigBuilder withInSubqueryThreshold(int inSubQueryThreshold) {
5717  return withInSubQueryThreshold(inSubQueryThreshold);
5718  }
5719 
5720  public ConfigBuilder withInSubQueryThreshold(int inSubQueryThreshold) {
5721  this.inSubQueryThreshold = inSubQueryThreshold;
5722  return this;
5723  }
5724 
5725  public ConfigBuilder withRelBuilderConfigTransform(
5726  UnaryOperator<RelBuilder.Config> relBuilderConfigTransform) {
5727  this.relBuilderConfigTransform = relBuilderConfigTransform;
5728  return this;
5729  }
5730 
5731  public ConfigBuilder withRelBuilderFactory(RelBuilderFactory relBuilderFactory) {
5732  this.relBuilderFactory = relBuilderFactory;
5733  return this;
5734  }
5735 
5736  public ConfigBuilder withHintStrategyTable(HintStrategyTable hintStrategyTable) {
5737  this.hintStrategyTable = hintStrategyTable;
5738  return this;
5739  }
5740 
5742  public Config build() {
5743  return new ConfigImpl(decorrelationEnabled,
5744  trimUnusedFields,
5745  createValuesRel,
5746  explain,
5747  expand,
5748  expandPredicate,
5749  inSubQueryThreshold,
5750  relBuilderConfigTransform,
5751  relBuilderFactory,
5752  hintStrategyTable);
5753  }
5754  }
5755 
5760  private static class ConfigImpl implements Config {
5761  private final boolean decorrelationEnabled;
5762  private final boolean trimUnusedFields;
5763  private final boolean createValuesRel;
5764  private final boolean explain;
5765  private final boolean expand;
5766  private final int inSubQueryThreshold;
5767  private final UnaryOperator<RelBuilder.Config> relBuilderConfigTransform;
5768  private final RelBuilderFactory relBuilderFactory;
5769  private final HintStrategyTable hintStrategyTable;
5770 
5771  private final BiPredicate<SqlNode, SqlNode> expandPredicate;
5772 
5773  // MAT 08 Jab 2021 HEAVY.AI code
5774  private class DefaultExpandPredicate implements BiPredicate<SqlNode, SqlNode> {
5775  @Override
5776  public boolean test(SqlNode t, SqlNode u) {
5777  return expand;
5778  }
5779  }
5780 
5781  // MAT 08 Jan 2021 HEAVY.AI code ends
5782 
5783  private ConfigImpl(boolean decorrelationEnabled,
5784  boolean trimUnusedFields,
5785  boolean createValuesRel,
5786  boolean explain,
5787  boolean expand,
5788  // MAT 08 Jan 2021 HEAVY.AI code
5789  BiPredicate<SqlNode, SqlNode> expandPredicate,
5790  // MAT 08 Jan 2021 HEAVY.AI code ends
5791  int inSubQueryThreshold,
5792  UnaryOperator<RelBuilder.Config> relBuilderConfigTransform,
5793  RelBuilderFactory relBuilderFactory,
5794  HintStrategyTable hintStrategyTable) {
5795  this.decorrelationEnabled = decorrelationEnabled;
5796  this.trimUnusedFields = trimUnusedFields;
5797  this.createValuesRel = createValuesRel;
5798  // MAT 08 Jan 2021 HEAVY.AI code
5799  this.explain = explain;
5800  this.expand = expand;
5801  // MAT 08 Jan 2021 HEAVY.AI code ends
5802  this.inSubQueryThreshold = inSubQueryThreshold;
5803  this.relBuilderConfigTransform = relBuilderConfigTransform;
5804  this.relBuilderFactory = relBuilderFactory;
5805  this.hintStrategyTable = hintStrategyTable;
5806  // MAT 08 Jan 2021 HEAVY.AI code
5807  if (null == expandPredicate) {
5808  expandPredicate = new DefaultExpandPredicate();
5809  }
5810  this.expandPredicate = expandPredicate;
5811 
5812  // MAT 08 Jan 2021 HEAVY.AI code ends
5813  }
5814 
5815  // TODO MAT 08 Jan 2021 add our added types to this equals
5816  @Override
5817  public boolean equals(Object obj) {
5818  return this == obj
5819  || obj instanceof ConfigImpl
5820  && decorrelationEnabled == ((ConfigImpl) obj).decorrelationEnabled
5821  && trimUnusedFields == ((ConfigImpl) obj).trimUnusedFields
5822  && createValuesRel == ((ConfigImpl) obj).createValuesRel
5823  && explain == ((ConfigImpl) obj).explain
5824  && expand == ((ConfigImpl) obj).expand
5825  && inSubQueryThreshold == ((ConfigImpl) obj).inSubQueryThreshold
5826  && relBuilderFactory == ((ConfigImpl) obj).relBuilderFactory
5827  && hintStrategyTable == ((ConfigImpl) obj).hintStrategyTable;
5828  }
5829 
5830  @Override
5831  public int hashCode() {
5832  return Objects.hash(decorrelationEnabled,
5833  trimUnusedFields,
5834  createValuesRel,
5835  explain,
5836  expand,
5837  inSubQueryThreshold,
5838  relBuilderFactory,
5839  hintStrategyTable);
5840  }
5841 
5842  public boolean isDecorrelationEnabled() {
5843  return decorrelationEnabled;
5844  }
5845 
5846  public boolean isTrimUnusedFields() {
5847  return trimUnusedFields;
5848  }
5849 
5850  public boolean isCreateValuesRel() {
5851  return createValuesRel;
5852  }
5853 
5854  public boolean isExplain() {
5855  return explain;
5856  }
5857 
5858  public boolean isExpand() {
5859  return expand;
5860  }
5861 
5862  public int getInSubQueryThreshold() {
5863  return inSubQueryThreshold;
5864  }
5865 
5866  public UnaryOperator<RelBuilder.Config> getRelBuilderConfigTransform() {
5867  return relBuilderConfigTransform;
5868  }
5869 
5870  public RelBuilderFactory getRelBuilderFactory() {
5871  return relBuilderFactory;
5872  }
5873 
5874  public HintStrategyTable getHintStrategyTable() {
5875  return hintStrategyTable;
5876  }
5877 
5878  // MAT 08 Jan 2021 HEAVY.AI code
5879  public BiPredicate<SqlNode, SqlNode> getExpandPredicate() {
5880  return expandPredicate;
5881  }
5882  // MAT 08 Jan 2021 HEAVY.AI code ends
5883  }
5884 }
RelRoot convertWith(SqlWith with, boolean top)
RelOptTable.ToRelContext createToRelContext(List< RelHint > hints)
SqlQuantifyOperator negate(SqlQuantifyOperator operator)
JoinType
Definition: sqldefs.h:174
void setDynamicParamCountInExplain(int explainParamCount)
static SqlNode pushDownNotForIn(SqlValidatorScope scope, SqlNode sqlNode)
void extraSelectItems(Blackboard bb, SqlSelect select, List< RexNode > exprList, List< String > nameList, Collection< String > aliasList, List< SqlMonotonicity > columnMonotonicityList)
void substituteSubQuery(Blackboard bb, SubQuery subQuery)
void convertCollectionTable(Blackboard bb, SqlCall call)
RexNode convertSubQuery(SqlCall subQuery, SqlToRelConverter parentConverter, boolean isExists, boolean isExplain)
void checkConvertedType(SqlNode query, RelNode result)
RexNode translateIn(RelOptUtil.Logic logic, RelNode root, final RexNode rex)
RelNode convertCursor(Blackboard bb, SubQuery subQuery)
RelNode convertQueryOrInList(Blackboard bb, SqlNode seek, RelDataType targetRowType)
std::string join(T const &container, std::string const &delim)
RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type)
RexNode ensureSqlType(RelDataType type, RexNode node)
tuple root
Definition: setup.in.py:14
void afterTableFunction(SqlToRelConverter.Blackboard bb, SqlCall call, LogicalTableFunctionScan callRel)
std::string toString(const QueryDescriptionType &type)
Definition: Types.h:64
RelNode toRel(final RelOptTable table, final List< RelHint > hints)
std::pair< FILE *, std::string > create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:55
SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable)
void convertValuesImpl(Blackboard bb, SqlCall values, RelDataType targetRowType)
void convertFrom(Blackboard bb, SqlNode from)
InitializerExpressionFactory getInitializerFactory(SqlValidatorTable validatorTable)
String deriveAlias(final SqlNode node, Collection< String > aliases, final int ordinal)
RelNode decorrelate(SqlNode query, RelNode rootRel)
void setRoot(List< RelNode > inputs, RelNode root, boolean hasSystemFields)
RexDynamicParam convertDynamicParam(final SqlDynamicParam dynamicParam)
tuple rows
Definition: report.py:114
RelNode convertRowValues(Blackboard bb, SqlNode rowList, Collection< SqlNode > rows, boolean allowLiteralsOnly, RelDataType targetRowType)
RexNode convertInToOr(final Blackboard bb, final List< RexNode > leftKeys, SqlNodeList valuesList, SqlInOperator op)
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:33
final Map< CorrelationId, DeferredLookup > mapCorrelToDeferred
RelOptUtil.Exists convertExists(SqlNode seek, RelOptUtil.SubQueryType subQueryType, RelOptUtil.Logic logic, boolean notIn, RelDataType targetDataType)
RelNode createSource(RelOptTable targetTable, RelNode source, ModifiableView modifiableView, RelDataType delegateRowType)
RelNode convertRowConstructor(Blackboard bb, SqlCall rowConstructor)
RelRoot convertQueryRecursive(SqlNode query, boolean top, RelDataType targetRowType)
SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptPlanner planner, RexBuilder rexBuilder, SqlRexConvertletTable convertletTable)
final List< SqlDynamicParam > dynamicParamSqlNodes
final Map< SqlNode, RexNode > mapConvertedNonCorrSubqs
RexLiteral convertLiteralInValuesList(SqlNode sqlNode, Blackboard bb, RelDataType rowType, int iField)
void convertUnnest(Blackboard bb, SqlCall call, List< String > fieldNames)
void distinctify(Blackboard bb, boolean checkForDupExprs)
RelNode createModify(RelOptTable targetTable, RelNode source)
RelNode trimUnusedFields(boolean ordered, RelNode rootRel)
RexNode convertUsing(SqlValidatorNamespace leftNamespace, SqlValidatorNamespace rightNamespace, List< String > nameList)
void convertSelectImpl(final Blackboard bb, SqlSelect select)
void addConvertedNonCorrSubqs(Map< SqlNode, RexNode > alreadyConvertedNonCorrSubqs)
RexNode convertJoinCondition(Blackboard bb, SqlValidatorNamespace leftNamespace, SqlValidatorNamespace rightNamespace, SqlNode condition, JoinConditionType conditionType, RelNode leftRel, RelNode rightRel)
void collectInsertTargets(SqlInsert call, final RexNode sourceRef, final List< String > targetColumnNames, List< RexNode > columnExprs)
void replaceSubQueries(final Blackboard bb, final SqlNode expr, RelOptUtil.Logic logic)
static< T > T unwrap(Object o, Class< T > clazz)
size_t indexOf(std::vector< T > &vec, T val)
SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable, Config config)
RelNode flattenTypes(RelNode rootRel, boolean restructure)
RelNode createAggregate(Blackboard bb, ImmutableBitSet groupSet, ImmutableList< ImmutableBitSet > groupSets, List< AggregateCall > aggCalls)
CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0)
RelNode convertValues(SqlCall values, RelDataType targetRowType)
void convertAgg(Blackboard bb, SqlSelect select, List< SqlNode > orderExprList)
RexNode convertOver(Blackboard bb, SqlNode node)
final void createAggImpl(Blackboard bb, final AggConverter aggConverter, SqlNodeList selectList, SqlNodeList groupList, SqlNode having, List< SqlNode > orderExprList)
Blackboard createInsertBlackboard(RelOptTable targetTable, RexNode sourceRef, List< String > targetColumnNames)
void convertIdentifier(Blackboard bb, SqlIdentifier id, SqlNodeList extendedColumns, SqlNodeList tableHints)
RelNode convertMultisets(final List< SqlNode > operands, Blackboard bb)
torch::Tensor f(torch::Tensor x, torch::Tensor W_target, torch::Tensor b_target)
static boolean containsNullLiteral(SqlNodeList valueList)
void gatherOrderExprs(Blackboard bb, SqlSelect select, SqlNodeList orderList, List< SqlNode > extraOrderExprs, List< RelFieldCollation > collationList)
RexNode convertExtendedExpression(SqlNode node, Blackboard bb)
dictionary params
Definition: report.py:27
void convertMatchRecognize(Blackboard bb, SqlCall call)
void convertWhere(final Blackboard bb, final SqlNode where)
RexNode convertExpression(SqlNode node, Map< String, RexNode > nameToNodeMap)
Blackboard(SqlValidatorScope scope, Map< String, RexNode > nameToNodeMap, boolean top)
RelNode rewriteAggregateWithGroupId(Blackboard bb, AggregatingSelectScope.Resolved r, AggConverter converter)
static SqlNode reg(SqlValidatorScope scope, SqlNode e)
boolean convertNonCorrelatedSubQuery(SubQuery subQuery, Blackboard bb, RelNode converted, boolean isExists)
void findSubQueries(Blackboard bb, SqlNode node, RelOptUtil.Logic logic, boolean registerOnlyScalarSubQueries)
Pair< RexNode, Map< String, Integer > > lookupExp(SqlQualified qualified)
boolean isSubQueryNonCorrelated(RelNode subq, Blackboard bb)
RexNode convertIdentifier(Blackboard bb, SqlIdentifier identifier)
RelNode createJoin(Blackboard bb, RelNode leftRel, RelNode rightRel, RexNode joinCond, JoinRelType joinType)
RexAccessShuttle(RexBuilder builder, RexCorrelVariable rexCorrel)
RelNode convertColumnList(final SqlInsert call, RelNode source)
void convertFrom(Blackboard bb, SqlNode from, List< String > fieldNames)
void convertTemporalTable(Blackboard bb, SqlCall call)
static boolean desc(RelFieldCollation.Direction direction)
string name
Definition: setup.in.py:72
constexpr double n
Definition: Utm.h:38
Set< RelColumnMapping > getColumnMappings(SqlOperator op)
Blackboard createBlackboard(SqlValidatorScope scope, Map< String, RexNode > nameToNodeMap, boolean top)
RelRoot convertQuery(SqlNode query, final boolean needsValidation, final boolean top)
RelFieldCollation convertOrderItem(SqlSelect select, SqlNode orderItem, List< SqlNode > extraExprs, RelFieldCollation.Direction direction, RelFieldCollation.NullDirection nullDirection)
final Map< RelNode, Map< Integer, Integer > > mapRootRelToFieldProjection
RelNode convertToSingleValueSubq(SqlNode query, RelNode plan)
static JoinRelType convertJoinType(JoinType joinType)
void setSubQueryConverter(SubQueryConverter converter)
RexNode adjustInputRef(Blackboard bb, RexInputRef inputRef)
void convertOrder(SqlSelect select, Blackboard bb, RelCollation collation, List< SqlNode > orderExprList, SqlNode offset, SqlNode fetch)
void convertSelectList(Blackboard bb, SqlSelect select, List< SqlNode > orderList)
RelNode convertSelect(SqlSelect select, boolean top)