OmniSciDB  a5dc49c757
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
HeavyDBPlanner.java
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one or more
3  * contributor license agreements. See the NOTICE file distributed with
4  * this work for additional information regarding copyright ownership.
5  * The ASF licenses this file to you under the Apache License, Version 2.0
6  * (the "License"); you may not use this file except in compliance with
7  * the License. You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 package org.apache.calcite.prepare;
18 
19 import com.google.common.collect.ImmutableSet;
24 
25 import org.apache.calcite.config.CalciteConnectionConfig;
26 import org.apache.calcite.config.CalciteConnectionConfigImpl;
27 import org.apache.calcite.config.CalciteConnectionProperty;
28 import org.apache.calcite.jdbc.CalciteSchema;
29 import org.apache.calcite.linq4j.function.Functions;
30 import org.apache.calcite.plan.Context;
31 import org.apache.calcite.plan.RelOptCluster;
32 import org.apache.calcite.plan.RelOptCostImpl;
33 import org.apache.calcite.plan.RelOptRule;
34 import org.apache.calcite.plan.hep.HepPlanner;
35 import org.apache.calcite.plan.hep.HepProgram;
36 import org.apache.calcite.plan.hep.HepProgramBuilder;
37 import org.apache.calcite.plan.volcano.VolcanoPlanner;
38 import org.apache.calcite.rel.RelNode;
39 import org.apache.calcite.rel.RelRoot;
40 import org.apache.calcite.rel.core.RelFactories;
42 import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
43 import org.apache.calcite.rel.rules.*;
44 import org.apache.calcite.rex.RexBuilder;
45 import org.apache.calcite.schema.SchemaPlus;
46 import org.apache.calcite.sql.SqlKind;
47 import org.apache.calcite.sql.SqlNode;
48 import org.apache.calcite.sql.advise.SqlAdvisor;
49 import org.apache.calcite.sql.parser.SqlParseException;
50 import org.apache.calcite.sql.validate.SqlConformanceEnum;
51 import org.apache.calcite.sql.validate.SqlMoniker;
52 import org.apache.calcite.sql.validate.SqlValidator;
53 import org.apache.calcite.tools.FrameworkConfig;
54 import org.slf4j.Logger;
55 import org.slf4j.LoggerFactory;
56 
57 import java.io.IOException;
58 import java.lang.reflect.InvocationTargetException;
59 import java.lang.reflect.Method;
60 import java.util.ArrayList;
61 import java.util.List;
62 import java.util.Properties;
63 
69 public class HeavyDBPlanner extends PlannerImpl {
70  FrameworkConfig config;
71  private List<HeavyDBParserOptions.FilterPushDownInfo> filterPushDownInfo =
72  new ArrayList<>();
73  private List<Restriction> restrictions = null;
74  final static Logger HEAVYDBLOGGER = LoggerFactory.getLogger(HeavyDBPlanner.class);
75 
76  public HeavyDBPlanner(FrameworkConfig config) {
77  super(config);
78  this.config = config;
79  }
80 
81  private static SchemaPlus rootSchema(SchemaPlus schema) {
82  for (;;) {
83  if (schema.getParentSchema() == null) {
84  return schema;
85  }
86  schema = schema.getParentSchema();
87  }
88  }
89 
90  public static class CompletionResult {
91  public List<SqlMoniker> hints;
92  public String replaced;
93 
94  CompletionResult(final List<SqlMoniker> hints, final String replaced) {
95  this.hints = hints;
96  this.replaced = replaced;
97  }
98  }
99 
100  private CalciteCatalogReader createCatalogReader() {
101  final SchemaPlus rootSchema = rootSchema(config.getDefaultSchema());
102  final Context context = config.getContext();
103  final CalciteConnectionConfig connectionConfig;
104 
105  if (context != null) {
106  connectionConfig = context.unwrap(CalciteConnectionConfig.class);
107  } else {
108  Properties properties = new Properties();
109  properties.setProperty(CalciteConnectionProperty.CASE_SENSITIVE.camelName(),
110  String.valueOf(config.getParserConfig().caseSensitive()));
111  connectionConfig = new CalciteConnectionConfigImpl(properties);
112  }
113 
114  return new CalciteCatalogReader(CalciteSchema.from(rootSchema),
115  CalciteSchema.from(config.getDefaultSchema()).path(null),
116  getTypeFactory(),
117  connectionConfig);
118  }
119 
120  public void advanceToValidate() {
121  try {
122  String dummySql = "SELECT 1";
123  super.parse(dummySql);
124  } catch (SqlParseException e) {
125  throw new RuntimeException(e);
126  }
127  }
128 
129  public void ready() {
130  // need to call ready on the super class, but that method is marked private
131  // circumventing via reflection for now
132  try {
133  Method readyMethod = getClass().getSuperclass().getDeclaredMethod("ready");
134  readyMethod.setAccessible(true);
135  readyMethod.invoke(this);
136  } catch (InvocationTargetException e) {
137  if (e.getCause() instanceof RuntimeException) {
138  throw(RuntimeException) e.getCause();
139  } else {
140  throw new RuntimeException(e.getCause());
141  }
142  } catch (Exception e) {
143  throw new RuntimeException(e);
144  }
145  }
146 
148  final String sql, final int cursor, final List<String> visibleTables) {
149  ready();
150 
151  SqlValidator.Config validatorConfig = SqlValidator.Config.DEFAULT;
152  validatorConfig = validatorConfig.withSqlConformance(SqlConformanceEnum.LENIENT);
153 
154  HeavyDBSqlAdvisorValidator advisor_validator =
155  new HeavyDBSqlAdvisorValidator(visibleTables,
156  config.getOperatorTable(),
158  getTypeFactory(),
159  validatorConfig);
160  SqlAdvisor advisor =
161  new HeavyDBSqlAdvisor(advisor_validator, config.getParserConfig());
162  String[] replaced = new String[1];
163  int adjusted_cursor = cursor < 0 ? sql.length() : cursor;
164  java.util.List<SqlMoniker> hints =
165  advisor.getCompletionHints(sql, adjusted_cursor, replaced);
166  return new CompletionResult(hints, replaced[0]);
167  }
168 
169  public static HepPlanner getHepPlanner(
170  HepProgram hepProgram, boolean doNotEliminateSharedNodesInQueryPlanDag) {
171  if (doNotEliminateSharedNodesInQueryPlanDag) {
172  return new HepPlanner(
173  hepProgram, null, true, Functions.ignore2(), RelOptCostImpl.FACTORY);
174  } else {
175  return new HepPlanner(hepProgram);
176  }
177  }
178 
179  @Override
180  public RelRoot rel(SqlNode sql) {
181  return super.rel(sql);
182  }
183 
184  public RelRoot getRelRoot(SqlNode sqlNode) {
185  return super.rel(sqlNode);
186  }
187 
188  public RelNode optimizeRATree(
189  RelNode rootNode, boolean viewOptimizationEnabled, boolean foundView) {
190  HepProgramBuilder firstOptPhaseProgram = HepProgram.builder();
191  firstOptPhaseProgram.addRuleInstance(CoreRules.AGGREGATE_MERGE)
192  .addRuleInstance(
193  new OuterJoinOptViaNullRejectionRule(RelFactories.LOGICAL_BUILDER))
194  .addRuleInstance(CoreRules.AGGREGATE_UNION_TRANSPOSE);
195  if (!viewOptimizationEnabled) {
196  firstOptPhaseProgram.addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE)
197  .addRuleInstance(
198  FilterTableFunctionMultiInputTransposeRule.Config.DEFAULT.toRule())
199  .addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE);
200  } else {
201  if (foundView) {
202  firstOptPhaseProgram.addRuleInstance(
203  CoreRules.JOIN_PROJECT_BOTH_TRANSPOSE_INCLUDE_OUTER);
204  firstOptPhaseProgram.addRuleInstance(CoreRules.FILTER_MERGE);
205  }
206  firstOptPhaseProgram.addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE);
207  firstOptPhaseProgram.addRuleInstance(
208  FilterTableFunctionMultiInputTransposeRule.Config.DEFAULT.toRule());
209  firstOptPhaseProgram.addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE);
210  if (foundView) {
211  firstOptPhaseProgram.addRuleInstance(CoreRules.PROJECT_MERGE);
212  firstOptPhaseProgram.addRuleInstance(ProjectProjectRemoveRule.INSTANCE);
213  }
214  }
215  HepProgram firstOptPhase = firstOptPhaseProgram.build();
216  HepPlanner firstPlanner = HeavyDBPlanner.getHepPlanner(firstOptPhase, true);
217  firstPlanner.setRoot(rootNode);
218  final RelNode firstOptimizedPlanRoot = firstPlanner.findBestExp();
219 
220  boolean hasRLSFilter = null != restrictions && !restrictions.isEmpty();
221  boolean needsSecondOptPhase = hasRLSFilter || !filterPushDownInfo.isEmpty();
222  if (needsSecondOptPhase) {
223  HepProgramBuilder secondOptPhaseProgram = HepProgram.builder();
224  if (hasRLSFilter) {
225  final InjectFilterRule injectFilterRule =
226  InjectFilterRule.Config.DEFAULT.toRule(restrictions);
227  secondOptPhaseProgram.addRuleInstance(injectFilterRule);
228  }
229  if (!filterPushDownInfo.isEmpty()) {
230  final DynamicFilterJoinRule dynamicFilterJoinRule =
231  new DynamicFilterJoinRule(true,
232  RelFactories.LOGICAL_BUILDER,
233  FilterJoinRule.TRUE_PREDICATE,
235  secondOptPhaseProgram.addRuleInstance(dynamicFilterJoinRule);
236  }
237 
238  HepProgram secondOptPhase = secondOptPhaseProgram.build();
239  HepPlanner secondPlanner = HeavyDBPlanner.getHepPlanner(secondOptPhase, true);
240  secondPlanner.setRoot(firstOptimizedPlanRoot);
241  final RelNode secondOptimizedPlanRoot = secondPlanner.findBestExp();
242  if (!filterPushDownInfo.isEmpty()) {
243  filterPushDownInfo.clear();
244  }
245  return secondOptimizedPlanRoot;
246  } else {
247  return firstOptimizedPlanRoot;
248  }
249  }
250 
251  private RelRoot applyInjectFilterRule(RelRoot root, List<Restriction> restrictions) {
252  // TODO consider doing these rules in one preplan pass
253 
254  final InjectFilterRule injectFilterRule =
255  InjectFilterRule.Config.DEFAULT.toRule(restrictions);
256 
257  final HepProgram program =
258  HepProgram.builder().addRuleInstance(injectFilterRule).build();
259  HepPlanner prePlanner = HeavyDBPlanner.getHepPlanner(program, false);
260  prePlanner.setRoot(root.rel);
261  final RelNode rootRelNode = prePlanner.findBestExp();
262  return root.withRel(rootRelNode);
263  }
264 
265  private RelRoot applyFilterPushdown(RelRoot root) {
266  if (filterPushDownInfo.isEmpty()) {
267  return root;
268  }
269  final DynamicFilterJoinRule dynamicFilterJoinRule = new DynamicFilterJoinRule(true,
270  RelFactories.LOGICAL_BUILDER,
271  FilterJoinRule.TRUE_PREDICATE,
273  final HepProgram program =
274  HepProgram.builder().addRuleInstance(dynamicFilterJoinRule).build();
275  HepPlanner prePlanner = HeavyDBPlanner.getHepPlanner(program, false);
276  prePlanner.setRoot(root.rel);
277  final RelNode rootRelNode = prePlanner.findBestExp();
278  filterPushDownInfo.clear();
279  return root.withRel(rootRelNode);
280  }
281 
282  private RelRoot applyOptimizationsRules(RelRoot root, ImmutableSet<RelOptRule> rules) {
283  HepProgramBuilder programBuilder = new HepProgramBuilder();
284  for (RelOptRule rule : rules) {
285  programBuilder.addRuleInstance(rule);
286  }
287  HepPlanner hepPlanner = HeavyDBPlanner.getHepPlanner(programBuilder.build(), false);
288  hepPlanner.setRoot(root.rel);
289  return root.withRel(hepPlanner.findBestExp());
290  }
291 
293  String query, HeavyDBSchema schema) throws IOException {
294  ready();
295  RexBuilder builder = new RexBuilder(getTypeFactory());
296  RelOptCluster cluster = RelOptCluster.create(new VolcanoPlanner(), builder);
297  CalciteCatalogReader catalogReader = createCatalogReader();
298  HeavyDBRelJsonReader reader =
299  new HeavyDBRelJsonReader(cluster, catalogReader, schema);
300 
301  RelRoot relR = RelRoot.of(reader.read(query), SqlKind.SELECT);
302 
303  if (restrictions != null) {
304  relR = applyInjectFilterRule(relR, restrictions);
305  }
306  QueryOptimizationRules outerJoinOptRule =
307  new OuterJoinOptViaNullRejectionRule(RelFactories.LOGICAL_BUILDER);
309  relR, ImmutableSet.of(CoreRules.AGGREGATE_MERGE, outerJoinOptRule));
310  relR = applyFilterPushdown(relR);
311  relR = applyOptimizationsRules(relR,
312  ImmutableSet.of(CoreRules.JOIN_PROJECT_BOTH_TRANSPOSE_INCLUDE_OUTER,
313  CoreRules.FILTER_REDUCE_EXPRESSIONS,
315  CoreRules.PROJECT_FILTER_TRANSPOSE));
316  relR = applyOptimizationsRules(relR, ImmutableSet.of(CoreRules.PROJECT_MERGE));
317  relR = applyOptimizationsRules(relR,
318  ImmutableSet.of(
319  CoreRules.FILTER_PROJECT_TRANSPOSE, CoreRules.PROJECT_REMOVE));
320  return RelRoot.of(relR.project(), relR.kind);
321  }
322 
324  final List<HeavyDBParserOptions.FilterPushDownInfo> filterPushDownInfo) {
325  this.filterPushDownInfo = filterPushDownInfo;
326  }
327 
328  public void setRestrictions(List<Restriction> restrictions) {
329  this.restrictions = restrictions;
330  }
331 }
332 
333 // End HeavyDBPlanner.java
void setFilterPushDownInfo(final List< HeavyDBParserOptions.FilterPushDownInfo > filterPushDownInfo)
List< HeavyDBParserOptions.FilterPushDownInfo > filterPushDownInfo
RelRoot applyOptimizationsRules(RelRoot root, ImmutableSet< RelOptRule > rules)
tuple root
Definition: setup.in.py:14
RelRoot applyInjectFilterRule(RelRoot root, List< Restriction > restrictions)
RelNode optimizeRATree(RelNode rootNode, boolean viewOptimizationEnabled, boolean foundView)
static SchemaPlus rootSchema(SchemaPlus schema)
static HepPlanner getHepPlanner(HepProgram hepProgram, boolean doNotEliminateSharedNodesInQueryPlanDag)
void setRestrictions(List< Restriction > restrictions)
RelRoot buildRATreeAndPerformQueryOptimization(String query, HeavyDBSchema schema)
CompletionResult getCompletionHints(final String sql, final int cursor, final List< String > visibleTables)
CompletionResult(final List< SqlMoniker > hints, final String replaced)