OmniSciDB  4201147b46
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
HeavyDBPlanner.java
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one or more
3  * contributor license agreements. See the NOTICE file distributed with
4  * this work for additional information regarding copyright ownership.
5  * The ASF licenses this file to you under the Apache License, Version 2.0
6  * (the "License"); you may not use this file except in compliance with
7  * the License. You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 package org.apache.calcite.prepare;
18 
19 import com.google.common.collect.ImmutableSet;
24 
25 import org.apache.calcite.config.CalciteConnectionConfig;
26 import org.apache.calcite.config.CalciteConnectionConfigImpl;
27 import org.apache.calcite.config.CalciteConnectionProperty;
28 import org.apache.calcite.jdbc.CalciteSchema;
29 import org.apache.calcite.linq4j.function.Functions;
30 import org.apache.calcite.plan.Context;
31 import org.apache.calcite.plan.RelOptCluster;
32 import org.apache.calcite.plan.RelOptCostImpl;
33 import org.apache.calcite.plan.RelOptRule;
34 import org.apache.calcite.plan.hep.HepPlanner;
35 import org.apache.calcite.plan.hep.HepProgram;
36 import org.apache.calcite.plan.hep.HepProgramBuilder;
37 import org.apache.calcite.plan.volcano.VolcanoPlanner;
38 import org.apache.calcite.rel.RelNode;
39 import org.apache.calcite.rel.RelRoot;
40 import org.apache.calcite.rel.core.RelFactories;
42 import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
43 import org.apache.calcite.rel.rules.*;
44 import org.apache.calcite.rex.RexBuilder;
45 import org.apache.calcite.schema.SchemaPlus;
46 import org.apache.calcite.sql.SqlKind;
47 import org.apache.calcite.sql.SqlNode;
48 import org.apache.calcite.sql.advise.SqlAdvisor;
49 import org.apache.calcite.sql.parser.SqlParseException;
50 import org.apache.calcite.sql.validate.SqlConformanceEnum;
51 import org.apache.calcite.sql.validate.SqlMoniker;
52 import org.apache.calcite.sql.validate.SqlValidator;
53 import org.apache.calcite.tools.FrameworkConfig;
54 import org.slf4j.Logger;
55 import org.slf4j.LoggerFactory;
56 
57 import java.io.IOException;
58 import java.lang.reflect.InvocationTargetException;
59 import java.lang.reflect.Method;
60 import java.util.ArrayList;
61 import java.util.List;
62 import java.util.Properties;
63 
69 public class HeavyDBPlanner extends PlannerImpl {
70  FrameworkConfig config;
71  private List<HeavyDBParserOptions.FilterPushDownInfo> filterPushDownInfo =
72  new ArrayList<>();
73  private List<Restriction> restrictions = null;
74  final static Logger HEAVYDBLOGGER = LoggerFactory.getLogger(HeavyDBPlanner.class);
75 
76  public HeavyDBPlanner(FrameworkConfig config) {
77  super(config);
78  this.config = config;
79  }
80 
81  private static SchemaPlus rootSchema(SchemaPlus schema) {
82  for (;;) {
83  if (schema.getParentSchema() == null) {
84  return schema;
85  }
86  schema = schema.getParentSchema();
87  }
88  }
89 
90  public static class CompletionResult {
91  public List<SqlMoniker> hints;
92  public String replaced;
93 
94  CompletionResult(final List<SqlMoniker> hints, final String replaced) {
95  this.hints = hints;
96  this.replaced = replaced;
97  }
98  }
99 
100  private CalciteCatalogReader createCatalogReader() {
101  final SchemaPlus rootSchema = rootSchema(config.getDefaultSchema());
102  final Context context = config.getContext();
103  final CalciteConnectionConfig connectionConfig;
104 
105  if (context != null) {
106  connectionConfig = context.unwrap(CalciteConnectionConfig.class);
107  } else {
108  Properties properties = new Properties();
109  properties.setProperty(CalciteConnectionProperty.CASE_SENSITIVE.camelName(),
110  String.valueOf(config.getParserConfig().caseSensitive()));
111  connectionConfig = new CalciteConnectionConfigImpl(properties);
112  }
113 
114  return new CalciteCatalogReader(CalciteSchema.from(rootSchema),
115  CalciteSchema.from(config.getDefaultSchema()).path(null),
116  getTypeFactory(),
117  connectionConfig);
118  }
119 
120  public void advanceToValidate() {
121  try {
122  String dummySql = "SELECT 1";
123  super.parse(dummySql);
124  } catch (SqlParseException e) {
125  throw new RuntimeException(e);
126  }
127  }
128 
129  public void ready() {
130  // need to call ready on the super class, but that method is marked private
131  // circumventing via reflection for now
132  try {
133  Method readyMethod = getClass().getSuperclass().getDeclaredMethod("ready");
134  readyMethod.setAccessible(true);
135  readyMethod.invoke(this);
136  } catch (InvocationTargetException e) {
137  if (e.getCause() instanceof RuntimeException) {
138  throw(RuntimeException) e.getCause();
139  } else {
140  throw new RuntimeException(e.getCause());
141  }
142  } catch (Exception e) {
143  throw new RuntimeException(e);
144  }
145  }
146 
148  final String sql, final int cursor, final List<String> visibleTables) {
149  ready();
150 
151  SqlValidator.Config validatorConfig = SqlValidator.Config.DEFAULT;
152  validatorConfig = validatorConfig.withSqlConformance(SqlConformanceEnum.LENIENT);
153 
154  HeavyDBSqlAdvisorValidator advisor_validator =
155  new HeavyDBSqlAdvisorValidator(visibleTables,
156  config.getOperatorTable(),
158  getTypeFactory(),
159  validatorConfig);
160  SqlAdvisor advisor =
161  new HeavyDBSqlAdvisor(advisor_validator, config.getParserConfig());
162  String[] replaced = new String[1];
163  int adjusted_cursor = cursor < 0 ? sql.length() : cursor;
164  java.util.List<SqlMoniker> hints =
165  advisor.getCompletionHints(sql, adjusted_cursor, replaced);
166  return new CompletionResult(hints, replaced[0]);
167  }
168 
169  public static HepPlanner getHepPlanner(HepProgram hepProgram, boolean noDag) {
170  if (noDag) {
171  return new HepPlanner(
172  hepProgram, null, true, Functions.ignore2(), RelOptCostImpl.FACTORY);
173  } else {
174  return new HepPlanner(hepProgram);
175  }
176  }
177 
178  @Override
179  public RelRoot rel(SqlNode sql) {
180  return super.rel(sql);
181  }
182 
183  public RelRoot getRelRoot(SqlNode sqlNode) {
184  return super.rel(sqlNode);
185  }
186 
187  public RelNode optimizeRATree(
188  RelNode rootNode, boolean viewOptimizationEnabled, boolean foundView) {
189  HepProgramBuilder firstOptPhaseProgram = HepProgram.builder();
190  firstOptPhaseProgram.addRuleInstance(CoreRules.AGGREGATE_MERGE)
191  .addRuleInstance(
192  new OuterJoinOptViaNullRejectionRule(RelFactories.LOGICAL_BUILDER))
193  .addRuleInstance(CoreRules.AGGREGATE_UNION_TRANSPOSE);
194  if (!viewOptimizationEnabled) {
195  firstOptPhaseProgram.addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE)
196  .addRuleInstance(
197  FilterTableFunctionMultiInputTransposeRule.Config.DEFAULT.toRule())
198  .addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE);
199  } else {
200  if (foundView) {
201  firstOptPhaseProgram.addRuleInstance(
202  CoreRules.JOIN_PROJECT_BOTH_TRANSPOSE_INCLUDE_OUTER);
203  firstOptPhaseProgram.addRuleInstance(CoreRules.FILTER_MERGE);
204  }
205  firstOptPhaseProgram.addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE);
206  firstOptPhaseProgram.addRuleInstance(
207  FilterTableFunctionMultiInputTransposeRule.Config.DEFAULT.toRule());
208  firstOptPhaseProgram.addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE);
209  if (foundView) {
210  firstOptPhaseProgram.addRuleInstance(CoreRules.PROJECT_MERGE);
211  firstOptPhaseProgram.addRuleInstance(ProjectProjectRemoveRule.INSTANCE);
212  }
213  }
214  HepProgram firstOptPhase = firstOptPhaseProgram.build();
215  HepPlanner firstPlanner = HeavyDBPlanner.getHepPlanner(firstOptPhase, true);
216  firstPlanner.setRoot(rootNode);
217  final RelNode firstOptimizedPlanRoot = firstPlanner.findBestExp();
218 
219  boolean hasRLSFilter = null != restrictions && !restrictions.isEmpty();
220  boolean needsSecondOptPhase = hasRLSFilter || !filterPushDownInfo.isEmpty();
221  if (needsSecondOptPhase) {
222  HepProgramBuilder secondOptPhaseProgram = HepProgram.builder();
223  if (hasRLSFilter) {
224  final InjectFilterRule injectFilterRule =
225  InjectFilterRule.Config.DEFAULT.toRule(restrictions);
226  secondOptPhaseProgram.addRuleInstance(injectFilterRule);
227  }
228  if (!filterPushDownInfo.isEmpty()) {
229  final DynamicFilterJoinRule dynamicFilterJoinRule =
230  new DynamicFilterJoinRule(true,
231  RelFactories.LOGICAL_BUILDER,
232  FilterJoinRule.TRUE_PREDICATE,
234  secondOptPhaseProgram.addRuleInstance(dynamicFilterJoinRule);
235  }
236 
237  HepProgram secondOptPhase = secondOptPhaseProgram.build();
238  HepPlanner secondPlanner = HeavyDBPlanner.getHepPlanner(secondOptPhase, false);
239  secondPlanner.setRoot(firstOptimizedPlanRoot);
240  final RelNode secondOptimizedPlanRoot = secondPlanner.findBestExp();
241  if (!filterPushDownInfo.isEmpty()) {
242  filterPushDownInfo.clear();
243  }
244  return secondOptimizedPlanRoot;
245  } else {
246  return firstOptimizedPlanRoot;
247  }
248  }
249 
250  private RelRoot applyInjectFilterRule(RelRoot root, List<Restriction> restrictions) {
251  // TODO consider doing these rules in one preplan pass
252 
253  final InjectFilterRule injectFilterRule =
254  InjectFilterRule.Config.DEFAULT.toRule(restrictions);
255 
256  final HepProgram program =
257  HepProgram.builder().addRuleInstance(injectFilterRule).build();
258  HepPlanner prePlanner = HeavyDBPlanner.getHepPlanner(program, false);
259  prePlanner.setRoot(root.rel);
260  final RelNode rootRelNode = prePlanner.findBestExp();
261  return root.withRel(rootRelNode);
262  }
263 
264  private RelRoot applyFilterPushdown(RelRoot root) {
265  if (filterPushDownInfo.isEmpty()) {
266  return root;
267  }
268  final DynamicFilterJoinRule dynamicFilterJoinRule = new DynamicFilterJoinRule(true,
269  RelFactories.LOGICAL_BUILDER,
270  FilterJoinRule.TRUE_PREDICATE,
272  final HepProgram program =
273  HepProgram.builder().addRuleInstance(dynamicFilterJoinRule).build();
274  HepPlanner prePlanner = HeavyDBPlanner.getHepPlanner(program, false);
275  prePlanner.setRoot(root.rel);
276  final RelNode rootRelNode = prePlanner.findBestExp();
277  filterPushDownInfo.clear();
278  return root.withRel(rootRelNode);
279  }
280 
281  private RelRoot applyOptimizationsRules(RelRoot root, ImmutableSet<RelOptRule> rules) {
282  HepProgramBuilder programBuilder = new HepProgramBuilder();
283  for (RelOptRule rule : rules) {
284  programBuilder.addRuleInstance(rule);
285  }
286  HepPlanner hepPlanner = HeavyDBPlanner.getHepPlanner(programBuilder.build(), false);
287  hepPlanner.setRoot(root.rel);
288  return root.withRel(hepPlanner.findBestExp());
289  }
290 
292  String query, HeavyDBSchema schema) throws IOException {
293  ready();
294  RexBuilder builder = new RexBuilder(getTypeFactory());
295  RelOptCluster cluster = RelOptCluster.create(new VolcanoPlanner(), builder);
296  CalciteCatalogReader catalogReader = createCatalogReader();
297  HeavyDBRelJsonReader reader =
298  new HeavyDBRelJsonReader(cluster, catalogReader, schema);
299 
300  RelRoot relR = RelRoot.of(reader.read(query), SqlKind.SELECT);
301 
302  if (restrictions != null) {
303  relR = applyInjectFilterRule(relR, restrictions);
304  }
305  QueryOptimizationRules outerJoinOptRule =
306  new OuterJoinOptViaNullRejectionRule(RelFactories.LOGICAL_BUILDER);
308  relR, ImmutableSet.of(CoreRules.AGGREGATE_MERGE, outerJoinOptRule));
309  relR = applyFilterPushdown(relR);
310  relR = applyOptimizationsRules(relR,
311  ImmutableSet.of(CoreRules.JOIN_PROJECT_BOTH_TRANSPOSE_INCLUDE_OUTER,
312  CoreRules.FILTER_REDUCE_EXPRESSIONS,
314  CoreRules.PROJECT_FILTER_TRANSPOSE));
315  relR = applyOptimizationsRules(relR, ImmutableSet.of(CoreRules.PROJECT_MERGE));
316  relR = applyOptimizationsRules(relR,
317  ImmutableSet.of(
318  CoreRules.FILTER_PROJECT_TRANSPOSE, CoreRules.PROJECT_REMOVE));
319  return RelRoot.of(relR.project(), relR.kind);
320  }
321 
323  final List<HeavyDBParserOptions.FilterPushDownInfo> filterPushDownInfo) {
324  this.filterPushDownInfo = filterPushDownInfo;
325  }
326 
327  public void setRestrictions(List<Restriction> restrictions) {
328  this.restrictions = restrictions;
329  }
330 }
331 
332 // End HeavyDBPlanner.java
void setFilterPushDownInfo(final List< HeavyDBParserOptions.FilterPushDownInfo > filterPushDownInfo)
List< HeavyDBParserOptions.FilterPushDownInfo > filterPushDownInfo
RelRoot applyOptimizationsRules(RelRoot root, ImmutableSet< RelOptRule > rules)
tuple root
Definition: setup.in.py:14
RelRoot applyInjectFilterRule(RelRoot root, List< Restriction > restrictions)
RelNode optimizeRATree(RelNode rootNode, boolean viewOptimizationEnabled, boolean foundView)
static SchemaPlus rootSchema(SchemaPlus schema)
static HepPlanner getHepPlanner(HepProgram hepProgram, boolean noDag)
void setRestrictions(List< Restriction > restrictions)
RelRoot buildRATreeAndPerformQueryOptimization(String query, HeavyDBSchema schema)
CompletionResult getCompletionHints(final String sql, final int cursor, final List< String > visibleTables)
CompletionResult(final List< SqlMoniker > hints, final String replaced)