16 package com.mapd.calcite.parser;
20 import com.google.common.collect.ImmutableList;
116 import org.slf4j.Logger;
117 import org.slf4j.LoggerFactory;
119 import java.io.IOException;
120 import java.lang.reflect.Field;
121 import java.util.ArrayList;
122 import java.util.Arrays;
123 import java.util.Collections;
124 import java.util.EnumSet;
125 import java.util.HashSet;
126 import java.util.List;
127 import java.util.Map;
128 import java.util.Properties;
129 import java.util.Set;
130 import java.util.concurrent.ConcurrentHashMap;
131 import java.util.function.BiPredicate;
132 import java.util.function.Supplier;
139 public static final ThreadLocal<MapDParser>
CURRENT_PARSER =
new ThreadLocal<>();
140 private static final EnumSet<SqlKind>
SCALAR =
141 EnumSet.of(SqlKind.SCALAR_QUERY, SqlKind.SELECT);
142 private static final EnumSet<SqlKind>
EXISTS = EnumSet.of(SqlKind.EXISTS);
143 private static final EnumSet<SqlKind>
DELETE = EnumSet.of(SqlKind.DELETE);
144 private static final EnumSet<SqlKind>
UPDATE = EnumSet.of(SqlKind.UPDATE);
145 private static final EnumSet<SqlKind>
IN = EnumSet.of(SqlKind.IN);
147 EnumSet.of(SqlKind.ARRAY_VALUE_CONSTRUCTOR);
163 final Supplier<MapDSqlOperatorTable> mapDSqlOperatorTable,
169 this.sock_transport_properties = skT;
173 SubqueryCorrMemo.clear();
178 CalciteConnectionConfig config =
new CalciteConnectionConfigImpl(
new Properties()) {
180 properties.put(CalciteConnectionProperty.CASE_SENSITIVE.camelName(),
181 String.valueOf(
false));
182 properties.put(CalciteConnectionProperty.CONFORMANCE.camelName(),
183 String.valueOf(SqlConformanceEnum.LENIENT));
186 @SuppressWarnings(
"unchecked")
187 public <T extends Object>
T typeSystem(
188 java.lang.Class<
T> typeSystemClass,
T defaultTypeSystem) {
189 return (
T) myTypeSystem;
192 public boolean caseSensitive() {
197 return SqlConformanceEnum.LENIENT;
202 public <C> C unwrap(Class<C> aClass) {
203 if (aClass.isInstance(config)) {
204 return aClass.cast(config);
215 String queryString = expression.toSqlString(CalciteSqlDialect.DEFAULT).getSql();
216 Boolean isCorrelatedSubquery = SubqueryCorrMemo.get(queryString);
217 if (null != isCorrelatedSubquery) {
218 return isCorrelatedSubquery;
223 dataDir, mapDSqlOperatorTable, mapdPort, sock_transport_properties);
227 }
catch (Exception e) {
229 SubqueryCorrMemo.put(queryString,
true);
232 SubqueryCorrMemo.put(queryString,
false);
237 final boolean allowPushdownJoinCondition) {
239 new MapDSchema(dataDir,
this, mapdPort, mapdUser, sock_transport_properties);
240 final SchemaPlus rootSchema = Frameworks.createRootSchema(
true);
242 BiPredicate<SqlNode, SqlNode> expandPredicate =
new BiPredicate<SqlNode, SqlNode>() {
244 public boolean test(SqlNode root, SqlNode expression) {
245 if (!allowSubQueryExpansion) {
250 if (expression.isA(SCALAR) || expression.isA(EXISTS) || expression.isA(IN)) {
253 if (expression.isA(EXISTS)) {
258 if (expression.isA(IN)) {
262 boolean found_expression =
false;
263 if (expression instanceof SqlCall) {
264 SqlCall call = (SqlCall) expression;
265 if (call.getOperandList().size() == 2) {
274 if (call.getOperandList().get(1) instanceof SqlSelect) {
275 expression = call.getOperandList().get(1);
276 SqlSelect select_call = (SqlSelect) expression;
277 if (select_call.hasWhere()) {
278 found_expression =
true;
283 if (!found_expression) {
289 SqlSelect select = null;
290 if (expression instanceof SqlCall) {
291 SqlCall call = (SqlCall) expression;
292 if (call.getOperator().equals(SqlStdOperatorTable.SCALAR_QUERY)) {
293 expression = call.getOperandList().get(0);
297 if (expression instanceof SqlSelect) {
298 select = (SqlSelect) expression;
301 if (null != select) {
302 if (null != select.getFetch() || null != select.getOffset()
303 || (null != select.getOrderList()
304 && select.getOrderList().size() != 0)) {
305 throw new CalciteException(
306 "Correlated sub-queries with ordering not supported.", null);
318 BiPredicate<SqlNode, Join> pushdownJoinPredicate =
new BiPredicate<SqlNode, Join>() {
320 public boolean test(SqlNode t, Join u) {
321 if (!allowPushdownJoinCondition) {
325 return !hasGeoColumns(u.getRowType());
328 private boolean hasGeoColumns(RelDataType
type) {
329 for (RelDataTypeField f : type.getFieldList()) {
330 if (
"any".equalsIgnoreCase(f.getType().getFamily().toString())) {
340 final FrameworkConfig config =
341 Frameworks.newConfigBuilder()
342 .defaultSchema(rootSchema.add(mapdUser.
getDB(), mapd))
343 .operatorTable(mapDSqlOperatorTable.get())
344 .parserConfig(SqlParser.configBuilder()
345 .setConformance(SqlConformanceEnum.LENIENT)
346 .setUnquotedCasing(Casing.UNCHANGED)
347 .setCaseSensitive(
false)
349 .setIdentifierMaxLength(512)
350 .setParserFactory(ExtendedSqlParser.FACTORY)
352 .sqlToRelConverterConfig(
356 .withExpandPredicate(expandPredicate)
358 .withInSubQueryThreshold(Integer.MAX_VALUE)
359 .withPushdownJoinCondition(pushdownJoinPredicate)
360 .withHintStrategyTable(
364 .context(MAPD_CONNECTION_CONTEXT)
373 public Pair<String, SqlIdentifierCapturer>
process(
375 throws SqlParseException, ValidationException, RelConversionException {
377 final SqlNode sqlNode =
parseSql(sql, parserOptions.isLegacySyntax(), planner);
381 return new Pair<String, SqlIdentifierCapturer>(
res, capture);
386 new MapDSchema(dataDir,
this, mapdPort, mapdUser, sock_transport_properties);
394 throws SqlParseException, ValidationException, RelConversionException {
398 final SqlNode sqlNode =
parseSql(sql, parserOptions.isLegacySyntax(), planner);
404 throws SqlParseException, ValidationException, RelConversionException {
408 return ((JsonSerializableDdl) sqlNode).toJsonString();
411 if (sqlNode instanceof SqlDdl) {
412 return sqlNode.toString();
419 RelNode project = sqlRel.project();
421 if (parserOptions.isExplain()) {
422 return RelOptUtil.toString(sqlRel.project());
431 String sql,
int cursor, List<String> visible_tables) {
437 new MapDSchema(dataDir,
this, mapdPort, mapdUser, sock_transport_properties);
438 HashSet<String> resolved =
new HashSet<>();
443 throw new RuntimeException(
"table/view not found: " + name);
447 MapDView view = (MapDView) table;
458 if (node.isA(EnumSet.of(SqlKind.AS))) {
459 node = ((SqlCall) node).getOperandList().get(1);
461 if (node instanceof SqlIdentifier) {
462 SqlIdentifier
id = (SqlIdentifier) node;
463 return id.names.get(
id.names.size() - 1);
469 SqlNode where = update.getCondition();
471 if (update.getSourceExpressionList().size() != 1) {
475 if (!(update.getSourceExpressionList().get(0) instanceof SqlSelect)) {
479 final SqlSelect inner = (SqlSelect) update.getSourceExpressionList().get(0);
481 if (null != inner.getGroup() || null != inner.getFetch() || null != inner.getOffset()
482 || (null != inner.getOrderList() && inner.getOrderList().size() != 0)
483 || (null != inner.getGroup() && inner.getGroup().size() != 0)
492 final String updateTableName =
getTableName(update.getTargetTable());
495 where = where.accept(
new SqlShuttle() {
497 public SqlNode visit(SqlIdentifier
id) {
499 id =
new SqlIdentifier(Arrays.asList(updateTableName,
id.getSimple()),
500 id.getParserPosition());
508 SqlJoin
join =
new SqlJoin(ZERO,
509 update.getTargetTable(),
510 SqlLiteral.createBoolean(
false, ZERO),
511 SqlLiteral.createSymbol(
JoinType.LEFT, ZERO),
513 SqlLiteral.createSymbol(JoinConditionType.ON, ZERO),
516 SqlNode select0 = inner.getSelectList().get(0);
518 boolean wrapInSingleValue =
true;
519 if (select0 instanceof SqlCall) {
520 SqlCall selectExprCall = (SqlCall) select0;
521 if (Util.isSingleValue(selectExprCall)) {
522 wrapInSingleValue =
false;
526 if (wrapInSingleValue) {
527 select0 =
new SqlBasicCall(
528 SqlStdOperatorTable.SINGLE_VALUE,
new SqlNode[] {select0}, ZERO);
531 SqlNodeList selectList =
new SqlNodeList(ZERO);
532 selectList.add(select0);
533 selectList.add(
new SqlBasicCall(SqlStdOperatorTable.AS,
534 new SqlNode[] {new SqlBasicCall(
535 new SqlUnresolvedFunction(
536 new SqlIdentifier(
"OFFSET_IN_FRAGMENT", ZERO),
541 SqlFunctionCategory.USER_DEFINED_FUNCTION),
544 new SqlIdentifier(
"EXPR$DELETE_OFFSET_IN_FRAGMENT", ZERO)},
547 SqlNodeList groupBy =
new SqlNodeList(ZERO);
548 groupBy.add(
new SqlIdentifier(
"EXPR$DELETE_OFFSET_IN_FRAGMENT", ZERO));
550 SqlSelect select =
new SqlSelect(ZERO,
566 throws SqlParseException, ValidationException, RelConversionException {
567 SqlIdentifier targetTable = (SqlIdentifier) update.getTargetTable();
568 String targetTableName = targetTable.names.get(targetTable.names.size() - 1);
570 String dummySql =
"DELETE FROM " + targetTableName;
571 SqlNode dummyNode = planner.parse(dummySql);
572 dummyNode = planner.validate(dummyNode);
573 RelRoot dummyRoot = planner.
rel(dummyNode);
574 LogicalTableModify dummyModify = (LogicalTableModify) dummyRoot.rel;
579 throws SqlParseException, ValidationException, RelConversionException {
580 int correlatedQueriesCount[] =
new int[1];
581 SqlBasicVisitor<Void> correlatedQueriesCounter =
new SqlBasicVisitor<Void>() {
583 public Void visit(SqlCall call) {
585 && ((call instanceof SqlBasicCall && call.operandCount() == 1
586 && !call.operand(0).isA(SCALAR))
587 || !(call instanceof SqlBasicCall))) {
589 correlatedQueriesCount[0]++;
592 return super.visit(call);
596 update.accept(correlatedQueriesCounter);
597 if (correlatedQueriesCount[0] > 1) {
598 throw new CalciteException(
599 "table modifications with multiple correlated sub-queries not supported.",
603 boolean allowPushdownJoinCondition =
false;
604 SqlNodeList sourceExpression =
new SqlNodeList(SqlParserPos.ZERO);
606 RelOptTable targetTable = dummyModify.getTable();
607 RelDataType targetTableType = targetTable.getRowType();
610 boolean applyRexCast = null == select;
612 if (null == select) {
613 for (
int i = 0; i < update.getSourceExpressionList().size(); i++) {
614 SqlNode targetColumn = update.getTargetColumnList().get(i);
615 SqlNode expression = update.getSourceExpressionList().get(i);
617 if (!(targetColumn instanceof SqlIdentifier)) {
618 throw new RuntimeException(
"Unknown identifier type!");
620 SqlIdentifier
id = (SqlIdentifier) targetColumn;
621 RelDataType fieldType =
622 targetTableType.getField(
id.names.get(
id.names.size() - 1),
false,
false)
625 if (expression.isA(ARRAY_VALUE) && null != fieldType.getComponentType()) {
628 SqlDataTypeSpec elementType =
new SqlDataTypeSpec(
629 new SqlBasicTypeNameSpec(fieldType.getComponentType().getSqlTypeName(),
630 fieldType.getPrecision(),
631 fieldType.getScale(),
632 null == fieldType.getCharset() ? null
633 : fieldType.getCharset().name(),
636 SqlCall array_expression = (SqlCall) expression;
637 ArrayList<SqlNode> values =
new ArrayList<>();
639 for (SqlNode value : array_expression.getOperandList()) {
640 if (value.isA(EnumSet.of(SqlKind.LITERAL))) {
641 SqlNode casted_value =
new SqlBasicCall(SqlStdOperatorTable.CAST,
642 new SqlNode[] {value, elementType},
643 value.getParserPosition());
644 values.add(casted_value);
651 values.toArray(
new SqlNode[0]),
652 expression.getParserPosition());
654 sourceExpression.add(expression);
657 sourceExpression.add(
new SqlBasicCall(SqlStdOperatorTable.AS,
659 new SqlBasicCall(new SqlUnresolvedFunction(
660 new SqlIdentifier(
"OFFSET_IN_FRAGMENT",
666 SqlFunctionCategory.USER_DEFINED_FUNCTION),
669 new SqlIdentifier(
"EXPR$DELETE_OFFSET_IN_FRAGMENT", ZERO)},
672 select =
new SqlSelect(SqlParserPos.ZERO,
675 update.getTargetTable(),
676 update.getCondition(),
687 SqlNode node = planner.parse(select.toSqlString(CalciteSqlDialect.DEFAULT).getSql());
688 node = planner.validate(node);
689 RelRoot root = planner.rel(node);
690 LogicalProject project = (LogicalProject) root.project();
692 ArrayList<String> fields =
new ArrayList<String>();
693 ArrayList<RexNode> nodes =
new ArrayList<RexNode>();
694 final RexBuilder builder =
new RexBuilder(planner.getTypeFactory());
696 for (SqlNode n : update.getTargetColumnList()) {
697 if (n instanceof SqlIdentifier) {
698 SqlIdentifier
id = (SqlIdentifier) n;
699 fields.add(
id.names.get(
id.names.size() - 1));
701 throw new RuntimeException(
"Unknown identifier type!");
706 for (RexNode exp : project.getChildExps()) {
707 if (applyRexCast && idx + 1 < project.getChildExps().size()) {
708 RelDataType expectedFieldType =
709 targetTableType.getField(fields.get(idx),
false,
false).getType();
710 if (!exp.getType().equals(expectedFieldType) && !exp.isA(ARRAY_VALUE)) {
711 exp = builder.makeCast(expectedFieldType, exp);
719 ArrayList<RexNode> inputs =
new ArrayList<RexNode>();
721 for (
int i = 0; i < fields.size(); i++) {
723 new RexInputRef(n, project.getRowType().getFieldList().get(n).getType()));
727 fields.add(
"EXPR$DELETE_OFFSET_IN_FRAGMENT");
728 inputs.add(
new RexInputRef(n, project.getRowType().getFieldList().get(n).getType()));
730 project = project.copy(
731 project.getTraitSet(), project.getInput(), nodes, project.getRowType());
733 LogicalTableModify modify = LogicalTableModify.create(targetTable,
734 dummyModify.getCatalogReader(),
740 return RelRoot.of(modify, SqlKind.UPDATE);
744 throws SqlParseException, ValidationException, RelConversionException {
746 final SqlNode sqlNode =
parseSql(sql, parserOptions.isLegacySyntax(), planner);
753 throws SqlParseException, ValidationException, RelConversionException {
754 SqlNode node = sqlNode;
756 boolean allowCorrelatedSubQueryExpansion =
true;
757 boolean allowPushdownJoinCondition =
true;
758 boolean patchUpdateToDelete =
false;
760 if (node.isA(DELETE)) {
761 SqlDelete sqlDelete = (SqlDelete) node;
762 node =
new SqlUpdate(node.getParserPosition(),
763 sqlDelete.getTargetTable(),
766 sqlDelete.getCondition(),
767 sqlDelete.getSourceSelect(),
768 sqlDelete.getAlias());
770 patchUpdateToDelete =
true;
773 if (node.isA(UPDATE)) {
774 SqlUpdate update = (SqlUpdate) node;
775 update = (SqlUpdate) planner.validate(update);
778 if (patchUpdateToDelete) {
779 LogicalTableModify modify = (LogicalTableModify) root.rel;
782 Field f = TableModify.class.getDeclaredField(
"operation");
783 f.setAccessible(
true);
784 f.set(modify, Operation.DELETE);
785 }
catch (Throwable e) {
786 throw new RuntimeException(e);
789 root = RelRoot.of(modify, SqlKind.DELETE);
795 if (parserOptions.isLegacySyntax()) {
799 planner =
getPlanner(allowCorrelatedSubQueryExpansion, allowPushdownJoinCondition);
801 node.toSqlString(CalciteSqlDialect.DEFAULT).toString(),
false, planner);
804 SqlNode validateR = planner.validate(node);
805 planner.setFilterPushDownInfo(parserOptions.getFilterPushDownInfo());
806 RelRoot relR = planner.rel(validateR);
810 if (!parserOptions.isViewOptimizeEnabled()) {
814 boolean foundView =
false;
816 dataDir,
this, mapdPort, mapdUser, sock_transport_properties);
821 throw new RuntimeException(
"table/view not found: " + name);
832 HepProgramBuilder builder =
new HepProgramBuilder();
833 builder.addRuleInstance(CoreRules.JOIN_PROJECT_BOTH_TRANSPOSE_INCLUDE_OUTER);
834 builder.addRuleInstance(CoreRules.FILTER_MERGE);
835 builder.addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE);
836 builder.addRuleInstance(CoreRules.PROJECT_MERGE);
839 HepPlanner hepPlanner =
new HepPlanner(builder.build());
840 final RelNode root = relR.project();
841 hepPlanner.setRoot(root);
842 final RelNode newRel = hepPlanner.findBestExp();
844 return RelRoot.of(newRel, relR.kind);
848 private RelRoot
replaceIsTrue(
final RelDataTypeFactory typeFactory, RelRoot root) {
849 final RexShuttle callShuttle =
new RexShuttle() {
850 RexBuilder builder =
new RexBuilder(typeFactory);
852 public RexNode visitCall(RexCall call) {
853 call = (RexCall) super.visitCall(call);
854 if (call.getKind() == SqlKind.IS_TRUE) {
855 return builder.makeCall(SqlStdOperatorTable.AND,
857 SqlStdOperatorTable.IS_NOT_NULL, call.getOperands().get(0)),
858 call.getOperands().get(0));
859 }
else if (call.getKind() == SqlKind.IS_NOT_TRUE) {
860 return builder.makeCall(SqlStdOperatorTable.OR,
862 SqlStdOperatorTable.IS_NULL, call.getOperands().get(0)),
863 builder.makeCall(SqlStdOperatorTable.NOT, call.getOperands().get(0)));
864 }
else if (call.getKind() == SqlKind.IS_FALSE) {
865 return builder.makeCall(SqlStdOperatorTable.AND,
867 SqlStdOperatorTable.IS_NOT_NULL, call.getOperands().get(0)),
868 builder.makeCall(SqlStdOperatorTable.NOT, call.getOperands().get(0)));
869 }
else if (call.getKind() == SqlKind.IS_NOT_FALSE) {
870 return builder.makeCall(SqlStdOperatorTable.OR,
872 SqlStdOperatorTable.IS_NULL, call.getOperands().get(0)),
873 call.getOperands().get(0));
880 RelNode node = root.rel.accept(
new RelShuttleImpl() {
882 protected RelNode visitChild(RelNode parent,
int i, RelNode child) {
883 RelNode node = super.visitChild(parent, i, child);
884 return node.accept(callShuttle);
888 return new RelRoot(node,
889 root.validatedRowType,
893 Collections.emptyList());
896 private SqlNode
parseSql(String sql,
final boolean legacy_syntax, Planner planner)
897 throws SqlParseException {
898 SqlNode parseR = null;
900 parseR = planner.parse(sql);
901 MAPDLOGGER.debug(
" node is \n" + parseR.toString());
902 }
catch (SqlParseException ex) {
903 MAPDLOGGER.error(
"failed to parse SQL '" + sql +
"' \n" + ex.toString());
907 if (!legacy_syntax) {
911 RelDataTypeFactory typeFactory = planner.getTypeFactory();
912 SqlSelect select_node = null;
913 if (parseR instanceof SqlSelect) {
914 select_node = (SqlSelect) parseR;
915 desugar(select_node, typeFactory);
916 }
else if (parseR instanceof SqlOrderBy) {
917 SqlOrderBy order_by_node = (SqlOrderBy) parseR;
918 if (order_by_node.query instanceof SqlSelect) {
919 select_node = (SqlSelect) order_by_node.query;
920 SqlOrderBy new_order_by_node =
desugar(select_node, order_by_node, typeFactory);
921 if (new_order_by_node != null) {
922 return new_order_by_node;
924 }
else if (order_by_node.query instanceof SqlWith) {
925 SqlWith old_with_node = (SqlWith) order_by_node.query;
926 if (old_with_node.body instanceof SqlSelect) {
927 select_node = (SqlSelect) old_with_node.body;
928 desugar(select_node, typeFactory);
931 }
else if (parseR instanceof SqlWith) {
932 SqlWith old_with_node = (SqlWith) parseR;
933 if (old_with_node.body instanceof SqlSelect) {
934 select_node = (SqlSelect) old_with_node.body;
935 desugar(select_node, typeFactory);
941 private void desugar(SqlSelect select_node, RelDataTypeFactory typeFactory) {
942 desugar(select_node, null, typeFactory);
945 private SqlNode
expandCase(SqlCase old_case_node, RelDataTypeFactory typeFactory) {
946 SqlNodeList newWhenList =
947 new SqlNodeList(old_case_node.getWhenOperands().getParserPosition());
948 SqlNodeList newThenList =
949 new SqlNodeList(old_case_node.getThenOperands().getParserPosition());
950 java.util.Map<String, SqlNode> id_to_expr =
new java.util.HashMap<String, SqlNode>();
951 for (SqlNode node : old_case_node.getWhenOperands()) {
952 SqlNode newCall =
expand(node, id_to_expr, typeFactory);
953 if (null != newCall) {
954 newWhenList.add(newCall);
956 newWhenList.add(node);
959 for (SqlNode node : old_case_node.getThenOperands()) {
960 SqlNode newCall =
expand(node, id_to_expr, typeFactory);
961 if (null != newCall) {
962 newThenList.add(newCall);
964 newThenList.add(node);
967 SqlNode new_else_operand = old_case_node.getElseOperand();
968 if (null != new_else_operand) {
969 SqlNode candidate_else_operand =
970 expand(old_case_node.getElseOperand(), id_to_expr, typeFactory);
971 if (null != candidate_else_operand) {
972 new_else_operand = candidate_else_operand;
975 SqlNode new_value_operand = old_case_node.getValueOperand();
976 if (null != new_value_operand) {
977 SqlNode candidate_value_operand =
978 expand(old_case_node.getValueOperand(), id_to_expr, typeFactory);
979 if (null != candidate_value_operand) {
980 new_value_operand = candidate_value_operand;
983 SqlNode newCaseNode = SqlCase.createSwitched(old_case_node.getParserPosition(),
991 private SqlOrderBy
desugar(SqlSelect select_node,
992 SqlOrderBy order_by_node,
993 RelDataTypeFactory typeFactory) {
994 MAPDLOGGER.debug(
"desugar: before: " + select_node.toString());
997 SqlNodeList select_list = select_node.getSelectList();
998 SqlNodeList new_select_list =
new SqlNodeList(select_list.getParserPosition());
999 java.util.Map<String, SqlNode> id_to_expr =
new java.util.HashMap<String, SqlNode>();
1000 for (SqlNode proj : select_list) {
1001 if (!(proj instanceof SqlBasicCall)) {
1002 if (proj instanceof SqlCase) {
1003 new_select_list.add(
expandCase((SqlCase) proj, typeFactory));
1005 new_select_list.add(proj);
1008 assert proj instanceof SqlBasicCall;
1009 SqlBasicCall proj_call = (SqlBasicCall) proj;
1010 if (proj_call.operands.length > 0) {
1011 for (
int i = 0; i < proj_call.operands.length; i++) {
1012 if (proj_call.operand(i) instanceof SqlCase) {
1013 SqlNode new_op =
expandCase(proj_call.operand(i), typeFactory);
1014 proj_call.setOperand(i, new_op);
1018 new_select_list.add(
expand(proj_call, id_to_expr, typeFactory));
1021 select_node.setSelectList(new_select_list);
1022 SqlNodeList group_by_list = select_node.getGroup();
1023 if (group_by_list != null) {
1024 select_node.setGroupBy(
expand(group_by_list, id_to_expr, typeFactory));
1026 SqlNode having = select_node.getHaving();
1027 if (having != null) {
1028 expand(having, id_to_expr, typeFactory);
1030 SqlOrderBy new_order_by_node = null;
1031 if (order_by_node != null && order_by_node.orderList != null
1032 && order_by_node.orderList.size() > 0) {
1033 SqlNodeList new_order_by_list =
1034 expand(order_by_node.orderList, id_to_expr, typeFactory);
1035 new_order_by_node =
new SqlOrderBy(order_by_node.getParserPosition(),
1038 order_by_node.offset,
1039 order_by_node.fetch);
1042 MAPDLOGGER.debug(
"desugar: after: " + select_node.toString());
1043 return new_order_by_node;
1047 if (node instanceof SqlSelect) {
1048 desugar((SqlSelect) node, typeFactory);
1051 if (!(node instanceof SqlBasicCall)) {
1054 SqlBasicCall basic_call = (SqlBasicCall) node;
1055 for (SqlNode
operator : basic_call.getOperands()) {
1056 if (
operator instanceof SqlOrderBy) {
1065 final java.util.Map<String, SqlNode> id_to_expr,
1066 RelDataTypeFactory typeFactory) {
1067 MAPDLOGGER.debug(
"expand: " + node.toString());
1068 if (node instanceof SqlBasicCall) {
1069 SqlBasicCall node_call = (SqlBasicCall) node;
1070 SqlNode[] operands = node_call.getOperands();
1071 for (
int i = 0; i < operands.length; ++i) {
1072 node_call.setOperand(i,
expand(operands[i], id_to_expr, typeFactory));
1074 SqlNode expanded_variance =
expandVariance(node_call, typeFactory);
1075 if (expanded_variance != null) {
1076 return expanded_variance;
1079 if (expanded_covariance != null) {
1080 return expanded_covariance;
1083 if (expanded_correlation != null) {
1084 return expanded_correlation;
1087 if (node instanceof SqlSelect) {
1088 SqlSelect select_node = (SqlSelect) node;
1089 desugar(select_node, typeFactory);
1094 private SqlNodeList
expand(
final SqlNodeList group_by_list,
1095 final java.util.Map<String, SqlNode> id_to_expr,
1096 RelDataTypeFactory typeFactory) {
1097 SqlNodeList new_group_by_list =
new SqlNodeList(
new SqlParserPos(-1, -1));
1098 for (SqlNode group_by : group_by_list) {
1099 if (!(group_by instanceof SqlIdentifier)) {
1100 new_group_by_list.add(
expand(group_by, id_to_expr, typeFactory));
1103 SqlIdentifier group_by_id = ((SqlIdentifier) group_by);
1104 if (id_to_expr.containsKey(group_by_id.toString())) {
1105 new_group_by_list.add(id_to_expr.get(group_by_id.toString()));
1107 new_group_by_list.add(group_by);
1110 return new_group_by_list;
1114 final SqlBasicCall proj_call, RelDataTypeFactory typeFactory) {
1116 if (proj_call.operandCount() != 1) {
1122 if (proj_call.getOperator().isName(
"STDDEV_POP",
false)) {
1126 }
else if (proj_call.getOperator().getName().equalsIgnoreCase(
"STDDEV_POP_FLOAT")) {
1130 }
else if (proj_call.getOperator().isName(
"STDDEV_SAMP",
false)
1131 || proj_call.getOperator().getName().equalsIgnoreCase(
"STDDEV")) {
1135 }
else if (proj_call.getOperator().getName().equalsIgnoreCase(
"STDDEV_SAMP_FLOAT")
1136 || proj_call.getOperator().getName().equalsIgnoreCase(
"STDDEV_FLOAT")) {
1140 }
else if (proj_call.getOperator().isName(
"VAR_POP",
false)) {
1144 }
else if (proj_call.getOperator().getName().equalsIgnoreCase(
"VAR_POP_FLOAT")) {
1148 }
else if (proj_call.getOperator().isName(
"VAR_SAMP",
false)
1149 || proj_call.getOperator().getName().equalsIgnoreCase(
"VARIANCE")) {
1153 }
else if (proj_call.getOperator().getName().equalsIgnoreCase(
"VAR_SAMP_FLOAT")
1154 || proj_call.getOperator().getName().equalsIgnoreCase(
"VARIANCE_FLOAT")) {
1161 final SqlNode operand = proj_call.operand(0);
1162 final SqlParserPos pos = proj_call.getParserPosition();
1163 SqlNode expanded_proj_call =
1165 MAPDLOGGER.debug(
"Expanded select_list SqlCall: " + proj_call.toString());
1166 MAPDLOGGER.debug(
"to : " + expanded_proj_call.toString());
1167 return expanded_proj_call;
1171 final SqlNode operand,
1175 RelDataTypeFactory typeFactory) {
1196 final SqlNode arg = SqlStdOperatorTable.CAST.createCall(pos,
1198 SqlTypeUtil.convertTypeToSpec(typeFactory.createSqlType(
1199 flt ? SqlTypeName.FLOAT : SqlTypeName.DOUBLE)));
1200 final SqlNode argSquared = SqlStdOperatorTable.MULTIPLY.createCall(pos, arg, arg);
1201 final SqlNode sumArgSquared = SqlStdOperatorTable.SUM.createCall(pos, argSquared);
1202 final SqlNode sum = SqlStdOperatorTable.SUM.createCall(pos, arg);
1203 final SqlNode sumSquared = SqlStdOperatorTable.MULTIPLY.createCall(pos, sum, sum);
1204 final SqlNode count = SqlStdOperatorTable.COUNT.createCall(pos, arg);
1205 final SqlLiteral nul = SqlLiteral.createNull(pos);
1206 final SqlNumericLiteral zero = SqlLiteral.createExactNumeric(
"0", pos);
1207 final SqlNode countEqZero = SqlStdOperatorTable.EQUALS.createCall(pos, count, zero);
1208 SqlNodeList whenList =
new SqlNodeList(pos);
1209 SqlNodeList thenList =
new SqlNodeList(pos);
1210 whenList.add(countEqZero);
1212 final SqlNode int_denominator = SqlStdOperatorTable.CASE.createCall(
1213 null, pos, null, whenList, thenList, count);
1214 final SqlNode denominator = SqlStdOperatorTable.CAST.createCall(pos,
1216 SqlTypeUtil.convertTypeToSpec(typeFactory.createSqlType(
1217 flt ? SqlTypeName.FLOAT : SqlTypeName.DOUBLE)));
1218 final SqlNode avgSumSquared =
1219 SqlStdOperatorTable.DIVIDE.createCall(pos, sumSquared, denominator);
1220 final SqlNode diff =
1221 SqlStdOperatorTable.MINUS.createCall(pos, sumArgSquared, avgSumSquared);
1222 final SqlNode denominator1;
1224 denominator1 = denominator;
1226 final SqlNumericLiteral one = SqlLiteral.createExactNumeric(
"1", pos);
1227 final SqlNode countEqOne = SqlStdOperatorTable.EQUALS.createCall(pos, count, one);
1228 final SqlNode countMinusOne = SqlStdOperatorTable.MINUS.createCall(pos, count, one);
1229 SqlNodeList whenList1 =
new SqlNodeList(pos);
1230 SqlNodeList thenList1 =
new SqlNodeList(pos);
1231 whenList1.add(countEqOne);
1233 final SqlNode int_denominator1 = SqlStdOperatorTable.CASE.createCall(
1234 null, pos, null, whenList1, thenList1, countMinusOne);
1235 denominator1 = SqlStdOperatorTable.CAST.createCall(pos,
1237 SqlTypeUtil.convertTypeToSpec(typeFactory.createSqlType(
1238 flt ? SqlTypeName.FLOAT : SqlTypeName.DOUBLE)));
1240 final SqlNode div = SqlStdOperatorTable.DIVIDE.createCall(pos, diff, denominator1);
1243 final SqlNumericLiteral half = SqlLiteral.createExactNumeric(
"0.5", pos);
1244 result = SqlStdOperatorTable.POWER.createCall(pos, div, half);
1246 return SqlStdOperatorTable.CAST.createCall(pos,
1248 SqlTypeUtil.convertTypeToSpec(typeFactory.createSqlType(
1249 flt ? SqlTypeName.FLOAT : SqlTypeName.DOUBLE)));
1253 final SqlBasicCall proj_call, RelDataTypeFactory typeFactory) {
1255 if (proj_call.operandCount() != 2) {
1260 if (proj_call.getOperator().isName(
"COVAR_POP",
false)) {
1263 }
else if (proj_call.getOperator().isName(
"COVAR_SAMP",
false)) {
1266 }
else if (proj_call.getOperator().getName().equalsIgnoreCase(
"COVAR_POP_FLOAT")) {
1269 }
else if (proj_call.getOperator().getName().equalsIgnoreCase(
"COVAR_SAMP_FLOAT")) {
1275 final SqlNode operand0 = proj_call.operand(0);
1276 final SqlNode operand1 = proj_call.operand(1);
1277 final SqlParserPos pos = proj_call.getParserPosition();
1278 SqlNode expanded_proj_call =
1280 MAPDLOGGER.debug(
"Expanded select_list SqlCall: " + proj_call.toString());
1281 MAPDLOGGER.debug(
"to : " + expanded_proj_call.toString());
1282 return expanded_proj_call;
1286 final SqlNode operand0,
1287 final SqlNode operand1,
1290 RelDataTypeFactory typeFactory) {
1294 final SqlNode arg0 = SqlStdOperatorTable.CAST.createCall(operand0.getParserPosition(),
1296 SqlTypeUtil.convertTypeToSpec(typeFactory.createSqlType(
1297 flt ? SqlTypeName.FLOAT : SqlTypeName.DOUBLE)));
1298 final SqlNode arg1 = SqlStdOperatorTable.CAST.createCall(operand1.getParserPosition(),
1300 SqlTypeUtil.convertTypeToSpec(typeFactory.createSqlType(
1301 flt ? SqlTypeName.FLOAT : SqlTypeName.DOUBLE)));
1302 final SqlNode mulArg = SqlStdOperatorTable.MULTIPLY.createCall(pos, arg0, arg1);
1303 final SqlNode avgArg1 = SqlStdOperatorTable.AVG.createCall(pos, arg1);
1305 final SqlNode avgMulArg = SqlStdOperatorTable.AVG.createCall(pos, mulArg);
1306 final SqlNode avgArg0 = SqlStdOperatorTable.AVG.createCall(pos, arg0);
1307 final SqlNode mulAvgAvg =
1308 SqlStdOperatorTable.MULTIPLY.createCall(pos, avgArg0, avgArg1);
1309 final SqlNode covarPop =
1310 SqlStdOperatorTable.MINUS.createCall(pos, avgMulArg, mulAvgAvg);
1311 return SqlStdOperatorTable.CAST.createCall(pos,
1313 SqlTypeUtil.convertTypeToSpec(typeFactory.createSqlType(
1314 flt ? SqlTypeName.FLOAT : SqlTypeName.DOUBLE)));
1316 final SqlNode sumMulArg = SqlStdOperatorTable.SUM.createCall(pos, mulArg);
1317 final SqlNode sumArg0 = SqlStdOperatorTable.SUM.createCall(pos, arg0);
1318 final SqlNode mulSumAvg =
1319 SqlStdOperatorTable.MULTIPLY.createCall(pos, sumArg0, avgArg1);
1320 final SqlNode sub = SqlStdOperatorTable.MINUS.createCall(pos, sumMulArg, mulSumAvg);
1321 final SqlNode count = SqlStdOperatorTable.COUNT.createCall(pos, operand0);
1322 final SqlNumericLiteral one = SqlLiteral.createExactNumeric(
"1", pos);
1323 final SqlNode countEqOne = SqlStdOperatorTable.EQUALS.createCall(pos, count, one);
1324 final SqlNode countMinusOne = SqlStdOperatorTable.MINUS.createCall(pos, count, one);
1325 final SqlLiteral nul = SqlLiteral.createNull(pos);
1326 SqlNodeList whenList1 =
new SqlNodeList(pos);
1327 SqlNodeList thenList1 =
new SqlNodeList(pos);
1328 whenList1.add(countEqOne);
1330 final SqlNode int_denominator = SqlStdOperatorTable.CASE.createCall(
1331 null, pos, null, whenList1, thenList1, countMinusOne);
1332 final SqlNode denominator = SqlStdOperatorTable.CAST.createCall(pos,
1334 SqlTypeUtil.convertTypeToSpec(typeFactory.createSqlType(
1335 flt ? SqlTypeName.FLOAT : SqlTypeName.DOUBLE)));
1336 final SqlNode covarSamp =
1337 SqlStdOperatorTable.DIVIDE.createCall(pos, sub, denominator);
1338 return SqlStdOperatorTable.CAST.createCall(pos,
1340 SqlTypeUtil.convertTypeToSpec(typeFactory.createSqlType(
1341 flt ? SqlTypeName.FLOAT : SqlTypeName.DOUBLE)));
1345 final SqlBasicCall proj_call, RelDataTypeFactory typeFactory) {
1347 if (proj_call.operandCount() != 2) {
1351 if (proj_call.getOperator().isName(
"CORR",
false)
1352 || proj_call.getOperator().getName().equalsIgnoreCase(
"CORRELATION")) {
1355 }
else if (proj_call.getOperator().getName().equalsIgnoreCase(
"CORR_FLOAT")
1356 || proj_call.getOperator().getName().equalsIgnoreCase(
"CORRELATION_FLOAT")) {
1365 final SqlNode operand0 = proj_call.operand(0);
1366 final SqlNode operand1 = proj_call.operand(1);
1367 final SqlParserPos pos = proj_call.getParserPosition();
1368 SqlNode covariance =
1370 SqlNode stddev0 =
expandVariance(pos, operand0,
true,
true, flt, typeFactory);
1371 SqlNode stddev1 =
expandVariance(pos, operand1,
true,
true, flt, typeFactory);
1372 final SqlNode mulStddev =
1373 SqlStdOperatorTable.MULTIPLY.createCall(pos, stddev0, stddev1);
1374 final SqlNumericLiteral zero = SqlLiteral.createExactNumeric(
"0.0", pos);
1375 final SqlNode mulStddevEqZero =
1376 SqlStdOperatorTable.EQUALS.createCall(pos, mulStddev, zero);
1377 final SqlLiteral nul = SqlLiteral.createNull(pos);
1378 SqlNodeList whenList1 =
new SqlNodeList(pos);
1379 SqlNodeList thenList1 =
new SqlNodeList(pos);
1380 whenList1.add(mulStddevEqZero);
1382 final SqlNode denominator = SqlStdOperatorTable.CASE.createCall(
1383 null, pos, null, whenList1, thenList1, mulStddev);
1384 final SqlNode expanded_proj_call =
1385 SqlStdOperatorTable.DIVIDE.createCall(pos, covariance, denominator);
1386 MAPDLOGGER.debug(
"Expanded select_list SqlCall: " + proj_call.toString());
1387 MAPDLOGGER.debug(
"to : " + expanded_proj_call.toString());
1388 return expanded_proj_call;
1392 throws SqlParseException {
1395 SqlNode node =
parseSql(sql, legacy_syntax, planner);
1397 }
catch (Exception | Error e) {
1398 MAPDLOGGER.error(
"Error parsing sql: " + sql, e);
1406 capturer.
scan(node);
1408 }
catch (Exception | Error e) {
1409 MAPDLOGGER.error(
"Error parsing sql: " + node, e);
1419 MAPDLOGGER.debug(
"schema :" + schema +
" table :" + table);
SqlNode expand(final SqlNode node, final java.util.Map< String, SqlNode > id_to_expr, RelDataTypeFactory typeFactory)
static final EnumSet< SqlKind > IN
static final EnumSet< SqlKind > SCALAR
RelRoot replaceIsTrue(final RelDataTypeFactory typeFactory, RelRoot root)
void desugarExpression(SqlNode node, RelDataTypeFactory typeFactory)
SqlNode expandCovariance(final SqlBasicCall proj_call, RelDataTypeFactory typeFactory)
SqlNode expandCorrelation(final SqlBasicCall proj_call, RelDataTypeFactory typeFactory)
RelRoot queryToRelNode(final String sql, final MapDParserOptions parserOptions)
static String toString(final RelNode rel)
Pair< String, SqlIdentifierCapturer > process(String sql, final MapDParserOptions parserOptions)
SqlNode expandCase(SqlCase old_case_node, RelDataTypeFactory typeFactory)
static final EnumSet< SqlKind > ARRAY_VALUE
void updateMetaData(String schema, String table)
final Supplier< MapDSqlOperatorTable > mapDSqlOperatorTable
SqlNode expandVariance(final SqlParserPos pos, final SqlNode operand, boolean biased, boolean sqrt, boolean flt, RelDataTypeFactory typeFactory)
SqlIdentifierCapturer getAccessedObjects()
void updateMetaData(String schema, String table)
MapDParser(String dataDir, final Supplier< MapDSqlOperatorTable > mapDSqlOperatorTable, int mapdPort, SockTransportProperties skT)
String processSql(String sql, final MapDParserOptions parserOptions)
SqlIdentifierCapturer captureIdentifiers(SqlNode node)
SqlSelect rewriteSimpleUpdateAsSelect(final SqlUpdate update)
SockTransportProperties sock_transport_properties
static final SqlArrayValueConstructorAllowingEmpty ARRAY_VALUE_CONSTRUCTOR
static final EnumSet< SqlKind > EXISTS
RelRoot optimizeRaQuery(String query, MapDSchema schema)
SqlNode parseSql(String sql, final boolean legacy_syntax, Planner planner)
static final EnumSet< SqlKind > UPDATE
SqlIdentifierCapturer captureIdentifiers(String sql, boolean legacy_syntax)
static Map< String, Boolean > SubqueryCorrMemo
static final Logger MAPDLOGGER
Set< String > resolveSelectIdentifiers(SqlIdentifierCapturer capturer)
RelRoot convertSqlToRelNode(final SqlNode sqlNode, final MapDPlanner mapDPlanner, final MapDParserOptions parserOptions)
String getTableName(SqlNode node)
boolean isCorrelated(SqlNode expression)
Table getTable(String string)
static final ProjectProjectRemoveRule INSTANCE
String processSql(final SqlNode sqlNode, final MapDParserOptions parserOptions)
void setUser(MapDUser mapdUser)
SqlNodeList expand(final SqlNodeList group_by_list, final java.util.Map< String, SqlNode > id_to_expr, RelDataTypeFactory typeFactory)
SqlNode expandCovariance(SqlParserPos pos, final SqlNode operand0, final SqlNode operand1, boolean pop, boolean flt, RelDataTypeFactory typeFactory)
static final HintStrategyTable HINT_STRATEGY_TABLE
void desugar(SqlSelect select_node, RelDataTypeFactory typeFactory)
CompletionResult getCompletionHints(final String sql, final int cursor, final List< String > visibleTables)
final Set< String > selects
String optimizeRAQuery(String query)
SqlOrderBy desugar(SqlSelect select_node, SqlOrderBy order_by_node, RelDataTypeFactory typeFactory)
static final ThreadLocal< MapDParser > CURRENT_PARSER
SqlNode expandVariance(final SqlBasicCall proj_call, RelDataTypeFactory typeFactory)
MapDPlanner.CompletionResult getCompletionHints(String sql, int cursor, List< String > visible_tables)
static final EnumSet< SqlKind > DELETE
MapDPlanner getPlanner(final boolean allowSubQueryExpansion, final boolean allowPushdownJoinCondition)
static final Context MAPD_CONNECTION_CONTEXT
RelDataTypeSystem createTypeSystem()
LogicalTableModify getDummyUpdate(SqlUpdate update)
RelRoot rewriteUpdateAsSelect(SqlUpdate update, MapDParserOptions parserOptions)