OmniSciDB  340b00dbf6
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
MapDRelJson.java
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package org.apache.calcite.rel.externalize;
18 
19 import com.fasterxml.jackson.databind.type.TypeFactory;
20 import com.google.common.collect.ImmutableList;
22 
23 import org.apache.calcite.avatica.AvaticaUtils;
24 import org.apache.calcite.avatica.util.TimeUnitRange;
25 import org.apache.calcite.plan.RelOptCluster;
26 import org.apache.calcite.plan.RelTraitSet;
27 import org.apache.calcite.rel.RelCollation;
28 import org.apache.calcite.rel.RelCollationImpl;
29 import org.apache.calcite.rel.RelCollations;
30 import org.apache.calcite.rel.RelDistribution;
31 import org.apache.calcite.rel.RelDistributions;
32 import org.apache.calcite.rel.RelFieldCollation;
33 import org.apache.calcite.rel.RelInput;
34 import org.apache.calcite.rel.RelNode;
35 import org.apache.calcite.rel.core.AggregateCall;
36 import org.apache.calcite.rel.core.CorrelationId;
37 import org.apache.calcite.rel.core.TableModify.Operation;
38 import org.apache.calcite.rel.type.RelDataType;
39 import org.apache.calcite.rel.type.RelDataTypeFactory;
40 import org.apache.calcite.rel.type.RelDataTypeField;
41 import org.apache.calcite.rex.RexBuilder;
42 import org.apache.calcite.rex.RexCall;
43 import org.apache.calcite.rex.RexCorrelVariable;
44 import org.apache.calcite.rex.RexFieldAccess;
45 import org.apache.calcite.rex.RexFieldCollation;
46 import org.apache.calcite.rex.RexInputRef;
47 import org.apache.calcite.rex.RexLiteral;
48 import org.apache.calcite.rex.RexNode;
49 import org.apache.calcite.rex.RexOver;
50 import org.apache.calcite.rex.RexSubQuery;
51 import org.apache.calcite.rex.RexWindow;
52 import org.apache.calcite.rex.RexWindowBound;
54 import org.apache.calcite.sql.SqlAggFunction;
55 import org.apache.calcite.sql.SqlFunction;
56 import org.apache.calcite.sql.SqlIdentifier;
57 import org.apache.calcite.sql.SqlKind;
58 import org.apache.calcite.sql.SqlOperator;
59 import org.apache.calcite.sql.SqlOperatorTable;
60 import org.apache.calcite.sql.SqlSyntax;
61 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
62 import org.apache.calcite.sql.type.SqlTypeName;
63 import org.apache.calcite.util.ImmutableBitSet;
64 import org.apache.calcite.util.JsonBuilder;
65 import org.apache.calcite.util.Util;
66 
67 import java.lang.reflect.Constructor;
68 import java.lang.reflect.InvocationTargetException;
69 import java.math.BigDecimal;
70 import java.util.ArrayList;
71 import java.util.HashMap;
72 import java.util.List;
73 import java.util.Map;
74 
79 public class MapDRelJson {
80  private final Map<String, Constructor> constructorMap =
81  new HashMap<String, Constructor>();
82  private final JsonBuilder jsonBuilder;
83 
84  public static final List<String> PACKAGES = ImmutableList.of("org.apache.calcite.rel.",
85  "org.apache.calcite.rel.core.",
86  "org.apache.calcite.rel.logical.",
87  "org.apache.calcite.adapter.jdbc.",
88  "org.apache.calcite.adapter.enumerable.",
89  "org.apache.calcite.adapter.jdbc.JdbcRules$");
90 
91  public MapDRelJson(JsonBuilder jsonBuilder) {
92  this.jsonBuilder = jsonBuilder;
93  }
94 
95  public RelNode create(Map<String, Object> map) {
96  String type = (String) map.get("type");
97  Constructor constructor = getConstructor(type);
98  try {
99  return (RelNode) constructor.newInstance(map);
100  } catch (InstantiationException e) {
101  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
102  } catch (IllegalAccessException e) {
103  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
104  } catch (InvocationTargetException e) {
105  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
106  } catch (ClassCastException e) {
107  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
108  }
109  }
110 
111  public Constructor getConstructor(String type) {
112  Constructor constructor = constructorMap.get(type);
113  if (constructor == null) {
114  Class clazz = typeNameToClass(type);
115  try {
116  // noinspection unchecked
117  constructor = clazz.getConstructor(RelInput.class);
118  } catch (NoSuchMethodException e) {
119  throw new RuntimeException(
120  "class does not have required constructor, " + clazz + "(RelInput)");
121  }
122  constructorMap.put(type, constructor);
123  }
124  return constructor;
125  }
126 
131  public Class typeNameToClass(String type) {
132  if (!type.contains(".")) {
133  for (String package_ : PACKAGES) {
134  try {
135  return Class.forName(package_ + type);
136  } catch (ClassNotFoundException e) {
137  // ignore
138  }
139  }
140  }
141  try {
142  return Class.forName(type);
143  } catch (ClassNotFoundException e) {
144  throw new RuntimeException("unknown type " + type);
145  }
146  }
147 
151  public String classToTypeName(Class<? extends RelNode> class_) {
152  final String canonicalName = class_.getName();
153  for (String package_ : PACKAGES) {
154  if (canonicalName.startsWith(package_)) {
155  String remaining = canonicalName.substring(package_.length());
156  if (remaining.indexOf('.') < 0 && remaining.indexOf('$') < 0) {
157  return remaining;
158  }
159  }
160  }
161  return canonicalName;
162  }
163 
164  public Object toJson(RelCollationImpl node) {
165  final List<Object> list = new ArrayList<Object>();
166  for (RelFieldCollation fieldCollation : node.getFieldCollations()) {
167  final Map<String, Object> map = jsonBuilder.map();
168  map.put("field", fieldCollation.getFieldIndex());
169  map.put("direction", fieldCollation.getDirection().name());
170  map.put("nulls", fieldCollation.nullDirection.name());
171  list.add(map);
172  }
173  return list;
174  }
175 
176  public Object toJson(RexFieldCollation node) {
177  final Map<String, Object> map = jsonBuilder.map();
178  map.put("field", toJson(node.left));
179  map.put("direction", node.getDirection().name());
180  map.put("nulls", node.getNullDirection().name());
181  return map;
182  }
183 
184  public RelCollation toCollation(List<Map<String, Object>> jsonFieldCollations) {
185  final List<RelFieldCollation> fieldCollations = new ArrayList<RelFieldCollation>();
186  for (Map<String, Object> map : jsonFieldCollations) {
187  fieldCollations.add(toFieldCollation(map));
188  }
189  return RelCollations.of(fieldCollations);
190  }
191 
192  public RelFieldCollation toFieldCollation(Map<String, Object> map) {
193  final Integer field = (Integer) map.get("field");
194  final RelFieldCollation.Direction direction = Util.enumVal(
195  RelFieldCollation.Direction.class, (String) map.get("direction"));
196  final RelFieldCollation.NullDirection nullDirection = Util.enumVal(
197  RelFieldCollation.NullDirection.class, (String) map.get("nulls"));
198  return new RelFieldCollation(field, direction, nullDirection);
199  }
200 
201  public RelDistribution toDistribution(Object o) {
202  return RelDistributions.ANY; // TODO:
203  }
204 
205  public RelDataType toType(RelDataTypeFactory typeFactory, Object o) {
206  if (o instanceof List) {
207  @SuppressWarnings("unchecked")
208  final List<Map<String, Object>> jsonList = (List<Map<String, Object>>) o;
209  final RelDataTypeFactory.FieldInfoBuilder builder = typeFactory.builder();
210  for (Map<String, Object> jsonMap : jsonList) {
211  builder.add((String) jsonMap.get("name"), toType(typeFactory, jsonMap));
212  }
213  return builder.build();
214  } else {
215  final Map<String, Object> map = (Map<String, Object>) o;
216  final SqlTypeName sqlTypeName =
217  Util.enumVal(SqlTypeName.class, (String) map.get("type"));
218  final Integer precision = (Integer) map.get("precision");
219  final Integer scale = (Integer) map.get("scale");
220  final RelDataType type;
221  if (precision == null) {
222  type = typeFactory.createSqlType(sqlTypeName);
223  } else if (scale == null) {
224  type = typeFactory.createSqlType(sqlTypeName, precision);
225  } else {
226  type = typeFactory.createSqlType(sqlTypeName, precision, scale);
227  }
228  final boolean nullable = (Boolean) map.get("nullable");
229  return typeFactory.createTypeWithNullability(type, nullable);
230  }
231  }
232 
233  public Object toJson(AggregateCall node) {
234  final Map<String, Object> map = jsonBuilder.map();
235  map.put("agg", toJson(node.getAggregation()));
236  map.put("type", toJson(node.getType()));
237  map.put("distinct", node.isDistinct());
238  map.put("operands", node.getArgList());
239  return map;
240  }
241 
242  Object toJson(Object value) {
243  if (value == null || value instanceof Number || value instanceof String
244  || value instanceof Boolean) {
245  return value;
246  } else if (value instanceof RexNode) {
247  return toJson((RexNode) value);
248  } else if (value instanceof CorrelationId) {
249  return toJson((CorrelationId) value);
250  } else if (value instanceof List) {
251  final List<Object> list = jsonBuilder.list();
252  for (Object o : (List) value) {
253  list.add(toJson(o));
254  }
255  return list;
256  } else if (value instanceof ImmutableBitSet) {
257  final List<Object> list = jsonBuilder.list();
258  for (Integer integer : (ImmutableBitSet) value) {
259  list.add(toJson(integer));
260  }
261  return list;
262  } else if (value instanceof AggregateCall) {
263  return toJson((AggregateCall) value);
264  } else if (value instanceof RelCollationImpl) {
265  return toJson((RelCollationImpl) value);
266  } else if (value instanceof RexFieldCollation) {
267  return toJson((RexFieldCollation) value);
268  } else if (value instanceof RelDataType) {
269  return toJson((RelDataType) value);
270  } else if (value instanceof RelDataTypeField) {
271  return toJson((RelDataTypeField) value);
272  } else if (value instanceof JoinType) {
273  return value.toString();
274  } else if (value instanceof Operation) {
275  return value.toString();
276  } else {
277  throw new UnsupportedOperationException("type not serializable: " + value
278  + " (type " + value.getClass().getCanonicalName() + ")");
279  }
280  }
281 
282  private Object toJson(RelDataType node) {
283  if (node.isStruct()) {
284  final List<Object> list = jsonBuilder.list();
285  for (RelDataTypeField field : node.getFieldList()) {
286  list.add(toJson(field));
287  }
288  return list;
289  } else {
290  final Map<String, Object> map = jsonBuilder.map();
291  map.put("type", node.getSqlTypeName().name());
292  map.put("nullable", node.isNullable());
293  if (node.getSqlTypeName().allowsPrec()) {
294  map.put("precision", node.getPrecision());
295  }
296  if (node.getSqlTypeName().allowsScale()) {
297  map.put("scale", node.getScale());
298  }
299  return map;
300  }
301  }
302 
303  private Object toJson(RelDataTypeField node) {
304  final Map<String, Object> map = (Map<String, Object>) toJson(node.getType());
305  map.put("name", node.getName());
306  return map;
307  }
308 
309  private Object toJson(CorrelationId node) {
310  return node.getId();
311  }
312 
313  private Object toJson(final RexWindowBound window_bound) {
314  final Map<String, Object> map = jsonBuilder.map();
315  map.put("unbounded", toJson(window_bound.isUnbounded()));
316  map.put("preceding", toJson(window_bound.isPreceding()));
317  map.put("following", toJson(window_bound.isFollowing()));
318  map.put("is_current_row", toJson(window_bound.isCurrentRow()));
319  map.put("offset",
320  window_bound.getOffset() != null ? toJson(window_bound.getOffset()) : null);
321  map.put("order_key", toJson(window_bound.getOrderKey()));
322  return map;
323  }
324 
325  private Object toJson(RexNode node) {
326  final Map<String, Object> map;
327  switch (node.getKind()) {
328  case FIELD_ACCESS:
329  map = jsonBuilder.map();
330  final RexFieldAccess fieldAccess = (RexFieldAccess) node;
331  map.put("field", fieldAccess.getField().getName());
332  map.put("expr", toJson(fieldAccess.getReferenceExpr()));
333  return map;
334  case LITERAL:
335  final RexLiteral literal = (RexLiteral) node;
336  final Object value2 = literal.getValue2();
337  map = jsonBuilder.map();
338  if (value2 instanceof TimeUnitRange) {
339  map.put("literal", value2.toString());
340  } else {
341  map.put("literal", value2);
342  }
343  map.put("type", literal.getTypeName().name());
344  map.put("target_type", literal.getType().getSqlTypeName().toString());
345  final Object value = literal.getValue();
346  if (value instanceof BigDecimal) {
347  map.put("scale", ((BigDecimal) value).scale());
348  map.put("precision", ((BigDecimal) value).precision());
349  } else {
350  map.put("scale", literal.getType().getScale());
351  map.put("precision", literal.getType().getPrecision());
352  }
353  map.put("type_scale", literal.getType().getScale());
354  map.put("type_precision", literal.getType().getPrecision());
355  return map;
356  case INPUT_REF:
357  map = jsonBuilder.map();
358  map.put("input", ((RexInputRef) node).getIndex());
359  return map;
360  case CORREL_VARIABLE:
361  map = jsonBuilder.map();
362  map.put("correl", ((RexCorrelVariable) node).getName());
363  map.put("type", toJson(node.getType()));
364  return map;
365  default:
366  if (node instanceof RexCall) {
367  final RexCall call = (RexCall) node;
368  map = jsonBuilder.map();
369  map.put("op", toJson(call.getOperator()));
370  final List<Object> list = jsonBuilder.list();
371  for (RexNode operand : call.getOperands()) {
372  list.add(toJson(operand));
373  }
374  map.put("operands", list);
375  map.put("type", toJson(node.getType()));
376  if (node instanceof RexSubQuery) {
377  final MapDRelJsonWriter subqueryWriter = new MapDRelJsonWriter();
378  ((RexSubQuery) node).rel.explain(subqueryWriter);
379  map.put("subquery", subqueryWriter.asJsonMap());
380  }
381  if (node instanceof RexOver) {
382  final RexWindow window = ((RexOver) node).getWindow();
383  final List<Object> partitionKeyList = jsonBuilder.list();
384  for (final RexNode partitionKey : window.partitionKeys) {
385  partitionKeyList.add(toJson(partitionKey));
386  }
387  map.put("partition_keys", partitionKeyList);
388  final List<Object> orderKeyList = jsonBuilder.list();
389  for (final RexFieldCollation orderKey : window.orderKeys) {
390  orderKeyList.add(toJson(orderKey));
391  }
392  map.put("order_keys", orderKeyList);
393  RexWindowBound lower_bound = window.getLowerBound();
394  RexWindowBound upper_bound = window.getUpperBound();
395  map.put("lower_bound", toJson(lower_bound));
396  map.put("upper_bound", toJson(upper_bound));
397  map.put("is_rows", toJson(window.isRows()));
398  }
399  if (call.getOperator() instanceof SqlFunction) {
400  switch (((SqlFunction) call.getOperator()).getFunctionType()) {
401  case USER_DEFINED_CONSTRUCTOR:
402  case USER_DEFINED_FUNCTION:
403  case USER_DEFINED_PROCEDURE:
404  case USER_DEFINED_SPECIFIC_FUNCTION:
405  map.put("class", call.getOperator().getClass().getName());
406  }
407  }
408  return map;
409  }
410  throw new UnsupportedOperationException("unknown rex " + node);
411  }
412  }
413 
414  RexNode toRex(RelInput relInput, Object o) {
415  final RelOptCluster cluster = relInput.getCluster();
416  final RexBuilder rexBuilder = cluster.getRexBuilder();
417  if (o == null) {
418  return null;
419  } else if (o instanceof Map) {
420  Map map = (Map) o;
421  final String op = (String) map.get("op");
422  if (op != null) {
423  final List operands = (List) map.get("operands");
424  final Object jsonType = map.get("type");
425  final SqlOperator operator = toOp(op, map);
426  final List<RexNode> rexOperands = toRexList(relInput, operands);
427  RelDataType type;
428  if (jsonType != null) {
429  type = toType(cluster.getTypeFactory(), jsonType);
430  } else {
431  type = rexBuilder.deriveReturnType(operator, rexOperands);
432  }
433  return rexBuilder.makeCall(type, operator, rexOperands);
434  }
435  final Integer input = (Integer) map.get("input");
436  if (input != null) {
437  List<RelNode> inputNodes = relInput.getInputs();
438  int i = input;
439  for (RelNode inputNode : inputNodes) {
440  final RelDataType rowType = inputNode.getRowType();
441  if (i < rowType.getFieldCount()) {
442  final RelDataTypeField field = rowType.getFieldList().get(i);
443  return rexBuilder.makeInputRef(field.getType(), input);
444  }
445  i -= rowType.getFieldCount();
446  }
447  throw new RuntimeException("input field " + input + " is out of range");
448  }
449  final String field = (String) map.get("field");
450  if (field != null) {
451  final Object jsonExpr = map.get("expr");
452  final RexNode expr = toRex(relInput, jsonExpr);
453  return rexBuilder.makeFieldAccess(expr, field, true);
454  }
455  final String correl = (String) map.get("correl");
456  if (correl != null) {
457  final Object jsonType = map.get("type");
458  RelDataType type = toType(cluster.getTypeFactory(), jsonType);
459  return rexBuilder.makeCorrel(type, new CorrelationId(correl));
460  }
461  if (map.containsKey("literal")) {
462  final Object literal = map.get("literal");
463  final SqlTypeName sqlTypeName =
464  Util.enumVal(SqlTypeName.class, (String) map.get("type"));
465  if (literal == null) {
466  return rexBuilder.makeNullLiteral(sqlTypeName);
467  }
468 
469  // omnisci serializes numeric literals differently, we need more data
470  // then the number. So, we need to have a special case for that.
471  if (literal instanceof Number) {
472  final SqlTypeName targetTypeName =
473  Util.enumVal(SqlTypeName.class, (String) map.get("target_type"));
474  final long scale = ((Number) map.get("scale")).longValue();
475  final long precision = ((Number) map.get("precision")).longValue();
476  final long typeScale = ((Number) map.get("type_scale")).longValue();
477  final long typePrecision = ((Number) map.get("type_precision")).longValue();
478  RelDataTypeFactory typeFactory = cluster.getTypeFactory();
479 
480  BigDecimal value =
481  BigDecimal.valueOf(((Number) literal).longValue(), (int) scale);
482 
483  if (typeScale != 0 && typeScale != -2147483648) {
484  return rexBuilder.makeLiteral(value,
485  typeFactory.createSqlType(
486  SqlTypeName.DECIMAL, (int) typePrecision, (int) typeScale),
487  false);
488  } else {
489  return rexBuilder.makeLiteral(
490  value, typeFactory.createSqlType(targetTypeName), false);
491  }
492  } else {
493  return toRex(relInput, literal);
494  }
495  }
496  throw new UnsupportedOperationException("cannot convert to rex " + o);
497  } else if (o instanceof Boolean) {
498  return rexBuilder.makeLiteral((Boolean) o);
499  } else if (o instanceof String) {
500  return rexBuilder.makeLiteral((String) o);
501  } else if (o instanceof Number) {
502  final Number number = (Number) o;
503  if (number instanceof Double || number instanceof Float) {
504  return rexBuilder.makeApproxLiteral(BigDecimal.valueOf(number.doubleValue()));
505  } else {
506  return rexBuilder.makeExactLiteral(BigDecimal.valueOf(number.longValue()));
507  }
508  } else {
509  throw new UnsupportedOperationException("cannot convert to rex " + o);
510  }
511  }
512 
513  private List<RexNode> toRexList(RelInput relInput, List operands) {
514  final List<RexNode> list = new ArrayList<RexNode>();
515  for (Object operand : operands) {
516  list.add(toRex(relInput, operand));
517  }
518  return list;
519  }
520 
521  private SqlOperator toOp(String op) {
522  return toOp(op, new HashMap<>());
523  }
524 
525  private SqlOperator toOp(String op, Map<String, Object> map) {
526  // TODO: build a map, for more efficient lookup
527  // TODO: look up based on SqlKind
528  MapDSqlOperatorTable operatorTable =
529  new MapDSqlOperatorTable(SqlStdOperatorTable.instance());
530  MapDSqlOperatorTable.addUDF(operatorTable, null);
531  final List<SqlOperator> operatorList = operatorTable.getOperatorList();
532  for (SqlOperator operator : operatorList) {
533  if (operator.getName().equals(op)) {
534  return operator;
535  }
536  }
537  String class_ = (String) map.get("class");
538  if (class_ != null) {
539  try {
540  return (SqlOperator) Class.forName(class_).newInstance();
541  } catch (InstantiationException e) {
542  throw new RuntimeException(e);
543  } catch (IllegalAccessException e) {
544  throw new RuntimeException(e);
545  } catch (ClassNotFoundException e) {
546  throw new RuntimeException(e);
547  }
548  }
549  throw new RuntimeException("Operator " + op + " does not supported");
550  }
551 
552  SqlOperator toOp(RelInput relInput, String name) {
553  // in case different operator has the same kind, check with both name and kind.
554  String kind = name;
555  String syntax = "FUNCTION";
556  SqlKind sqlKind = SqlKind.valueOf(kind);
557  SqlSyntax sqlSyntax = SqlSyntax.valueOf(syntax);
558  MapDSqlOperatorTable operatorTable =
559  new MapDSqlOperatorTable(SqlStdOperatorTable.instance());
560  MapDSqlOperatorTable.addUDF(operatorTable, null);
561  final List<SqlOperator> operators = operatorTable.getOperatorList();
562  List<String> names = new ArrayList<>();
563  for (SqlOperator operator : operators) {
564  names.add(operator.toString());
565  if (operator.getName().equals(name)) {
566  return operator;
567  }
568  }
569  throw new RuntimeException("Aggregation function with name " + name
570  + " not found, search in " + names.toString());
571  }
572 
573  SqlAggFunction toAggregation(String agg) {
574  return (SqlAggFunction) toOp(agg);
575  }
576 
577  SqlAggFunction toAggregation(RelInput relInput, String agg) {
578  return (SqlAggFunction) toOp(relInput, agg);
579  }
580 
581  private String toJson(SqlOperator operator) {
582  // User-defined operators are not yet handled.
583  return operator.getName();
584  }
585 }
586 
587 // End RelJson.java
JoinType
Definition: sqldefs.h:107
RexNode toRex(RelInput relInput, Object o)
RelDataType toType(RelDataTypeFactory typeFactory, Object o)
RelNode create(Map< String, Object > map)
String classToTypeName(Class<?extends RelNode > class_)
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31
Object toJson(RexFieldCollation node)
SqlOperator toOp(String op, Map< String, Object > map)
RelCollation toCollation(List< Map< String, Object >> jsonFieldCollations)
final Map< String, Constructor > constructorMap
string name
Definition: setup.py:35
SqlOperator toOp(RelInput relInput, String name)
SqlAggFunction toAggregation(RelInput relInput, String agg)
Object toJson(final RexWindowBound window_bound)
List< RexNode > toRexList(RelInput relInput, List operands)
RelFieldCollation toFieldCollation(Map< String, Object > map)