OmniSciDB  21ac014ffc
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
MapDRelJson.java
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package org.apache.calcite.rel.externalize;
18 
19 import com.fasterxml.jackson.databind.type.TypeFactory;
20 import com.google.common.collect.ImmutableList;
22 
23 import org.apache.calcite.avatica.AvaticaUtils;
24 import org.apache.calcite.avatica.util.TimeUnitRange;
25 import org.apache.calcite.plan.RelOptCluster;
26 import org.apache.calcite.plan.RelTraitSet;
27 import org.apache.calcite.rel.RelCollation;
28 import org.apache.calcite.rel.RelCollationImpl;
29 import org.apache.calcite.rel.RelCollations;
30 import org.apache.calcite.rel.RelDistribution;
31 import org.apache.calcite.rel.RelDistributions;
32 import org.apache.calcite.rel.RelFieldCollation;
33 import org.apache.calcite.rel.RelInput;
34 import org.apache.calcite.rel.RelNode;
35 import org.apache.calcite.rel.core.AggregateCall;
36 import org.apache.calcite.rel.core.CorrelationId;
37 import org.apache.calcite.rel.core.TableModify.Operation;
38 import org.apache.calcite.rel.type.RelDataType;
39 import org.apache.calcite.rel.type.RelDataTypeFactory;
40 import org.apache.calcite.rel.type.RelDataTypeField;
41 import org.apache.calcite.rex.RexBuilder;
42 import org.apache.calcite.rex.RexCall;
43 import org.apache.calcite.rex.RexCorrelVariable;
44 import org.apache.calcite.rex.RexFieldAccess;
45 import org.apache.calcite.rex.RexFieldCollation;
46 import org.apache.calcite.rex.RexInputRef;
47 import org.apache.calcite.rex.RexLiteral;
48 import org.apache.calcite.rex.RexNode;
49 import org.apache.calcite.rex.RexOver;
50 import org.apache.calcite.rex.RexSubQuery;
51 import org.apache.calcite.rex.RexWindow;
52 import org.apache.calcite.rex.RexWindowBound;
54 import org.apache.calcite.sql.SqlAggFunction;
55 import org.apache.calcite.sql.SqlFunction;
56 import org.apache.calcite.sql.SqlIdentifier;
57 import org.apache.calcite.sql.SqlKind;
58 import org.apache.calcite.sql.SqlOperator;
59 import org.apache.calcite.sql.SqlOperatorTable;
60 import org.apache.calcite.sql.SqlSyntax;
61 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
62 import org.apache.calcite.sql.type.SqlTypeName;
63 import org.apache.calcite.util.ImmutableBitSet;
64 import org.apache.calcite.util.JsonBuilder;
65 import org.apache.calcite.util.Util;
66 
67 import java.lang.reflect.Constructor;
68 import java.lang.reflect.InvocationTargetException;
69 import java.math.BigDecimal;
70 import java.util.ArrayList;
71 import java.util.HashMap;
72 import java.util.List;
73 import java.util.Map;
74 
79 public class MapDRelJson {
80  private final Map<String, Constructor> constructorMap =
81  new HashMap<String, Constructor>();
82  private final JsonBuilder jsonBuilder;
83 
84  public static final List<String> PACKAGES = ImmutableList.of("org.apache.calcite.rel.",
85  "org.apache.calcite.rel.core.",
86  "org.apache.calcite.rel.logical.",
87  "org.apache.calcite.adapter.jdbc.",
88  "org.apache.calcite.adapter.enumerable.",
89  "org.apache.calcite.adapter.jdbc.JdbcRules$");
90 
91  public MapDRelJson(JsonBuilder jsonBuilder) {
92  this.jsonBuilder = jsonBuilder;
93  }
94 
95  public RelNode create(Map<String, Object> map) {
96  String type = (String) map.get("type");
97  Constructor constructor = getConstructor(type);
98  try {
99  return (RelNode) constructor.newInstance(map);
100  } catch (InstantiationException e) {
101  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
102  } catch (IllegalAccessException e) {
103  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
104  } catch (InvocationTargetException e) {
105  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
106  } catch (ClassCastException e) {
107  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
108  }
109  }
110 
111  public Constructor getConstructor(String type) {
112  Constructor constructor = constructorMap.get(type);
113  if (constructor == null) {
114  Class clazz = typeNameToClass(type);
115  try {
116  // noinspection unchecked
117  constructor = clazz.getConstructor(RelInput.class);
118  } catch (NoSuchMethodException e) {
119  throw new RuntimeException(
120  "class does not have required constructor, " + clazz + "(RelInput)");
121  }
122  constructorMap.put(type, constructor);
123  }
124  return constructor;
125  }
126 
131  public Class typeNameToClass(String type) {
132  if (!type.contains(".")) {
133  for (String package_ : PACKAGES) {
134  try {
135  return Class.forName(package_ + type);
136  } catch (ClassNotFoundException e) {
137  // ignore
138  }
139  }
140  }
141  try {
142  return Class.forName(type);
143  } catch (ClassNotFoundException e) {
144  throw new RuntimeException("unknown type " + type);
145  }
146  }
147 
151  public String classToTypeName(Class<? extends RelNode> class_) {
152  final String canonicalName = class_.getName();
153  for (String package_ : PACKAGES) {
154  if (canonicalName.startsWith(package_)) {
155  String remaining = canonicalName.substring(package_.length());
156  if (remaining.indexOf('.') < 0 && remaining.indexOf('$') < 0) {
157  return remaining;
158  }
159  }
160  }
161  return canonicalName;
162  }
163 
164  public Object toJson(RelCollationImpl node) {
165  final List<Object> list = new ArrayList<Object>();
166  for (RelFieldCollation fieldCollation : node.getFieldCollations()) {
167  final Map<String, Object> map = jsonBuilder.map();
168  map.put("field", fieldCollation.getFieldIndex());
169  map.put("direction", fieldCollation.getDirection().name());
170  map.put("nulls", fieldCollation.nullDirection.name());
171  list.add(map);
172  }
173  return list;
174  }
175 
176  public Object toJson(RexFieldCollation node) {
177  final Map<String, Object> map = jsonBuilder.map();
178  map.put("field", toJson(node.left));
179  map.put("direction", node.getDirection().name());
180  map.put("nulls", node.getNullDirection().name());
181  return map;
182  }
183 
184  public RelCollation toCollation(List<Map<String, Object>> jsonFieldCollations) {
185  final List<RelFieldCollation> fieldCollations = new ArrayList<RelFieldCollation>();
186  for (Map<String, Object> map : jsonFieldCollations) {
187  fieldCollations.add(toFieldCollation(map));
188  }
189  return RelCollations.of(fieldCollations);
190  }
191 
192  public RelFieldCollation toFieldCollation(Map<String, Object> map) {
193  final Integer field = (Integer) map.get("field");
194  final RelFieldCollation.Direction direction = Util.enumVal(
195  RelFieldCollation.Direction.class, (String) map.get("direction"));
196  final RelFieldCollation.NullDirection nullDirection = Util.enumVal(
197  RelFieldCollation.NullDirection.class, (String) map.get("nulls"));
198  return new RelFieldCollation(field, direction, nullDirection);
199  }
200 
201  public RelDistribution toDistribution(Object o) {
202  return RelDistributions.ANY; // TODO:
203  }
204 
205  public RelDataType toType(RelDataTypeFactory typeFactory, Object o) {
206  if (o instanceof List) {
207  @SuppressWarnings("unchecked")
208  final List<Map<String, Object>> jsonList = (List<Map<String, Object>>) o;
209  final RelDataTypeFactory.Builder builder = typeFactory.builder();
210  for (Map<String, Object> jsonMap : jsonList) {
211  builder.add((String) jsonMap.get("name"), toType(typeFactory, jsonMap));
212  }
213  return builder.build();
214  } else {
215  final Map<String, Object> map = (Map<String, Object>) o;
216  final SqlTypeName sqlTypeName =
217  Util.enumVal(SqlTypeName.class, (String) map.get("type"));
218  final Integer precision = (Integer) map.get("precision");
219  final Integer scale = (Integer) map.get("scale");
220  final RelDataType type;
221  if (precision == null) {
222  type = typeFactory.createSqlType(sqlTypeName);
223  } else if (scale == null) {
224  type = typeFactory.createSqlType(sqlTypeName, precision);
225  } else {
226  type = typeFactory.createSqlType(sqlTypeName, precision, scale);
227  }
228  final boolean nullable = (Boolean) map.get("nullable");
229  return typeFactory.createTypeWithNullability(type, nullable);
230  }
231  }
232 
233  public Object toJson(AggregateCall node) {
234  final Map<String, Object> map = jsonBuilder.map();
235  map.put("agg", toJson(node.getAggregation()));
236  map.put("type", toJson(node.getType()));
237  map.put("distinct", node.isDistinct());
238  map.put("operands", node.getArgList());
239  return map;
240  }
241 
242  Object toJson(Object value) {
243  if (value == null || value instanceof Number || value instanceof String
244  || value instanceof Boolean) {
245  return value;
246  } else if (value instanceof RexNode) {
247  return toJson((RexNode) value);
248  } else if (value instanceof CorrelationId) {
249  return toJson((CorrelationId) value);
250  } else if (value instanceof List) {
251  final List<Object> list = jsonBuilder.list();
252  for (Object o : (List) value) {
253  list.add(toJson(o));
254  }
255  return list;
256  } else if (value instanceof ImmutableBitSet) {
257  final List<Object> list = jsonBuilder.list();
258  for (Integer integer : (ImmutableBitSet) value) {
259  list.add(toJson(integer));
260  }
261  return list;
262  } else if (value instanceof AggregateCall) {
263  return toJson((AggregateCall) value);
264  } else if (value instanceof RelCollationImpl) {
265  return toJson((RelCollationImpl) value);
266  } else if (value instanceof RexFieldCollation) {
267  return toJson((RexFieldCollation) value);
268  } else if (value instanceof RelDataType) {
269  return toJson((RelDataType) value);
270  } else if (value instanceof RelDataTypeField) {
271  return toJson((RelDataTypeField) value);
272  } else if (value instanceof JoinType) {
273  return value.toString();
274  } else if (value instanceof Operation) {
275  return value.toString();
276  } else {
277  throw new UnsupportedOperationException("type not serializable: " + value
278  + " (type " + value.getClass().getCanonicalName() + ")");
279  }
280  }
281 
282  private Object toJson(RelDataType node) {
283  if (node.isStruct()) {
284  final List<Object> list = jsonBuilder.list();
285  for (RelDataTypeField field : node.getFieldList()) {
286  list.add(toJson(field));
287  }
288  return list;
289  } else {
290  final Map<String, Object> map = jsonBuilder.map();
291  map.put("type", node.getSqlTypeName().name());
292  map.put("nullable", node.isNullable());
293  if (node.getSqlTypeName().allowsPrec()) {
294  map.put("precision", node.getPrecision());
295  }
296  if (node.getSqlTypeName().allowsScale()) {
297  map.put("scale", node.getScale());
298  }
299  return map;
300  }
301  }
302 
303  private Object toJson(RelDataTypeField node) {
304  final Map<String, Object> map = (Map<String, Object>) toJson(node.getType());
305  map.put("name", node.getName());
306  return map;
307  }
308 
309  private Object toJson(CorrelationId node) {
310  return node.getId();
311  }
312 
313  private Object toJson(final RexWindowBound window_bound) {
314  final Map<String, Object> map = jsonBuilder.map();
315  map.put("unbounded", toJson(window_bound.isUnbounded()));
316  map.put("preceding", toJson(window_bound.isPreceding()));
317  map.put("following", toJson(window_bound.isFollowing()));
318  map.put("is_current_row", toJson(window_bound.isCurrentRow()));
319  map.put("offset",
320  window_bound.getOffset() != null ? toJson(window_bound.getOffset()) : null);
321  map.put("order_key", toJson(window_bound.getOrderKey()));
322  return map;
323  }
324 
325  private Object toJson(RexNode node) {
326  final Map<String, Object> map;
327  switch (node.getKind()) {
328  case FIELD_ACCESS:
329  map = jsonBuilder.map();
330  final RexFieldAccess fieldAccess = (RexFieldAccess) node;
331  map.put("field", fieldAccess.getField().getName());
332  map.put("expr", toJson(fieldAccess.getReferenceExpr()));
333  return map;
334  case LITERAL:
335  final RexLiteral literal = (RexLiteral) node;
336  final Object value2 = literal.getValue2();
337  map = jsonBuilder.map();
338  if (value2 instanceof TimeUnitRange) {
339  map.put("literal", value2.toString());
340  } else {
341  map.put("literal", value2);
342  }
343  map.put("type", literal.getTypeName().name());
344  map.put("target_type", literal.getType().getSqlTypeName().toString());
345  final Object value = literal.getValue();
346  if (value instanceof BigDecimal) {
347  map.put("scale", ((BigDecimal) value).scale());
348  map.put("precision", ((BigDecimal) value).precision());
349  } else {
350  map.put("scale", literal.getType().getScale());
351  map.put("precision", literal.getType().getPrecision());
352  }
353  map.put("type_scale", literal.getType().getScale());
354  map.put("type_precision", literal.getType().getPrecision());
355  return map;
356  case INPUT_REF:
357  map = jsonBuilder.map();
358  map.put("input", ((RexInputRef) node).getIndex());
359  return map;
360  case CORREL_VARIABLE:
361  map = jsonBuilder.map();
362  map.put("correl", ((RexCorrelVariable) node).getName());
363  map.put("type", toJson(node.getType()));
364  return map;
365  default:
366  if (node instanceof RexCall) {
367  final RexCall call = (RexCall) node;
368  map = jsonBuilder.map();
369  map.put("op", toJson(call.getOperator()));
370  final List<Object> list = jsonBuilder.list();
371  for (RexNode operand : call.getOperands()) {
372  list.add(toJson(operand));
373  }
374  map.put("operands", list);
375  map.put("type", toJson(node.getType()));
376  if (node instanceof RexSubQuery) {
377  final MapDRelJsonWriter subqueryWriter = new MapDRelJsonWriter();
378  ((RexSubQuery) node).rel.explain(subqueryWriter);
379  map.put("subquery", subqueryWriter.asJsonMap());
380  }
381  if (node instanceof RexOver) {
382  final RexWindow window = ((RexOver) node).getWindow();
383  final List<Object> partitionKeyList = jsonBuilder.list();
384  for (final RexNode partitionKey : window.partitionKeys) {
385  partitionKeyList.add(toJson(partitionKey));
386  }
387  map.put("partition_keys", partitionKeyList);
388  final List<Object> orderKeyList = jsonBuilder.list();
389  for (final RexFieldCollation orderKey : window.orderKeys) {
390  orderKeyList.add(toJson(orderKey));
391  }
392  map.put("order_keys", orderKeyList);
393  RexWindowBound lower_bound = window.getLowerBound();
394  RexWindowBound upper_bound = window.getUpperBound();
395  map.put("lower_bound", toJson(lower_bound));
396  map.put("upper_bound", toJson(upper_bound));
397  map.put("is_rows", toJson(window.isRows()));
398  }
399  if (call.getOperator() instanceof SqlFunction) {
400  switch (((SqlFunction) call.getOperator()).getFunctionType()) {
401  case USER_DEFINED_CONSTRUCTOR:
402  case USER_DEFINED_FUNCTION:
403  case USER_DEFINED_PROCEDURE:
404  case USER_DEFINED_SPECIFIC_FUNCTION:
405  map.put("class", call.getOperator().getClass().getName());
406  }
407  }
408  return map;
409  }
410  throw new UnsupportedOperationException("unknown rex " + node);
411  }
412  }
413 
414  RexNode toRex(RelInput relInput, Object o) {
415  final RelOptCluster cluster = relInput.getCluster();
416  final RexBuilder rexBuilder = cluster.getRexBuilder();
417  if (o == null) {
418  return null;
419  } else if (o instanceof Map) {
420  Map map = (Map) o;
421  final String op = (String) map.get("op");
422  if (op != null) {
423  final List operands = (List) map.get("operands");
424  final Object jsonType = map.get("type");
425  final SqlOperator operator = toOp(op, map);
426  final List<RexNode> rexOperands = toRexList(relInput, operands);
427  RelDataType type;
428  if (jsonType != null) {
429  type = toType(cluster.getTypeFactory(), jsonType);
430  } else {
431  type = rexBuilder.deriveReturnType(operator, rexOperands);
432  }
433  return rexBuilder.makeCall(type, operator, rexOperands);
434  }
435  final Integer input = (Integer) map.get("input");
436  if (input != null) {
437  List<RelNode> inputNodes = relInput.getInputs();
438  int i = input;
439  for (RelNode inputNode : inputNodes) {
440  final RelDataType rowType = inputNode.getRowType();
441  if (i < rowType.getFieldCount()) {
442  final RelDataTypeField field = rowType.getFieldList().get(i);
443  return rexBuilder.makeInputRef(field.getType(), input);
444  }
445  i -= rowType.getFieldCount();
446  }
447  throw new RuntimeException("input field " + input + " is out of range");
448  }
449  final String field = (String) map.get("field");
450  if (field != null) {
451  final Object jsonExpr = map.get("expr");
452  final RexNode expr = toRex(relInput, jsonExpr);
453  return rexBuilder.makeFieldAccess(expr, field, true);
454  }
455  final String correl = (String) map.get("correl");
456  if (correl != null) {
457  final Object jsonType = map.get("type");
458  RelDataType type = toType(cluster.getTypeFactory(), jsonType);
459  return rexBuilder.makeCorrel(type, new CorrelationId(correl));
460  }
461  if (map.containsKey("literal")) {
462  final Object literal = map.get("literal");
463  final SqlTypeName sqlTypeName =
464  Util.enumVal(SqlTypeName.class, (String) map.get("type"));
465  if (literal == null) {
466  return rexBuilder.makeNullLiteral(
467  cluster.getTypeFactory().createSqlType(sqlTypeName));
468  }
469 
470  // omnisci serializes numeric literals differently, we need more data
471  // then the number. So, we need to have a special case for that.
472  if (literal instanceof Number) {
473  final SqlTypeName targetTypeName =
474  Util.enumVal(SqlTypeName.class, (String) map.get("target_type"));
475  final long scale = ((Number) map.get("scale")).longValue();
476  final long precision = ((Number) map.get("precision")).longValue();
477  final long typeScale = ((Number) map.get("type_scale")).longValue();
478  final long typePrecision = ((Number) map.get("type_precision")).longValue();
479  RelDataTypeFactory typeFactory = cluster.getTypeFactory();
480 
481  BigDecimal value =
482  BigDecimal.valueOf(((Number) literal).longValue(), (int) scale);
483 
484  if (typeScale != 0 && typeScale != -2147483648) {
485  return rexBuilder.makeLiteral(value,
486  typeFactory.createSqlType(
487  SqlTypeName.DECIMAL, (int) typePrecision, (int) typeScale),
488  false);
489  } else {
490  return rexBuilder.makeLiteral(
491  value, typeFactory.createSqlType(targetTypeName), false);
492  }
493  } else {
494  return toRex(relInput, literal);
495  }
496  }
497  throw new UnsupportedOperationException("cannot convert to rex " + o);
498  } else if (o instanceof Boolean) {
499  return rexBuilder.makeLiteral((Boolean) o);
500  } else if (o instanceof String) {
501  return rexBuilder.makeLiteral((String) o);
502  } else if (o instanceof Number) {
503  final Number number = (Number) o;
504  if (number instanceof Double || number instanceof Float) {
505  return rexBuilder.makeApproxLiteral(BigDecimal.valueOf(number.doubleValue()));
506  } else {
507  return rexBuilder.makeExactLiteral(BigDecimal.valueOf(number.longValue()));
508  }
509  } else {
510  throw new UnsupportedOperationException("cannot convert to rex " + o);
511  }
512  }
513 
514  private List<RexNode> toRexList(RelInput relInput, List operands) {
515  final List<RexNode> list = new ArrayList<RexNode>();
516  for (Object operand : operands) {
517  list.add(toRex(relInput, operand));
518  }
519  return list;
520  }
521 
522  private SqlOperator toOp(String op) {
523  return toOp(op, new HashMap<>());
524  }
525 
526  private SqlOperator toOp(String op, Map<String, Object> map) {
527  // TODO: build a map, for more efficient lookup
528  // TODO: look up based on SqlKind
529  MapDSqlOperatorTable operatorTable =
530  new MapDSqlOperatorTable(SqlStdOperatorTable.instance());
531  MapDSqlOperatorTable.addUDF(operatorTable, null);
532  final List<SqlOperator> operatorList = operatorTable.getOperatorList();
533  for (SqlOperator operator : operatorList) {
534  if (operator.getName().equals(op)) {
535  return operator;
536  }
537  }
538  String class_ = (String) map.get("class");
539  if (class_ != null) {
540  try {
541  return (SqlOperator) Class.forName(class_).newInstance();
542  } catch (InstantiationException e) {
543  throw new RuntimeException(e);
544  } catch (IllegalAccessException e) {
545  throw new RuntimeException(e);
546  } catch (ClassNotFoundException e) {
547  throw new RuntimeException(e);
548  }
549  }
550  throw new RuntimeException("Operator " + op + " does not supported");
551  }
552 
553  SqlOperator toOp(RelInput relInput, String name) {
554  // in case different operator has the same kind, check with both name and kind.
555  String kind = name;
556  String syntax = "FUNCTION";
557  SqlKind sqlKind = SqlKind.valueOf(kind);
558  SqlSyntax sqlSyntax = SqlSyntax.valueOf(syntax);
559  MapDSqlOperatorTable operatorTable =
560  new MapDSqlOperatorTable(SqlStdOperatorTable.instance());
561  MapDSqlOperatorTable.addUDF(operatorTable, null);
562  final List<SqlOperator> operators = operatorTable.getOperatorList();
563  List<String> names = new ArrayList<>();
564  for (SqlOperator operator : operators) {
565  names.add(operator.toString());
566  if (operator.getName().equals(name)) {
567  return operator;
568  }
569  }
570  throw new RuntimeException("Aggregation function with name " + name
571  + " not found, search in " + names.toString());
572  }
573 
574  SqlAggFunction toAggregation(String agg) {
575  return (SqlAggFunction) toOp(agg);
576  }
577 
578  SqlAggFunction toAggregation(RelInput relInput, String agg) {
579  return (SqlAggFunction) toOp(relInput, agg);
580  }
581 
582  private String toJson(SqlOperator operator) {
583  // User-defined operators are not yet handled.
584  return operator.getName();
585  }
586 }
587 
588 // End RelJson.java
DEVICE auto upper_bound(ARGS &&...args)
Definition: gpu_enabled.h:123
JoinType
Definition: sqldefs.h:108
RexNode toRex(RelInput relInput, Object o)
RelDataType toType(RelDataTypeFactory typeFactory, Object o)
RelNode create(Map< String, Object > map)
String classToTypeName(Class<?extends RelNode > class_)
string name
Definition: setup.in.py:72
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31
Object toJson(RexFieldCollation node)
SqlOperator toOp(String op, Map< String, Object > map)
RelCollation toCollation(List< Map< String, Object >> jsonFieldCollations)
final Map< String, Constructor > constructorMap
#define ANY
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78
#define DECIMAL
SqlOperator toOp(RelInput relInput, String name)
SqlAggFunction toAggregation(RelInput relInput, String agg)
Object toJson(final RexWindowBound window_bound)
List< RexNode > toRexList(RelInput relInput, List operands)
RelFieldCollation toFieldCollation(Map< String, Object > map)