OmniSciDB  a987f07e93
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
HeavyDBRelJson.java
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package org.apache.calcite.rel.externalize;
18 
19 import com.fasterxml.jackson.databind.type.TypeFactory;
20 import com.google.common.collect.ImmutableList;
22 
23 import org.apache.calcite.avatica.AvaticaUtils;
24 import org.apache.calcite.avatica.util.TimeUnitRange;
25 import org.apache.calcite.plan.RelOptCluster;
26 import org.apache.calcite.plan.RelTraitSet;
27 import org.apache.calcite.rel.RelCollation;
28 import org.apache.calcite.rel.RelCollationImpl;
29 import org.apache.calcite.rel.RelCollations;
30 import org.apache.calcite.rel.RelDistribution;
31 import org.apache.calcite.rel.RelDistributions;
32 import org.apache.calcite.rel.RelFieldCollation;
33 import org.apache.calcite.rel.RelInput;
34 import org.apache.calcite.rel.RelNode;
35 import org.apache.calcite.rel.core.AggregateCall;
36 import org.apache.calcite.rel.core.CorrelationId;
37 import org.apache.calcite.rel.core.TableModify.Operation;
38 import org.apache.calcite.rel.type.RelDataType;
39 import org.apache.calcite.rel.type.RelDataTypeFactory;
40 import org.apache.calcite.rel.type.RelDataTypeField;
41 import org.apache.calcite.rex.RexBuilder;
42 import org.apache.calcite.rex.RexCall;
43 import org.apache.calcite.rex.RexCorrelVariable;
44 import org.apache.calcite.rex.RexFieldAccess;
45 import org.apache.calcite.rex.RexFieldCollation;
46 import org.apache.calcite.rex.RexInputRef;
47 import org.apache.calcite.rex.RexLiteral;
48 import org.apache.calcite.rex.RexNode;
49 import org.apache.calcite.rex.RexOver;
50 import org.apache.calcite.rex.RexSubQuery;
51 import org.apache.calcite.rex.RexWindow;
52 import org.apache.calcite.rex.RexWindowBound;
54 import org.apache.calcite.sql.SqlAggFunction;
55 import org.apache.calcite.sql.SqlFunction;
56 import org.apache.calcite.sql.SqlIdentifier;
57 import org.apache.calcite.sql.SqlKind;
59 import org.apache.calcite.sql.SqlOperatorTable;
60 import org.apache.calcite.sql.SqlSyntax;
61 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
62 import org.apache.calcite.sql.fun.SqlTrimFunction;
63 import org.apache.calcite.sql.type.SqlTypeName;
64 import org.apache.calcite.util.ImmutableBitSet;
65 import org.apache.calcite.util.JsonBuilder;
66 import org.apache.calcite.util.Util;
67 
68 import java.lang.reflect.Constructor;
69 import java.lang.reflect.InvocationTargetException;
70 import java.math.BigDecimal;
71 import java.util.ArrayList;
72 import java.util.HashMap;
73 import java.util.List;
74 import java.util.Map;
75 
80 public class HeavyDBRelJson {
81  private final Map<String, Constructor> constructorMap =
82  new HashMap<String, Constructor>();
83  private final JsonBuilder jsonBuilder;
84 
85  public static final List<String> PACKAGES = ImmutableList.of("org.apache.calcite.rel.",
86  "org.apache.calcite.rel.core.",
87  "org.apache.calcite.rel.logical.",
88  "org.apache.calcite.adapter.jdbc.",
89  "org.apache.calcite.adapter.enumerable.",
90  "org.apache.calcite.adapter.jdbc.JdbcRules$");
91 
92  public HeavyDBRelJson(JsonBuilder jsonBuilder) {
93  this.jsonBuilder = jsonBuilder;
94  }
95 
96  public RelNode create(Map<String, Object> map) {
97  String type = (String) map.get("type");
98  Constructor constructor = getConstructor(type);
99  try {
100  return (RelNode) constructor.newInstance(map);
101  } catch (InstantiationException e) {
102  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
103  } catch (IllegalAccessException e) {
104  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
105  } catch (InvocationTargetException e) {
106  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
107  } catch (ClassCastException e) {
108  throw new RuntimeException("while invoking constructor for type '" + type + "'", e);
109  }
110  }
111 
112  public Constructor getConstructor(String type) {
113  Constructor constructor = constructorMap.get(type);
114  if (constructor == null) {
115  Class clazz = typeNameToClass(type);
116  try {
117  // noinspection unchecked
118  constructor = clazz.getConstructor(RelInput.class);
119  } catch (NoSuchMethodException e) {
120  throw new RuntimeException(
121  "class does not have required constructor, " + clazz + "(RelInput)");
122  }
123  constructorMap.put(type, constructor);
124  }
125  return constructor;
126  }
127 
132  public Class typeNameToClass(String type) {
133  if (!type.contains(".")) {
134  for (String package_ : PACKAGES) {
135  try {
136  return Class.forName(package_ + type);
137  } catch (ClassNotFoundException e) {
138  // ignore
139  }
140  }
141  }
142  try {
143  return Class.forName(type);
144  } catch (ClassNotFoundException e) {
145  throw new RuntimeException("unknown type " + type);
146  }
147  }
148 
152  public String classToTypeName(Class<? extends RelNode> class_) {
153  final String canonicalName = class_.getName();
154  for (String package_ : PACKAGES) {
155  if (canonicalName.startsWith(package_)) {
156  String remaining = canonicalName.substring(package_.length());
157  if (remaining.indexOf('.') < 0 && remaining.indexOf('$') < 0) {
158  return remaining;
159  }
160  }
161  }
162  return canonicalName;
163  }
164 
165  public Object toJson(RelCollationImpl node) {
166  final List<Object> list = new ArrayList<Object>();
167  for (RelFieldCollation fieldCollation : node.getFieldCollations()) {
168  final Map<String, Object> map = jsonBuilder.map();
169  map.put("field", fieldCollation.getFieldIndex());
170  map.put("direction", fieldCollation.getDirection().name());
171  map.put("nulls", fieldCollation.nullDirection.name());
172  list.add(map);
173  }
174  return list;
175  }
176 
177  public Object toJson(RexFieldCollation node) {
178  final Map<String, Object> map = jsonBuilder.map();
179  map.put("field", toJson(node.left));
180  map.put("direction", node.getDirection().name());
181  map.put("nulls", node.getNullDirection().name());
182  return map;
183  }
184 
185  public RelCollation toCollation(List<Map<String, Object>> jsonFieldCollations) {
186  final List<RelFieldCollation> fieldCollations = new ArrayList<RelFieldCollation>();
187  for (Map<String, Object> map : jsonFieldCollations) {
188  fieldCollations.add(toFieldCollation(map));
189  }
190  return RelCollations.of(fieldCollations);
191  }
192 
193  public RelFieldCollation toFieldCollation(Map<String, Object> map) {
194  final Integer field = (Integer) map.get("field");
195  final RelFieldCollation.Direction direction = Util.enumVal(
196  RelFieldCollation.Direction.class, (String) map.get("direction"));
197  final RelFieldCollation.NullDirection nullDirection = Util.enumVal(
198  RelFieldCollation.NullDirection.class, (String) map.get("nulls"));
199  return new RelFieldCollation(field, direction, nullDirection);
200  }
201 
202  public RelDistribution toDistribution(Object o) {
203  return RelDistributions.ANY; // TODO:
204  }
205 
206  public RelDataType toType(RelDataTypeFactory typeFactory, Object o) {
207  if (o instanceof List) {
208  @SuppressWarnings("unchecked")
209  final List<Map<String, Object>> jsonList = (List<Map<String, Object>>) o;
210  final RelDataTypeFactory.Builder builder = typeFactory.builder();
211  for (Map<String, Object> jsonMap : jsonList) {
212  builder.add((String) jsonMap.get("name"), toType(typeFactory, jsonMap));
213  }
214  return builder.build();
215  } else {
216  final Map<String, Object> map = (Map<String, Object>) o;
217  final SqlTypeName sqlTypeName =
218  Util.enumVal(SqlTypeName.class, (String) map.get("type"));
219  final Integer precision = (Integer) map.get("precision");
220  final Integer scale = (Integer) map.get("scale");
221  final RelDataType type;
222  if (precision == null) {
223  type = typeFactory.createSqlType(sqlTypeName);
224  } else if (scale == null) {
225  type = typeFactory.createSqlType(sqlTypeName, precision);
226  } else {
227  type = typeFactory.createSqlType(sqlTypeName, precision, scale);
228  }
229  final boolean nullable = (Boolean) map.get("nullable");
230  return typeFactory.createTypeWithNullability(type, nullable);
231  }
232  }
233 
234  public Object toJson(AggregateCall node) {
235  final Map<String, Object> map = jsonBuilder.map();
236  map.put("agg", toJson(node.getAggregation()));
237  map.put("type", toJson(node.getType()));
238  map.put("distinct", node.isDistinct());
239  map.put("operands", node.getArgList());
240  return map;
241  }
242 
243  Object toJson(Object value) {
244  if (value == null || value instanceof Number || value instanceof String
245  || value instanceof Boolean) {
246  return value;
247  } else if (value instanceof RexNode) {
248  return toJson((RexNode) value);
249  } else if (value instanceof CorrelationId) {
250  return toJson((CorrelationId) value);
251  } else if (value instanceof List) {
252  final List<Object> list = jsonBuilder.list();
253  for (Object o : (List) value) {
254  list.add(toJson(o));
255  }
256  return list;
257  } else if (value instanceof ImmutableBitSet) {
258  final List<Object> list = jsonBuilder.list();
259  for (Integer integer : (ImmutableBitSet) value) {
260  list.add(toJson(integer));
261  }
262  return list;
263  } else if (value instanceof AggregateCall) {
264  return toJson((AggregateCall) value);
265  } else if (value instanceof RelCollationImpl) {
266  return toJson((RelCollationImpl) value);
267  } else if (value instanceof RexFieldCollation) {
268  return toJson((RexFieldCollation) value);
269  } else if (value instanceof RelDataType) {
270  return toJson((RelDataType) value);
271  } else if (value instanceof RelDataTypeField) {
272  return toJson((RelDataTypeField) value);
273  } else if (value instanceof JoinType) {
274  return value.toString();
275  } else if (value instanceof Operation) {
276  return value.toString();
277  } else {
278  throw new UnsupportedOperationException("type not serializable: " + value
279  + " (type " + value.getClass().getCanonicalName() + ")");
280  }
281  }
282 
283  private Object toJson(RelDataType node) {
284  if (node.isStruct()) {
285  final List<Object> list = jsonBuilder.list();
286  for (RelDataTypeField field : node.getFieldList()) {
287  list.add(toJson(field));
288  }
289  return list;
290  } else {
291  final Map<String, Object> map = jsonBuilder.map();
292  map.put("type", node.getSqlTypeName().name());
293  map.put("nullable", node.isNullable());
294  if (node.getSqlTypeName().allowsPrec()) {
295  map.put("precision", node.getPrecision());
296  }
297  if (node.getSqlTypeName().allowsScale()) {
298  map.put("scale", node.getScale());
299  }
300  return map;
301  }
302  }
303 
304  private Object toJson(RelDataTypeField node) {
305  final Map<String, Object> map = (Map<String, Object>) toJson(node.getType());
306  map.put("name", node.getName());
307  return map;
308  }
309 
310  private Object toJson(CorrelationId node) {
311  return node.getId();
312  }
313 
314  private Object toJson(final RexWindowBound window_bound) {
315  final Map<String, Object> map = jsonBuilder.map();
316  map.put("unbounded", toJson(window_bound.isUnbounded()));
317  map.put("preceding", toJson(window_bound.isPreceding()));
318  map.put("following", toJson(window_bound.isFollowing()));
319  map.put("is_current_row", toJson(window_bound.isCurrentRow()));
320  map.put("offset",
321  window_bound.getOffset() != null ? toJson(window_bound.getOffset()) : null);
322  map.put("order_key", toJson(window_bound.getOrderKey()));
323  return map;
324  }
325 
326  private Object toJson(RexNode node) {
327  final Map<String, Object> map;
328  switch (node.getKind()) {
329  case FIELD_ACCESS:
330  map = jsonBuilder.map();
331  final RexFieldAccess fieldAccess = (RexFieldAccess) node;
332  map.put("field", fieldAccess.getField().getName());
333  map.put("expr", toJson(fieldAccess.getReferenceExpr()));
334  return map;
335  case LITERAL:
336  final RexLiteral literal = (RexLiteral) node;
337  final Object value2 = literal.getValue2();
338  map = jsonBuilder.map();
339  if (value2 instanceof TimeUnitRange || value2 instanceof SqlTrimFunction.Flag) {
340  map.put("literal", value2.toString());
341  } else {
342  map.put("literal", value2);
343  }
344  map.put("type", literal.getTypeName().name());
345  map.put("target_type", literal.getType().getSqlTypeName().toString());
346  final Object value = literal.getValue();
347  if (value instanceof BigDecimal) {
348  map.put("scale", ((BigDecimal) value).scale());
349  map.put("precision", ((BigDecimal) value).precision());
350  } else {
351  map.put("scale", literal.getType().getScale());
352  map.put("precision", literal.getType().getPrecision());
353  }
354  map.put("type_scale", literal.getType().getScale());
355  map.put("type_precision", literal.getType().getPrecision());
356  return map;
357  case INPUT_REF:
358  map = jsonBuilder.map();
359  map.put("input", ((RexInputRef) node).getIndex());
360  return map;
361  case CORREL_VARIABLE:
362  map = jsonBuilder.map();
363  map.put("correl", ((RexCorrelVariable) node).getName());
364  map.put("type", toJson(node.getType()));
365  return map;
366  default:
367  if (node instanceof RexCall) {
368  final RexCall call = (RexCall) node;
369  map = jsonBuilder.map();
370  map.put("op", toJson(call.getOperator()));
371  final List<Object> list = jsonBuilder.list();
372  for (RexNode operand : call.getOperands()) {
373  list.add(toJson(operand));
374  }
375  map.put("operands", list);
376  map.put("type", toJson(node.getType()));
377  if (node instanceof RexSubQuery) {
378  final HeavyDBRelJsonWriter subqueryWriter = new HeavyDBRelJsonWriter();
379  ((RexSubQuery) node).rel.explain(subqueryWriter);
380  map.put("subquery", subqueryWriter.asJsonMap());
381  }
382  if (node instanceof RexOver) {
383  final RexWindow window = ((RexOver) node).getWindow();
384  final List<Object> partitionKeyList = jsonBuilder.list();
385  for (final RexNode partitionKey : window.partitionKeys) {
386  partitionKeyList.add(toJson(partitionKey));
387  }
388  map.put("partition_keys", partitionKeyList);
389  final List<Object> orderKeyList = jsonBuilder.list();
390  for (final RexFieldCollation orderKey : window.orderKeys) {
391  orderKeyList.add(toJson(orderKey));
392  }
393  map.put("order_keys", orderKeyList);
394  RexWindowBound lower_bound = window.getLowerBound();
395  RexWindowBound upper_bound = window.getUpperBound();
396  map.put("lower_bound", toJson(lower_bound));
397  map.put("upper_bound", toJson(upper_bound));
398  map.put("is_rows", toJson(window.isRows()));
399  }
400  if (call.getOperator() instanceof SqlFunction) {
401  switch (((SqlFunction) call.getOperator()).getFunctionType()) {
402  case USER_DEFINED_CONSTRUCTOR:
403  case USER_DEFINED_FUNCTION:
404  case USER_DEFINED_PROCEDURE:
405  case USER_DEFINED_SPECIFIC_FUNCTION:
406  map.put("class", call.getOperator().getClass().getName());
407  }
408  }
409  return map;
410  }
411  throw new UnsupportedOperationException("unknown rex " + node);
412  }
413  }
414 
415  RexNode toRex(RelInput relInput, Object o) {
416  final RelOptCluster cluster = relInput.getCluster();
417  final RexBuilder rexBuilder = cluster.getRexBuilder();
418  if (o == null) {
419  return null;
420  } else if (o instanceof Map) {
421  Map map = (Map) o;
422  final String op = (String) map.get("op");
423  if (op != null) {
424  final List operands = (List) map.get("operands");
425  final Object jsonType = map.get("type");
426  final SqlOperator operator = toOp(op, map);
427  final List<RexNode> rexOperands = toRexList(relInput, operands);
428  RelDataType type;
429  if (jsonType != null) {
430  type = toType(cluster.getTypeFactory(), jsonType);
431  } else {
432  type = rexBuilder.deriveReturnType(operator, rexOperands);
433  }
434  return rexBuilder.makeCall(type, operator, rexOperands);
435  }
436  final Integer input = (Integer) map.get("input");
437  if (input != null) {
438  List<RelNode> inputNodes = relInput.getInputs();
439  int i = input;
440  for (RelNode inputNode : inputNodes) {
441  final RelDataType rowType = inputNode.getRowType();
442  if (i < rowType.getFieldCount()) {
443  final RelDataTypeField field = rowType.getFieldList().get(i);
444  return rexBuilder.makeInputRef(field.getType(), input);
445  }
446  i -= rowType.getFieldCount();
447  }
448  throw new RuntimeException("input field " + input + " is out of range");
449  }
450  final String field = (String) map.get("field");
451  if (field != null) {
452  final Object jsonExpr = map.get("expr");
453  final RexNode expr = toRex(relInput, jsonExpr);
454  return rexBuilder.makeFieldAccess(expr, field, true);
455  }
456  final String correl = (String) map.get("correl");
457  if (correl != null) {
458  final Object jsonType = map.get("type");
459  RelDataType type = toType(cluster.getTypeFactory(), jsonType);
460  return rexBuilder.makeCorrel(type, new CorrelationId(correl));
461  }
462  if (map.containsKey("literal")) {
463  final Object literal = map.get("literal");
464  final SqlTypeName sqlTypeName =
465  Util.enumVal(SqlTypeName.class, (String) map.get("type"));
466  if (literal == null) {
467  return rexBuilder.makeNullLiteral(
468  cluster.getTypeFactory().createSqlType(sqlTypeName));
469  }
470 
471  // HEAVY.AI serializes numeric literals differently, we need more data
472  // then the number. So, we need to have a special case for that.
473  if (literal instanceof Number) {
474  final SqlTypeName targetTypeName =
475  Util.enumVal(SqlTypeName.class, (String) map.get("target_type"));
476  final long scale = ((Number) map.get("scale")).longValue();
477  final long precision = ((Number) map.get("precision")).longValue();
478  final long typeScale = ((Number) map.get("type_scale")).longValue();
479  final long typePrecision = ((Number) map.get("type_precision")).longValue();
480  RelDataTypeFactory typeFactory = cluster.getTypeFactory();
481 
482  BigDecimal value =
483  BigDecimal.valueOf(((Number) literal).longValue(), (int) scale);
484 
485  if (typeScale != 0 && typeScale != -2147483648) {
486  return rexBuilder.makeLiteral(value,
487  typeFactory.createSqlType(
488  SqlTypeName.DECIMAL, (int) typePrecision, (int) typeScale),
489  false);
490  } else {
491  return rexBuilder.makeLiteral(
492  value, typeFactory.createSqlType(targetTypeName), false);
493  }
494  } else {
495  return toRex(relInput, literal);
496  }
497  }
498  throw new UnsupportedOperationException("cannot convert to rex " + o);
499  } else if (o instanceof Boolean) {
500  return rexBuilder.makeLiteral((Boolean) o);
501  } else if (o instanceof String) {
502  return rexBuilder.makeLiteral((String) o);
503  } else if (o instanceof Number) {
504  final Number number = (Number) o;
505  if (number instanceof Double || number instanceof Float) {
506  return rexBuilder.makeApproxLiteral(BigDecimal.valueOf(number.doubleValue()));
507  } else {
508  return rexBuilder.makeExactLiteral(BigDecimal.valueOf(number.longValue()));
509  }
510  } else {
511  throw new UnsupportedOperationException("cannot convert to rex " + o);
512  }
513  }
514 
515  private List<RexNode> toRexList(RelInput relInput, List operands) {
516  final List<RexNode> list = new ArrayList<RexNode>();
517  for (Object operand : operands) {
518  list.add(toRex(relInput, operand));
519  }
520  return list;
521  }
522 
523  private SqlOperator toOp(String op) {
524  return toOp(op, new HashMap<>());
525  }
526 
527  private SqlOperator toOp(String op, Map<String, Object> map) {
528  // TODO: build a map, for more efficient lookup
529  // TODO: look up based on SqlKind
530  HeavyDBSqlOperatorTable operatorTable =
531  new HeavyDBSqlOperatorTable(SqlStdOperatorTable.instance());
532  operatorTable.addUDF(null);
533  final List<SqlOperator> operatorList = operatorTable.getOperatorList();
534  for (SqlOperator operator : operatorList) {
535  if (operator.getName().equals(op)) {
536  return operator;
537  }
538  }
539  String class_ = (String) map.get("class");
540  if (class_ != null) {
541  try {
542  return (SqlOperator) Class.forName(class_).newInstance();
543  } catch (InstantiationException e) {
544  throw new RuntimeException(e);
545  } catch (IllegalAccessException e) {
546  throw new RuntimeException(e);
547  } catch (ClassNotFoundException e) {
548  throw new RuntimeException(e);
549  }
550  }
551  throw new RuntimeException("Operator " + op + " does not supported");
552  }
553 
554  SqlOperator toOp(RelInput relInput, String name) {
555  // in case different operator has the same kind, check with both name and kind.
556  String kind = name;
557  String syntax = "FUNCTION";
558  SqlKind sqlKind = SqlKind.valueOf(kind);
559  SqlSyntax sqlSyntax = SqlSyntax.valueOf(syntax);
560  HeavyDBSqlOperatorTable operatorTable =
561  new HeavyDBSqlOperatorTable(SqlStdOperatorTable.instance());
562  operatorTable.addUDF(null);
563  final List<SqlOperator> operators = operatorTable.getOperatorList();
564  List<String> names = new ArrayList<>();
565  for (SqlOperator operator : operators) {
566  names.add(operator.toString());
567  if (operator.getName().equals(name)) {
568  return operator;
569  }
570  }
571  throw new RuntimeException("Aggregation function with name " + name
572  + " not found, search in " + names.toString());
573  }
574 
575  SqlAggFunction toAggregation(String agg) {
576  return (SqlAggFunction) toOp(agg);
577  }
578 
579  SqlAggFunction toAggregation(RelInput relInput, String agg) {
580  return (SqlAggFunction) toOp(relInput, agg);
581  }
582 
583  private String toJson(SqlOperator operator) {
584  // User-defined operators are not yet handled.
585  return operator.getName();
586  }
587 }
588 
589 // End RelJson.java
DEVICE auto upper_bound(ARGS &&...args)
Definition: gpu_enabled.h:123
RelCollation toCollation(List< Map< String, Object >> jsonFieldCollations)
JoinType
Definition: sqldefs.h:164
List< RexNode > toRexList(RelInput relInput, List operands)
final Map< String, Constructor > constructorMap
SqlOperator toOp(String op, Map< String, Object > map)
SqlAggFunction toAggregation(RelInput relInput, String agg)
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31
String classToTypeName(Class<?extends RelNode > class_)
SqlOperator toOp(RelInput relInput, String name)
RexNode toRex(RelInput relInput, Object o)
RelDataType toType(RelDataTypeFactory typeFactory, Object o)
RelFieldCollation toFieldCollation(Map< String, Object > map)
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78
RelNode create(Map< String, Object > map)
string name
Definition: setup.in.py:72
Object toJson(final RexWindowBound window_bound)