OmniSciDB  0bd2ec9cf4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
CalciteServerHandler.java
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.mapd.parser.server;
17 
18 import static com.mapd.calcite.parser.MapDParser.CURRENT_PARSER;
19 
24 import com.mapd.thrift.calciteserver.CalciteServer;
25 import com.mapd.thrift.calciteserver.InvalidParseRequest;
26 import com.mapd.thrift.calciteserver.TAccessedQueryObjects;
27 import com.mapd.thrift.calciteserver.TCompletionHint;
28 import com.mapd.thrift.calciteserver.TCompletionHintType;
29 import com.mapd.thrift.calciteserver.TExtArgumentType;
30 import com.mapd.thrift.calciteserver.TFilterPushDownInfo;
31 import com.mapd.thrift.calciteserver.TPlanResult;
32 import com.mapd.thrift.calciteserver.TUserDefinedFunction;
33 import com.mapd.thrift.calciteserver.TUserDefinedTableFunction;
34 
37 import org.apache.calcite.runtime.CalciteContextException;
38 import org.apache.calcite.sql.parser.SqlParseException;
39 import org.apache.calcite.sql.validate.SqlMoniker;
40 import org.apache.calcite.sql.validate.SqlMonikerType;
41 import org.apache.calcite.tools.RelConversionException;
42 import org.apache.calcite.tools.ValidationException;
43 import org.apache.commons.pool.PoolableObjectFactory;
44 import org.apache.commons.pool.impl.GenericObjectPool;
45 import org.apache.thrift.TException;
46 import org.apache.thrift.server.TServer;
47 import org.slf4j.Logger;
48 import org.slf4j.LoggerFactory;
49 
50 import java.io.IOException;
51 import java.util.ArrayList;
52 import java.util.HashMap;
53 import java.util.List;
54 import java.util.Map;
55 
60 class CalciteServerHandler implements CalciteServer.Iface {
61  final static Logger MAPDLOGGER = LoggerFactory.getLogger(CalciteServerHandler.class);
62  private TServer server;
63 
64  private final int mapdPort;
65 
66  private volatile long callCount;
67 
68  private final GenericObjectPool parserPool;
69 
70  private final String extSigsJson;
71 
72  private final String udfSigsJson;
73 
74  private String udfRTSigsJson = "";
75  Map<String, ExtensionFunction> udfRTSigs = null;
76 
78  private Map<String, ExtensionFunction> extSigs = null;
79  private String dataDir;
80 
81  // TODO MAT we need to merge this into common code base for these functions with
82  // CalciteDirect since we are not deprecating this stuff yet
85  String extensionFunctionsAstFile,
87  String udfAstFile) {
88  this.mapdPort = mapdPort;
89  this.dataDir = dataDir;
90 
91  Map<String, ExtensionFunction> udfSigs = null;
92 
93  try {
94  extSigs = ExtensionFunctionSignatureParser.parse(extensionFunctionsAstFile);
95  } catch (IOException ex) {
96  MAPDLOGGER.error(
97  "Could not load extension function signatures: " + ex.getMessage());
98  }
99  extSigsJson = ExtensionFunctionSignatureParser.signaturesToJson(extSigs);
100 
101  try {
102  if (!udfAstFile.isEmpty()) {
103  udfSigs = ExtensionFunctionSignatureParser.parseUdfAst(udfAstFile);
104  }
105  } catch (IOException ex) {
106  MAPDLOGGER.error("Could not load udf function signatures: " + ex.getMessage());
107  }
108  udfSigsJson = ExtensionFunctionSignatureParser.signaturesToJson(udfSigs);
109 
110  // Put all the udf functions signatures in extSigs so Calcite has a view of
111  // extension functions and udf functions
112  if (!udfAstFile.isEmpty()) {
113  extSigs.putAll(udfSigs);
114  }
115 
116  PoolableObjectFactory parserFactory =
117  new CalciteParserFactory(dataDir, extSigs, mapdPort, skT);
118  // GenericObjectPool::setFactory is deprecated
119  this.parserPool = new GenericObjectPool(parserFactory);
120  }
121 
122  @Override
123  public void ping() throws TException {
124  MAPDLOGGER.debug("Ping hit");
125  }
126 
127  @Override
128  public TPlanResult process(String user,
129  String session,
130  String catalog,
131  String sqlText,
132  java.util.List<TFilterPushDownInfo> thriftFilterPushDownInfo,
133  boolean legacySyntax,
134  boolean isExplain,
135  boolean isViewOptimize) throws InvalidParseRequest, TException {
136  long timer = System.currentTimeMillis();
137  callCount++;
139  try {
140  parser = (MapDParser) parserPool.borrowObject();
141  parser.clearMemo();
142  } catch (Exception ex) {
143  String msg = "Could not get Parse Item from pool: " + ex.getMessage();
144  MAPDLOGGER.error(msg);
145  throw new InvalidParseRequest(-1, msg);
146  }
147  MapDUser mapDUser = new MapDUser(user, session, catalog, mapdPort);
148  MAPDLOGGER.debug("process was called User: " + user + " Catalog: " + catalog
149  + " sql: " + sqlText);
150  parser.setUser(mapDUser);
151  CURRENT_PARSER.set(parser);
152 
153  // need to trim the sql string as it seems it is not trimed prior to here
154  sqlText = sqlText.trim();
155  // remove last charcter if it is a ;
156  if (sqlText.length() > 0 && sqlText.charAt(sqlText.length() - 1) == ';') {
157  sqlText = sqlText.substring(0, sqlText.length() - 1);
158  }
159  String relAlgebra;
160  SqlIdentifierCapturer capturer;
161  TAccessedQueryObjects primaryAccessedObjects = new TAccessedQueryObjects();
162  TAccessedQueryObjects resolvedAccessedObjects = new TAccessedQueryObjects();
163  try {
164  final List<MapDParserOptions.FilterPushDownInfo> filterPushDownInfo =
165  new ArrayList<>();
166  for (final TFilterPushDownInfo req : thriftFilterPushDownInfo) {
167  filterPushDownInfo.add(new MapDParserOptions.FilterPushDownInfo(
168  req.input_prev, req.input_start, req.input_next));
169  }
170  try {
171  MapDParserOptions parserOptions = new MapDParserOptions(
172  filterPushDownInfo, legacySyntax, isExplain, isViewOptimize);
173  relAlgebra = parser.getRelAlgebra(sqlText, parserOptions);
174  } catch (ValidationException ex) {
175  String msg = "Validation: " + ex.getMessage();
176  MAPDLOGGER.error(msg);
177  throw ex;
178  } catch (RelConversionException ex) {
179  String msg = " RelConversion failed: " + ex.getMessage();
180  MAPDLOGGER.error(msg);
181  throw ex;
182  }
183  capturer = parser.captureIdentifiers(sqlText, legacySyntax);
184 
185  primaryAccessedObjects.tables_selected_from = new ArrayList<>(capturer.selects);
186  primaryAccessedObjects.tables_inserted_into = new ArrayList<>(capturer.inserts);
187  primaryAccessedObjects.tables_updated_in = new ArrayList<>(capturer.updates);
188  primaryAccessedObjects.tables_deleted_from = new ArrayList<>(capturer.deletes);
189 
190  // also resolve all the views in the select part
191  // resolution of the other parts is not
192  // necessary as these cannot be views
193  resolvedAccessedObjects.tables_selected_from =
194  new ArrayList<>(parser.resolveSelectIdentifiers(capturer));
195  resolvedAccessedObjects.tables_inserted_into = new ArrayList<>(capturer.inserts);
196  resolvedAccessedObjects.tables_updated_in = new ArrayList<>(capturer.updates);
197  resolvedAccessedObjects.tables_deleted_from = new ArrayList<>(capturer.deletes);
198 
199  } catch (SqlParseException ex) {
200  String msg = "Parse failed: " + ex.getMessage();
201  MAPDLOGGER.error(msg);
202  throw new InvalidParseRequest(-2, msg);
203  } catch (CalciteContextException ex) {
204  String msg = "Validate failed: " + ex.getMessage();
205  MAPDLOGGER.error(msg);
206  throw new InvalidParseRequest(-3, msg);
207  } catch (Throwable ex) {
208  String msg = "Exception occurred: " + ex.getMessage();
209  MAPDLOGGER.error(msg);
210  throw new InvalidParseRequest(-4, msg);
211  } finally {
212  CURRENT_PARSER.set(null);
213  try {
214  // put parser object back in pool for others to use
215  parserPool.returnObject(parser);
216  } catch (Exception ex) {
217  String msg = "Could not return parse object: " + ex.getMessage();
218  MAPDLOGGER.error(msg);
219  throw new InvalidParseRequest(-4, msg);
220  }
221  }
222 
223  TPlanResult result = new TPlanResult();
224  result.primary_accessed_objects = primaryAccessedObjects;
225  result.resolved_accessed_objects = resolvedAccessedObjects;
226  result.plan_result = relAlgebra;
227  result.execution_time_ms = System.currentTimeMillis() - timer;
228 
229  return result;
230  }
231 
232  @Override
233  public void shutdown() throws TException {
234  // received request to shutdown
235  MAPDLOGGER.debug("Shutdown calcite java server");
236  server.stop();
237  }
238 
239  @Override
241  return this.extSigsJson;
242  }
243 
244  @Override
246  return this.udfSigsJson;
247  }
248 
249  @Override
251  return this.udfRTSigsJson;
252  }
253 
254  void setServer(TServer s) {
255  server = s;
256  }
257 
258  @Override
259  public void updateMetadata(String catalog, String table) throws TException {
260  MAPDLOGGER.debug("Received invalidation from server for " + catalog + " : " + table);
261  long timer = System.currentTimeMillis();
262  callCount++;
264  try {
265  parser = (MapDParser) parserPool.borrowObject();
266  } catch (Exception ex) {
267  String msg = "Could not get Parse Item from pool: " + ex.getMessage();
268  MAPDLOGGER.error(msg);
269  return;
270  }
271  CURRENT_PARSER.set(parser);
272  try {
273  parser.updateMetaData(catalog, table);
274  } finally {
275  CURRENT_PARSER.set(null);
276  try {
277  // put parser object back in pool for others to use
278  MAPDLOGGER.debug("Returning object to pool");
279  parserPool.returnObject(parser);
280  } catch (Exception ex) {
281  String msg = "Could not return parse object: " + ex.getMessage();
282  MAPDLOGGER.error(msg);
283  }
284  }
285  }
286 
287  @Override
288  public List<TCompletionHint> getCompletionHints(String user,
289  String session,
290  String catalog,
291  List<String> visible_tables,
292  String sql,
293  int cursor) throws TException {
294  callCount++;
296  try {
297  parser = (MapDParser) parserPool.borrowObject();
298  } catch (Exception ex) {
299  String msg = "Could not get Parse Item from pool: " + ex.getMessage();
300  MAPDLOGGER.error(msg);
301  throw new TException(msg);
302  }
303  MapDUser mapDUser = new MapDUser(user, session, catalog, mapdPort);
304  MAPDLOGGER.debug("getCompletionHints was called User: " + user
305  + " Catalog: " + catalog + " sql: " + sql);
306  parser.setUser(mapDUser);
307  CURRENT_PARSER.set(parser);
308 
309  MapDPlanner.CompletionResult completion_result;
310  try {
311  completion_result = parser.getCompletionHints(sql, cursor, visible_tables);
312  } catch (Exception ex) {
313  String msg = "Could not retrieve completion hints: " + ex.getMessage();
314  MAPDLOGGER.error(msg);
315  return new ArrayList<>();
316  } finally {
317  CURRENT_PARSER.set(null);
318  try {
319  // put parser object back in pool for others to use
320  parserPool.returnObject(parser);
321  } catch (Exception ex) {
322  String msg = "Could not return parse object: " + ex.getMessage();
323  MAPDLOGGER.error(msg);
324  throw new InvalidParseRequest(-4, msg);
325  }
326  }
327  List<TCompletionHint> result = new ArrayList<>();
328  for (final SqlMoniker hint : completion_result.hints) {
329  result.add(new TCompletionHint(hintTypeToThrift(hint.getType()),
330  hint.getFullyQualifiedNames(),
331  completion_result.replaced));
332  }
333  return result;
334  }
335 
336  @Override
338  List<TUserDefinedFunction> udfs, List<TUserDefinedTableFunction> udtfs) {
339  // Clean up previously defined Runtime UDFs
340  if (udfRTSigs != null) {
341  for (String name : udfRTSigs.keySet()) extSigs.remove(name);
342  udfRTSigsJson = "";
343  udfRTSigs.clear();
344  } else {
345  udfRTSigs = new HashMap<String, ExtensionFunction>();
346  }
347 
348  for (TUserDefinedFunction udf : udfs) {
349  udfRTSigs.put(udf.name, toExtensionFunction(udf));
350  }
351 
352  for (TUserDefinedTableFunction udtf : udtfs) {
353  udfRTSigs.put(udtf.name, toExtensionFunction(udtf));
354  }
355 
356  // Avoid overwritting compiled and Loadtime UDFs:
357  for (String name : udfRTSigs.keySet()) {
358  if (extSigs.containsKey(name)) {
359  MAPDLOGGER.error("Extension function `" + name
360  + "` exists. Skipping runtime extenension function with the same name.");
361  udfRTSigs.remove(name);
362  }
363  }
364 
365  udfRTSigsJson = ExtensionFunctionSignatureParser.signaturesToJson(udfRTSigs);
366  // Expose RT UDFs to Calcite server:
367  extSigs.putAll(udfRTSigs);
368  }
369 
370  private static ExtensionFunction toExtensionFunction(TUserDefinedFunction udf) {
371  List<ExtensionFunction.ExtArgumentType> args =
372  new ArrayList<ExtensionFunction.ExtArgumentType>();
373  for (TExtArgumentType atype : udf.argTypes) {
374  final ExtensionFunction.ExtArgumentType arg_type = toExtArgumentType(atype);
375  if (arg_type != ExtensionFunction.ExtArgumentType.Void) {
376  args.add(arg_type);
377  }
378  }
379  return new ExtensionFunction(args, toExtArgumentType(udf.retType), true);
380  }
381 
382  private static ExtensionFunction toExtensionFunction(TUserDefinedTableFunction udtf) {
383  List<ExtensionFunction.ExtArgumentType> args =
384  new ArrayList<ExtensionFunction.ExtArgumentType>();
385  for (TExtArgumentType atype : udtf.sqlArgTypes) {
386  args.add(toExtArgumentType(atype));
387  }
388  return new ExtensionFunction(args, ExtensionFunction.ExtArgumentType.Void, false);
389  }
390 
391  private static ExtensionFunction.ExtArgumentType toExtArgumentType(
392  TExtArgumentType type) {
393  switch (type) {
394  case Int8:
395  return ExtensionFunction.ExtArgumentType.Int8;
396  case Int16:
397  return ExtensionFunction.ExtArgumentType.Int16;
398  case Int32:
399  return ExtensionFunction.ExtArgumentType.Int32;
400  case Int64:
401  return ExtensionFunction.ExtArgumentType.Int64;
402  case Float:
403  return ExtensionFunction.ExtArgumentType.Float;
404  case Double:
405  return ExtensionFunction.ExtArgumentType.Double;
406  case Void:
407  return ExtensionFunction.ExtArgumentType.Void;
408  case PInt8:
409  return ExtensionFunction.ExtArgumentType.PInt8;
410  case PInt16:
411  return ExtensionFunction.ExtArgumentType.PInt16;
412  case PInt32:
413  return ExtensionFunction.ExtArgumentType.PInt32;
414  case PInt64:
415  return ExtensionFunction.ExtArgumentType.PInt64;
416  case PFloat:
417  return ExtensionFunction.ExtArgumentType.PFloat;
418  case PDouble:
419  return ExtensionFunction.ExtArgumentType.PDouble;
420  case Bool:
421  return ExtensionFunction.ExtArgumentType.Bool;
422  case ArrayInt8:
423  return ExtensionFunction.ExtArgumentType.ArrayInt8;
424  case ArrayInt16:
425  return ExtensionFunction.ExtArgumentType.ArrayInt16;
426  case ArrayInt32:
427  return ExtensionFunction.ExtArgumentType.ArrayInt32;
428  case ArrayInt64:
429  return ExtensionFunction.ExtArgumentType.ArrayInt64;
430  case ArrayFloat:
431  return ExtensionFunction.ExtArgumentType.ArrayFloat;
432  case ArrayDouble:
433  return ExtensionFunction.ExtArgumentType.ArrayDouble;
434  case GeoPoint:
435  return ExtensionFunction.ExtArgumentType.GeoPoint;
436  case Cursor:
437  return ExtensionFunction.ExtArgumentType.Cursor;
438  default:
439  MAPDLOGGER.error("toExtArgumentType: unknown type " + type);
440  return null;
441  }
442  }
443 
444  private static TCompletionHintType hintTypeToThrift(final SqlMonikerType type) {
445  switch (type) {
446  case COLUMN:
447  return TCompletionHintType.COLUMN;
448  case TABLE:
449  return TCompletionHintType.TABLE;
450  case VIEW:
451  return TCompletionHintType.VIEW;
452  case SCHEMA:
453  return TCompletionHintType.SCHEMA;
454  case CATALOG:
455  return TCompletionHintType.CATALOG;
456  case REPOSITORY:
457  return TCompletionHintType.REPOSITORY;
458  case FUNCTION:
459  return TCompletionHintType.FUNCTION;
460  case KEYWORD:
461  return TCompletionHintType.KEYWORD;
462  default:
463  return null;
464  }
465  }
466 }
static ExtensionFunction.ExtArgumentType toExtArgumentType(TExtArgumentType type)
void updateMetadata(String catalog, String table)
Map< String, ExtensionFunction > udfRTSigs
static TCompletionHintType hintTypeToThrift(final SqlMonikerType type)
static ExtensionFunction toExtensionFunction(TUserDefinedTableFunction udtf)
static ExtensionFunction toExtensionFunction(TUserDefinedFunction udf)
TPlanResult process(String user, String session, String catalog, String sqlText, java.util.List< TFilterPushDownInfo > thriftFilterPushDownInfo, boolean legacySyntax, boolean isExplain, boolean isViewOptimize)
CalciteServerHandler(int mapdPort, String dataDir, String extensionFunctionsAstFile, SockTransportProperties skT, String udfAstFile)
void setRuntimeExtensionFunctions(List< TUserDefinedFunction > udfs, List< TUserDefinedTableFunction > udtfs)
List< TCompletionHint > getCompletionHints(String user, String session, String catalog, List< String > visible_tables, String sql, int cursor)