OmniSciDB  04ee39c94c
CalciteServerHandler.java
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.mapd.parser.server;
17 
19 
24 import com.mapd.thrift.calciteserver.CalciteServer;
25 import com.mapd.thrift.calciteserver.InvalidParseRequest;
26 import com.mapd.thrift.calciteserver.TAccessedQueryObjects;
27 import com.mapd.thrift.calciteserver.TCompletionHint;
28 import com.mapd.thrift.calciteserver.TCompletionHintType;
29 import com.mapd.thrift.calciteserver.TFilterPushDownInfo;
30 import com.mapd.thrift.calciteserver.TPlanResult;
31 
34 import org.apache.calcite.runtime.CalciteContextException;
35 import org.apache.calcite.sql.parser.SqlParseException;
36 import org.apache.calcite.sql.validate.SqlMoniker;
37 import org.apache.calcite.sql.validate.SqlMonikerType;
38 import org.apache.calcite.tools.RelConversionException;
39 import org.apache.calcite.tools.ValidationException;
40 import org.apache.commons.pool.PoolableObjectFactory;
41 import org.apache.commons.pool.impl.GenericObjectPool;
42 import org.apache.thrift.TException;
43 import org.apache.thrift.server.TServer;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46 
47 import java.io.IOException;
48 import java.util.ArrayList;
49 import java.util.List;
50 import java.util.Map;
51 
56 class CalciteServerHandler implements CalciteServer.Iface {
57  final static Logger MAPDLOGGER = LoggerFactory.getLogger(CalciteServerHandler.class);
58  private TServer server;
59 
60  private final int mapdPort;
61 
62  private volatile long callCount;
63 
64  private final GenericObjectPool parserPool;
65 
66  private final String extSigsJson;
67 
68  private final String udfSigsJson;
69 
70  private String udfRTSigsJson = "";
71  Map<String, ExtensionFunction> udfRTSigs = null;
72 
74  private Map<String, ExtensionFunction> extSigs = null;
75  private String dataDir;
76 
77  // TODO MAT we need to merge this into common code base for these functions with
78  // CalciteDirect since we are not deprecating this stuff yet
79  CalciteServerHandler(int mapdPort,
80  String dataDir,
81  String extensionFunctionsAstFile,
83  String udfAstFile) {
84  this.mapdPort = mapdPort;
85  this.dataDir = dataDir;
86 
87  Map<String, ExtensionFunction> udfSigs = null;
88 
89  try {
90  extSigs = ExtensionFunctionSignatureParser.parse(extensionFunctionsAstFile);
91  } catch (IOException ex) {
92  MAPDLOGGER.error(
93  "Could not load extension function signatures: " + ex.getMessage());
94  }
96 
97  try {
98  if (!udfAstFile.isEmpty()) {
99  udfSigs = ExtensionFunctionSignatureParser.parse(udfAstFile);
100  }
101  } catch (IOException ex) {
102  MAPDLOGGER.error("Could not load udf function signatures: " + ex.getMessage());
103  }
104  udfSigsJson = ExtensionFunctionSignatureParser.signaturesToJson(udfSigs);
105 
106  // Put all the udf functions signatures in extSigs so Calcite has a view of
107  // extension functions and udf functions
108  if (!udfAstFile.isEmpty()) {
109  extSigs.putAll(udfSigs);
110  }
111 
112  PoolableObjectFactory parserFactory =
113  new CalciteParserFactory(dataDir, extSigs, mapdPort, skT);
114  // GenericObjectPool::setFactory is deprecated
115  this.parserPool = new GenericObjectPool(parserFactory);
116  }
117 
118  @Override
119  public void ping() throws TException {
120  MAPDLOGGER.debug("Ping hit");
121  }
122 
123  @Override
124  public TPlanResult process(String user,
125  String session,
126  String catalog,
127  String sqlText,
128  java.util.List<TFilterPushDownInfo> thriftFilterPushDownInfo,
129  boolean legacySyntax,
130  boolean isExplain,
131  boolean isViewOptimize) throws InvalidParseRequest, TException {
132  long timer = System.currentTimeMillis();
133  callCount++;
135  try {
136  parser = (MapDParser) parserPool.borrowObject();
137  } catch (Exception ex) {
138  String msg = "Could not get Parse Item from pool: " + ex.getMessage();
139  MAPDLOGGER.error(msg);
140  throw new InvalidParseRequest(-1, msg);
141  }
142  MapDUser mapDUser = new MapDUser(user, session, catalog, mapdPort);
143  MAPDLOGGER.debug("process was called User: " + user + " Catalog: " + catalog
144  + " sql: " + sqlText);
145  parser.setUser(mapDUser);
146  CURRENT_PARSER.set(parser);
147 
148  // need to trim the sql string as it seems it is not trimed prior to here
149  sqlText = sqlText.trim();
150  // remove last charcter if it is a ;
151  if (sqlText.length() > 0 && sqlText.charAt(sqlText.length() - 1) == ';') {
152  sqlText = sqlText.substring(0, sqlText.length() - 1);
153  }
154  String relAlgebra;
155  SqlIdentifierCapturer capturer;
156  TAccessedQueryObjects primaryAccessedObjects = new TAccessedQueryObjects();
157  TAccessedQueryObjects resolvedAccessedObjects = new TAccessedQueryObjects();
158  try {
159  final List<MapDParserOptions.FilterPushDownInfo> filterPushDownInfo =
160  new ArrayList<>();
161  for (final TFilterPushDownInfo req : thriftFilterPushDownInfo) {
162  filterPushDownInfo.add(new MapDParserOptions.FilterPushDownInfo(
163  req.input_prev, req.input_start, req.input_next));
164  }
165  try {
166  MapDParserOptions parserOptions = new MapDParserOptions(
167  filterPushDownInfo, legacySyntax, isExplain, isViewOptimize);
168  relAlgebra = parser.getRelAlgebra(sqlText, parserOptions, mapDUser);
169  } catch (ValidationException ex) {
170  String msg = "Validation: " + ex.getMessage();
171  MAPDLOGGER.error(msg);
172  throw ex;
173  } catch (RelConversionException ex) {
174  String msg = " RelConversion failed: " + ex.getMessage();
175  MAPDLOGGER.error(msg);
176  throw ex;
177  }
178  capturer = parser.captureIdentifiers(sqlText, legacySyntax);
179 
180  primaryAccessedObjects.tables_selected_from = new ArrayList<>(capturer.selects);
181  primaryAccessedObjects.tables_inserted_into = new ArrayList<>(capturer.inserts);
182  primaryAccessedObjects.tables_updated_in = new ArrayList<>(capturer.updates);
183  primaryAccessedObjects.tables_deleted_from = new ArrayList<>(capturer.deletes);
184 
185  // also resolve all the views in the select part
186  // resolution of the other parts is not
187  // necessary as these cannot be views
188  resolvedAccessedObjects.tables_selected_from =
189  new ArrayList<>(parser.resolveSelectIdentifiers(capturer));
190  resolvedAccessedObjects.tables_inserted_into = new ArrayList<>(capturer.inserts);
191  resolvedAccessedObjects.tables_updated_in = new ArrayList<>(capturer.updates);
192  resolvedAccessedObjects.tables_deleted_from = new ArrayList<>(capturer.deletes);
193 
194  } catch (SqlParseException ex) {
195  String msg = "Parse failed: " + ex.getMessage();
196  MAPDLOGGER.error(msg);
197  throw new InvalidParseRequest(-2, msg);
198  } catch (CalciteContextException ex) {
199  String msg = "Validate failed: " + ex.getMessage();
200  MAPDLOGGER.error(msg);
201  throw new InvalidParseRequest(-3, msg);
202  } catch (Exception ex) {
203  String msg = "Exception occurred: " + ex.getMessage();
204  MAPDLOGGER.error(msg);
205  throw new InvalidParseRequest(-4, msg);
206  } finally {
207  CURRENT_PARSER.set(null);
208  try {
209  // put parser object back in pool for others to use
210  parserPool.returnObject(parser);
211  } catch (Exception ex) {
212  String msg = "Could not return parse object: " + ex.getMessage();
213  MAPDLOGGER.error(msg);
214  throw new InvalidParseRequest(-4, msg);
215  }
216  }
217 
218  TPlanResult result = new TPlanResult();
219  result.primary_accessed_objects = primaryAccessedObjects;
220  result.resolved_accessed_objects = resolvedAccessedObjects;
221  result.plan_result = relAlgebra;
222  result.execution_time_ms = System.currentTimeMillis() - timer;
223 
224  return result;
225  }
226 
227  @Override
228  public void shutdown() throws TException {
229  // received request to shutdown
230  MAPDLOGGER.debug("Shutdown calcite java server");
231  server.stop();
232  }
233 
234  @Override
236  return this.extSigsJson;
237  }
238 
239  @Override
241  return this.udfSigsJson;
242  }
243 
244  @Override
246  return this.udfRTSigsJson;
247  }
248 
249  void setServer(TServer s) {
250  server = s;
251  }
252 
253  @Override
254  public void updateMetadata(String catalog, String table) throws TException {
255  MAPDLOGGER.debug("Received invalidation from server for " + catalog + " : " + table);
256  long timer = System.currentTimeMillis();
257  callCount++;
259  try {
260  parser = (MapDParser) parserPool.borrowObject();
261  } catch (Exception ex) {
262  String msg = "Could not get Parse Item from pool: " + ex.getMessage();
263  MAPDLOGGER.error(msg);
264  return;
265  }
266  CURRENT_PARSER.set(parser);
267  try {
268  parser.updateMetaData(catalog, table);
269  } finally {
270  CURRENT_PARSER.set(null);
271  try {
272  // put parser object back in pool for others to use
273  MAPDLOGGER.debug("Returning object to pool");
274  parserPool.returnObject(parser);
275  } catch (Exception ex) {
276  String msg = "Could not return parse object: " + ex.getMessage();
277  MAPDLOGGER.error(msg);
278  }
279  }
280  }
281 
282  @Override
283  public List<TCompletionHint> getCompletionHints(String user,
284  String session,
285  String catalog,
286  List<String> visible_tables,
287  String sql,
288  int cursor) throws TException {
289  callCount++;
291  try {
292  parser = (MapDParser) parserPool.borrowObject();
293  } catch (Exception ex) {
294  String msg = "Could not get Parse Item from pool: " + ex.getMessage();
295  MAPDLOGGER.error(msg);
296  throw new TException(msg);
297  }
298  MapDUser mapDUser = new MapDUser(user, session, catalog, mapdPort);
299  MAPDLOGGER.debug("getCompletionHints was called User: " + user
300  + " Catalog: " + catalog + " sql: " + sql);
301  parser.setUser(mapDUser);
302  CURRENT_PARSER.set(parser);
303 
304  MapDPlanner.CompletionResult completion_result;
305  try {
306  completion_result = parser.getCompletionHints(sql, cursor, visible_tables);
307  } catch (Exception ex) {
308  String msg = "Could not retrieve completion hints: " + ex.getMessage();
309  MAPDLOGGER.error(msg);
310  return new ArrayList<>();
311  } finally {
312  CURRENT_PARSER.set(null);
313  try {
314  // put parser object back in pool for others to use
315  parserPool.returnObject(parser);
316  } catch (Exception ex) {
317  String msg = "Could not return parse object: " + ex.getMessage();
318  MAPDLOGGER.error(msg);
319  throw new InvalidParseRequest(-4, msg);
320  }
321  }
322  List<TCompletionHint> result = new ArrayList<>();
323  for (final SqlMoniker hint : completion_result.hints) {
324  result.add(new TCompletionHint(hintTypeToThrift(hint.getType()),
325  hint.getFullyQualifiedNames(),
326  completion_result.replaced));
327  }
328  return result;
329  }
330 
331  @Override
332  public void setRuntimeUserDefinedFunction(String udfString) {
333  // Clean up previously defined Runtime UDFs
334  if (udfRTSigs != null) {
335  for (String name : udfRTSigs.keySet()) extSigs.remove(name);
336  udfRTSigsJson = "";
337  }
338 
339  if (!udfString.isEmpty()) {
340  try {
341  udfRTSigs = ExtensionFunctionSignatureParser.parseFromString(udfString);
342  } catch (IOException ex) {
343  MAPDLOGGER.error(
344  "Could not parse extension function signatures: " + ex.getMessage());
345  }
346  // Avoid overwritting compiled and Loadtime UDFs:
347  for (String name : udfRTSigs.keySet()) {
348  if (extSigs.containsKey(name)) {
349  MAPDLOGGER.error("Extension function `" + name
350  + "` exists. Skipping Runtime UDF with the same name.");
351  udfRTSigs.remove(name);
352  }
353  }
354  udfRTSigsJson = ExtensionFunctionSignatureParser.signaturesToJson(udfRTSigs);
355  // Expose RT UDFs to Calcite server:
356  extSigs.putAll(udfRTSigs);
357  }
358  }
359 
360  private static TCompletionHintType hintTypeToThrift(final SqlMonikerType type) {
361  switch (type) {
362  case COLUMN:
363  return TCompletionHintType.COLUMN;
364  case TABLE:
365  return TCompletionHintType.TABLE;
366  case VIEW:
367  return TCompletionHintType.VIEW;
368  case SCHEMA:
369  return TCompletionHintType.SCHEMA;
370  case CATALOG:
371  return TCompletionHintType.CATALOG;
372  case REPOSITORY:
373  return TCompletionHintType.REPOSITORY;
374  case FUNCTION:
375  return TCompletionHintType.FUNCTION;
376  case KEYWORD:
377  return TCompletionHintType.KEYWORD;
378  default:
379  return null;
380  }
381  }
382 }
static Map< String, ExtensionFunction > parseFromString(final String udf_string)
auto sql(const std::string &sql_stmts)
void updateMetaData(String schema, String table)
void updateMetadata(String catalog, String table)
SqlIdentifierCapturer captureIdentifiers(String sql, boolean legacy_syntax)
static Map< String, ExtensionFunction > parse(final String file_path)
Set< String > resolveSelectIdentifiers(SqlIdentifierCapturer capturer)
Map< String, ExtensionFunction > udfRTSigs
void setUser(MapDUser mapdUser)
static TCompletionHintType hintTypeToThrift(final SqlMonikerType type)
static String signaturesToJson(final Map< String, ExtensionFunction > sigs)
static final ThreadLocal< MapDParser > CURRENT_PARSER
Definition: MapDParser.java:90
TSessionId session
MapDPlanner.CompletionResult getCompletionHints(String sql, int cursor, List< String > visible_tables)
String getRelAlgebra(String sql, final MapDParserOptions parserOptions, final MapDUser mapDUser)
TPlanResult process(String user, String session, String catalog, String sqlText, java.util.List< TFilterPushDownInfo > thriftFilterPushDownInfo, boolean legacySyntax, boolean isExplain, boolean isViewOptimize)
CalciteServerHandler(int mapdPort, String dataDir, String extensionFunctionsAstFile, SockTransportProperties skT, String udfAstFile)
List< TCompletionHint > getCompletionHints(String user, String session, String catalog, List< String > visible_tables, String sql, int cursor)