17 package com.mapd.parser.server;
19 import static com.mapd.calcite.parser.HeavyDBParser.CURRENT_PARSER;
29 import org.apache.calcite.runtime.CalciteContextException;
30 import org.apache.calcite.sql.SqlNode;
31 import org.apache.calcite.sql.parser.SqlParseException;
32 import org.apache.calcite.sql.type.SqlTypeName;
33 import org.apache.calcite.sql.validate.SqlMoniker;
34 import org.apache.calcite.sql.validate.SqlMonikerType;
35 import org.apache.calcite.tools.RelConversionException;
36 import org.apache.calcite.tools.ValidationException;
37 import org.apache.calcite.util.Pair;
38 import org.apache.commons.pool.PoolableObjectFactory;
39 import org.apache.commons.pool.impl.GenericObjectPool;
40 import org.apache.thrift.TException;
41 import org.apache.thrift.server.TServer;
42 import org.slf4j.Logger;
43 import org.slf4j.LoggerFactory;
45 import java.io.IOException;
46 import java.util.ArrayList;
47 import java.util.Arrays;
48 import java.util.HashMap;
49 import java.util.List;
51 import java.util.stream.Collectors;
53 import ai.heavy.thrift.calciteserver.CalciteServer;
54 import ai.heavy.thrift.calciteserver.InvalidParseRequest;
55 import ai.heavy.thrift.calciteserver.TAccessedQueryObjects;
56 import ai.heavy.thrift.calciteserver.TCompletionHint;
57 import ai.heavy.thrift.calciteserver.TCompletionHintType;
58 import ai.heavy.thrift.calciteserver.TExtArgumentType;
59 import ai.heavy.thrift.calciteserver.TFilterPushDownInfo;
60 import ai.heavy.thrift.calciteserver.TOptimizationOption;
61 import ai.heavy.thrift.calciteserver.TPlanResult;
62 import ai.heavy.thrift.calciteserver.TQueryParsingOption;
63 import ai.heavy.thrift.calciteserver.TRestriction;
64 import ai.heavy.thrift.calciteserver.TUserDefinedFunction;
65 import ai.heavy.thrift.calciteserver.TUserDefinedTableFunction;
68 final static Logger
HEAVYDBLOGGER = LoggerFactory.getLogger(CalciteServerHandler.class);
86 Map<String, ExtensionFunction>
udtfSigs = null;
89 private Map<String, ExtensionFunction>
extSigs = null;
96 String extensionFunctionsAstFile,
102 Map<String, ExtensionFunction> udfSigs = null;
105 extSigs = ExtensionFunctionSignatureParser.parse(extensionFunctionsAstFile);
106 }
catch (IOException ex) {
108 "Could not load extension function signatures: " + ex.getMessage(), ex);
113 if (!udfAstFile.isEmpty()) {
114 udfSigs = ExtensionFunctionSignatureParser.parseUdfAst(udfAstFile);
116 }
catch (IOException ex) {
118 "Could not load udf function signatures: " + ex.getMessage(), ex);
120 udfSigsJson = ExtensionFunctionSignatureParser.signaturesToJson(udfSigs);
124 if (!udfAstFile.isEmpty()) {
125 extSigs.putAll(udfSigs);
135 public void ping() throws TException {
136 HEAVYDBLOGGER.debug(
"Ping hit");
144 TQueryParsingOption queryParsingOption,
145 TOptimizationOption optimizationOption,
146 List<TRestriction> trestrictions)
throws InvalidParseRequest, TException {
147 long timer = System.currentTimeMillis();
154 }
catch (Exception ex) {
155 String msg =
"Could not get Parse Item from pool: " + ex.getMessage();
156 HEAVYDBLOGGER.error(msg, ex);
157 throw new InvalidParseRequest(-1, msg);
159 List<Restriction> rests = null;
160 if (trestrictions != null && !trestrictions.isEmpty()) {
161 rests =
new ArrayList<>();
162 for (TRestriction trestriction : trestrictions) {
167 trestriction.values);
172 HEAVYDBLOGGER.debug(
"process was called User: " + user +
" Catalog: " + catalog
173 +
" sql: " + queryText);
174 parser.setUser(dbUser);
175 CURRENT_PARSER.set(
parser);
179 boolean buildRATreeFromRAString =
false;
180 if (queryText.startsWith(
"execute calcite")) {
181 queryText = queryText.replaceFirst(
"execute calcite",
"");
182 buildRATreeFromRAString =
true;
186 queryText = queryText.trim();
188 if (queryText.length() > 0 && queryText.charAt(queryText.length() - 1) ==
';') {
189 queryText = queryText.substring(0, queryText.length() - 1);
193 TAccessedQueryObjects primaryAccessedObjects =
new TAccessedQueryObjects();
194 TAccessedQueryObjects resolvedAccessedObjects =
new TAccessedQueryObjects();
196 final List<HeavyDBParserOptions.FilterPushDownInfo> filterPushDownInfo =
198 for (
final TFilterPushDownInfo req : optimizationOption.filter_push_down_info) {
199 filterPushDownInfo.add(
new HeavyDBParserOptions.FilterPushDownInfo(
200 req.input_prev, req.input_start, req.input_next));
203 queryParsingOption.legacy_syntax,
204 queryParsingOption.is_explain,
205 optimizationOption.is_view_optimize,
206 optimizationOption.enable_watchdog,
207 optimizationOption.distributed_mode);
209 if (!buildRATreeFromRAString) {
210 Pair<String, SqlIdentifierCapturer>
res;
213 res = parser.process(queryText, parserOptions);
214 jsonResult = res.left;
215 capturer = res.right;
217 primaryAccessedObjects.tables_selected_from =
new ArrayList<>(capturer.selects);
218 primaryAccessedObjects.tables_inserted_into =
new ArrayList<>(capturer.inserts);
219 primaryAccessedObjects.tables_updated_in =
new ArrayList<>(capturer.updates);
220 primaryAccessedObjects.tables_deleted_from =
new ArrayList<>(capturer.deletes);
225 resolvedAccessedObjects.tables_selected_from =
226 new ArrayList<>(parser.resolveSelectIdentifiers(capturer));
227 resolvedAccessedObjects.tables_inserted_into =
new ArrayList<>(capturer.inserts);
228 resolvedAccessedObjects.tables_updated_in =
new ArrayList<>(capturer.updates);
229 resolvedAccessedObjects.tables_deleted_from =
new ArrayList<>(capturer.deletes);
234 parser.buildRATreeAndPerformQueryOptimization(queryText, parserOptions);
236 }
catch (SqlParseException ex) {
237 String msg =
"SQL Error: " + ex.getMessage();
238 HEAVYDBLOGGER.error(msg);
239 throw new InvalidParseRequest(-2, msg);
240 }
catch (org.apache.calcite.tools.ValidationException ex) {
241 String msg =
"SQL Error: " + ex.getMessage();
242 if (ex.getCause() != null
243 && (ex.getCause().getClass() == CalciteContextException.class)) {
244 msg =
"SQL Error: " + ex.getCause().getMessage();
246 HEAVYDBLOGGER.error(msg);
247 throw new InvalidParseRequest(-3, msg);
248 }
catch (CalciteContextException ex) {
249 String msg = ex.getMessage();
250 HEAVYDBLOGGER.error(msg);
251 throw new InvalidParseRequest(-6, msg);
252 }
catch (RelConversionException ex) {
253 String msg =
"Failed to generate relational algebra for query " + ex.getMessage();
254 HEAVYDBLOGGER.error(msg, ex);
255 throw new InvalidParseRequest(-5, msg);
256 }
catch (Throwable ex) {
257 HEAVYDBLOGGER.error(ex.getClass().
toString());
258 String msg = ex.getMessage();
259 HEAVYDBLOGGER.error(msg, ex);
260 throw new InvalidParseRequest(-4, msg);
262 CURRENT_PARSER.set(null);
265 parserPool.returnObject(
parser);
266 }
catch (Exception ex) {
267 String msg =
"Could not return parse object: " + ex.getMessage();
268 HEAVYDBLOGGER.error(msg, ex);
269 throw new InvalidParseRequest(-7, msg);
273 TPlanResult
result =
new TPlanResult();
274 result.primary_accessed_objects = primaryAccessedObjects;
275 result.resolved_accessed_objects = resolvedAccessedObjects;
276 result.plan_result = jsonResult;
277 result.execution_time_ms = System.currentTimeMillis() - timer;
285 HEAVYDBLOGGER.debug(
"Shutdown calcite java server");
291 return this.extSigsJson;
296 return this.udfSigsJson;
301 return this.udfRTSigsJson;
312 "Received invalidation from server for " + catalog +
" : " + table);
313 long timer = System.currentTimeMillis();
318 }
catch (Exception ex) {
319 String msg =
"Could not get Parse Item from pool: " + ex.getMessage();
320 HEAVYDBLOGGER.error(msg, ex);
323 CURRENT_PARSER.set(
parser);
325 parser.updateMetaData(catalog, table);
327 CURRENT_PARSER.set(null);
330 HEAVYDBLOGGER.debug(
"Returning object to pool");
331 parserPool.returnObject(
parser);
332 }
catch (Exception ex) {
333 String msg =
"Could not return parse object: " + ex.getMessage();
334 HEAVYDBLOGGER.error(msg, ex);
343 List<String> visible_tables,
345 int cursor)
throws TException {
350 }
catch (Exception ex) {
351 String msg =
"Could not get Parse Item from pool: " + ex.getMessage();
352 HEAVYDBLOGGER.error(msg, ex);
353 throw new TException(msg);
356 HEAVYDBLOGGER.debug(
"getCompletionHints was called User: " + user
357 +
" Catalog: " + catalog +
" sql: " + sql);
358 parser.setUser(dbUser);
359 CURRENT_PARSER.set(
parser);
361 HeavyDBPlanner.CompletionResult completion_result;
363 completion_result = parser.getCompletionHints(sql, cursor, visible_tables);
364 }
catch (Exception ex) {
365 String msg =
"Could not retrieve completion hints: " + ex.getMessage();
366 HEAVYDBLOGGER.error(msg, ex);
367 return new ArrayList<>();
369 CURRENT_PARSER.set(null);
372 parserPool.returnObject(
parser);
373 }
catch (Exception ex) {
374 String msg =
"Could not return parse object: " + ex.getMessage();
375 HEAVYDBLOGGER.error(msg, ex);
376 throw new InvalidParseRequest(-4, msg);
379 List<TCompletionHint>
result =
new ArrayList<>();
380 for (
final SqlMoniker hint : completion_result.hints) {
382 hint.getFullyQualifiedNames(),
383 completion_result.replaced));
390 List<TUserDefinedTableFunction> udtfs,
399 udfRTSigs =
new HashMap<String, ExtensionFunction>();
402 for (TUserDefinedFunction udf : udfs) {
406 for (TUserDefinedTableFunction udtf : udtfs) {
413 HEAVYDBLOGGER.error(
"Extension function `" +
name
414 +
"` exists. Skipping runtime extenension function with the same name.");
415 udfRTSigs.remove(
name);
425 udtfSigs =
new HashMap<String, ExtensionFunction>();
428 for (TUserDefinedTableFunction udtf : udtfs) {
435 calciteParserFactory.updateOperatorTable();
439 TUserDefinedFunction udf,
boolean isruntime) {
440 List<ExtensionFunction.ExtArgumentType>
args =
441 new ArrayList<ExtensionFunction.ExtArgumentType>();
442 for (TExtArgumentType atype : udf.argTypes) {
452 TUserDefinedTableFunction udtf,
boolean isruntime) {
453 int sqlInputArgIdx = 0;
455 int outputArgIdx = 0;
456 List<String> names =
new ArrayList<String>();
457 List<ExtensionFunction.ExtArgumentType>
args =
new ArrayList<>();
458 Map<String, List<ExtensionFunction.ExtArgumentType>> cursor_field_types =
460 for (TExtArgumentType atype : udtf.sqlArgTypes) {
462 Map<String, String> annot = udtf.annotations.get(sqlInputArgIdx);
463 String
name = annot.getOrDefault(
"name",
"inp" + sqlInputArgIdx);
464 if (atype == TExtArgumentType.Cursor) {
465 String field_names_annot = annot.getOrDefault(
"fields",
"");
466 List<ExtensionFunction.ExtArgumentType> field_types =
new ArrayList<>();
467 if (field_names_annot.length() > 0) {
468 String[] field_names =
469 field_names_annot.substring(1, field_names_annot.length() - 1)
471 for (
int i = 0; i < field_names.length; i++) {
481 name = name + field_names_annot;
482 cursor_field_types.put(
name, field_types);
490 List<ExtensionFunction.ExtArgumentType> outs =
new ArrayList<>();
491 for (TExtArgumentType otype : udtf.outputArgTypes) {
493 Map<String, String> annot = udtf.annotations.get(sqlInputArgIdx);
494 names.add(annot.getOrDefault(
"name",
"out" + outputArgIdx));
501 udtf.annotations.get(udtf.annotations.size() - 1),
506 TExtArgumentType
type) {
509 return ExtensionFunction.ExtArgumentType.Int8;
511 return ExtensionFunction.ExtArgumentType.Int16;
513 return ExtensionFunction.ExtArgumentType.Int32;
515 return ExtensionFunction.ExtArgumentType.Int64;
517 return ExtensionFunction.ExtArgumentType.Float;
521 return ExtensionFunction.ExtArgumentType.Void;
523 return ExtensionFunction.ExtArgumentType.PInt8;
525 return ExtensionFunction.ExtArgumentType.PInt16;
527 return ExtensionFunction.ExtArgumentType.PInt32;
529 return ExtensionFunction.ExtArgumentType.PInt64;
531 return ExtensionFunction.ExtArgumentType.PFloat;
533 return ExtensionFunction.ExtArgumentType.PDouble;
535 return ExtensionFunction.ExtArgumentType.PBool;
537 return ExtensionFunction.ExtArgumentType.Bool;
539 return ExtensionFunction.ExtArgumentType.ArrayInt8;
541 return ExtensionFunction.ExtArgumentType.ArrayInt16;
543 return ExtensionFunction.ExtArgumentType.ArrayInt32;
545 return ExtensionFunction.ExtArgumentType.ArrayInt64;
547 return ExtensionFunction.ExtArgumentType.ArrayFloat;
549 return ExtensionFunction.ExtArgumentType.ArrayDouble;
551 return ExtensionFunction.ExtArgumentType.ArrayBool;
553 return ExtensionFunction.ExtArgumentType.ArrayTextEncodingDict;
555 return ExtensionFunction.ExtArgumentType.ColumnInt8;
557 return ExtensionFunction.ExtArgumentType.ColumnInt16;
559 return ExtensionFunction.ExtArgumentType.ColumnInt32;
561 return ExtensionFunction.ExtArgumentType.ColumnInt64;
563 return ExtensionFunction.ExtArgumentType.ColumnFloat;
565 return ExtensionFunction.ExtArgumentType.ColumnDouble;
567 return ExtensionFunction.ExtArgumentType.ColumnBool;
569 return ExtensionFunction.ExtArgumentType.ColumnTextEncodingDict;
571 return ExtensionFunction.ExtArgumentType.ColumnTimestamp;
573 return ExtensionFunction.ExtArgumentType.GeoPoint;
575 return ExtensionFunction.ExtArgumentType.GeoMultiPoint;
577 return ExtensionFunction.ExtArgumentType.GeoLineString;
579 return ExtensionFunction.ExtArgumentType.GeoMultiLineString;
581 return ExtensionFunction.ExtArgumentType.Cursor;
583 return ExtensionFunction.ExtArgumentType.GeoPolygon;
585 return ExtensionFunction.ExtArgumentType.GeoMultiPolygon;
587 return ExtensionFunction.ExtArgumentType.TextEncodingNone;
589 return ExtensionFunction.ExtArgumentType.TextEncodingDict;
591 return ExtensionFunction.ExtArgumentType.Timestamp;
593 return ExtensionFunction.ExtArgumentType.ColumnListInt8;
595 return ExtensionFunction.ExtArgumentType.ColumnListInt16;
597 return ExtensionFunction.ExtArgumentType.ColumnListInt32;
599 return ExtensionFunction.ExtArgumentType.ColumnListInt64;
601 return ExtensionFunction.ExtArgumentType.ColumnListFloat;
603 return ExtensionFunction.ExtArgumentType.ColumnListDouble;
605 return ExtensionFunction.ExtArgumentType.ColumnListBool;
607 return ExtensionFunction.ExtArgumentType.ColumnListTextEncodingDict;
609 return ExtensionFunction.ExtArgumentType.ColumnArrayInt8;
611 return ExtensionFunction.ExtArgumentType.ColumnArrayInt16;
613 return ExtensionFunction.ExtArgumentType.ColumnArrayInt32;
615 return ExtensionFunction.ExtArgumentType.ColumnArrayInt64;
617 return ExtensionFunction.ExtArgumentType.ColumnArrayFloat;
619 return ExtensionFunction.ExtArgumentType.ColumnArrayDouble;
621 return ExtensionFunction.ExtArgumentType.ColumnArrayBool;
623 return ExtensionFunction.ExtArgumentType.ColumnArrayTextEncodingDict;
625 return ExtensionFunction.ExtArgumentType.ColumnListArrayInt8;
627 return ExtensionFunction.ExtArgumentType.ColumnListArrayInt16;
629 return ExtensionFunction.ExtArgumentType.ColumnListArrayInt32;
631 return ExtensionFunction.ExtArgumentType.ColumnListArrayInt64;
633 return ExtensionFunction.ExtArgumentType.ColumnListArrayFloat;
635 return ExtensionFunction.ExtArgumentType.ColumnListArrayDouble;
637 return ExtensionFunction.ExtArgumentType.ColumnListArrayBool;
639 return ExtensionFunction.ExtArgumentType.ColumnListArrayTextEncodingDict;
641 return ExtensionFunction.ExtArgumentType.DayTimeInterval;
643 return ExtensionFunction.ExtArgumentType.YearMonthTimeInterval;
645 HEAVYDBLOGGER.error(
"toExtArgumentType: unknown type " +
type);
653 return TCompletionHintType.COLUMN;
655 return TCompletionHintType.TABLE;
657 return TCompletionHintType.VIEW;
659 return TCompletionHintType.SCHEMA;
661 return TCompletionHintType.CATALOG;
663 return TCompletionHintType.REPOSITORY;
665 return TCompletionHintType.FUNCTION;
667 return TCompletionHintType.KEYWORD;
Simplified core of GeoJSON Polygon coordinates definition.
final CalciteParserFactory calciteParserFactory
CalciteServerHandler(int dbPort, String dataDir, String extensionFunctionsAstFile, SockTransportProperties skT, String udfAstFile)
void setRuntimeExtensionFunctions(List< TUserDefinedFunction > udfs, List< TUserDefinedTableFunction > udtfs, boolean isruntime)
Map< String, ExtensionFunction > udtfSigs
Simplified core of GeoJSON MultiPolygon coordinates definition.
static ExtensionFunction.ExtArgumentType toExtArgumentType(TExtArgumentType type)
String getExtensionFunctionWhitelist()
void setServer(TServer s)
void updateMetadata(String catalog, String table)
String getUserDefinedFunctionWhitelist()
final GenericObjectPool parserPool
std::string toString(const ExecutorDeviceType &device_type)
Map< String, ExtensionFunction > extSigs
Map< String, ExtensionFunction > udfRTSigs
static TCompletionHintType hintTypeToThrift(final SqlMonikerType type)
final List< Map< String, String > > annotations
static ExtensionFunction toExtensionFunction(TUserDefinedFunction udf, boolean isruntime)
static ExtensionFunction toExtensionFunction(TUserDefinedTableFunction udtf, boolean isruntime)
static final Logger HEAVYDBLOGGER
String getRuntimeExtensionFunctionWhitelist()
SockTransportProperties skT
TPlanResult process(String user, String session, String catalog, String queryText, TQueryParsingOption queryParsingOption, TOptimizationOption optimizationOption, List< TRestriction > trestrictions)
List< TCompletionHint > getCompletionHints(String user, String session, String catalog, List< String > visible_tables, String sql, int cursor)