16 package com.mapd.metadata;
18 import com.google.common.collect.ImmutableList;
19 import com.google.gson.Gson;
20 import com.google.gson.JsonArray;
21 import com.google.gson.JsonElement;
22 import com.google.gson.JsonObject;
28 import com.omnisci.thrift.server.OmniSci;
29 import com.omnisci.thrift.server.TColumnType;
30 import com.omnisci.thrift.server.TDatumType;
31 import com.omnisci.thrift.server.TEncodingType;
32 import com.omnisci.thrift.server.TOmniSciException;
33 import com.omnisci.thrift.server.TTableDetails;
34 import com.omnisci.thrift.server.TTypeInfo;
36 import org.apache.calcite.schema.Table;
37 import org.apache.thrift.TException;
38 import org.apache.thrift.protocol.TBinaryProtocol;
39 import org.apache.thrift.protocol.TProtocol;
40 import org.apache.thrift.transport.TServerSocket;
41 import org.apache.thrift.transport.TSocket;
42 import org.apache.thrift.transport.TTransport;
43 import org.apache.thrift.transport.TTransportException;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
48 import java.io.FileInputStream;
49 import java.io.IOException;
50 import java.sql.Connection;
51 import java.sql.DriverManager;
52 import java.sql.ResultSet;
53 import java.sql.SQLException;
54 import java.sql.Statement;
55 import java.util.HashSet;
56 import java.util.List;
58 import java.util.Map.Entry;
60 import java.util.concurrent.ConcurrentHashMap;
67 final static Logger
MAPDLOGGER = LoggerFactory.getLogger(MetaConnect.class);
69 private final String
db;
76 private static final int KCHAR = 2;
80 private static final int KINT = 6;
84 private static final int KTIME = 10;
87 private static final int KTEXT = 13;
88 private static final int KDATE = 14;
89 private static final int KARRAY = 15;
92 private static final int KPOINT = 18;
99 new ConcurrentHashMap<>();
101 new ConcurrentHashMap<>();
110 if (currentMapDUser != null) {
111 this.db = currentMapDUser.getDB();
115 this.currentUser = currentMapDUser;
118 this.sock_transport_properties = skT;
124 Class.forName(
"org.sqlite.JDBC");
125 }
catch (ClassNotFoundException ex) {
126 String err =
"Could not find class for metadata connection; DB: '" +
db
127 +
"' data dir '" +
dataDir +
"', error was " + ex.getMessage();
128 MAPDLOGGER.error(err);
129 throw new RuntimeException(err);
131 String connectURL =
"jdbc:sqlite:" +
dataDir +
"/mapd_catalogs/" +
db;
133 catConn = DriverManager.getConnection(connectURL);
134 }
catch (SQLException ex) {
135 String err =
"Could not establish a connection for metadata; DB: '" + db
136 +
"' data dir '" +
dataDir +
"', error was " + ex.getMessage();
137 MAPDLOGGER.error(err);
138 throw new RuntimeException(err);
140 MAPDLOGGER.debug(
"Opened database successfully");
146 }
catch (SQLException ex) {
147 String err =
"Could not disconnect for metadata; DB: '" +
db +
"' data dir '"
148 +
dataDir +
"', error was " + ex.getMessage();
149 MAPDLOGGER.error(err);
150 throw new RuntimeException(err);
155 List<String> dbTable = ImmutableList.of(db.toUpperCase(), tableName.toUpperCase());
156 Table cTable = MAPD_TABLE_DETAILS.get(dbTable);
157 if (cTable != null) {
163 if (td.getView_sql() == null || td.getView_sql().isEmpty()) {
164 MAPDLOGGER.debug(
"Processing a table");
166 MAPD_TABLE_DETAILS.putIfAbsent(dbTable, rTable);
169 MAPDLOGGER.debug(
"Processing a view");
171 MAPD_TABLE_DETAILS.putIfAbsent(dbTable, rTable);
177 Set<String> mSet = MAPD_DATABASE_TO_TABLES.get(db.toUpperCase());
187 MAPD_DATABASE_TO_TABLES.putIfAbsent(db.toUpperCase(), ts);
192 TProtocol protocol = null;
193 TTransport transport =
194 sock_transport_properties.openClientTransport(
"localhost",
mapdPort);
195 if (!transport.isOpen()) transport.open();
196 protocol =
new TBinaryProtocol(transport);
198 OmniSci.Client client =
new OmniSci.Client(protocol);
200 List<String> tablesList = client.get_tables(currentUser.getSession());
201 Set<String> ts =
new HashSet<String>(tablesList.size());
202 for (String tableName : tablesList) {
207 MAPD_DATABASE_TO_TABLES.putIfAbsent(db.toUpperCase(), ts);
210 }
catch (TTransportException ex) {
211 MAPDLOGGER.error(
"TTransportException on port [" +
mapdPort +
"]");
212 MAPDLOGGER.error(ex.toString());
213 throw new RuntimeException(ex.toString());
214 }
catch (TOmniSciException ex) {
215 MAPDLOGGER.error(ex.toString());
216 throw new RuntimeException(ex.toString());
217 }
catch (TException ex) {
218 MAPDLOGGER.error(ex.toString());
219 throw new RuntimeException(ex.toString());
225 Set<String> tableSet =
new HashSet<String>();
226 Statement stmt = null;
230 stmt = catConn.createStatement();
233 rs = stmt.executeQuery(
"SELECT name FROM mapd_tables ");
235 tableSet.add(rs.getString(
"name"));
237 MAPDLOGGER.debug(
"Object name = " + rs.getString(
"name"));
242 }
catch (Exception e) {
243 String err =
"error trying to get all the tables, error was " + e.getMessage();
244 MAPDLOGGER.error(err);
245 throw new RuntimeException(err);
251 final String filePath =
dataDir +
"/mapd_catalogs/" +
db +
"_temp_tables.json";
252 MAPDLOGGER.debug(
"Opening temp table file at " + filePath);
253 String tempTablesJsonStr;
255 File tempTablesFile =
new File(filePath);
256 FileInputStream tempTablesStream =
new FileInputStream(tempTablesFile);
257 byte[] data =
new byte[(int) tempTablesFile.length()];
258 tempTablesStream.read(data);
259 tempTablesStream.close();
261 tempTablesJsonStr =
new String(data,
"UTF-8");
262 }
catch (java.io.FileNotFoundException e) {
266 Gson gson =
new Gson();
267 JsonObject fileParentObject = gson.fromJson(tempTablesJsonStr, JsonObject.class);
268 for (Entry<String, JsonElement> member : fileParentObject.entrySet()) {
269 String tableName = member.getKey();
270 tableSet.add(tableName);
272 MAPDLOGGER.debug(
"Temp table object name = " + tableName);
275 }
catch (Exception e) {
276 String err =
"error trying to load temporary tables from json file, error was "
278 MAPDLOGGER.error(err);
279 throw new RuntimeException(err);
295 TProtocol protocol = null;
297 TTransport transport =
298 sock_transport_properties.openClientTransport(
"localhost",
mapdPort);
299 if (!transport.isOpen()) transport.open();
300 protocol =
new TBinaryProtocol(transport);
302 OmniSci.Client client =
new OmniSci.Client(protocol);
305 client.get_internal_table_details(currentUser.getSession(), tableName);
311 }
catch (TTransportException ex) {
312 MAPDLOGGER.error(ex.toString());
313 throw new RuntimeException(ex.toString());
314 }
catch (TOmniSciException ex) {
315 MAPDLOGGER.error(ex.toString());
316 throw new RuntimeException(ex.toString());
317 }
catch (TException ex) {
318 MAPDLOGGER.error(ex.toString());
319 throw new RuntimeException(ex.toString());
345 TTableDetails td =
new TTableDetails();
346 td.getRow_descIterator();
351 }
catch (Exception e) {
352 String err =
"Table '" + tableName +
"' does not exist for DB '" +
db +
"'";
353 MAPDLOGGER.error(err);
354 throw new RuntimeException(err);
359 Statement stmt = null;
362 stmt = catConn.createStatement();
363 MAPDLOGGER.debug(
"table id is " + id);
364 MAPDLOGGER.debug(
"table name is " + tableName);
365 String query = String.format(
366 "SELECT * FROM mapd_columns where tableid = %d and not is_deletedcol order by columnid;",
368 MAPDLOGGER.debug(query);
369 rs = stmt.executeQuery(query);
370 int skip_physical_cols = 0;
372 String colName = rs.getString(
"name");
373 MAPDLOGGER.debug(
"name = " + colName);
374 int colType = rs.getInt(
"coltype");
375 MAPDLOGGER.debug(
"coltype = " + colType);
376 int colSubType = rs.getInt(
"colsubtype");
377 MAPDLOGGER.debug(
"colsubtype = " + colSubType);
378 int colDim = rs.getInt(
"coldim");
379 MAPDLOGGER.debug(
"coldim = " + colDim);
380 int colScale = rs.getInt(
"colscale");
381 MAPDLOGGER.debug(
"colscale = " + colScale);
382 boolean isNotNull = rs.getBoolean(
"is_notnull");
383 MAPDLOGGER.debug(
"is_notnull = " + isNotNull);
384 boolean isSystemCol = rs.getBoolean(
"is_systemcol");
385 MAPDLOGGER.debug(
"is_systemcol = " + isSystemCol);
386 boolean isVirtualCol = rs.getBoolean(
"is_virtualcol");
387 MAPDLOGGER.debug(
"is_vitrualcol = " + isVirtualCol);
388 MAPDLOGGER.debug(
"");
389 TColumnType tct =
new TColumnType();
390 TTypeInfo tti =
new TTypeInfo();
397 tti.is_array =
false;
401 tti.nullable = !isNotNull;
402 tti.encoding = TEncodingType.NONE;
404 tti.scale = colScale;
405 tti.precision = colDim;
407 tct.col_name = colName;
409 tct.is_system = isSystemCol;
412 if (
is_geometry(colType) || skip_physical_cols-- <= 0) td.addToRow_desc(tct);
414 }
catch (Exception e) {
415 String err =
"error trying to read from mapd_columns, error was " + e.getMessage();
416 MAPDLOGGER.error(err);
417 throw new RuntimeException(err);
422 }
catch (SQLException ex) {
423 String err =
"Could not close resultset, error was " + ex.getMessage();
424 MAPDLOGGER.error(err);
425 throw new RuntimeException(err);
431 }
catch (SQLException ex) {
432 String err =
"Could not close stmt, error was " + ex.getMessage();
433 MAPDLOGGER.error(err);
434 throw new RuntimeException(err);
439 td.setView_sqlIsSet(
true);
446 throws IOException, RuntimeException {
447 TTableDetails td =
new TTableDetails();
448 td.getRow_descIterator();
451 final String filePath =
dataDir +
"/mapd_catalogs/" +
db +
"_temp_tables.json";
452 MAPDLOGGER.debug(
"Opening temp table file at " + filePath);
454 String tempTablesJsonStr;
456 File tempTablesFile =
new File(filePath);
457 FileInputStream tempTablesStream =
new FileInputStream(tempTablesFile);
458 byte[] data =
new byte[(int) tempTablesFile.length()];
459 tempTablesStream.read(data);
460 tempTablesStream.close();
462 tempTablesJsonStr =
new String(data,
"UTF-8");
463 }
catch (java.io.FileNotFoundException e) {
464 throw new RuntimeException(
"Failed to read temporary tables file.");
467 Gson gson =
new Gson();
468 JsonObject fileParentObject = gson.fromJson(tempTablesJsonStr, JsonObject.class);
469 if (fileParentObject == null) {
470 throw new IOException(
"Malformed temporary tables file.");
473 JsonObject tableObject = fileParentObject.getAsJsonObject(tableName);
474 if (tableObject == null) {
475 throw new RuntimeException(
476 "Failed to find table " + tableName +
" in temporary tables file.");
479 String jsonTableName = tableObject.get(
"name").getAsString();
480 assert (tableName == jsonTableName);
481 int id = tableObject.get(
"id").getAsInt();
482 MAPDLOGGER.debug(
"table id is " + id);
483 MAPDLOGGER.debug(
"table name is " + tableName);
485 JsonArray jsonColumns = tableObject.getAsJsonArray(
"columns");
486 assert (jsonColumns != null);
488 int skip_physical_cols = 0;
489 for (JsonElement columnElement : jsonColumns) {
490 JsonObject columnObject = columnElement.getAsJsonObject();
492 String colName = columnObject.get(
"name").getAsString();
493 MAPDLOGGER.debug(
"name = " + colName);
494 int colType = columnObject.get(
"coltype").getAsInt();
495 MAPDLOGGER.debug(
"coltype = " + colType);
496 int colSubType = columnObject.get(
"colsubtype").getAsInt();
497 MAPDLOGGER.debug(
"colsubtype = " + colSubType);
498 int colDim = columnObject.get(
"coldim").getAsInt();
499 MAPDLOGGER.debug(
"coldim = " + colDim);
500 int colScale = columnObject.get(
"colscale").getAsInt();
501 MAPDLOGGER.debug(
"colscale = " + colScale);
502 boolean isNotNull = columnObject.get(
"is_notnull").getAsBoolean();
503 MAPDLOGGER.debug(
"is_notnull = " + isNotNull);
504 boolean isSystemCol = columnObject.get(
"is_systemcol").getAsBoolean();
505 MAPDLOGGER.debug(
"is_systemcol = " + isSystemCol);
506 boolean isVirtualCol = columnObject.get(
"is_virtualcol").getAsBoolean();
507 MAPDLOGGER.debug(
"is_vitrualcol = " + isVirtualCol);
508 boolean isDeletedCol = columnObject.get(
"is_deletedcol").getAsBoolean();
509 MAPDLOGGER.debug(
"is_deletedcol = " + isDeletedCol);
510 MAPDLOGGER.debug(
"");
513 MAPDLOGGER.debug(
"Skipping delete column.");
517 TColumnType tct =
new TColumnType();
518 TTypeInfo tti =
new TTypeInfo();
525 tti.is_array =
false;
529 tti.nullable = !isNotNull;
530 tti.encoding = TEncodingType.NONE;
532 tti.scale = colScale;
533 tti.precision = colDim;
535 tct.col_name = colName;
537 tct.is_system = isSystemCol;
540 if (
is_geometry(colType) || skip_physical_cols-- <= 0) td.addToRow_desc(tct);
547 Statement stmt = null;
551 stmt = catConn.createStatement();
552 rs = stmt.executeQuery(String.format(
553 "SELECT tableid FROM mapd_tables where name = '%s' COLLATE NOCASE;",
556 tableId = rs.getInt(
"tableid");
557 MAPDLOGGER.debug(
"tableId = " + tableId);
558 MAPDLOGGER.debug(
"");
562 }
catch (Exception e) {
563 String err =
"Error trying to read from metadata table mapd_tables;DB: " +
db
564 +
" data dir " +
dataDir +
", error was " + e.getMessage();
565 MAPDLOGGER.error(err);
566 throw new RuntimeException(err);
571 }
catch (SQLException ex) {
572 String err =
"Could not close resultset, error was " + ex.getMessage();
573 MAPDLOGGER.error(err);
574 throw new RuntimeException(err);
580 }
catch (SQLException ex) {
581 String err =
"Could not close stmt, error was " + ex.getMessage();
582 MAPDLOGGER.error(err);
583 throw new RuntimeException(err);
590 private boolean isView(String tableName) {
595 stmt = catConn.createStatement();
596 rs = stmt.executeQuery(String.format(
597 "SELECT isview FROM mapd_tables where name = '%s' COLLATE NOCASE;",
600 viewFlag = rs.getInt(
"isview");
601 MAPDLOGGER.debug(
"viewFlag = " + viewFlag);
602 MAPDLOGGER.debug(
"");
606 }
catch (Exception e) {
607 String err =
"error trying to read from mapd_views, error was " + e.getMessage();
608 MAPDLOGGER.error(err);
609 throw new RuntimeException(err);
611 return (viewFlag == 1);
624 TProtocol protocol = null;
626 TTransport transport =
627 sock_transport_properties.openClientTransport(
"localhost",
mapdPort);
628 if (!transport.isOpen()) transport.open();
629 protocol =
new TBinaryProtocol(transport);
631 OmniSci.Client client =
new OmniSci.Client(protocol);
633 TTableDetails td = client.get_table_details(currentUser.getSession(), tableName);
637 sqlText = td.getView_sql();
639 }
catch (TTransportException ex) {
640 MAPDLOGGER.error(ex.toString());
641 throw new RuntimeException(ex.toString());
642 }
catch (TOmniSciException ex) {
643 MAPDLOGGER.error(ex.toString());
644 throw new RuntimeException(ex.toString());
645 }
catch (TException ex) {
646 MAPDLOGGER.error(ex.toString());
647 throw new RuntimeException(ex.toString());
651 if (sqlText.charAt(sqlText.length() - 1) ==
';') {
652 return (sqlText.substring(0, sqlText.length() - 1));
664 stmt = catConn.createStatement();
665 rs = stmt.executeQuery(String.format(
666 "SELECT sql FROM mapd_views where tableid = '%s' COLLATE NOCASE;",
669 sqlText = rs.getString(
"sql");
670 MAPDLOGGER.debug(
"View definition = " + sqlText);
671 MAPDLOGGER.debug(
"");
675 }
catch (Exception e) {
676 String err =
"error trying to read from mapd_views, error was " + e.getMessage();
677 MAPDLOGGER.error(err);
678 throw new RuntimeException(err);
680 if (sqlText == null || sqlText.length() == 0) {
681 String err =
"No view text found";
682 MAPDLOGGER.error(err);
683 throw new RuntimeException(err);
691 return TDatumType.BOOL;
693 return TDatumType.TINYINT;
695 return TDatumType.SMALLINT;
697 return TDatumType.INT;
699 return TDatumType.BIGINT;
701 return TDatumType.FLOAT;
704 return TDatumType.DECIMAL;
706 return TDatumType.DOUBLE;
710 return TDatumType.STR;
712 return TDatumType.TIME;
714 return TDatumType.TIMESTAMP;
716 return TDatumType.DATE;
718 return TDatumType.INTERVAL_DAY_TIME;
720 return TDatumType.INTERVAL_YEAR_MONTH;
722 return TDatumType.POINT;
724 return TDatumType.LINESTRING;
726 return TDatumType.POLYGON;
728 return TDatumType.MULTIPOLYGON;
738 if (table.equals(
"")) {
741 Set<List<String>>
all =
new HashSet<>(MAPD_TABLE_DETAILS.keySet());
742 for (List<String> keys : all) {
743 if (keys.get(0).equals(schema.toUpperCase())) {
744 MAPDLOGGER.debug(
"removing schema " + keys.get(0) +
" table " + keys.get(1));
745 MAPD_TABLE_DETAILS.remove(keys);
749 MAPDLOGGER.debug(
"removing schema " + schema.toUpperCase() +
" table "
750 + table.toUpperCase());
751 MAPD_TABLE_DETAILS.remove(
752 ImmutableList.of(schema.toUpperCase(), table.toUpperCase()));
755 Set<List<String>>
all =
new HashSet<>(MAPD_TABLE_DETAILS.keySet());
756 for (List<String> keys : all) {
757 if (keys.get(0).equals(schema.toUpperCase())) {
758 Table ttable = MAPD_TABLE_DETAILS.get(keys);
761 "removing view in schema " + keys.get(0) +
" view " + keys.get(1));
762 MAPD_TABLE_DETAILS.remove(keys);
767 MAPDLOGGER.debug(
"removing schema " + schema.toUpperCase());
768 MAPD_DATABASE_TO_TABLES.remove(schema.toUpperCase());