OmniSciDB  fe05a0c208
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
MetaConnect.java
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.mapd.metadata;
17 
18 import com.google.common.collect.ImmutableList;
19 import com.google.gson.Gson;
20 import com.google.gson.JsonArray;
21 import com.google.gson.JsonElement;
22 import com.google.gson.JsonObject;
28 import com.omnisci.thrift.server.OmniSci;
29 import com.omnisci.thrift.server.TColumnType;
30 import com.omnisci.thrift.server.TDBInfo;
31 import com.omnisci.thrift.server.TDatumType;
32 import com.omnisci.thrift.server.TEncodingType;
33 import com.omnisci.thrift.server.TOmniSciException;
34 import com.omnisci.thrift.server.TTableDetails;
35 import com.omnisci.thrift.server.TTypeInfo;
36 
37 import org.apache.calcite.schema.Table;
38 import org.apache.thrift.TException;
39 import org.apache.thrift.protocol.TBinaryProtocol;
40 import org.apache.thrift.protocol.TProtocol;
41 import org.apache.thrift.transport.TServerSocket;
42 import org.apache.thrift.transport.TSocket;
43 import org.apache.thrift.transport.TTransport;
44 import org.apache.thrift.transport.TTransportException;
45 import org.slf4j.Logger;
46 import org.slf4j.LoggerFactory;
47 
48 import java.io.File;
49 import java.io.FileInputStream;
50 import java.io.IOException;
51 import java.sql.Connection;
52 import java.sql.DriverManager;
53 import java.sql.ResultSet;
54 import java.sql.SQLException;
55 import java.sql.Statement;
56 import java.util.ArrayList;
57 import java.util.HashSet;
58 import java.util.List;
59 import java.util.Map;
60 import java.util.Map.Entry;
61 import java.util.Set;
62 import java.util.concurrent.ConcurrentHashMap;
63 
68 public class MetaConnect {
69  final static Logger MAPDLOGGER = LoggerFactory.getLogger(MetaConnect.class);
70  private final String dataDir;
71  private final String default_db;
72  private final MapDUser currentUser;
73  private final int mapdPort;
74  private Connection catConn;
75  private final MapDParser parser;
76 
77  private static final int KBOOLEAN = 1;
78  private static final int KCHAR = 2;
79  private static final int KVARCHAR = 3;
80  private static final int KNUMERIC = 4;
81  private static final int KDECIMAL = 5;
82  private static final int KINT = 6;
83  private static final int KSMALLINT = 7;
84  private static final int KFLOAT = 8;
85  private static final int KDOUBLE = 9;
86  private static final int KTIME = 10;
87  private static final int KTIMESTAMP = 11;
88  private static final int KBIGINT = 12;
89  private static final int KTEXT = 13;
90  private static final int KDATE = 14;
91  private static final int KARRAY = 15;
92  private static final int KINTERVAL_DAY_TIME = 16;
93  private static final int KINTERVAL_YEAR_MONTH = 17;
94  private static final int KPOINT = 18;
95  private static final int KLINESTRING = 19;
96  private static final int KPOLYGON = 20;
97  private static final int KMULTIPOLYGON = 21;
98  private static final int KTINYINT = 22;
99 
100  private static volatile Map<String, Set<String>> DATABASE_TO_TABLES =
101  new ConcurrentHashMap<>();
102  private static volatile Map<List<String>, Table> MAPD_TABLE_DETAILS =
103  new ConcurrentHashMap<>();
105 
106  public MetaConnect(int mapdPort,
107  String dataDir,
108  MapDUser currentMapDUser,
111  String db) {
112  this.dataDir = dataDir;
113  if (db != null) {
114  this.default_db = db;
115  } else {
116  if (currentMapDUser != null) {
117  this.default_db = currentMapDUser.getDB();
118  } else {
119  this.default_db = null;
120  }
121  }
122  this.currentUser = currentMapDUser;
123  this.mapdPort = mapdPort;
124  this.parser = parser;
125  this.sock_transport_properties = skT;
126 
127  // check to see if we have a populated DATABASE_TO_TABLES structure
128  // first time in we need to make sure this gets populated
129  if (DATABASE_TO_TABLES.size() == 0) {
130  // get all databases
132  }
133  }
134 
135  public MetaConnect(int mapdPort,
136  String dataDir,
137  MapDUser currentMapDUser,
140  this(mapdPort, dataDir, currentMapDUser, parser, skT, null);
141  }
142 
143  public List<String> getDatabases() {
144  List<String> dbList = new ArrayList<String>(DATABASE_TO_TABLES.size());
145  for (String db : DATABASE_TO_TABLES.keySet()) {
146  dbList.add(db);
147  }
148  return dbList;
149  }
150 
151  private void connectToCatalog(String catalog) {
152  try {
153  // try {
154  Class.forName("org.sqlite.JDBC");
155  } catch (ClassNotFoundException ex) {
156  String err = "Could not find class for metadata connection; DB: '" + catalog
157  + "' data dir '" + dataDir + "', error was " + ex.getMessage();
158  MAPDLOGGER.error(err);
159  throw new RuntimeException(err);
160  }
161  String connectURL = "jdbc:sqlite:" + dataDir + "/mapd_catalogs/" + catalog;
162  try {
163  catConn = DriverManager.getConnection(connectURL);
164  } catch (SQLException ex) {
165  String err = "Could not establish a connection for metadata; DB: '" + catalog
166  + "' data dir '" + dataDir + "', error was " + ex.getMessage();
167  MAPDLOGGER.error(err);
168  throw new RuntimeException(err);
169  }
170  MAPDLOGGER.debug("Opened database successfully");
171  }
172 
173  private void disconnectFromCatalog() {
174  try {
175  catConn.close();
176  } catch (SQLException ex) {
177  String err = "Could not disconnect from metadata "
178  + " data dir '" + dataDir + "', error was " + ex.getMessage();
179  MAPDLOGGER.error(err);
180  throw new RuntimeException(err);
181  }
182  }
183 
184  private void connectToDBCatalog() {
186  }
187 
188  public Table getTable(String tableName) {
189  List<String> dbTable =
190  ImmutableList.of(default_db.toUpperCase(), tableName.toUpperCase());
191  Table cTable = MAPD_TABLE_DETAILS.get(dbTable);
192  if (cTable != null) {
193  MAPDLOGGER.debug("Metaconnect DB " + default_db + " get table " + tableName
194  + " details " + cTable);
195  return cTable;
196  }
197 
198  TTableDetails td = get_table_details(tableName);
199 
200  if (td.getView_sql() == null || td.getView_sql().isEmpty()) {
201  MAPDLOGGER.debug("Processing a table");
202  Table rTable = new MapDTable(td);
203  MAPD_TABLE_DETAILS.putIfAbsent(dbTable, rTable);
204  MAPDLOGGER.debug("Metaconnect DB " + default_db + " get table " + tableName
205  + " details " + rTable + " Not in buffer");
206  return rTable;
207  } else {
208  MAPDLOGGER.debug("Processing a view");
209  Table rTable = new MapDView(getViewSql(tableName), td, parser);
210  MAPD_TABLE_DETAILS.putIfAbsent(dbTable, rTable);
211  MAPDLOGGER.debug("Metaconnect DB " + default_db + " get view " + tableName
212  + " details " + rTable + " Not in buffer");
213  return rTable;
214  }
215  }
216 
217  public Set<String> getTables() {
218  Set<String> mSet = DATABASE_TO_TABLES.get(default_db.toUpperCase());
219  if (mSet != null && mSet.size() > 0) {
220  MAPDLOGGER.debug("Metaconnect DB getTables " + default_db + " tables " + mSet);
221  return mSet;
222  }
223 
224  if (mapdPort == -1) {
225  // use sql
227  Set<String> ts = getTables_SQL();
229  DATABASE_TO_TABLES.put(default_db.toUpperCase(), ts);
230  MAPDLOGGER.debug(
231  "Metaconnect DB getTables " + default_db + " tables " + ts + " from catDB");
232  return ts;
233  }
234  // use thrift direct to local server
235  try {
236  TProtocol protocol = null;
237  TTransport transport =
238  sock_transport_properties.openClientTransport("localhost", mapdPort);
239  if (!transport.isOpen()) transport.open();
240  protocol = new TBinaryProtocol(transport);
241 
242  OmniSci.Client client = new OmniSci.Client(protocol);
243  List<String> tablesList =
244  client.get_tables_for_database(currentUser.getSession(), default_db);
245  Set<String> ts = new HashSet<String>(tablesList.size());
246  for (String tableName : tablesList) {
247  ts.add(tableName);
248  }
249 
250  transport.close();
251  DATABASE_TO_TABLES.put(default_db.toUpperCase(), ts);
252  MAPDLOGGER.debug("Metaconnect DB getTables " + default_db + " tables " + ts
253  + " from server");
254  return ts;
255 
256  } catch (TTransportException ex) {
257  MAPDLOGGER.error("TTransportException on port [" + mapdPort + "]");
258  MAPDLOGGER.error(ex.toString());
259  throw new RuntimeException(ex.toString());
260  } catch (TOmniSciException ex) {
261  MAPDLOGGER.error(ex.toString());
262  throw new RuntimeException(ex.toString());
263  } catch (TException ex) {
264  MAPDLOGGER.error(ex.toString());
265  throw new RuntimeException(ex.toString());
266  }
267  }
268 
269  private Set<String> getTables_SQL() {
271  Set<String> tableSet = new HashSet<String>();
272  Statement stmt = null;
273  ResultSet rs = null;
274  String sqlText = "";
275  try {
276  stmt = catConn.createStatement();
277 
278  // get the tables
279  rs = stmt.executeQuery("SELECT name FROM mapd_tables ");
280  while (rs.next()) {
281  tableSet.add(rs.getString("name"));
282  /*--*/
283  MAPDLOGGER.debug("Object name = " + rs.getString("name"));
284  }
285  rs.close();
286  stmt.close();
287 
288  } catch (Exception e) {
289  String err = "error trying to get all the tables, error was " + e.getMessage();
290  MAPDLOGGER.error(err);
291  throw new RuntimeException(err);
292  }
294 
295  try {
296  // open temp table json file
297  final String filePath =
298  dataDir + "/mapd_catalogs/" + default_db + "_temp_tables.json";
299  MAPDLOGGER.debug("Opening temp table file at " + filePath);
300  String tempTablesJsonStr;
301  try {
302  File tempTablesFile = new File(filePath);
303  FileInputStream tempTablesStream = new FileInputStream(tempTablesFile);
304  byte[] data = new byte[(int) tempTablesFile.length()];
305  tempTablesStream.read(data);
306  tempTablesStream.close();
307 
308  tempTablesJsonStr = new String(data, "UTF-8");
309  } catch (java.io.FileNotFoundException e) {
310  return tableSet;
311  }
312 
313  Gson gson = new Gson();
314  JsonObject fileParentObject = gson.fromJson(tempTablesJsonStr, JsonObject.class);
315  for (Entry<String, JsonElement> member : fileParentObject.entrySet()) {
316  String tableName = member.getKey();
317  tableSet.add(tableName);
318  /*--*/
319  MAPDLOGGER.debug("Temp table object name = " + tableName);
320  }
321 
322  } catch (Exception e) {
323  String err = "error trying to load temporary tables from json file, error was "
324  + e.getMessage();
325  MAPDLOGGER.error(err);
326  throw new RuntimeException(err);
327  }
328 
329  return tableSet;
330  }
331 
332  public TTableDetails get_table_details(String tableName) {
333  if (mapdPort == -1) {
334  // use sql
336  TTableDetails td = get_table_detail_SQL(tableName);
338  return td;
339  }
340  // use thrift direct to local server
341  try {
342  TProtocol protocol = null;
343 
344  TTransport transport =
345  sock_transport_properties.openClientTransport("localhost", mapdPort);
346  if (!transport.isOpen()) transport.open();
347  protocol = new TBinaryProtocol(transport);
348 
349  OmniSci.Client client = new OmniSci.Client(protocol);
350  TTableDetails td = client.get_internal_table_details_for_database(
351  currentUser.getSession(), tableName, default_db);
352  transport.close();
353 
354  return td;
355 
356  } catch (TTransportException ex) {
357  MAPDLOGGER.error(ex.toString());
358  throw new RuntimeException(ex.toString());
359  } catch (TOmniSciException ex) {
360  MAPDLOGGER.error(ex.toString());
361  throw new RuntimeException(ex.toString());
362  } catch (TException ex) {
363  MAPDLOGGER.error(ex.toString());
364  throw new RuntimeException(ex.toString());
365  }
366  }
367 
368  public static final int get_physical_cols(int type) {
369  switch (type) {
370  case KPOINT:
371  return 1; // coords
372  case KLINESTRING:
373  return 2; // coords, bounds
374  case KPOLYGON:
375  return 4; // coords, ring_sizes, bounds, render_group
376  case KMULTIPOLYGON:
377  return 5; // coords, ring_sizes, poly_rings, bounds, render_group
378  default:
379  break;
380  }
381  return 0;
382  }
383 
384  public static final boolean is_geometry(int type) {
385  return type == KPOINT || type == KLINESTRING || type == KPOLYGON
386  || type == KMULTIPOLYGON;
387  }
388 
389  private TTableDetails get_table_detail_SQL(String tableName) {
390  TTableDetails td = new TTableDetails();
391  td.getRow_descIterator();
392  int id = getTableId(tableName);
393  if (id == -1) {
394  try {
395  return get_table_detail_JSON(tableName);
396  } catch (Exception e) {
397  String err =
398  "Table '" + tableName + "' does not exist for DB '" + default_db + "'";
399  MAPDLOGGER.error(err);
400  throw new RuntimeException(err);
401  }
402  }
403 
404  // read data from table
405  Statement stmt = null;
406  ResultSet rs = null;
407  try {
408  stmt = catConn.createStatement();
409  MAPDLOGGER.debug("table id is " + id);
410  MAPDLOGGER.debug("table name is " + tableName);
411  String query = String.format(
412  "SELECT * FROM mapd_columns where tableid = %d and not is_deletedcol order by columnid;",
413  id);
414  MAPDLOGGER.debug(query);
415  rs = stmt.executeQuery(query);
416  int skip_physical_cols = 0;
417  while (rs.next()) {
418  String colName = rs.getString("name");
419  MAPDLOGGER.debug("name = " + colName);
420  int colType = rs.getInt("coltype");
421  MAPDLOGGER.debug("coltype = " + colType);
422  int colSubType = rs.getInt("colsubtype");
423  MAPDLOGGER.debug("colsubtype = " + colSubType);
424  int colDim = rs.getInt("coldim");
425  MAPDLOGGER.debug("coldim = " + colDim);
426  int colScale = rs.getInt("colscale");
427  MAPDLOGGER.debug("colscale = " + colScale);
428  boolean isNotNull = rs.getBoolean("is_notnull");
429  MAPDLOGGER.debug("is_notnull = " + isNotNull);
430  boolean isSystemCol = rs.getBoolean("is_systemcol");
431  MAPDLOGGER.debug("is_systemcol = " + isSystemCol);
432  boolean isVirtualCol = rs.getBoolean("is_virtualcol");
433  MAPDLOGGER.debug("is_vitrualcol = " + isVirtualCol);
434  MAPDLOGGER.debug("");
435  TColumnType tct = new TColumnType();
436  TTypeInfo tti = new TTypeInfo();
437  TDatumType tdt;
438 
439  if (colType == KARRAY) {
440  tti.is_array = true;
441  tdt = typeToThrift(colSubType);
442  } else {
443  tti.is_array = false;
444  tdt = typeToThrift(colType);
445  }
446 
447  tti.nullable = !isNotNull;
448  tti.encoding = TEncodingType.NONE;
449  tti.type = tdt;
450  tti.scale = colScale;
451  tti.precision = colDim;
452 
453  tct.col_name = colName;
454  tct.col_type = tti;
455  tct.is_system = isSystemCol;
456 
457  if (skip_physical_cols <= 0) skip_physical_cols = get_physical_cols(colType);
458  if (is_geometry(colType) || skip_physical_cols-- <= 0) td.addToRow_desc(tct);
459  }
460  } catch (Exception e) {
461  String err = "error trying to read from mapd_columns, error was " + e.getMessage();
462  MAPDLOGGER.error(err);
463  throw new RuntimeException(err);
464  } finally {
465  if (rs != null) {
466  try {
467  rs.close();
468  } catch (SQLException ex) {
469  String err = "Could not close resultset, error was " + ex.getMessage();
470  MAPDLOGGER.error(err);
471  throw new RuntimeException(err);
472  }
473  }
474  if (stmt != null) {
475  try {
476  stmt.close();
477  } catch (SQLException ex) {
478  String err = "Could not close stmt, error was " + ex.getMessage();
479  MAPDLOGGER.error(err);
480  throw new RuntimeException(err);
481  }
482  }
483  }
484  if (isView(tableName)) {
485  td.setView_sqlIsSet(true);
486  td.setView_sql(getViewSqlViaSql(id));
487  }
488  return td;
489  }
490 
491  private TTableDetails get_table_detail_JSON(String tableName)
492  throws IOException, RuntimeException {
493  TTableDetails td = new TTableDetails();
494  td.getRow_descIterator();
495 
496  // open table json file
497  final String filePath =
498  dataDir + "/mapd_catalogs/" + default_db + "_temp_tables.json";
499  MAPDLOGGER.debug("Opening temp table file at " + filePath);
500 
501  String tempTablesJsonStr;
502  try {
503  File tempTablesFile = new File(filePath);
504  FileInputStream tempTablesStream = new FileInputStream(tempTablesFile);
505  byte[] data = new byte[(int) tempTablesFile.length()];
506  tempTablesStream.read(data);
507  tempTablesStream.close();
508 
509  tempTablesJsonStr = new String(data, "UTF-8");
510  } catch (java.io.FileNotFoundException e) {
511  throw new RuntimeException("Failed to read temporary tables file.");
512  }
513 
514  Gson gson = new Gson();
515  JsonObject fileParentObject = gson.fromJson(tempTablesJsonStr, JsonObject.class);
516  if (fileParentObject == null) {
517  throw new IOException("Malformed temporary tables file.");
518  }
519 
520  JsonObject tableObject = fileParentObject.getAsJsonObject(tableName);
521  if (tableObject == null) {
522  throw new RuntimeException(
523  "Failed to find table " + tableName + " in temporary tables file.");
524  }
525 
526  String jsonTableName = tableObject.get("name").getAsString();
527  assert (tableName == jsonTableName);
528  int id = tableObject.get("id").getAsInt();
529  MAPDLOGGER.debug("table id is " + id);
530  MAPDLOGGER.debug("table name is " + tableName);
531 
532  JsonArray jsonColumns = tableObject.getAsJsonArray("columns");
533  assert (jsonColumns != null);
534 
535  int skip_physical_cols = 0;
536  for (JsonElement columnElement : jsonColumns) {
537  JsonObject columnObject = columnElement.getAsJsonObject();
538 
539  String colName = columnObject.get("name").getAsString();
540  MAPDLOGGER.debug("name = " + colName);
541  int colType = columnObject.get("coltype").getAsInt();
542  MAPDLOGGER.debug("coltype = " + colType);
543  int colSubType = columnObject.get("colsubtype").getAsInt();
544  MAPDLOGGER.debug("colsubtype = " + colSubType);
545  int colDim = columnObject.get("coldim").getAsInt();
546  MAPDLOGGER.debug("coldim = " + colDim);
547  int colScale = columnObject.get("colscale").getAsInt();
548  MAPDLOGGER.debug("colscale = " + colScale);
549  boolean isNotNull = columnObject.get("is_notnull").getAsBoolean();
550  MAPDLOGGER.debug("is_notnull = " + isNotNull);
551  boolean isSystemCol = columnObject.get("is_systemcol").getAsBoolean();
552  MAPDLOGGER.debug("is_systemcol = " + isSystemCol);
553  boolean isVirtualCol = columnObject.get("is_virtualcol").getAsBoolean();
554  MAPDLOGGER.debug("is_vitrualcol = " + isVirtualCol);
555  boolean isDeletedCol = columnObject.get("is_deletedcol").getAsBoolean();
556  MAPDLOGGER.debug("is_deletedcol = " + isDeletedCol);
557  MAPDLOGGER.debug("");
558 
559  if (isDeletedCol) {
560  MAPDLOGGER.debug("Skipping delete column.");
561  continue;
562  }
563 
564  TColumnType tct = new TColumnType();
565  TTypeInfo tti = new TTypeInfo();
566  TDatumType tdt;
567 
568  if (colType == KARRAY) {
569  tti.is_array = true;
570  tdt = typeToThrift(colSubType);
571  } else {
572  tti.is_array = false;
573  tdt = typeToThrift(colType);
574  }
575 
576  tti.nullable = !isNotNull;
577  tti.encoding = TEncodingType.NONE;
578  tti.type = tdt;
579  tti.scale = colScale;
580  tti.precision = colDim;
581 
582  tct.col_name = colName;
583  tct.col_type = tti;
584  tct.is_system = isSystemCol;
585 
586  if (skip_physical_cols <= 0) skip_physical_cols = get_physical_cols(colType);
587  if (is_geometry(colType) || skip_physical_cols-- <= 0) td.addToRow_desc(tct);
588  }
589 
590  return td;
591  }
592 
593  private int getTableId(String tableName) {
594  Statement stmt = null;
595  ResultSet rs = null;
596  int tableId = -1;
597  try {
598  stmt = catConn.createStatement();
599  rs = stmt.executeQuery(String.format(
600  "SELECT tableid FROM mapd_tables where name = '%s' COLLATE NOCASE;",
601  tableName));
602  while (rs.next()) {
603  tableId = rs.getInt("tableid");
604  MAPDLOGGER.debug("tableId = " + tableId);
605  MAPDLOGGER.debug("");
606  }
607  rs.close();
608  stmt.close();
609  } catch (Exception e) {
610  String err = "Error trying to read from metadata table mapd_tables;DB: "
611  + default_db + " data dir " + dataDir + ", error was " + e.getMessage();
612  MAPDLOGGER.error(err);
613  throw new RuntimeException(err);
614  } finally {
615  if (rs != null) {
616  try {
617  rs.close();
618  } catch (SQLException ex) {
619  String err = "Could not close resultset, error was " + ex.getMessage();
620  MAPDLOGGER.error(err);
621  throw new RuntimeException(err);
622  }
623  }
624  if (stmt != null) {
625  try {
626  stmt.close();
627  } catch (SQLException ex) {
628  String err = "Could not close stmt, error was " + ex.getMessage();
629  MAPDLOGGER.error(err);
630  throw new RuntimeException(err);
631  }
632  }
633  }
634  return (tableId);
635  }
636 
637  private boolean isView(String tableName) {
638  Statement stmt;
639  ResultSet rs;
640  int viewFlag = 0;
641  try {
642  stmt = catConn.createStatement();
643  rs = stmt.executeQuery(String.format(
644  "SELECT isview FROM mapd_tables where name = '%s' COLLATE NOCASE;",
645  tableName));
646  while (rs.next()) {
647  viewFlag = rs.getInt("isview");
648  MAPDLOGGER.debug("viewFlag = " + viewFlag);
649  MAPDLOGGER.debug("");
650  }
651  rs.close();
652  stmt.close();
653  } catch (Exception e) {
654  String err = "error trying to read from mapd_views, error was " + e.getMessage();
655  MAPDLOGGER.error(err);
656  throw new RuntimeException(err);
657  }
658  return (viewFlag == 1);
659  }
660 
661  private String getViewSql(String tableName) {
662  String sqlText;
663  if (mapdPort == -1) {
664  // use sql
666  sqlText = getViewSqlViaSql(getTableId(tableName));
668  } else {
669  // use thrift direct to local server
670  try {
671  TProtocol protocol = null;
672 
673  TTransport transport =
674  sock_transport_properties.openClientTransport("localhost", mapdPort);
675  if (!transport.isOpen()) transport.open();
676  protocol = new TBinaryProtocol(transport);
677 
678  OmniSci.Client client = new OmniSci.Client(protocol);
679  TTableDetails td = client.get_table_details_for_database(
680  currentUser.getSession(), tableName, default_db);
681  transport.close();
682 
683  sqlText = td.getView_sql();
684 
685  } catch (TTransportException ex) {
686  MAPDLOGGER.error(ex.toString());
687  throw new RuntimeException(ex.toString());
688  } catch (TOmniSciException ex) {
689  MAPDLOGGER.error(ex.toString());
690  throw new RuntimeException(ex.toString());
691  } catch (TException ex) {
692  MAPDLOGGER.error(ex.toString());
693  throw new RuntimeException(ex.toString());
694  }
695  }
696  /* return string without the sqlite's trailing semicolon */
697  if (sqlText.charAt(sqlText.length() - 1) == ';') {
698  return (sqlText.substring(0, sqlText.length() - 1));
699  } else {
700  return (sqlText);
701  }
702  }
703 
704  // we assume there is already a DB connection here
705  private String getViewSqlViaSql(int tableId) {
706  Statement stmt;
707  ResultSet rs;
708  String sqlText = "";
709  try {
710  stmt = catConn.createStatement();
711  rs = stmt.executeQuery(String.format(
712  "SELECT sql FROM mapd_views where tableid = '%s' COLLATE NOCASE;",
713  tableId));
714  while (rs.next()) {
715  sqlText = rs.getString("sql");
716  MAPDLOGGER.debug("View definition = " + sqlText);
717  MAPDLOGGER.debug("");
718  }
719  rs.close();
720  stmt.close();
721  } catch (Exception e) {
722  String err = "error trying to read from mapd_views, error was " + e.getMessage();
723  MAPDLOGGER.error(err);
724  throw new RuntimeException(err);
725  }
726  if (sqlText == null || sqlText.length() == 0) {
727  String err = "No view text found";
728  MAPDLOGGER.error(err);
729  throw new RuntimeException(err);
730  }
731  return sqlText;
732  }
733 
734  private TDatumType typeToThrift(int type) {
735  switch (type) {
736  case KBOOLEAN:
737  return TDatumType.BOOL;
738  case KTINYINT:
739  return TDatumType.TINYINT;
740  case KSMALLINT:
741  return TDatumType.SMALLINT;
742  case KINT:
743  return TDatumType.INT;
744  case KBIGINT:
745  return TDatumType.BIGINT;
746  case KFLOAT:
747  return TDatumType.FLOAT;
748  case KNUMERIC:
749  case KDECIMAL:
750  return TDatumType.DECIMAL;
751  case KDOUBLE:
752  return TDatumType.DOUBLE;
753  case KTEXT:
754  case KVARCHAR:
755  case KCHAR:
756  return TDatumType.STR;
757  case KTIME:
758  return TDatumType.TIME;
759  case KTIMESTAMP:
760  return TDatumType.TIMESTAMP;
761  case KDATE:
762  return TDatumType.DATE;
763  case KINTERVAL_DAY_TIME:
764  return TDatumType.INTERVAL_DAY_TIME;
766  return TDatumType.INTERVAL_YEAR_MONTH;
767  case KPOINT:
768  return TDatumType.POINT;
769  case KLINESTRING:
770  return TDatumType.LINESTRING;
771  case KPOLYGON:
772  return TDatumType.POLYGON;
773  case KMULTIPOLYGON:
775  default:
776  return null;
777  }
778  }
779 
780  private void populateDatabases() {
781  // TODO 13 Mar 2021 MAT
782  // this probably has to come across from the server on first start up rather
783  // than lazy instantiation here
784  // as a user may not be able to see all schemas and this sets it for the life
785  // of the server.
786  // Proceeding this way as a WIP
787  if (mapdPort == 0) {
788  // seems to be a condition that is expected
789  // for FSI testing
790  return;
791  }
792  if (mapdPort == -1) {
793  // use sql
794  connectToCatalog("omnisci_system_catalog"); // hardcoded sys catalog
795  Set<String> dbNames = getDatabases_SQL();
797  for (String dbName : dbNames) {
798  Set<String> ts = new HashSet<String>();
799  DATABASE_TO_TABLES.putIfAbsent(dbName, ts);
800  }
801  return;
802  }
803  // use thrift direct to local server
804  try {
805  TProtocol protocol = null;
806  TTransport transport =
807  sock_transport_properties.openClientTransport("localhost", mapdPort);
808  if (!transport.isOpen()) transport.open();
809  protocol = new TBinaryProtocol(transport);
810 
811  OmniSci.Client client = new OmniSci.Client(protocol);
812 
813  List<TDBInfo> dbList = client.get_databases(currentUser.getSession());
814  for (TDBInfo dbInfo : dbList) {
815  Set<String> ts = new HashSet<String>();
816  DATABASE_TO_TABLES.putIfAbsent(dbInfo.db_name, ts);
817  }
818  transport.close();
819 
820  } catch (TTransportException ex) {
821  MAPDLOGGER.error("TTransportException on port [" + mapdPort + "]");
822  MAPDLOGGER.error(ex.toString());
823  throw new RuntimeException(ex.toString());
824  } catch (TOmniSciException ex) {
825  MAPDLOGGER.error(ex.toString());
826  throw new RuntimeException(ex.toString());
827  } catch (TException ex) {
828  MAPDLOGGER.error(ex.toString());
829  throw new RuntimeException(ex.toString());
830  }
831  }
832 
833  private Set<String> getDatabases_SQL() {
834  Set<String> dbSet = new HashSet<String>();
835  Statement stmt = null;
836  ResultSet rs = null;
837  String sqlText = "";
838  try {
839  stmt = catConn.createStatement();
840 
841  // get the tables
842  rs = stmt.executeQuery("SELECT name FROM mapd_databases ");
843  while (rs.next()) {
844  dbSet.add(rs.getString("name"));
845  /*--*/
846  MAPDLOGGER.debug("Object name = " + rs.getString("name"));
847  }
848  rs.close();
849  stmt.close();
850 
851  } catch (Exception e) {
852  String err = "error trying to get all the databases, error was " + e.getMessage();
853  MAPDLOGGER.error(err);
854  throw new RuntimeException(err);
855  }
856  return dbSet;
857  }
858 
859  public void updateMetaData(String schema, String table) {
860  // Check if table is specified, if not we are dropping an entire DB so need to
861  // remove all
862  // tables for that DB
863  if (table.equals("")) {
864  // Drop db and all tables
865  // iterate through all and remove matching schema
866  Set<List<String>> all = new HashSet<>(MAPD_TABLE_DETAILS.keySet());
867  for (List<String> keys : all) {
868  if (keys.get(0).equals(schema.toUpperCase())) {
869  MAPDLOGGER.debug(
870  "removing all for schema " + keys.get(0) + " table " + keys.get(1));
871  MAPD_TABLE_DETAILS.remove(keys);
872  }
873  }
874  } else {
875  MAPDLOGGER.debug("removing schema " + schema.toUpperCase() + " table "
876  + table.toUpperCase());
877  MAPD_TABLE_DETAILS.remove(
878  ImmutableList.of(schema.toUpperCase(), table.toUpperCase()));
879  }
880  // Invalidate views
881  Set<List<String>> all = new HashSet<>(MAPD_TABLE_DETAILS.keySet());
882  for (List<String> keys : all) {
883  if (keys.get(0).equals(schema.toUpperCase())) {
884  Table ttable = MAPD_TABLE_DETAILS.get(keys);
885  if (ttable instanceof MapDView) {
886  MAPDLOGGER.debug(
887  "removing view in schema " + keys.get(0) + " view " + keys.get(1));
888  MAPD_TABLE_DETAILS.remove(keys);
889  }
890  }
891  }
892  // Could be a removal or an add request for a DB
893  Set<String> mSet = DATABASE_TO_TABLES.get(schema.toUpperCase());
894  if (mSet != null) {
895  MAPDLOGGER.debug("removing schema " + schema.toUpperCase());
896  DATABASE_TO_TABLES.remove(schema.toUpperCase());
897  } else {
898  // add a empty database descriptor for new DB, it will be lazily populated when
899  // required
900  Set<String> ts = new HashSet<String>();
901  DATABASE_TO_TABLES.putIfAbsent(schema.toUpperCase(), ts);
902  }
903  }
904 }
#define LINESTRING
static volatile Map< String, Set< String > > DATABASE_TO_TABLES
Table getTable(String tableName)
static final int get_physical_cols(int type)
MetaConnect(int mapdPort, String dataDir, MapDUser currentMapDUser, MapDParser parser, SockTransportProperties skT)
#define SMALLINT
static final int KMULTIPOLYGON
#define DOUBLE
#define BIGINT
void updateMetaData(String schema, String table)
#define DATE
static volatile Map< List< String >, Table > MAPD_TABLE_DETAILS
String getViewSql(String tableName)
static final boolean is_geometry(int type)
String getViewSqlViaSql(int tableId)
#define MULTIPOLYGON
#define POINT
TTableDetails get_table_details(String tableName)
static final int KINTERVAL_YEAR_MONTH
static final Logger MAPDLOGGER
final SockTransportProperties sock_transport_properties
static final int KLINESTRING
#define TIME
#define TINYINT
void connectToCatalog(String catalog)
#define TIMESTAMP
TTableDetails get_table_detail_SQL(String tableName)
TDatumType typeToThrift(int type)
Set< String > getDatabases_SQL()
MetaConnect(int mapdPort, String dataDir, MapDUser currentMapDUser, MapDParser parser, SockTransportProperties skT, String db)
int getTableId(String tableName)
#define DECIMAL
TTableDetails get_table_detail_JSON(String tableName)
static final int KINTERVAL_DAY_TIME
boolean isView(String tableName)
#define POLYGON
#define FLOAT