OmniSciDB  471d68cefb
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
MetaConnect.java
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.mapd.metadata;
17 
18 import com.google.common.collect.ImmutableList;
19 import com.google.gson.Gson;
20 import com.google.gson.JsonArray;
21 import com.google.gson.JsonElement;
22 import com.google.gson.JsonObject;
28 import com.omnisci.thrift.server.OmniSci;
29 import com.omnisci.thrift.server.TColumnType;
30 import com.omnisci.thrift.server.TDBInfo;
31 import com.omnisci.thrift.server.TDatumType;
32 import com.omnisci.thrift.server.TEncodingType;
33 import com.omnisci.thrift.server.TOmniSciException;
34 import com.omnisci.thrift.server.TTableDetails;
35 import com.omnisci.thrift.server.TTypeInfo;
36 
37 import org.apache.calcite.schema.Table;
38 import org.apache.thrift.TException;
39 import org.apache.thrift.protocol.TBinaryProtocol;
40 import org.apache.thrift.protocol.TProtocol;
41 import org.apache.thrift.transport.TServerSocket;
42 import org.apache.thrift.transport.TSocket;
43 import org.apache.thrift.transport.TTransport;
44 import org.apache.thrift.transport.TTransportException;
45 import org.slf4j.Logger;
46 import org.slf4j.LoggerFactory;
47 
48 import java.io.File;
49 import java.io.FileInputStream;
50 import java.io.IOException;
51 import java.sql.Connection;
52 import java.sql.DriverManager;
53 import java.sql.ResultSet;
54 import java.sql.SQLException;
55 import java.sql.Statement;
56 import java.util.ArrayList;
57 import java.util.HashSet;
58 import java.util.List;
59 import java.util.Map;
60 import java.util.Map.Entry;
61 import java.util.Set;
62 import java.util.concurrent.ConcurrentHashMap;
63 
68 public class MetaConnect {
69  final static Logger MAPDLOGGER = LoggerFactory.getLogger(MetaConnect.class);
70  private final String dataDir;
71  private final String default_db;
72  private final MapDUser currentUser;
73  private final int mapdPort;
74  private Connection catConn;
75  private final MapDParser parser;
76 
77  private static final int KBOOLEAN = 1;
78  private static final int KCHAR = 2;
79  private static final int KVARCHAR = 3;
80  private static final int KNUMERIC = 4;
81  private static final int KDECIMAL = 5;
82  private static final int KINT = 6;
83  private static final int KSMALLINT = 7;
84  private static final int KFLOAT = 8;
85  private static final int KDOUBLE = 9;
86  private static final int KTIME = 10;
87  private static final int KTIMESTAMP = 11;
88  private static final int KBIGINT = 12;
89  private static final int KTEXT = 13;
90  private static final int KDATE = 14;
91  private static final int KARRAY = 15;
92  private static final int KINTERVAL_DAY_TIME = 16;
93  private static final int KINTERVAL_YEAR_MONTH = 17;
94  private static final int KPOINT = 18;
95  private static final int KLINESTRING = 19;
96  private static final int KPOLYGON = 20;
97  private static final int KMULTIPOLYGON = 21;
98  private static final int KTINYINT = 22;
99 
100  private static volatile Map<String, Set<String>> DATABASE_TO_TABLES =
101  new ConcurrentHashMap<>();
102  private static volatile Map<List<String>, Table> MAPD_TABLE_DETAILS =
103  new ConcurrentHashMap<>();
105 
106  public MetaConnect(int mapdPort,
107  String dataDir,
108  MapDUser currentMapDUser,
111  String db) {
112  this.dataDir = dataDir;
113  if (db != null) {
114  this.default_db = db;
115  } else {
116  if (currentMapDUser != null) {
117  this.default_db = currentMapDUser.getDB();
118  } else {
119  this.default_db = null;
120  }
121  }
122  this.currentUser = currentMapDUser;
123  this.mapdPort = mapdPort;
124  this.parser = parser;
125  this.sock_transport_properties = skT;
126 
127  // check to see if we have a populated DATABASE_TO_TABLES structure
128  // first time in we need to make sure this gets populated
129  // It is OK to use a MetaConnect without a user
130  // but it should not attempt to populate the DB
131  if (currentUser != null && DATABASE_TO_TABLES.size() == 0) {
132  // get all databases
134  }
135  }
136 
137  public MetaConnect(int mapdPort,
138  String dataDir,
139  MapDUser currentMapDUser,
142  this(mapdPort, dataDir, currentMapDUser, parser, skT, null);
143  }
144 
145  public List<String> getDatabases() {
146  List<String> dbList = new ArrayList<String>(DATABASE_TO_TABLES.size());
147  for (String db : DATABASE_TO_TABLES.keySet()) {
148  dbList.add(db);
149  }
150  return dbList;
151  }
152 
153  private void connectToCatalog(String catalog) {
154  try {
155  // try {
156  Class.forName("org.sqlite.JDBC");
157  } catch (ClassNotFoundException ex) {
158  String err = "Could not find class for metadata connection; DB: '" + catalog
159  + "' data dir '" + dataDir + "', error was " + ex.getMessage();
160  MAPDLOGGER.error(err);
161  throw new RuntimeException(err);
162  }
163  String connectURL = "jdbc:sqlite:" + dataDir + "/mapd_catalogs/" + catalog;
164  try {
165  catConn = DriverManager.getConnection(connectURL);
166  } catch (SQLException ex) {
167  String err = "Could not establish a connection for metadata; DB: '" + catalog
168  + "' data dir '" + dataDir + "', error was " + ex.getMessage();
169  MAPDLOGGER.error(err);
170  throw new RuntimeException(err);
171  }
172  MAPDLOGGER.debug("Opened database successfully");
173  }
174 
175  private void disconnectFromCatalog() {
176  try {
177  catConn.close();
178  } catch (SQLException ex) {
179  String err = "Could not disconnect from metadata "
180  + " data dir '" + dataDir + "', error was " + ex.getMessage();
181  MAPDLOGGER.error(err);
182  throw new RuntimeException(err);
183  }
184  }
185 
186  private void connectToDBCatalog() {
188  }
189 
190  public Table getTable(String tableName) {
191  List<String> dbTable =
192  ImmutableList.of(default_db.toUpperCase(), tableName.toUpperCase());
193  Table cTable = MAPD_TABLE_DETAILS.get(dbTable);
194  if (cTable != null) {
195  MAPDLOGGER.debug("Metaconnect DB " + default_db + " get table " + tableName
196  + " details " + cTable);
197  return cTable;
198  }
199 
200  TTableDetails td = get_table_details(tableName);
201 
202  if (td.getView_sql() == null || td.getView_sql().isEmpty()) {
203  MAPDLOGGER.debug("Processing a table");
204  Table rTable = new MapDTable(td);
205  MAPD_TABLE_DETAILS.putIfAbsent(dbTable, rTable);
206  MAPDLOGGER.debug("Metaconnect DB " + default_db + " get table " + tableName
207  + " details " + rTable + " Not in buffer");
208  return rTable;
209  } else {
210  MAPDLOGGER.debug("Processing a view");
211  Table rTable = new MapDView(getViewSql(tableName), td, parser);
212  MAPD_TABLE_DETAILS.putIfAbsent(dbTable, rTable);
213  MAPDLOGGER.debug("Metaconnect DB " + default_db + " get view " + tableName
214  + " details " + rTable + " Not in buffer");
215  return rTable;
216  }
217  }
218 
219  public Set<String> getTables() {
220  Set<String> mSet = DATABASE_TO_TABLES.get(default_db.toUpperCase());
221  if (mSet != null && mSet.size() > 0) {
222  MAPDLOGGER.debug("Metaconnect DB getTables " + default_db + " tables " + mSet);
223  return mSet;
224  }
225 
226  if (mapdPort == -1) {
227  // use sql
229  Set<String> ts = getTables_SQL();
231  DATABASE_TO_TABLES.put(default_db.toUpperCase(), ts);
232  MAPDLOGGER.debug(
233  "Metaconnect DB getTables " + default_db + " tables " + ts + " from catDB");
234  return ts;
235  }
236  // use thrift direct to local server
237  try {
238  TProtocol protocol = null;
239  TTransport transport =
240  sock_transport_properties.openClientTransport("localhost", mapdPort);
241  if (!transport.isOpen()) transport.open();
242  protocol = new TBinaryProtocol(transport);
243 
244  OmniSci.Client client = new OmniSci.Client(protocol);
245  List<String> tablesList =
246  client.get_tables_for_database(currentUser.getSession(), default_db);
247  Set<String> ts = new HashSet<String>(tablesList.size());
248  for (String tableName : tablesList) {
249  ts.add(tableName);
250  }
251 
252  transport.close();
253  DATABASE_TO_TABLES.put(default_db.toUpperCase(), ts);
254  MAPDLOGGER.debug("Metaconnect DB getTables " + default_db + " tables " + ts
255  + " from server");
256  return ts;
257 
258  } catch (TTransportException ex) {
259  MAPDLOGGER.error("TTransportException on port [" + mapdPort + "]");
260  MAPDLOGGER.error(ex.toString());
261  throw new RuntimeException(ex.toString());
262  } catch (TOmniSciException ex) {
263  MAPDLOGGER.error(ex.getError_msg());
264  throw new RuntimeException(ex.getError_msg());
265  } catch (TException ex) {
266  MAPDLOGGER.error(ex.toString());
267  throw new RuntimeException(ex.toString());
268  }
269  }
270 
271  private Set<String> getTables_SQL() {
273  Set<String> tableSet = new HashSet<String>();
274  Statement stmt = null;
275  ResultSet rs = null;
276  String sqlText = "";
277  try {
278  stmt = catConn.createStatement();
279 
280  // get the tables
281  rs = stmt.executeQuery("SELECT name FROM mapd_tables ");
282  while (rs.next()) {
283  tableSet.add(rs.getString("name"));
284  /*--*/
285  MAPDLOGGER.debug("Object name = " + rs.getString("name"));
286  }
287  rs.close();
288  stmt.close();
289 
290  } catch (Exception e) {
291  String err = "error trying to get all the tables, error was " + e.getMessage();
292  MAPDLOGGER.error(err);
293  throw new RuntimeException(err);
294  }
296 
297  try {
298  // open temp table json file
299  final String filePath =
300  dataDir + "/mapd_catalogs/" + default_db + "_temp_tables.json";
301  MAPDLOGGER.debug("Opening temp table file at " + filePath);
302  String tempTablesJsonStr;
303  try {
304  File tempTablesFile = new File(filePath);
305  FileInputStream tempTablesStream = new FileInputStream(tempTablesFile);
306  byte[] data = new byte[(int) tempTablesFile.length()];
307  tempTablesStream.read(data);
308  tempTablesStream.close();
309 
310  tempTablesJsonStr = new String(data, "UTF-8");
311  } catch (java.io.FileNotFoundException e) {
312  return tableSet;
313  }
314 
315  Gson gson = new Gson();
316  JsonObject fileParentObject = gson.fromJson(tempTablesJsonStr, JsonObject.class);
317  for (Entry<String, JsonElement> member : fileParentObject.entrySet()) {
318  String tableName = member.getKey();
319  tableSet.add(tableName);
320  /*--*/
321  MAPDLOGGER.debug("Temp table object name = " + tableName);
322  }
323 
324  } catch (Exception e) {
325  String err = "error trying to load temporary tables from json file, error was "
326  + e.getMessage();
327  MAPDLOGGER.error(err);
328  throw new RuntimeException(err);
329  }
330 
331  return tableSet;
332  }
333 
334  public TTableDetails get_table_details(String tableName) {
335  if (mapdPort == -1) {
336  // use sql
338  TTableDetails td = get_table_detail_SQL(tableName);
340  return td;
341  }
342  // use thrift direct to local server
343  try {
344  TProtocol protocol = null;
345 
346  TTransport transport =
347  sock_transport_properties.openClientTransport("localhost", mapdPort);
348  if (!transport.isOpen()) transport.open();
349  protocol = new TBinaryProtocol(transport);
350 
351  OmniSci.Client client = new OmniSci.Client(protocol);
352  TTableDetails td = client.get_internal_table_details_for_database(
353  currentUser.getSession(), tableName, default_db);
354  transport.close();
355 
356  return td;
357 
358  } catch (TTransportException ex) {
359  MAPDLOGGER.error(ex.toString());
360  throw new RuntimeException(ex.toString());
361  } catch (TOmniSciException ex) {
362  MAPDLOGGER.error(ex.getError_msg());
363  throw new RuntimeException(ex.getError_msg());
364  } catch (TException ex) {
365  MAPDLOGGER.error(ex.toString());
366  throw new RuntimeException(ex.toString());
367  }
368  }
369 
370  public static final int get_physical_cols(int type) {
371  switch (type) {
372  case KPOINT:
373  return 1; // coords
374  case KLINESTRING:
375  return 2; // coords, bounds
376  case KPOLYGON:
377  return 4; // coords, ring_sizes, bounds, render_group
378  case KMULTIPOLYGON:
379  return 5; // coords, ring_sizes, poly_rings, bounds, render_group
380  default:
381  break;
382  }
383  return 0;
384  }
385 
386  public static final boolean is_geometry(int type) {
387  return type == KPOINT || type == KLINESTRING || type == KPOLYGON
388  || type == KMULTIPOLYGON;
389  }
390 
391  private TTableDetails get_table_detail_SQL(String tableName) {
392  TTableDetails td = new TTableDetails();
393  td.getRow_descIterator();
394  int id = getTableId(tableName);
395  if (id == -1) {
396  try {
397  return get_table_detail_JSON(tableName);
398  } catch (Exception e) {
399  String err =
400  "Table '" + tableName + "' does not exist for DB '" + default_db + "'";
401  MAPDLOGGER.error(err);
402  throw new RuntimeException(err);
403  }
404  }
405 
406  // read data from table
407  Statement stmt = null;
408  ResultSet rs = null;
409  try {
410  stmt = catConn.createStatement();
411  MAPDLOGGER.debug("table id is " + id);
412  MAPDLOGGER.debug("table name is " + tableName);
413  String query = String.format(
414  "SELECT * FROM mapd_columns where tableid = %d and not is_deletedcol order by columnid;",
415  id);
416  MAPDLOGGER.debug(query);
417  rs = stmt.executeQuery(query);
418  int skip_physical_cols = 0;
419  while (rs.next()) {
420  String colName = rs.getString("name");
421  MAPDLOGGER.debug("name = " + colName);
422  int colType = rs.getInt("coltype");
423  MAPDLOGGER.debug("coltype = " + colType);
424  int colSubType = rs.getInt("colsubtype");
425  MAPDLOGGER.debug("colsubtype = " + colSubType);
426  int colDim = rs.getInt("coldim");
427  MAPDLOGGER.debug("coldim = " + colDim);
428  int colScale = rs.getInt("colscale");
429  MAPDLOGGER.debug("colscale = " + colScale);
430  boolean isNotNull = rs.getBoolean("is_notnull");
431  MAPDLOGGER.debug("is_notnull = " + isNotNull);
432  boolean isSystemCol = rs.getBoolean("is_systemcol");
433  MAPDLOGGER.debug("is_systemcol = " + isSystemCol);
434  boolean isVirtualCol = rs.getBoolean("is_virtualcol");
435  MAPDLOGGER.debug("is_vitrualcol = " + isVirtualCol);
436  MAPDLOGGER.debug("");
437  TColumnType tct = new TColumnType();
438  TTypeInfo tti = new TTypeInfo();
439  TDatumType tdt;
440 
441  if (colType == KARRAY) {
442  tti.is_array = true;
443  tdt = typeToThrift(colSubType);
444  } else {
445  tti.is_array = false;
446  tdt = typeToThrift(colType);
447  }
448 
449  tti.nullable = !isNotNull;
450  tti.encoding = TEncodingType.NONE;
451  tti.type = tdt;
452  tti.scale = colScale;
453  tti.precision = colDim;
454 
455  tct.col_name = colName;
456  tct.col_type = tti;
457  tct.is_system = isSystemCol;
458 
459  if (skip_physical_cols <= 0) skip_physical_cols = get_physical_cols(colType);
460  if (is_geometry(colType) || skip_physical_cols-- <= 0) td.addToRow_desc(tct);
461  }
462  } catch (Exception e) {
463  String err = "error trying to read from mapd_columns, error was " + e.getMessage();
464  MAPDLOGGER.error(err);
465  throw new RuntimeException(err);
466  } finally {
467  if (rs != null) {
468  try {
469  rs.close();
470  } catch (SQLException ex) {
471  String err = "Could not close resultset, error was " + ex.getMessage();
472  MAPDLOGGER.error(err);
473  throw new RuntimeException(err);
474  }
475  }
476  if (stmt != null) {
477  try {
478  stmt.close();
479  } catch (SQLException ex) {
480  String err = "Could not close stmt, error was " + ex.getMessage();
481  MAPDLOGGER.error(err);
482  throw new RuntimeException(err);
483  }
484  }
485  }
486  if (isView(tableName)) {
487  td.setView_sqlIsSet(true);
488  td.setView_sql(getViewSqlViaSql(id));
489  }
490  return td;
491  }
492 
493  private TTableDetails get_table_detail_JSON(String tableName)
494  throws IOException, RuntimeException {
495  TTableDetails td = new TTableDetails();
496  td.getRow_descIterator();
497 
498  // open table json file
499  final String filePath =
500  dataDir + "/mapd_catalogs/" + default_db + "_temp_tables.json";
501  MAPDLOGGER.debug("Opening temp table file at " + filePath);
502 
503  String tempTablesJsonStr;
504  try {
505  File tempTablesFile = new File(filePath);
506  FileInputStream tempTablesStream = new FileInputStream(tempTablesFile);
507  byte[] data = new byte[(int) tempTablesFile.length()];
508  tempTablesStream.read(data);
509  tempTablesStream.close();
510 
511  tempTablesJsonStr = new String(data, "UTF-8");
512  } catch (java.io.FileNotFoundException e) {
513  throw new RuntimeException("Failed to read temporary tables file.");
514  }
515 
516  Gson gson = new Gson();
517  JsonObject fileParentObject = gson.fromJson(tempTablesJsonStr, JsonObject.class);
518  if (fileParentObject == null) {
519  throw new IOException("Malformed temporary tables file.");
520  }
521 
522  JsonObject tableObject = fileParentObject.getAsJsonObject(tableName);
523  if (tableObject == null) {
524  throw new RuntimeException(
525  "Failed to find table " + tableName + " in temporary tables file.");
526  }
527 
528  String jsonTableName = tableObject.get("name").getAsString();
529  assert (tableName == jsonTableName);
530  int id = tableObject.get("id").getAsInt();
531  MAPDLOGGER.debug("table id is " + id);
532  MAPDLOGGER.debug("table name is " + tableName);
533 
534  JsonArray jsonColumns = tableObject.getAsJsonArray("columns");
535  assert (jsonColumns != null);
536 
537  int skip_physical_cols = 0;
538  for (JsonElement columnElement : jsonColumns) {
539  JsonObject columnObject = columnElement.getAsJsonObject();
540 
541  String colName = columnObject.get("name").getAsString();
542  MAPDLOGGER.debug("name = " + colName);
543  int colType = columnObject.get("coltype").getAsInt();
544  MAPDLOGGER.debug("coltype = " + colType);
545  int colSubType = columnObject.get("colsubtype").getAsInt();
546  MAPDLOGGER.debug("colsubtype = " + colSubType);
547  int colDim = columnObject.get("coldim").getAsInt();
548  MAPDLOGGER.debug("coldim = " + colDim);
549  int colScale = columnObject.get("colscale").getAsInt();
550  MAPDLOGGER.debug("colscale = " + colScale);
551  boolean isNotNull = columnObject.get("is_notnull").getAsBoolean();
552  MAPDLOGGER.debug("is_notnull = " + isNotNull);
553  boolean isSystemCol = columnObject.get("is_systemcol").getAsBoolean();
554  MAPDLOGGER.debug("is_systemcol = " + isSystemCol);
555  boolean isVirtualCol = columnObject.get("is_virtualcol").getAsBoolean();
556  MAPDLOGGER.debug("is_vitrualcol = " + isVirtualCol);
557  boolean isDeletedCol = columnObject.get("is_deletedcol").getAsBoolean();
558  MAPDLOGGER.debug("is_deletedcol = " + isDeletedCol);
559  MAPDLOGGER.debug("");
560 
561  if (isDeletedCol) {
562  MAPDLOGGER.debug("Skipping delete column.");
563  continue;
564  }
565 
566  TColumnType tct = new TColumnType();
567  TTypeInfo tti = new TTypeInfo();
568  TDatumType tdt;
569 
570  if (colType == KARRAY) {
571  tti.is_array = true;
572  tdt = typeToThrift(colSubType);
573  } else {
574  tti.is_array = false;
575  tdt = typeToThrift(colType);
576  }
577 
578  tti.nullable = !isNotNull;
579  tti.encoding = TEncodingType.NONE;
580  tti.type = tdt;
581  tti.scale = colScale;
582  tti.precision = colDim;
583 
584  tct.col_name = colName;
585  tct.col_type = tti;
586  tct.is_system = isSystemCol;
587 
588  if (skip_physical_cols <= 0) skip_physical_cols = get_physical_cols(colType);
589  if (is_geometry(colType) || skip_physical_cols-- <= 0) td.addToRow_desc(tct);
590  }
591 
592  return td;
593  }
594 
595  private int getTableId(String tableName) {
596  Statement stmt = null;
597  ResultSet rs = null;
598  int tableId = -1;
599  try {
600  stmt = catConn.createStatement();
601  rs = stmt.executeQuery(String.format(
602  "SELECT tableid FROM mapd_tables where name = '%s' COLLATE NOCASE;",
603  tableName));
604  while (rs.next()) {
605  tableId = rs.getInt("tableid");
606  MAPDLOGGER.debug("tableId = " + tableId);
607  MAPDLOGGER.debug("");
608  }
609  rs.close();
610  stmt.close();
611  } catch (Exception e) {
612  String err = "Error trying to read from metadata table mapd_tables;DB: "
613  + default_db + " data dir " + dataDir + ", error was " + e.getMessage();
614  MAPDLOGGER.error(err);
615  throw new RuntimeException(err);
616  } finally {
617  if (rs != null) {
618  try {
619  rs.close();
620  } catch (SQLException ex) {
621  String err = "Could not close resultset, error was " + ex.getMessage();
622  MAPDLOGGER.error(err);
623  throw new RuntimeException(err);
624  }
625  }
626  if (stmt != null) {
627  try {
628  stmt.close();
629  } catch (SQLException ex) {
630  String err = "Could not close stmt, error was " + ex.getMessage();
631  MAPDLOGGER.error(err);
632  throw new RuntimeException(err);
633  }
634  }
635  }
636  return (tableId);
637  }
638 
639  private boolean isView(String tableName) {
640  Statement stmt;
641  ResultSet rs;
642  int viewFlag = 0;
643  try {
644  stmt = catConn.createStatement();
645  rs = stmt.executeQuery(String.format(
646  "SELECT isview FROM mapd_tables where name = '%s' COLLATE NOCASE;",
647  tableName));
648  while (rs.next()) {
649  viewFlag = rs.getInt("isview");
650  MAPDLOGGER.debug("viewFlag = " + viewFlag);
651  MAPDLOGGER.debug("");
652  }
653  rs.close();
654  stmt.close();
655  } catch (Exception e) {
656  String err = "error trying to read from mapd_views, error was " + e.getMessage();
657  MAPDLOGGER.error(err);
658  throw new RuntimeException(err);
659  }
660  return (viewFlag == 1);
661  }
662 
663  private String getViewSql(String tableName) {
664  String sqlText;
665  if (mapdPort == -1) {
666  // use sql
668  sqlText = getViewSqlViaSql(getTableId(tableName));
670  } else {
671  // use thrift direct to local server
672  try {
673  TProtocol protocol = null;
674 
675  TTransport transport =
676  sock_transport_properties.openClientTransport("localhost", mapdPort);
677  if (!transport.isOpen()) transport.open();
678  protocol = new TBinaryProtocol(transport);
679 
680  OmniSci.Client client = new OmniSci.Client(protocol);
681  TTableDetails td = client.get_table_details_for_database(
682  currentUser.getSession(), tableName, default_db);
683  transport.close();
684 
685  sqlText = td.getView_sql();
686 
687  } catch (TTransportException ex) {
688  MAPDLOGGER.error(ex.toString());
689  throw new RuntimeException(ex.toString());
690  } catch (TOmniSciException ex) {
691  MAPDLOGGER.error(ex.getError_msg());
692  throw new RuntimeException(ex.getError_msg());
693  } catch (TException ex) {
694  MAPDLOGGER.error(ex.toString());
695  throw new RuntimeException(ex.toString());
696  }
697  }
698  /* return string without the sqlite's trailing semicolon */
699  if (sqlText.charAt(sqlText.length() - 1) == ';') {
700  return (sqlText.substring(0, sqlText.length() - 1));
701  } else {
702  return (sqlText);
703  }
704  }
705 
706  // we assume there is already a DB connection here
707  private String getViewSqlViaSql(int tableId) {
708  Statement stmt;
709  ResultSet rs;
710  String sqlText = "";
711  try {
712  stmt = catConn.createStatement();
713  rs = stmt.executeQuery(String.format(
714  "SELECT sql FROM mapd_views where tableid = '%s' COLLATE NOCASE;",
715  tableId));
716  while (rs.next()) {
717  sqlText = rs.getString("sql");
718  MAPDLOGGER.debug("View definition = " + sqlText);
719  MAPDLOGGER.debug("");
720  }
721  rs.close();
722  stmt.close();
723  } catch (Exception e) {
724  String err = "error trying to read from mapd_views, error was " + e.getMessage();
725  MAPDLOGGER.error(err);
726  throw new RuntimeException(err);
727  }
728  if (sqlText == null || sqlText.length() == 0) {
729  String err = "No view text found";
730  MAPDLOGGER.error(err);
731  throw new RuntimeException(err);
732  }
733  return sqlText;
734  }
735 
736  private TDatumType typeToThrift(int type) {
737  switch (type) {
738  case KBOOLEAN:
739  return TDatumType.BOOL;
740  case KTINYINT:
741  return TDatumType.TINYINT;
742  case KSMALLINT:
743  return TDatumType.SMALLINT;
744  case KINT:
745  return TDatumType.INT;
746  case KBIGINT:
747  return TDatumType.BIGINT;
748  case KFLOAT:
749  return TDatumType.FLOAT;
750  case KNUMERIC:
751  case KDECIMAL:
752  return TDatumType.DECIMAL;
753  case KDOUBLE:
754  return TDatumType.DOUBLE;
755  case KTEXT:
756  case KVARCHAR:
757  case KCHAR:
758  return TDatumType.STR;
759  case KTIME:
760  return TDatumType.TIME;
761  case KTIMESTAMP:
762  return TDatumType.TIMESTAMP;
763  case KDATE:
764  return TDatumType.DATE;
765  case KINTERVAL_DAY_TIME:
766  return TDatumType.INTERVAL_DAY_TIME;
768  return TDatumType.INTERVAL_YEAR_MONTH;
769  case KPOINT:
770  return TDatumType.POINT;
771  case KLINESTRING:
772  return TDatumType.LINESTRING;
773  case KPOLYGON:
774  return TDatumType.POLYGON;
775  case KMULTIPOLYGON:
777  default:
778  return null;
779  }
780  }
781 
782  private void populateDatabases() {
783  // TODO 13 Mar 2021 MAT
784  // this probably has to come across from the server on first start up rather
785  // than lazy instantiation here
786  // as a user may not be able to see all schemas and this sets it for the life
787  // of the server.
788  // Proceeding this way as a WIP
789  if (mapdPort == 0) {
790  // seems to be a condition that is expected
791  // for FSI testing
792  return;
793  }
794  if (mapdPort == -1) {
795  // use sql
796  connectToCatalog("omnisci_system_catalog"); // hardcoded sys catalog
797  Set<String> dbNames = getDatabases_SQL();
799  for (String dbName : dbNames) {
800  Set<String> ts = new HashSet<String>();
801  DATABASE_TO_TABLES.putIfAbsent(dbName, ts);
802  }
803  return;
804  }
805  // use thrift direct to local server
806  try {
807  TProtocol protocol = null;
808  TTransport transport =
809  sock_transport_properties.openClientTransport("localhost", mapdPort);
810  if (!transport.isOpen()) transport.open();
811  protocol = new TBinaryProtocol(transport);
812 
813  OmniSci.Client client = new OmniSci.Client(protocol);
814 
815  List<TDBInfo> dbList = client.get_databases(currentUser.getSession());
816  for (TDBInfo dbInfo : dbList) {
817  Set<String> ts = new HashSet<String>();
818  DATABASE_TO_TABLES.putIfAbsent(dbInfo.db_name, ts);
819  }
820  transport.close();
821 
822  } catch (TTransportException ex) {
823  MAPDLOGGER.error("TTransportException on port [" + mapdPort + "]");
824  MAPDLOGGER.error(ex.toString());
825  throw new RuntimeException(ex.toString());
826  } catch (TOmniSciException ex) {
827  MAPDLOGGER.error(ex.getError_msg());
828  throw new RuntimeException(ex.getError_msg());
829  } catch (TException ex) {
830  MAPDLOGGER.error(ex.toString());
831  throw new RuntimeException(ex.toString());
832  }
833  }
834 
835  private Set<String> getDatabases_SQL() {
836  Set<String> dbSet = new HashSet<String>();
837  Statement stmt = null;
838  ResultSet rs = null;
839  String sqlText = "";
840  try {
841  stmt = catConn.createStatement();
842 
843  // get the tables
844  rs = stmt.executeQuery("SELECT name FROM mapd_databases ");
845  while (rs.next()) {
846  dbSet.add(rs.getString("name"));
847  /*--*/
848  MAPDLOGGER.debug("Object name = " + rs.getString("name"));
849  }
850  rs.close();
851  stmt.close();
852 
853  } catch (Exception e) {
854  String err = "error trying to get all the databases, error was " + e.getMessage();
855  MAPDLOGGER.error(err);
856  throw new RuntimeException(err);
857  }
858  return dbSet;
859  }
860 
861  public void updateMetaData(String schema, String table) {
862  // Check if table is specified, if not we are dropping an entire DB so need to
863  // remove all
864  // tables for that DB
865  if (table.equals("")) {
866  // Drop db and all tables
867  // iterate through all and remove matching schema
868  Set<List<String>> all = new HashSet<>(MAPD_TABLE_DETAILS.keySet());
869  for (List<String> keys : all) {
870  if (keys.get(0).equals(schema.toUpperCase())) {
871  MAPDLOGGER.debug(
872  "removing all for schema " + keys.get(0) + " table " + keys.get(1));
873  MAPD_TABLE_DETAILS.remove(keys);
874  }
875  }
876  } else {
877  MAPDLOGGER.debug("removing schema " + schema.toUpperCase() + " table "
878  + table.toUpperCase());
879  MAPD_TABLE_DETAILS.remove(
880  ImmutableList.of(schema.toUpperCase(), table.toUpperCase()));
881  }
882  // Invalidate views
883  Set<List<String>> all = new HashSet<>(MAPD_TABLE_DETAILS.keySet());
884  for (List<String> keys : all) {
885  if (keys.get(0).equals(schema.toUpperCase())) {
886  Table ttable = MAPD_TABLE_DETAILS.get(keys);
887  if (ttable instanceof MapDView) {
888  MAPDLOGGER.debug(
889  "removing view in schema " + keys.get(0) + " view " + keys.get(1));
890  MAPD_TABLE_DETAILS.remove(keys);
891  }
892  }
893  }
894  // Could be a removal or an add request for a DB
895  Set<String> mSet = DATABASE_TO_TABLES.get(schema.toUpperCase());
896  if (mSet != null) {
897  MAPDLOGGER.debug("removing schema " + schema.toUpperCase());
898  DATABASE_TO_TABLES.remove(schema.toUpperCase());
899  } else {
900  // add a empty database descriptor for new DB, it will be lazily populated when
901  // required
902  Set<String> ts = new HashSet<String>();
903  DATABASE_TO_TABLES.putIfAbsent(schema.toUpperCase(), ts);
904  }
905  }
906 }
#define LINESTRING
static volatile Map< String, Set< String > > DATABASE_TO_TABLES
Table getTable(String tableName)
static final int get_physical_cols(int type)
MetaConnect(int mapdPort, String dataDir, MapDUser currentMapDUser, MapDParser parser, SockTransportProperties skT)
#define SMALLINT
static final int KMULTIPOLYGON
#define DOUBLE
#define BIGINT
void updateMetaData(String schema, String table)
#define DATE
static volatile Map< List< String >, Table > MAPD_TABLE_DETAILS
String getViewSql(String tableName)
static final boolean is_geometry(int type)
String getViewSqlViaSql(int tableId)
#define MULTIPOLYGON
#define POINT
TTableDetails get_table_details(String tableName)
static final int KINTERVAL_YEAR_MONTH
static final Logger MAPDLOGGER
final SockTransportProperties sock_transport_properties
static final int KLINESTRING
#define TIME
#define TINYINT
void connectToCatalog(String catalog)
#define TIMESTAMP
TTableDetails get_table_detail_SQL(String tableName)
TDatumType typeToThrift(int type)
Set< String > getDatabases_SQL()
MetaConnect(int mapdPort, String dataDir, MapDUser currentMapDUser, MapDParser parser, SockTransportProperties skT, String db)
int getTableId(String tableName)
#define DECIMAL
TTableDetails get_table_detail_JSON(String tableName)
static final int KINTERVAL_DAY_TIME
boolean isView(String tableName)
#define POLYGON
#define FLOAT