OmniSciDB  471d68cefb
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
com.mapd.utility.SQLImporter Class Reference
+ Collaboration diagram for com.mapd.utility.SQLImporter:

Static Public Member Functions

static void main (String[] args)
 

Protected Attributes

String session = null
 
OmniSci.Client client = null
 

Package Functions

void doWork (String[] args)
 
void executeQuery ()
 

Package Attributes

Db_vendor_types vendor_types = null
 

Static Package Attributes

static final Logger LOGGER = LoggerFactory.getLogger(SQLImporter.class)
 

Private Member Functions

void run_init (Connection conn)
 
void help (Options options)
 
void checkMapDTable (Connection otherdb_conn, ResultSetMetaData md) throws SQLException
 
void verifyColumnSignaturesMatch (Connection otherdb_conn, List< TColumnType > dstColumns, ResultSetMetaData srcColumns) throws SQLException
 
void createMapDTable (Connection otherdb_conn, ResultSetMetaData metaData)
 
void createMapDConnection ()
 
List< TColumnType > getColumnInfo (String tName)
 
boolean tableExists (String tName)
 
void executeMapDCommand (String sql)
 
String getColType (int cType, int precision, int scale)
 
TColumn setupBinaryColumn (int i, ResultSetMetaData md, int bufferSize) throws SQLException
 
void setColValue (ResultSet rs, TColumn col, int columnType, int colNum, int scale, String colTypeName) throws SQLException
 
void resetBinaryColumn (int i, ResultSetMetaData md, int bufferSize, TColumn col) throws SQLException
 

Private Attributes

CommandLine cmd = null
 
DateTimeUtils dateTimeUtils
 

Detailed Description

Definition at line 313 of file SQLImporter.java.

Member Function Documentation

void com.mapd.utility.SQLImporter.checkMapDTable ( Connection  otherdb_conn,
ResultSetMetaData  md 
) throws SQLException
inlineprivate

Definition at line 536 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.cmd, com.mapd.utility.SQLImporter.createMapDConnection(), com.mapd.utility.SQLImporter.createMapDTable(), com.mapd.utility.SQLImporter.executeMapDCommand(), com.mapd.utility.SQLImporter.getColumnInfo(), com.mapd.utility.SQLImporter.tableExists(), and com.mapd.utility.SQLImporter.verifyColumnSignaturesMatch().

Referenced by com.mapd.utility.SQLImporter.executeQuery().

537  {
539  String tName = cmd.getOptionValue("targetTable");
540 
541  if (tableExists(tName)) {
542  // check if we want to truncate
543  if (cmd.hasOption("truncate")) {
544  executeMapDCommand("Drop table " + tName);
545  createMapDTable(otherdb_conn, md);
546  } else {
547  List<TColumnType> columnInfo = getColumnInfo(tName);
548  verifyColumnSignaturesMatch(otherdb_conn, columnInfo, md);
549  }
550  } else {
551  createMapDTable(otherdb_conn, md);
552  }
553  }
List< TColumnType > getColumnInfo(String tName)
void createMapDTable(Connection otherdb_conn, ResultSetMetaData metaData)
void verifyColumnSignaturesMatch(Connection otherdb_conn, List< TColumnType > dstColumns, ResultSetMetaData srcColumns)
boolean tableExists(String tName)
void executeMapDCommand(String sql)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.createMapDConnection ( )
inlineprivate

Definition at line 716 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.client, com.mapd.utility.SQLImporter.cmd, and com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable().

716  {
717  TTransport transport = null;
718  TProtocol protocol = new TBinaryProtocol(transport);
719  int port = Integer.valueOf(cmd.getOptionValue("port", "6274"));
720  String server = cmd.getOptionValue("server", "localhost");
721  try {
722  // Uses default certificate stores.
723  boolean load_trust_store = cmd.hasOption("https");
724  SockTransportProperties skT = null;
725  if (cmd.hasOption("https")) {
726  skT = SockTransportProperties.getEncryptedClientDefaultTrustStore(
727  !cmd.hasOption("insecure"));
728  transport = skT.openHttpsClientTransport(server, port);
729  transport.open();
730  protocol = new TJSONProtocol(transport);
731  } else if (cmd.hasOption("http")) {
732  skT = SockTransportProperties.getUnencryptedClient();
733  transport = skT.openHttpClientTransport(server, port);
734  protocol = new TJSONProtocol(transport);
735  } else {
736  skT = SockTransportProperties.getUnencryptedClient();
737  transport = skT.openClientTransport(server, port);
738  transport.open();
739  protocol = new TBinaryProtocol(transport);
740  }
741 
742  client = new OmniSci.Client(protocol);
743  // This if will be useless until PKI signon
744  if (cmd.hasOption("user")) {
745  session = client.connect(cmd.getOptionValue("user", "admin"),
746  cmd.getOptionValue("passwd", "HyperInteractive"),
747  cmd.getOptionValue("database", "omnisci"));
748  }
749  LOGGER.debug("Connected session is " + session);
750 
751  } catch (TTransportException ex) {
752  LOGGER.error("Connection failed - " + ex.toString());
753  exit(1);
754  } catch (TOmniSciException ex) {
755  LOGGER.error("Connection failed - " + ex.getError_msg());
756  exit(2);
757  } catch (TException ex) {
758  LOGGER.error("Connection failed - " + ex.toString());
759  exit(3);
760  } catch (Exception ex) {
761  LOGGER.error("General exception - " + ex.toString());
762  exit(4);
763  }
764  }

+ Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.createMapDTable ( Connection  otherdb_conn,
ResultSetMetaData  metaData 
)
inlineprivate

Definition at line 674 of file SQLImporter.java.

References File_Namespace.append(), com.mapd.utility.SQLImporter.cmd, com.mapd.utility.SQLImporter.executeMapDCommand(), com.mapd.utility.SQLImporter.getColType(), i, Integer, and run_benchmark_import.type.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable().

674  {
675  StringBuilder sb = new StringBuilder();
676  sb.append("Create table ").append(cmd.getOptionValue("targetTable")).append("(");
677 
678  // Now iterate the metadata
679  try {
680  for (int i = 1; i <= metaData.getColumnCount(); i++) {
681  if (i > 1) {
682  sb.append(",");
683  }
684  LOGGER.debug("Column name is " + metaData.getColumnName(i));
685  LOGGER.debug("Column type is " + metaData.getColumnTypeName(i));
686  LOGGER.debug("Column type is " + metaData.getColumnType(i));
687 
688  sb.append(metaData.getColumnName(i)).append(" ");
689  int col_type = metaData.getColumnType(i);
690  if (col_type == java.sql.Types.OTHER) {
691  Db_vendor_types.GisType type =
692  vendor_types.find_gis_type(otherdb_conn, metaData, i);
693  sb.append(Db_vendor_types.gis_type_to_str(type));
694  } else {
695  sb.append(getColType(metaData.getColumnType(i),
696  metaData.getPrecision(i),
697  metaData.getScale(i)));
698  }
699  }
700  sb.append(")");
701 
702  if (Integer.valueOf(cmd.getOptionValue("fragmentSize", "0")) > 0) {
703  sb.append(" with (fragment_size = ");
704  sb.append(cmd.getOptionValue("fragmentSize", "0"));
705  sb.append(")");
706  }
707 
708  } catch (SQLException ex) {
709  LOGGER.error("Error processing the metadata - " + ex.toString());
710  exit(1);
711  }
712 
713  executeMapDCommand(sb.toString());
714  }
size_t append(FILE *f, const size_t size, const int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:159
String getColType(int cType, int precision, int scale)
void executeMapDCommand(String sql)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.doWork ( String[]  args)
inlinepackage

Definition at line 329 of file SQLImporter.java.

References run_benchmark_import.args, com.mapd.utility.SQLImporter.cmd, and com.mapd.utility.SQLImporter.executeQuery().

329  {
330  // create Options object
331 
332  SQLImporter_args s_args = new SQLImporter_args();
333 
334  try {
335  cmd = s_args.parse(args);
336  } catch (ParseException ex) {
337  LOGGER.error(ex.getLocalizedMessage());
338  s_args.printHelpMessage();
339  exit(0);
340  }
341  executeQuery();
342  }

+ Here is the call graph for this function:

void com.mapd.utility.SQLImporter.executeMapDCommand ( String  sql)
inlineprivate

Definition at line 801 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable(), and com.mapd.utility.SQLImporter.createMapDTable().

801  {
802  LOGGER.info("Run Command - " + sql);
803 
804  try {
805  TQueryResult sqlResult = client.sql_execute(session, sql + ";", true, null, -1, -1);
806  } catch (TOmniSciException ex) {
807  LOGGER.error("SQL Execute failed - " + ex.getError_msg());
808  exit(1);
809  } catch (TException ex) {
810  LOGGER.error("SQL Execute failed - " + ex.toString());
811  exit(1);
812  }
813  }

+ Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.executeQuery ( )
inlinepackage

Definition at line 344 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.checkMapDTable(), com.mapd.utility.SQLImporter.cmd, i, com.mapd.utility.db_vendors.Db_vendor_types.isAutoCommitDisabledRequired(), com.mapd.utility.SQLImporter.resetBinaryColumn(), com.mapd.utility.SQLImporter.run_init(), com.mapd.utility.SQLImporter.session, com.mapd.utility.SQLImporter.setColValue(), com.mapd.utility.SQLImporter.setupBinaryColumn(), and com.mapd.utility.SQLImporter.vendor_types.

Referenced by com.mapd.utility.SQLImporter.doWork().

344  {
345  Connection conn = null;
346  Statement stmt = null;
347 
348  long totalTime = 0;
349 
350  try {
351  // Open a connection
352  LOGGER.info("Connecting to database url :" + cmd.getOptionValue("jdbcConnect"));
353  conn = DriverManager.getConnection(cmd.getOptionValue("jdbcConnect"),
354  cmd.getOptionValue("sourceUser"),
355  cmd.getOptionValue("sourcePasswd"));
356  vendor_types = Db_vendor_types.Db_vendor_factory(cmd.getOptionValue("jdbcConnect"));
357  long startTime = System.currentTimeMillis();
358 
359  // run init file script on targe DB if present
360  if (cmd.hasOption("initializeFile")) {
361  run_init(conn);
362  }
363 
364  try {
366  conn.setAutoCommit(false);
367  }
368  } catch (SQLException se) {
369  LOGGER.warn(
370  "SQLException when attempting to setAutoCommit to false, jdbc driver probably doesnt support it. Error is "
371  + se.toString());
372  }
373 
374  // Execute a query
375  stmt = conn.createStatement();
376 
377  int bufferSize = Integer.valueOf(cmd.getOptionValue("bufferSize", "10000"));
378  // set the jdbc fetch buffer size to reduce the amount of records being moved to
379  // java from postgress
380  stmt.setFetchSize(bufferSize);
381  long timer;
382 
383  ResultSet rs = stmt.executeQuery(cmd.getOptionValue("sqlStmt"));
384 
385  // check if table already exists and is compatible in OmniSci with the query
386  // metadata
387  ResultSetMetaData md = rs.getMetaData();
388  checkMapDTable(conn, md);
389 
390  timer = System.currentTimeMillis();
391 
392  long resultCount = 0;
393  int bufferCount = 0;
394  long total = 0;
395 
396  List<TColumn> cols = new ArrayList(md.getColumnCount());
397  for (int i = 1; i <= md.getColumnCount(); i++) {
398  TColumn col = setupBinaryColumn(i, md, bufferSize);
399  cols.add(col);
400  }
401 
402  boolean assignRenderGroups = !cmd.hasOption("noPolyRenderGroups");
403 
404  // read data from old DB
405  while (rs.next()) {
406  for (int i = 1; i <= md.getColumnCount(); i++) {
407  setColValue(rs,
408  cols.get(i - 1),
409  md.getColumnType(i),
410  i,
411  md.getScale(i),
412  md.getColumnTypeName(i));
413  }
414  resultCount++;
415  bufferCount++;
416  if (bufferCount == bufferSize) {
417  bufferCount = 0;
418  // send the buffer to mapD
419  if (assignRenderGroups) {
420  client.load_table_binary_columnar_polys(
421  session, cmd.getOptionValue("targetTable"), cols, null, true);
422  } else {
423  client.load_table_binary_columnar(
424  session, cmd.getOptionValue("targetTable"), cols, null);
425  }
426  // recreate columnar store for use
427  for (int i = 1; i <= md.getColumnCount(); i++) {
428  resetBinaryColumn(i, md, bufferSize, cols.get(i - 1));
429  }
430 
431  if (resultCount % 100000 == 0) {
432  LOGGER.info("Imported " + resultCount + " records");
433  }
434  }
435  }
436  if (bufferCount > 0) {
437  // send the LAST buffer to mapD
438  if (assignRenderGroups) {
439  client.load_table_binary_columnar_polys(
440  session, cmd.getOptionValue("targetTable"), cols, null, true);
441  } else {
442  client.load_table_binary_columnar(
443  session, cmd.getOptionValue("targetTable"), cols, null);
444  }
445  bufferCount = 0;
446  }
447 
448  // dump render group assignment data immediately
449  if (assignRenderGroups) {
450  client.load_table_binary_columnar_polys(
451  session, cmd.getOptionValue("targetTable"), null, null, false);
452  }
453 
454  LOGGER.info("result set count is " + resultCount + " read time is "
455  + (System.currentTimeMillis() - timer) + "ms");
456 
457  // Clean-up environment
458  rs.close();
459  stmt.close();
460  conn.close();
461 
462  totalTime = System.currentTimeMillis() - startTime;
463  } catch (SQLException se) {
464  LOGGER.error("SQLException - " + se.toString());
465  se.printStackTrace();
466  } catch (TOmniSciException ex) {
467  LOGGER.error("TOmniSciException - " + ex.getError_msg());
468  ex.printStackTrace();
469  } catch (TException ex) {
470  LOGGER.error("TException failed - " + ex.toString());
471  ex.printStackTrace();
472  } finally {
473  // finally block used to close resources
474  try {
475  if (stmt != null) {
476  stmt.close();
477  }
478  } catch (SQLException se2) {
479  } // nothing we can do
480  try {
481  if (conn != null) {
482  conn.close();
483  }
484  } catch (SQLException se) {
485  LOGGER.error("SQlException in close - " + se.toString());
486  se.printStackTrace();
487  }
488  try {
489  if (session != null) {
490  client.disconnect(session);
491  }
492  } catch (TOmniSciException ex) {
493  LOGGER.error("TOmniSciException - in finalization " + ex.getError_msg());
494  ex.printStackTrace();
495  } catch (TException ex) {
496  LOGGER.error("TException - in finalization" + ex.toString());
497  ex.printStackTrace();
498  }
499  }
500  }
void resetBinaryColumn(int i, ResultSetMetaData md, int bufferSize, TColumn col)
void checkMapDTable(Connection otherdb_conn, ResultSetMetaData md)
void setColValue(ResultSet rs, TColumn col, int columnType, int colNum, int scale, String colTypeName)
TColumn setupBinaryColumn(int i, ResultSetMetaData md, int bufferSize)
void run_init(Connection conn)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

String com.mapd.utility.SQLImporter.getColType ( int  cType,
int  precision,
int  scale 
)
inlineprivate

Definition at line 815 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.createMapDTable().

815  {
816  // Note - if cType is OTHER a earlier call will have been made
817  // to try and work out the db vendors specific type.
818  if (precision > 19) {
819  precision = 19;
820  }
821  if (scale > 19) {
822  scale = 18;
823  }
824  switch (cType) {
825  case java.sql.Types.TINYINT:
826  return ("TINYINT");
827  case java.sql.Types.SMALLINT:
828  return ("SMALLINT");
829  case java.sql.Types.INTEGER:
830  return ("INTEGER");
831  case java.sql.Types.BIGINT:
832  return ("BIGINT");
833  case java.sql.Types.FLOAT:
834  return ("FLOAT");
835  case java.sql.Types.DECIMAL:
836  return ("DECIMAL(" + precision + "," + scale + ")");
837  case java.sql.Types.DOUBLE:
838  return ("DOUBLE");
839  case java.sql.Types.REAL:
840  return ("REAL");
841  case java.sql.Types.NUMERIC:
842  return ("NUMERIC(" + precision + "," + scale + ")");
843  case java.sql.Types.TIME:
844  return ("TIME");
845  case java.sql.Types.TIMESTAMP:
846  return ("TIMESTAMP");
847  case java.sql.Types.DATE:
848  return ("DATE");
849  case java.sql.Types.BOOLEAN:
850  case java.sql.Types
851  .BIT: // deal with postgress treating boolean as bit... this will bite me
852  return ("BOOLEAN");
853  case java.sql.Types.NVARCHAR:
854  case java.sql.Types.VARCHAR:
855  case java.sql.Types.NCHAR:
856  case java.sql.Types.CHAR:
857  case java.sql.Types.LONGVARCHAR:
858  case java.sql.Types.LONGNVARCHAR:
859  return ("TEXT ENCODING DICT");
860  default:
861  throw new AssertionError("Column type " + cType + " not Supported");
862  }
863  }

+ Here is the caller graph for this function:

List<TColumnType> com.mapd.utility.SQLImporter.getColumnInfo ( String  tName)
inlineprivate

Definition at line 766 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable().

766  {
767  LOGGER.debug("Getting columns for " + tName);
768  List<TColumnType> row_descriptor = null;
769  try {
770  TTableDetails table_details = client.get_table_details(session, tName);
771  row_descriptor = table_details.row_desc;
772  } catch (TOmniSciException ex) {
773  LOGGER.error("column check failed - " + ex.getError_msg());
774  exit(3);
775  } catch (TException ex) {
776  LOGGER.error("column check failed - " + ex.toString());
777  exit(3);
778  }
779  return row_descriptor;
780  }

+ Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.help ( Options  options)
inlineprivate

Definition at line 529 of file SQLImporter.java.

529  {
530  // automatically generate the help statement
531  HelpFormatter formatter = new HelpFormatter();
532  formatter.setOptionComparator(null); // get options in the order they are created
533  formatter.printHelp("SQLImporter", options);
534  }
static void com.mapd.utility.SQLImporter.main ( String[]  args)
inlinestatic

Definition at line 324 of file SQLImporter.java.

References run_benchmark_import.args.

324  {
325  SQLImporter sq = new SQLImporter();
326  sq.doWork(args);
327  }
void com.mapd.utility.SQLImporter.resetBinaryColumn ( int  i,
ResultSetMetaData  md,
int  bufferSize,
TColumn  col 
) throws SQLException
inlineprivate

Definition at line 1037 of file SQLImporter.java.

References i.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

1038  {
1039  col.nulls.clear();
1040 
1041  switch (md.getColumnType(i)) {
1042  case java.sql.Types.TINYINT:
1043  case java.sql.Types.SMALLINT:
1044  case java.sql.Types.INTEGER:
1045  case java.sql.Types.BIGINT:
1046  case java.sql.Types.TIME:
1047  case java.sql.Types.TIMESTAMP:
1048  case java.sql.Types
1049  .BIT: // deal with postgress treating boolean as bit... this will bite me
1050  case java.sql.Types.BOOLEAN:
1051  case java.sql.Types.DATE:
1052  case java.sql.Types.DECIMAL:
1053  case java.sql.Types.NUMERIC:
1054  col.data.int_col.clear();
1055  break;
1056 
1057  case java.sql.Types.FLOAT:
1058  case java.sql.Types.DOUBLE:
1059  case java.sql.Types.REAL:
1060  col.data.real_col.clear();
1061  break;
1062 
1063  case java.sql.Types.NVARCHAR:
1064  case java.sql.Types.VARCHAR:
1065  case java.sql.Types.NCHAR:
1066  case java.sql.Types.CHAR:
1067  case java.sql.Types.LONGVARCHAR:
1068  case java.sql.Types.LONGNVARCHAR:
1069  case java.sql.Types.OTHER:
1070  col.data.str_col.clear();
1071  break;
1072  default:
1073  throw new AssertionError("Column type " + md.getColumnType(i) + " not Supported");
1074  }
1075  }

+ Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.run_init ( Connection  conn)
inlineprivate

Definition at line 502 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.cmd, and parse_ast.line.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

502  {
503  // attempt to open file
504  String line = "";
505  try {
506  BufferedReader reader =
507  new BufferedReader(new FileReader(cmd.getOptionValue("initializeFile")));
508  Statement stmt = conn.createStatement();
509  while ((line = reader.readLine()) != null) {
510  if (line.isEmpty()) {
511  continue;
512  }
513  LOGGER.info("Running : " + line);
514  stmt.execute(line);
515  }
516  stmt.close();
517  reader.close();
518  } catch (IOException e) {
519  LOGGER.error("Exception occurred trying to read initialize file: "
520  + cmd.getOptionValue("initFile"));
521  exit(1);
522  } catch (SQLException e) {
523  LOGGER.error(
524  "Exception occurred trying to execute initialize file entry : " + line);
525  exit(1);
526  }
527  }
tuple line
Definition: parse_ast.py:10

+ Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.setColValue ( ResultSet  rs,
TColumn  col,
int  columnType,
int  colNum,
int  scale,
String  colTypeName 
) throws SQLException
inlineprivate

Definition at line 911 of file SQLImporter.java.

References omnisci.dtypes.Date, Double, t, omnisci.dtypes.Time, and omnisci.dtypes.Timestamp.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

916  {
917  switch (columnType) {
918  case java.sql.Types
919  .BIT: // deal with postgress treating boolean as bit... this will bite me
920  case java.sql.Types.BOOLEAN:
921  Boolean b = rs.getBoolean(colNum);
922  if (rs.wasNull()) {
923  col.nulls.add(Boolean.TRUE);
924  col.data.int_col.add(0L);
925  } else {
926  col.nulls.add(Boolean.FALSE);
927  col.data.int_col.add(b ? 1L : 0L);
928  }
929  break;
930 
931  case java.sql.Types.DECIMAL:
932  case java.sql.Types.NUMERIC:
933  BigDecimal bd = rs.getBigDecimal(colNum);
934  if (rs.wasNull()) {
935  col.nulls.add(Boolean.TRUE);
936  col.data.int_col.add(0L);
937  } else {
938  col.nulls.add(Boolean.FALSE);
939  col.data.int_col.add(bd.multiply(new BigDecimal(pow(10L, scale))).longValue());
940  }
941  break;
942 
943  case java.sql.Types.TINYINT:
944  case java.sql.Types.SMALLINT:
945  case java.sql.Types.INTEGER:
946  case java.sql.Types.BIGINT:
947  Long l = rs.getLong(colNum);
948  if (rs.wasNull()) {
949  col.nulls.add(Boolean.TRUE);
950  col.data.int_col.add(new Long(0));
951  } else {
952  col.nulls.add(Boolean.FALSE);
953  col.data.int_col.add(l);
954  }
955  break;
956 
957  case java.sql.Types.TIME:
958  Time t = rs.getTime(colNum);
959  if (rs.wasNull()) {
960  col.nulls.add(Boolean.TRUE);
961  col.data.int_col.add(0L);
962 
963  } else {
964  col.data.int_col.add(dateTimeUtils.getSecondsFromMilliseconds(t.getTime()));
965  col.nulls.add(Boolean.FALSE);
966  }
967 
968  break;
969  case java.sql.Types.TIMESTAMP:
970  Timestamp ts = rs.getTimestamp(colNum);
971  if (rs.wasNull()) {
972  col.nulls.add(Boolean.TRUE);
973  col.data.int_col.add(0L);
974 
975  } else {
976  col.data.int_col.add(dateTimeUtils.getSecondsFromMilliseconds(ts.getTime()));
977  col.nulls.add(Boolean.FALSE);
978  }
979 
980  break;
981  case java.sql.Types.DATE:
982  Date d = rs.getDate(colNum);
983  if (rs.wasNull()) {
984  col.nulls.add(Boolean.TRUE);
985  col.data.int_col.add(0L);
986 
987  } else {
988  col.data.int_col.add(dateTimeUtils.getSecondsFromMilliseconds(d.getTime()));
989  col.nulls.add(Boolean.FALSE);
990  }
991  break;
992  case java.sql.Types.FLOAT:
993  case java.sql.Types.DOUBLE:
994  case java.sql.Types.REAL:
995  Double db = rs.getDouble(colNum);
996  if (rs.wasNull()) {
997  col.nulls.add(Boolean.TRUE);
998  col.data.real_col.add(new Double(0));
999 
1000  } else {
1001  col.nulls.add(Boolean.FALSE);
1002  col.data.real_col.add(db);
1003  }
1004  break;
1005 
1006  case java.sql.Types.NVARCHAR:
1007  case java.sql.Types.VARCHAR:
1008  case java.sql.Types.NCHAR:
1009  case java.sql.Types.CHAR:
1010  case java.sql.Types.LONGVARCHAR:
1011  case java.sql.Types.LONGNVARCHAR:
1012  String strVal = rs.getString(colNum);
1013  if (rs.wasNull()) {
1014  col.nulls.add(Boolean.TRUE);
1015  col.data.str_col.add("");
1016 
1017  } else {
1018  col.data.str_col.add(strVal);
1019  col.nulls.add(Boolean.FALSE);
1020  }
1021  break;
1022  case java.sql.Types.OTHER:
1023  Object objVal = rs.getObject(colNum);
1024  if (rs.wasNull()) {
1025  col.nulls.add(Boolean.TRUE);
1026  col.data.str_col.add("");
1027  } else {
1028  col.data.str_col.add(vendor_types.get_wkt(rs, colNum, colTypeName));
1029  col.nulls.add(Boolean.FALSE);
1030  }
1031  break;
1032  default:
1033  throw new AssertionError("Column type " + columnType + " not Supported");
1034  }
1035  }
char * t

+ Here is the caller graph for this function:

TColumn com.mapd.utility.SQLImporter.setupBinaryColumn ( int  i,
ResultSetMetaData  md,
int  bufferSize 
) throws SQLException
inlineprivate

Definition at line 865 of file SQLImporter.java.

References i.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

866  {
867  TColumn col = new TColumn();
868 
869  col.nulls = new ArrayList<Boolean>(bufferSize);
870 
871  col.data = new TColumnData();
872 
873  switch (md.getColumnType(i)) {
874  case java.sql.Types.TINYINT:
875  case java.sql.Types.SMALLINT:
876  case java.sql.Types.INTEGER:
877  case java.sql.Types.BIGINT:
878  case java.sql.Types.TIME:
879  case java.sql.Types.TIMESTAMP:
880  case java.sql.Types
881  .BIT: // deal with postgress treating boolean as bit... this will bite me
882  case java.sql.Types.BOOLEAN:
883  case java.sql.Types.DATE:
884  case java.sql.Types.DECIMAL:
885  case java.sql.Types.NUMERIC:
886  col.data.int_col = new ArrayList<Long>(bufferSize);
887  break;
888 
889  case java.sql.Types.FLOAT:
890  case java.sql.Types.DOUBLE:
891  case java.sql.Types.REAL:
892  col.data.real_col = new ArrayList<Double>(bufferSize);
893  break;
894 
895  case java.sql.Types.NVARCHAR:
896  case java.sql.Types.VARCHAR:
897  case java.sql.Types.NCHAR:
898  case java.sql.Types.CHAR:
899  case java.sql.Types.LONGVARCHAR:
900  case java.sql.Types.LONGNVARCHAR:
901  case java.sql.Types.OTHER:
902  col.data.str_col = new ArrayList<String>(bufferSize);
903  break;
904 
905  default:
906  throw new AssertionError("Column type " + md.getColumnType(i) + " not Supported");
907  }
908  return col;
909  }

+ Here is the caller graph for this function:

boolean com.mapd.utility.SQLImporter.tableExists ( String  tName)
inlineprivate

Definition at line 782 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable().

782  {
783  LOGGER.debug("Check for table " + tName);
784  try {
785  List<String> recv_get_tables = client.get_tables(session);
786  for (String s : recv_get_tables) {
787  if (s.equals(tName)) {
788  return true;
789  }
790  }
791  } catch (TOmniSciException ex) {
792  LOGGER.error("Table check failed - " + ex.getError_msg());
793  exit(3);
794  } catch (TException ex) {
795  LOGGER.error("Table check failed - " + ex.toString());
796  exit(3);
797  }
798  return false;
799  }

+ Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.verifyColumnSignaturesMatch ( Connection  otherdb_conn,
List< TColumnType >  dstColumns,
ResultSetMetaData  srcColumns 
) throws SQLException
inlineprivate

Definition at line 555 of file SQLImporter.java.

References BIGINT, com.mapd.utility.SQLImporter.cmd, DATE, DOUBLE, FLOAT, i, LINESTRING, MULTIPOLYGON, POINT, POLYGON, SMALLINT, TIME, TIMESTAMP, and TINYINT.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable().

557  {
558  if (srcColumns.getColumnCount() != dstColumns.size()) {
559  LOGGER.error("Table sizes do not match: Destination " + dstColumns.size()
560  + " versus Source " + srcColumns.getColumnCount());
561  exit(1);
562  }
563  for (int i = 1; i <= dstColumns.size(); ++i) {
564  if (!dstColumns.get(i - 1).getCol_name().equalsIgnoreCase(
565  srcColumns.getColumnName(i))) {
566  LOGGER.error(
567  "Destination table does not have matching column in same order for column number "
568  + i + " destination column name is " + dstColumns.get(i - 1).col_name
569  + " versus target column " + srcColumns.getColumnName(i));
570  exit(1);
571  }
572  TDatumType dstType = dstColumns.get(i - 1).getCol_type().getType();
573  int dstPrecision = dstColumns.get(i - 1).getCol_type().getPrecision();
574  int dstScale = dstColumns.get(i - 1).getCol_type().getScale();
575  int srcType = srcColumns.getColumnType(i);
576  int srcPrecision = srcColumns.getPrecision(i);
577  int srcScale = srcColumns.getScale(i);
578 
579  boolean match = false;
580  switch (srcType) {
581  case java.sql.Types.TINYINT:
582  match |= dstType == TDatumType.TINYINT;
583  // NOTE: it's okay to import smaller type to a bigger one,
584  // so we just fall through and try to match the next type.
585  // But the order of case statements is important here!
586  case java.sql.Types.SMALLINT:
587  match |= dstType == TDatumType.SMALLINT;
588  case java.sql.Types.INTEGER:
589  match |= dstType == TDatumType.INT;
590  case java.sql.Types.BIGINT:
591  match |= dstType == TDatumType.BIGINT;
592  break;
593  case java.sql.Types.DECIMAL:
594  case java.sql.Types.NUMERIC:
595  match = dstType == TDatumType.DECIMAL && dstPrecision == srcPrecision
596  && dstScale == srcScale;
597  break;
598  case java.sql.Types.FLOAT:
599  case java.sql.Types.REAL:
600  match |= dstType == TDatumType.FLOAT;
601  // Fall through and try double
602  case java.sql.Types.DOUBLE:
603  match |= dstType == TDatumType.DOUBLE;
604  if (cmd.hasOption("AllowDoubleToFloat")) {
605  match |= dstType == TDatumType.FLOAT;
606  }
607  break;
608  case java.sql.Types.TIME:
609  match = dstType == TDatumType.TIME;
610  break;
611  case java.sql.Types.TIMESTAMP:
612  match = dstType == TDatumType.TIMESTAMP;
613  break;
614  case java.sql.Types.DATE:
615  match = dstType == TDatumType.DATE;
616  break;
617  case java.sql.Types.BOOLEAN:
618  case java.sql.Types
619  .BIT: // deal with postgres treating boolean as bit... this will bite me
620  match = dstType == TDatumType.BOOL;
621  break;
622  case java.sql.Types.NVARCHAR:
623  case java.sql.Types.VARCHAR:
624  case java.sql.Types.NCHAR:
625  case java.sql.Types.CHAR:
626  case java.sql.Types.LONGVARCHAR:
627  case java.sql.Types.LONGNVARCHAR:
628  match = (dstType == TDatumType.STR || dstType == TDatumType.POINT
629  || dstType == TDatumType.POLYGON || dstType == TDatumType.MULTIPOLYGON
630  || dstType == TDatumType.LINESTRING);
631  break;
632  case java.sql.Types.OTHER:
633  // NOTE: I ignore subtypes (geography vs geopetry vs none) here just because
634  // it makes no difference for OmniSciDB at the moment
635  Db_vendor_types.GisType gisType =
636  vendor_types.find_gis_type(otherdb_conn, srcColumns, i);
637  if (gisType.srid != dstScale) {
638  match = false;
639  break;
640  }
641  switch (dstType) {
642  case POINT:
643  match = gisType.type.equalsIgnoreCase("POINT");
644  break;
645  case LINESTRING:
646  match = gisType.type.equalsIgnoreCase("LINESTRING");
647  break;
648  case POLYGON:
649  match = gisType.type.equalsIgnoreCase("POLYGON");
650  break;
651  case MULTIPOLYGON:
652  match = gisType.type.equalsIgnoreCase("MULTIPOLYGON");
653  break;
654  default:
655  LOGGER.error("Column type " + JDBCType.valueOf(srcType).getName()
656  + " not Supported");
657  exit(1);
658  }
659  break;
660  default:
661  LOGGER.error("Column type " + JDBCType.valueOf(srcType).getName()
662  + " not Supported");
663  exit(1);
664  }
665  if (!match) {
666  LOGGER.error("Source and destination types for column "
667  + srcColumns.getColumnName(i)
668  + " do not match. Please make sure that type, precision and scale are exactly the same");
669  exit(1);
670  }
671  }
672  }
#define LINESTRING
#define SMALLINT
#define DOUBLE
#define BIGINT
#define DATE
#define MULTIPOLYGON
#define POINT
#define TIME
#define TINYINT
#define TIMESTAMP
#define POLYGON
#define FLOAT

+ Here is the caller graph for this function:

Member Data Documentation

OmniSci.Client com.mapd.utility.SQLImporter.client = null
protected

Definition at line 315 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.createMapDConnection().

DateTimeUtils com.mapd.utility.SQLImporter.dateTimeUtils
private
Initial value:
= (milliseconds) -> {
return milliseconds / 1000;
}

Definition at line 318 of file SQLImporter.java.

final Logger com.mapd.utility.SQLImporter.LOGGER = LoggerFactory.getLogger(SQLImporter.class)
staticpackage

Definition at line 317 of file SQLImporter.java.

String com.mapd.utility.SQLImporter.session = null
protected

Definition at line 314 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.createMapDConnection(), com.mapd.utility.SQLImporter.executeMapDCommand(), com.mapd.utility.SQLImporter.executeQuery(), com.mapd.utility.SQLImporter.getColumnInfo(), omnisci.thrift.OmniSci.disconnect_args.read(), omnisci.thrift.OmniSci.switch_database_args.read(), omnisci.thrift.OmniSci.clone_session_args.read(), omnisci.thrift.OmniSci.get_server_status_args.read(), omnisci.thrift.OmniSci.get_status_args.read(), omnisci.thrift.OmniSci.get_hardware_info_args.read(), omnisci.thrift.OmniSci.get_tables_args.read(), omnisci.thrift.OmniSci.get_tables_for_database_args.read(), omnisci.thrift.OmniSci.get_physical_tables_args.read(), omnisci.thrift.OmniSci.get_views_args.read(), omnisci.thrift.OmniSci.get_tables_meta_args.read(), omnisci.thrift.OmniSci.get_table_details_args.read(), omnisci.thrift.OmniSci.get_table_details_for_database_args.read(), omnisci.thrift.OmniSci.get_internal_table_details_args.read(), omnisci.thrift.OmniSci.get_internal_table_details_for_database_args.read(), omnisci.thrift.OmniSci.get_users_args.read(), omnisci.thrift.OmniSci.get_databases_args.read(), omnisci.thrift.OmniSci.start_heap_profile_args.read(), omnisci.thrift.OmniSci.stop_heap_profile_args.read(), omnisci.thrift.OmniSci.get_heap_profile_args.read(), omnisci.thrift.OmniSci.get_memory_args.read(), omnisci.thrift.OmniSci.clear_cpu_memory_args.read(), omnisci.thrift.OmniSci.clear_gpu_memory_args.read(), omnisci.thrift.OmniSci.set_table_epoch_args.read(), omnisci.thrift.OmniSci.set_table_epoch_by_name_args.read(), omnisci.thrift.OmniSci.get_table_epoch_args.read(), omnisci.thrift.OmniSci.get_table_epoch_by_name_args.read(), omnisci.thrift.OmniSci.get_table_epochs_args.read(), omnisci.thrift.OmniSci.set_table_epochs_args.read(), omnisci.thrift.OmniSci.get_session_info_args.read(), omnisci.thrift.OmniSci.sql_execute_args.read(), omnisci.thrift.OmniSci.sql_execute_df_args.read(), omnisci.thrift.OmniSci.sql_execute_gdf_args.read(), omnisci.thrift.OmniSci.deallocate_df_args.read(), omnisci.thrift.OmniSci.sql_validate_args.read(), omnisci.thrift.OmniSci.get_completion_hints_args.read(), omnisci.thrift.OmniSci.set_execution_mode_args.read(), omnisci.thrift.OmniSci.render_vega_args.read(), omnisci.thrift.OmniSci.get_result_row_for_pixel_args.read(), omnisci.thrift.OmniSci.create_custom_expression_args.read(), omnisci.thrift.OmniSci.get_custom_expressions_args.read(), omnisci.thrift.OmniSci.update_custom_expression_args.read(), omnisci.thrift.OmniSci.delete_custom_expressions_args.read(), omnisci.thrift.OmniSci.get_dashboard_args.read(), omnisci.thrift.OmniSci.get_dashboards_args.read(), omnisci.thrift.OmniSci.create_dashboard_args.read(), omnisci.thrift.OmniSci.replace_dashboard_args.read(), omnisci.thrift.OmniSci.delete_dashboard_args.read(), omnisci.thrift.OmniSci.share_dashboards_args.read(), omnisci.thrift.OmniSci.delete_dashboards_args.read(), omnisci.thrift.OmniSci.share_dashboard_args.read(), omnisci.thrift.OmniSci.unshare_dashboard_args.read(), omnisci.thrift.OmniSci.unshare_dashboards_args.read(), omnisci.thrift.OmniSci.get_dashboard_grantees_args.read(), omnisci.thrift.OmniSci.get_link_view_args.read(), omnisci.thrift.OmniSci.create_link_args.read(), omnisci.thrift.OmniSci.load_table_binary_args.read(), omnisci.thrift.OmniSci.load_table_binary_columnar_args.read(), omnisci.thrift.OmniSci.load_table_binary_columnar_polys_args.read(), omnisci.thrift.OmniSci.load_table_binary_arrow_args.read(), omnisci.thrift.OmniSci.load_table_args.read(), omnisci.thrift.OmniSci.detect_column_types_args.read(), omnisci.thrift.OmniSci.create_table_args.read(), omnisci.thrift.OmniSci.import_table_args.read(), omnisci.thrift.OmniSci.import_geo_table_args.read(), omnisci.thrift.OmniSci.import_table_status_args.read(), omnisci.thrift.OmniSci.get_first_geo_file_in_archive_args.read(), omnisci.thrift.OmniSci.get_all_files_in_archive_args.read(), omnisci.thrift.OmniSci.get_layers_in_geo_file_args.read(), omnisci.thrift.OmniSci.query_get_outer_fragment_count_args.read(), omnisci.thrift.OmniSci.check_table_consistency_args.read(), omnisci.thrift.OmniSci.start_render_query_args.read(), omnisci.thrift.OmniSci.insert_data_args.read(), omnisci.thrift.OmniSci.checkpoint_args.read(), omnisci.thrift.OmniSci.get_roles_args.read(), omnisci.thrift.OmniSci.get_db_objects_for_grantee_args.read(), omnisci.thrift.OmniSci.get_db_object_privs_args.read(), omnisci.thrift.OmniSci.get_all_roles_for_user_args.read(), omnisci.thrift.OmniSci.has_role_args.read(), omnisci.thrift.OmniSci.has_object_privilege_args.read(), omnisci.thrift.OmniSci.set_license_key_args.read(), omnisci.thrift.OmniSci.get_license_claims_args.read(), omnisci.thrift.OmniSci.get_device_parameters_args.read(), omnisci.thrift.OmniSci.register_runtime_extension_functions_args.read(), com.mapd.utility.SQLImporter.tableExists(), omnisci.thrift.OmniSci.disconnect_args.write(), omnisci.thrift.OmniSci.switch_database_args.write(), omnisci.thrift.OmniSci.clone_session_args.write(), omnisci.thrift.OmniSci.get_server_status_args.write(), omnisci.thrift.OmniSci.get_status_args.write(), omnisci.thrift.OmniSci.get_hardware_info_args.write(), omnisci.thrift.OmniSci.get_tables_args.write(), omnisci.thrift.OmniSci.get_tables_for_database_args.write(), omnisci.thrift.OmniSci.get_physical_tables_args.write(), omnisci.thrift.OmniSci.get_views_args.write(), omnisci.thrift.OmniSci.get_tables_meta_args.write(), omnisci.thrift.OmniSci.get_table_details_args.write(), omnisci.thrift.OmniSci.get_table_details_for_database_args.write(), omnisci.thrift.OmniSci.get_internal_table_details_args.write(), omnisci.thrift.OmniSci.get_internal_table_details_for_database_args.write(), omnisci.thrift.OmniSci.get_users_args.write(), omnisci.thrift.OmniSci.get_databases_args.write(), omnisci.thrift.OmniSci.start_heap_profile_args.write(), omnisci.thrift.OmniSci.stop_heap_profile_args.write(), omnisci.thrift.OmniSci.get_heap_profile_args.write(), omnisci.thrift.OmniSci.get_memory_args.write(), omnisci.thrift.OmniSci.clear_cpu_memory_args.write(), omnisci.thrift.OmniSci.clear_gpu_memory_args.write(), omnisci.thrift.OmniSci.set_table_epoch_args.write(), omnisci.thrift.OmniSci.set_table_epoch_by_name_args.write(), omnisci.thrift.OmniSci.get_table_epoch_args.write(), omnisci.thrift.OmniSci.get_table_epoch_by_name_args.write(), omnisci.thrift.OmniSci.get_table_epochs_args.write(), omnisci.thrift.OmniSci.set_table_epochs_args.write(), omnisci.thrift.OmniSci.get_session_info_args.write(), omnisci.thrift.OmniSci.sql_execute_args.write(), omnisci.thrift.OmniSci.sql_execute_df_args.write(), omnisci.thrift.OmniSci.sql_execute_gdf_args.write(), omnisci.thrift.OmniSci.deallocate_df_args.write(), omnisci.thrift.OmniSci.sql_validate_args.write(), omnisci.thrift.OmniSci.get_completion_hints_args.write(), omnisci.thrift.OmniSci.set_execution_mode_args.write(), omnisci.thrift.OmniSci.render_vega_args.write(), omnisci.thrift.OmniSci.get_result_row_for_pixel_args.write(), omnisci.thrift.OmniSci.create_custom_expression_args.write(), omnisci.thrift.OmniSci.get_custom_expressions_args.write(), omnisci.thrift.OmniSci.update_custom_expression_args.write(), omnisci.thrift.OmniSci.delete_custom_expressions_args.write(), omnisci.thrift.OmniSci.get_dashboard_args.write(), omnisci.thrift.OmniSci.get_dashboards_args.write(), omnisci.thrift.OmniSci.create_dashboard_args.write(), omnisci.thrift.OmniSci.replace_dashboard_args.write(), omnisci.thrift.OmniSci.delete_dashboard_args.write(), omnisci.thrift.OmniSci.share_dashboards_args.write(), omnisci.thrift.OmniSci.delete_dashboards_args.write(), omnisci.thrift.OmniSci.share_dashboard_args.write(), omnisci.thrift.OmniSci.unshare_dashboard_args.write(), omnisci.thrift.OmniSci.unshare_dashboards_args.write(), omnisci.thrift.OmniSci.get_dashboard_grantees_args.write(), omnisci.thrift.OmniSci.get_link_view_args.write(), omnisci.thrift.OmniSci.create_link_args.write(), omnisci.thrift.OmniSci.load_table_binary_args.write(), omnisci.thrift.OmniSci.load_table_binary_columnar_args.write(), omnisci.thrift.OmniSci.load_table_binary_columnar_polys_args.write(), omnisci.thrift.OmniSci.load_table_binary_arrow_args.write(), omnisci.thrift.OmniSci.load_table_args.write(), omnisci.thrift.OmniSci.detect_column_types_args.write(), omnisci.thrift.OmniSci.create_table_args.write(), omnisci.thrift.OmniSci.import_table_args.write(), omnisci.thrift.OmniSci.import_geo_table_args.write(), omnisci.thrift.OmniSci.import_table_status_args.write(), omnisci.thrift.OmniSci.get_first_geo_file_in_archive_args.write(), omnisci.thrift.OmniSci.get_all_files_in_archive_args.write(), omnisci.thrift.OmniSci.get_layers_in_geo_file_args.write(), omnisci.thrift.OmniSci.query_get_outer_fragment_count_args.write(), omnisci.thrift.OmniSci.check_table_consistency_args.write(), omnisci.thrift.OmniSci.start_render_query_args.write(), omnisci.thrift.OmniSci.insert_data_args.write(), omnisci.thrift.OmniSci.checkpoint_args.write(), omnisci.thrift.OmniSci.get_roles_args.write(), omnisci.thrift.OmniSci.get_db_objects_for_grantee_args.write(), omnisci.thrift.OmniSci.get_db_object_privs_args.write(), omnisci.thrift.OmniSci.get_all_roles_for_user_args.write(), omnisci.thrift.OmniSci.has_role_args.write(), omnisci.thrift.OmniSci.has_object_privilege_args.write(), omnisci.thrift.OmniSci.set_license_key_args.write(), omnisci.thrift.OmniSci.get_license_claims_args.write(), omnisci.thrift.OmniSci.get_device_parameters_args.write(), and omnisci.thrift.OmniSci.register_runtime_extension_functions_args.write().

Db_vendor_types com.mapd.utility.SQLImporter.vendor_types = null
package

Definition at line 322 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.executeQuery().


The documentation for this class was generated from the following file: