com.mapd.utility.SQLImporter Class Reference

Collaboration diagram for com.mapd.utility.SQLImporter:

Static Public Member Functions
static void	main (String[] args)

Protected Attributes
String	session = null
OmniSci.Client	client = null

Package Functions
void	doWork (String[] args)
void	executeQuery ()

Package Attributes
Db_vendor_types	vendor_types = null

Static Package Attributes
static final Logger	LOGGER = LoggerFactory.getLogger(SQLImporter.class)

Private Member Functions
void	run_init (Connection conn)
void	help (Options options)
void	checkMapDTable (Connection otherdb_conn, ResultSetMetaData md) throws SQLException
void	verifyColumnSignaturesMatch (Connection otherdb_conn, List< TColumnType > dstColumns, ResultSetMetaData srcColumns) throws SQLException
void	createMapDTable (Connection otherdb_conn, ResultSetMetaData metaData)
void	createMapDConnection ()
List< TColumnType >	getColumnInfo (String tName)
boolean	tableExists (String tName)
void	executeMapDCommand (String sql)
String	getColType (int cType, int precision, int scale)
TColumn	setupBinaryColumn (int i, ResultSetMetaData md, int bufferSize) throws SQLException
void	setColValue (ResultSet rs, TColumn col, int columnType, int colNum, int scale, String colTypeName) throws SQLException
void	resetBinaryColumn (int i, ResultSetMetaData md, int bufferSize, TColumn col) throws SQLException

Private Attributes
CommandLine	cmd = null
DateTimeUtils	dateTimeUtils

Detailed Description

Definition at line 307 of file SQLImporter.java.

Member Function Documentation

void com.mapd.utility.SQLImporter.checkMapDTable ( Connection  otherdb_conn, ResultSetMetaData  md  ) throws SQLException

inlineprivate

Definition at line 516 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.cmd, com.mapd.utility.SQLImporter.createMapDConnection(), com.mapd.utility.SQLImporter.createMapDTable(), com.mapd.utility.SQLImporter.executeMapDCommand(), com.mapd.utility.SQLImporter.getColumnInfo(), com.mapd.utility.SQLImporter.tableExists(), and com.mapd.utility.SQLImporter.verifyColumnSignaturesMatch().

Referenced by com.mapd.utility.SQLImporter.executeQuery().

                              {
    createMapDConnection();
    String tName = cmd.getOptionValue("targetTable");

    if (tableExists(tName)) {
      // check if we want to truncate
      if (cmd.hasOption("truncate")) {
        executeMapDCommand("Drop table " + tName);
        createMapDTable(otherdb_conn, md);
      } else {
        List<TColumnType> columnInfo = getColumnInfo(tName);
        verifyColumnSignaturesMatch(otherdb_conn, columnInfo, md);
      }
    } else {
      createMapDTable(otherdb_conn, md);
    }
  }

Here is the call graph for this function:

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.createMapDConnection ( )

inlineprivate

Definition at line 696 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.client, com.mapd.utility.SQLImporter.cmd, and com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable().

                                      {
    TTransport transport = null;
    TProtocol protocol = new TBinaryProtocol(transport);
    int port = Integer.valueOf(cmd.getOptionValue("port", "6274"));
    String server = cmd.getOptionValue("server", "localhost");
    try {
      // Uses default certificate stores.
      boolean load_trust_store = cmd.hasOption("https");
      SockTransportProperties skT = null;
      if (cmd.hasOption("https")) {
        skT = SockTransportProperties.getEncryptedClientDefaultTrustStore(
                !cmd.hasOption("insecure"));
        transport = skT.openHttpsClientTransport(server, port);
        transport.open();
        protocol = new TJSONProtocol(transport);
      } else if (cmd.hasOption("http")) {
        skT = SockTransportProperties.getUnencryptedClient();
        transport = skT.openHttpClientTransport(server, port);
        protocol = new TJSONProtocol(transport);
      } else {
        skT = SockTransportProperties.getUnencryptedClient();
        transport = skT.openClientTransport(server, port);
        transport.open();
        protocol = new TBinaryProtocol(transport);
      }

      client = new OmniSci.Client(protocol);
      // This if will be useless until PKI signon
      if (cmd.hasOption("user")) {
        session = client.connect(cmd.getOptionValue("user", "admin"),
                cmd.getOptionValue("passwd", "HyperInteractive"),
                cmd.getOptionValue("database", "omnisci"));
      }
      LOGGER.debug("Connected session is " + session);

    } catch (TTransportException ex) {
      LOGGER.error("Connection failed - " + ex.toString());
      exit(1);
    } catch (TOmniSciException ex) {
      LOGGER.error("Connection failed - " + ex.toString());
      exit(2);
    } catch (TException ex) {
      LOGGER.error("Connection failed - " + ex.toString());
      exit(3);
    } catch (Exception ex) {
      LOGGER.error("General exception - " + ex.toString());
      exit(4);
    }
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.createMapDTable ( Connection  otherdb_conn, ResultSetMetaData  metaData  )

inlineprivate

Definition at line 654 of file SQLImporter.java.

References File_Namespace.append(), com.mapd.utility.SQLImporter.cmd, com.mapd.utility.SQLImporter.executeMapDCommand(), com.mapd.utility.SQLImporter.getColType(), i, Integer, and run_benchmark_import.type.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable().

                                                                                    {
    StringBuilder sb = new StringBuilder();
    sb.append("Create table ").append(cmd.getOptionValue("targetTable")).append("(");

    // Now iterate the metadata
    try {
      for (int i = 1; i <= metaData.getColumnCount(); i++) {
        if (i > 1) {
          sb.append(",");
        }
        LOGGER.debug("Column name is " + metaData.getColumnName(i));
        LOGGER.debug("Column type is " + metaData.getColumnTypeName(i));
        LOGGER.debug("Column type is " + metaData.getColumnType(i));

        sb.append(metaData.getColumnName(i)).append(" ");
        int col_type = metaData.getColumnType(i);
        if (col_type == java.sql.Types.OTHER) {
          Db_vendor_types.GisType type =
                  vendor_types.find_gis_type(otherdb_conn, metaData, i);
          sb.append(Db_vendor_types.gis_type_to_str(type));
        } else {
          sb.append(getColType(metaData.getColumnType(i),
                  metaData.getPrecision(i),
                  metaData.getScale(i)));
        }
      }
      sb.append(")");

      if (Integer.valueOf(cmd.getOptionValue("fragmentSize", "0")) > 0) {
        sb.append(" with (fragment_size = ");
        sb.append(cmd.getOptionValue("fragmentSize", "0"));
        sb.append(")");
      }

    } catch (SQLException ex) {
      LOGGER.error("Error processing the metadata - " + ex.toString());
      exit(1);
    }

    executeMapDCommand(sb.toString());
  }

Here is the call graph for this function:

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.doWork ( String[]  args )

inlinepackage

Definition at line 323 of file SQLImporter.java.

References run_benchmark_import.args, com.mapd.utility.SQLImporter.cmd, and com.mapd.utility.SQLImporter.executeQuery().

                             {
    // create Options object

    SQLImporter_args s_args = new SQLImporter_args();

    try {
      cmd = s_args.parse(args);
    } catch (ParseException ex) {
      LOGGER.error(ex.getLocalizedMessage());
      s_args.printHelpMessage();
      exit(0);
    }
    executeQuery();
  }

Here is the call graph for this function:

void com.mapd.utility.SQLImporter.executeMapDCommand ( String  sql )

inlineprivate

Definition at line 781 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable(), and com.mapd.utility.SQLImporter.createMapDTable().

                                              {
    LOGGER.info("Run Command - " + sql);

    try {
      TQueryResult sqlResult = client.sql_execute(session, sql + ";", true, null, -1, -1);
    } catch (TOmniSciException ex) {
      LOGGER.error("SQL Execute failed - " + ex.toString());
      exit(1);
    } catch (TException ex) {
      LOGGER.error("SQL Execute failed - " + ex.toString());
      exit(1);
    }
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.executeQuery ( )

inlinepackage

Definition at line 338 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.checkMapDTable(), com.mapd.utility.SQLImporter.cmd, i, com.mapd.utility.db_vendors.Db_vendor_types.isAutoCommitDisabledRequired(), com.mapd.utility.SQLImporter.resetBinaryColumn(), com.mapd.utility.SQLImporter.run_init(), com.mapd.utility.SQLImporter.session, com.mapd.utility.SQLImporter.setColValue(), com.mapd.utility.SQLImporter.setupBinaryColumn(), and com.mapd.utility.SQLImporter.vendor_types.

Referenced by com.mapd.utility.SQLImporter.doWork().

                      {
    Connection conn = null;
    Statement stmt = null;

    long totalTime = 0;

    try {
      // Open a connection
      LOGGER.info("Connecting to database url :" + cmd.getOptionValue("jdbcConnect"));
      conn = DriverManager.getConnection(cmd.getOptionValue("jdbcConnect"),
              cmd.getOptionValue("sourceUser"),
              cmd.getOptionValue("sourcePasswd"));
      vendor_types = Db_vendor_types.Db_vendor_factory(cmd.getOptionValue("jdbcConnect"));
      long startTime = System.currentTimeMillis();

      // run init file script on targe DB if present
      if (cmd.hasOption("initializeFile")) {
        run_init(conn);
      }

      try {
        if (vendor_types.isAutoCommitDisabledRequired()) {
          conn.setAutoCommit(false);
        }
      } catch (SQLException se) {
        LOGGER.warn(
                "SQLException when attempting to setAutoCommit to false, jdbc driver probably doesnt support it.  Error is "
                + se.toString());
      }

      // Execute a query
      stmt = conn.createStatement();

      int bufferSize = Integer.valueOf(cmd.getOptionValue("bufferSize", "10000"));
      // set the jdbc fetch buffer size to reduce the amount of records being moved to
      // java from postgress
      stmt.setFetchSize(bufferSize);
      long timer;

      ResultSet rs = stmt.executeQuery(cmd.getOptionValue("sqlStmt"));

      // check if table already exists and is compatible in OmniSci with the query
      // metadata
      ResultSetMetaData md = rs.getMetaData();
      checkMapDTable(conn, md);

      timer = System.currentTimeMillis();

      long resultCount = 0;
      int bufferCount = 0;
      long total = 0;

      List<TColumn> cols = new ArrayList(md.getColumnCount());
      for (int i = 1; i <= md.getColumnCount(); i++) {
        TColumn col = setupBinaryColumn(i, md, bufferSize);
        cols.add(col);
      }

      // read data from old DB
      while (rs.next()) {
        for (int i = 1; i <= md.getColumnCount(); i++) {
          setColValue(rs,
                  cols.get(i - 1),
                  md.getColumnType(i),
                  i,
                  md.getScale(i),
                  md.getColumnTypeName(i));
        }
        resultCount++;
        bufferCount++;
        if (bufferCount == bufferSize) {
          bufferCount = 0;
          // send the buffer to mapD
          client.load_table_binary_columnar_polys(
                  session, cmd.getOptionValue("targetTable"), cols, null, true);
          // recreate columnar store for use
          for (int i = 1; i <= md.getColumnCount(); i++) {
            resetBinaryColumn(i, md, bufferSize, cols.get(i - 1));
          }

          if (resultCount % 100000 == 0) {
            LOGGER.info("Imported " + resultCount + " records");
          }
        }
      }
      if (bufferCount > 0) {
        // send the LAST buffer to mapD
        client.load_table_binary_columnar_polys(
                session, cmd.getOptionValue("targetTable"), cols, null, true);
        bufferCount = 0;
      }

      // dump render group assignment data immediately
      client.load_table_binary_columnar_polys(
              session, cmd.getOptionValue("targetTable"), null, null, false);

      LOGGER.info("result set count is " + resultCount + " read time is "
              + (System.currentTimeMillis() - timer) + "ms");

      // Clean-up environment
      rs.close();
      stmt.close();
      conn.close();

      totalTime = System.currentTimeMillis() - startTime;
    } catch (SQLException se) {
      LOGGER.error("SQLException - " + se.toString());
      se.printStackTrace();
    } catch (TOmniSciException ex) {
      LOGGER.error("TOmniSciException - " + ex.toString());
      ex.printStackTrace();
    } catch (TException ex) {
      LOGGER.error("TException failed - " + ex.toString());
      ex.printStackTrace();
    } finally {
      // finally block used to close resources
      try {
        if (stmt != null) {
          stmt.close();
        }
      } catch (SQLException se2) {
      } // nothing we can do
      try {
        if (conn != null) {
          conn.close();
        }
      } catch (SQLException se) {
        LOGGER.error("SQlException in close - " + se.toString());
        se.printStackTrace();
      }
      try {
        if (session != null) {
          client.disconnect(session);
        }
      } catch (TOmniSciException ex) {
        LOGGER.error("TOmniSciException - in finalization " + ex.toString());
        ex.printStackTrace();
      } catch (TException ex) {
        LOGGER.error("TException - in finalization" + ex.toString());
        ex.printStackTrace();
      }
    }
  }

Here is the call graph for this function:

Here is the caller graph for this function:

String com.mapd.utility.SQLImporter.getColType ( int  cType, int  precision, int  scale  )

inlineprivate

Definition at line 795 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.createMapDTable().

                                                                 {
    // Note - if cType is OTHER a earlier call will have been made
    // to try and work out the db vendors specific type.
    if (precision > 19) {
      precision = 19;
    }
    if (scale > 19) {
      scale = 18;
    }
    switch (cType) {
      case java.sql.Types.TINYINT:
        return ("TINYINT");
      case java.sql.Types.SMALLINT:
        return ("SMALLINT");
      case java.sql.Types.INTEGER:
        return ("INTEGER");
      case java.sql.Types.BIGINT:
        return ("BIGINT");
      case java.sql.Types.FLOAT:
        return ("FLOAT");
      case java.sql.Types.DECIMAL:
        return ("DECIMAL(" + precision + "," + scale + ")");
      case java.sql.Types.DOUBLE:
        return ("DOUBLE");
      case java.sql.Types.REAL:
        return ("REAL");
      case java.sql.Types.NUMERIC:
        return ("NUMERIC(" + precision + "," + scale + ")");
      case java.sql.Types.TIME:
        return ("TIME");
      case java.sql.Types.TIMESTAMP:
        return ("TIMESTAMP");
      case java.sql.Types.DATE:
        return ("DATE");
      case java.sql.Types.BOOLEAN:
      case java.sql.Types
              .BIT: // deal with postgress treating boolean as bit... this will bite me
        return ("BOOLEAN");
      case java.sql.Types.NVARCHAR:
      case java.sql.Types.VARCHAR:
      case java.sql.Types.NCHAR:
      case java.sql.Types.CHAR:
      case java.sql.Types.LONGVARCHAR:
      case java.sql.Types.LONGNVARCHAR:
        return ("TEXT ENCODING DICT");
      default:
        throw new AssertionError("Column type " + cType + " not Supported");
    }
  }

Here is the caller graph for this function:

List<TColumnType> com.mapd.utility.SQLImporter.getColumnInfo ( String  tName )

inlineprivate

Definition at line 746 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable().

                                                        {
    LOGGER.debug("Getting columns for  " + tName);
    List<TColumnType> row_descriptor = null;
    try {
      TTableDetails table_details = client.get_table_details(session, tName);
      row_descriptor = table_details.row_desc;
    } catch (TOmniSciException ex) {
      LOGGER.error("column check failed - " + ex.toString());
      exit(3);
    } catch (TException ex) {
      LOGGER.error("column check failed - " + ex.toString());
      exit(3);
    }
    return row_descriptor;
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.help ( Options  options )

inlineprivate

Definition at line 509 of file SQLImporter.java.

                                     {
    // automatically generate the help statement
    HelpFormatter formatter = new HelpFormatter();
    formatter.setOptionComparator(null); // get options in the order they are created
    formatter.printHelp("SQLImporter", options);
  }

static void com.mapd.utility.SQLImporter.main ( String[]  args )

inlinestatic

Definition at line 318 of file SQLImporter.java.

References run_benchmark_import.args.

                                         {
    SQLImporter sq = new SQLImporter();
    sq.doWork(args);
  }

void com.mapd.utility.SQLImporter.resetBinaryColumn ( int  i, ResultSetMetaData  md, int  bufferSize, TColumn  col  ) throws SQLException

inlineprivate

Definition at line 1016 of file SQLImporter.java.

References i.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

                              {
    col.nulls.clear();

    switch (md.getColumnType(i)) {
      case java.sql.Types.TINYINT:
      case java.sql.Types.SMALLINT:
      case java.sql.Types.INTEGER:
      case java.sql.Types.BIGINT:
      case java.sql.Types.TIME:
      case java.sql.Types.TIMESTAMP:
      case java.sql.Types
              .BIT: // deal with postgress treating boolean as bit... this will bite me
      case java.sql.Types.BOOLEAN:
      case java.sql.Types.DATE:
      case java.sql.Types.DECIMAL:
      case java.sql.Types.NUMERIC:
        col.data.int_col.clear();
        break;

      case java.sql.Types.FLOAT:
      case java.sql.Types.DOUBLE:
      case java.sql.Types.REAL:
        col.data.real_col.clear();
        break;

      case java.sql.Types.NVARCHAR:
      case java.sql.Types.VARCHAR:
      case java.sql.Types.NCHAR:
      case java.sql.Types.CHAR:
      case java.sql.Types.LONGVARCHAR:
      case java.sql.Types.LONGNVARCHAR:
      case java.sql.Types.OTHER:
        col.data.str_col.clear();
        break;
      default:
        throw new AssertionError("Column type " + md.getColumnType(i) + " not Supported");
    }
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.run_init ( Connection  conn )

inlineprivate

Definition at line 482 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.cmd, and generate_TableFunctionsFactory_init.line.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

                                         {
    // attempt to open file
    String line = "";
    try {
      BufferedReader reader =
              new BufferedReader(new FileReader(cmd.getOptionValue("initializeFile")));
      Statement stmt = conn.createStatement();
      while ((line = reader.readLine()) != null) {
        if (line.isEmpty()) {
          continue;
        }
        LOGGER.info("Running : " + line);
        stmt.execute(line);
      }
      stmt.close();
      reader.close();
    } catch (IOException e) {
      LOGGER.error("Exception occurred trying to read initialize file: "
              + cmd.getOptionValue("initFile"));
      exit(1);
    } catch (SQLException e) {
      LOGGER.error(
              "Exception occurred trying to execute initialize file entry : " + line);
      exit(1);
    }
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.setColValue ( ResultSet  rs, TColumn  col, int  columnType, int  colNum, int  scale, String  colTypeName  ) throws SQLException

inlineprivate

Definition at line 891 of file SQLImporter.java.

References test_fsi.d, omnisci.dtypes.Date, Double, t, omnisci.dtypes.Time, and omnisci.dtypes.Timestamp.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

                                                  {
    switch (columnType) {
      case java.sql.Types
              .BIT: // deal with postgress treating boolean as bit... this will bite me
      case java.sql.Types.BOOLEAN:
        Boolean b = rs.getBoolean(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(0L);
        } else {
          col.nulls.add(Boolean.FALSE);
          col.data.int_col.add(b ? 1L : 0L);
        }
        break;

      case java.sql.Types.DECIMAL:
      case java.sql.Types.NUMERIC:
        BigDecimal bd = rs.getBigDecimal(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(0L);
        } else {
          col.nulls.add(Boolean.FALSE);
          col.data.int_col.add(bd.multiply(new BigDecimal(pow(10L, scale))).longValue());
        }
        break;

      case java.sql.Types.TINYINT:
      case java.sql.Types.SMALLINT:
      case java.sql.Types.INTEGER:
      case java.sql.Types.BIGINT:
        Long l = rs.getLong(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(new Long(0));
        } else {
          col.nulls.add(Boolean.FALSE);
          col.data.int_col.add(l);
        }
        break;

      case java.sql.Types.TIME:
        Time t = rs.getTime(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(0L);

        } else {
          col.data.int_col.add(dateTimeUtils.getSecondsFromMilliseconds(t.getTime()));
          col.nulls.add(Boolean.FALSE);
        }

        break;
      case java.sql.Types.TIMESTAMP:
        Timestamp ts = rs.getTimestamp(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(0L);

        } else {
          col.data.int_col.add(dateTimeUtils.getSecondsFromMilliseconds(ts.getTime()));
          col.nulls.add(Boolean.FALSE);
        }

        break;
      case java.sql.Types.DATE:
        Date d = rs.getDate(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.int_col.add(0L);

        } else {
          col.data.int_col.add(dateTimeUtils.getSecondsFromMilliseconds(d.getTime()));
          col.nulls.add(Boolean.FALSE);
        }
        break;
      case java.sql.Types.FLOAT:
      case java.sql.Types.DOUBLE:
      case java.sql.Types.REAL:
        Double db = rs.getDouble(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.real_col.add(new Double(0));

        } else {
          col.nulls.add(Boolean.FALSE);
          col.data.real_col.add(db);
        }
        break;

      case java.sql.Types.NVARCHAR:
      case java.sql.Types.VARCHAR:
      case java.sql.Types.NCHAR:
      case java.sql.Types.CHAR:
      case java.sql.Types.LONGVARCHAR:
      case java.sql.Types.LONGNVARCHAR:
        String strVal = rs.getString(colNum);
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.str_col.add("");

        } else {
          col.data.str_col.add(strVal);
          col.nulls.add(Boolean.FALSE);
        }
        break;
      case java.sql.Types.OTHER:
        if (rs.wasNull()) {
          col.nulls.add(Boolean.TRUE);
          col.data.str_col.add("");
        } else {
          col.data.str_col.add(vendor_types.get_wkt(rs, colNum, colTypeName));
          col.nulls.add(Boolean.FALSE);
        }
        break;
      default:
        throw new AssertionError("Column type " + columnType + " not Supported");
    }
  }

Here is the caller graph for this function:

TColumn com.mapd.utility.SQLImporter.setupBinaryColumn ( int  i, ResultSetMetaData  md, int  bufferSize  ) throws SQLException

inlineprivate

Definition at line 845 of file SQLImporter.java.

References i.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

                              {
    TColumn col = new TColumn();

    col.nulls = new ArrayList<Boolean>(bufferSize);

    col.data = new TColumnData();

    switch (md.getColumnType(i)) {
      case java.sql.Types.TINYINT:
      case java.sql.Types.SMALLINT:
      case java.sql.Types.INTEGER:
      case java.sql.Types.BIGINT:
      case java.sql.Types.TIME:
      case java.sql.Types.TIMESTAMP:
      case java.sql.Types
              .BIT: // deal with postgress treating boolean as bit... this will bite me
      case java.sql.Types.BOOLEAN:
      case java.sql.Types.DATE:
      case java.sql.Types.DECIMAL:
      case java.sql.Types.NUMERIC:
        col.data.int_col = new ArrayList<Long>(bufferSize);
        break;

      case java.sql.Types.FLOAT:
      case java.sql.Types.DOUBLE:
      case java.sql.Types.REAL:
        col.data.real_col = new ArrayList<Double>(bufferSize);
        break;

      case java.sql.Types.NVARCHAR:
      case java.sql.Types.VARCHAR:
      case java.sql.Types.NCHAR:
      case java.sql.Types.CHAR:
      case java.sql.Types.LONGVARCHAR:
      case java.sql.Types.LONGNVARCHAR:
      case java.sql.Types.OTHER:
        col.data.str_col = new ArrayList<String>(bufferSize);
        break;

      default:
        throw new AssertionError("Column type " + md.getColumnType(i) + " not Supported");
    }
    return col;
  }

Here is the caller graph for this function:

boolean com.mapd.utility.SQLImporter.tableExists ( String  tName )

inlineprivate

Definition at line 762 of file SQLImporter.java.

References com.mapd.utility.SQLImporter.session.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable().

                                            {
    LOGGER.debug("Check for table " + tName);
    try {
      List<String> recv_get_tables = client.get_tables(session);
      for (String s : recv_get_tables) {
        if (s.equals(tName)) {
          return true;
        }
      }
    } catch (TOmniSciException ex) {
      LOGGER.error("Table check failed - " + ex.toString());
      exit(3);
    } catch (TException ex) {
      LOGGER.error("Table check failed - " + ex.toString());
      exit(3);
    }
    return false;
  }

Here is the caller graph for this function:

void com.mapd.utility.SQLImporter.verifyColumnSignaturesMatch ( Connection  otherdb_conn, List< TColumnType >  dstColumns, ResultSetMetaData  srcColumns  ) throws SQLException

inlineprivate

Definition at line 535 of file SQLImporter.java.

References BIGINT, com.mapd.utility.SQLImporter.cmd, DATE, DOUBLE, FLOAT, i, LINESTRING, MULTIPOLYGON, POINT, POLYGON, SMALLINT, TIME, TIMESTAMP, and TINYINT.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable().

                                                            {
    if (srcColumns.getColumnCount() != dstColumns.size()) {
      LOGGER.error("Table sizes do not match: Destination " + dstColumns.size()
              + " versus Source " + srcColumns.getColumnCount());
      exit(1);
    }
    for (int i = 1; i <= dstColumns.size(); ++i) {
      if (!dstColumns.get(i - 1).getCol_name().equalsIgnoreCase(
                  srcColumns.getColumnName(i))) {
        LOGGER.error(
                "Destination table does not have matching column in same order for column number "
                + i + " destination column name is " + dstColumns.get(i - 1).col_name
                + " versus target column " + srcColumns.getColumnName(i));
        exit(1);
      }
      TDatumType dstType = dstColumns.get(i - 1).getCol_type().getType();
      int dstPrecision = dstColumns.get(i - 1).getCol_type().getPrecision();
      int dstScale = dstColumns.get(i - 1).getCol_type().getScale();
      int srcType = srcColumns.getColumnType(i);
      int srcPrecision = srcColumns.getPrecision(i);
      int srcScale = srcColumns.getScale(i);

      boolean match = false;
      switch (srcType) {
        case java.sql.Types.TINYINT:
          match |= dstType == TDatumType.TINYINT;
          // NOTE: it's okay to import smaller type to a bigger one,
          // so we just fall through and try to match the next type.
          // But the order of case statements is important here!
        case java.sql.Types.SMALLINT:
          match |= dstType == TDatumType.SMALLINT;
        case java.sql.Types.INTEGER:
          match |= dstType == TDatumType.INT;
        case java.sql.Types.BIGINT:
          match |= dstType == TDatumType.BIGINT;
          break;
        case java.sql.Types.DECIMAL:
        case java.sql.Types.NUMERIC:
          match = dstType == TDatumType.DECIMAL && dstPrecision == srcPrecision
                  && dstScale == srcScale;
          break;
        case java.sql.Types.FLOAT:
        case java.sql.Types.REAL:
          match |= dstType == TDatumType.FLOAT;
          // Fall through and try double
        case java.sql.Types.DOUBLE:
          match |= dstType == TDatumType.DOUBLE;
          if (cmd.hasOption("AllowDoubleToFloat")) {
            match |= dstType == TDatumType.FLOAT;
          }
          break;
        case java.sql.Types.TIME:
          match = dstType == TDatumType.TIME;
          break;
        case java.sql.Types.TIMESTAMP:
          match = dstType == TDatumType.TIMESTAMP;
          break;
        case java.sql.Types.DATE:
          match = dstType == TDatumType.DATE;
          break;
        case java.sql.Types.BOOLEAN:
        case java.sql.Types
                .BIT: // deal with postgres treating boolean as bit... this will bite me
          match = dstType == TDatumType.BOOL;
          break;
        case java.sql.Types.NVARCHAR:
        case java.sql.Types.VARCHAR:
        case java.sql.Types.NCHAR:
        case java.sql.Types.CHAR:
        case java.sql.Types.LONGVARCHAR:
        case java.sql.Types.LONGNVARCHAR:
          match = (dstType == TDatumType.STR || dstType == TDatumType.POINT
                  || dstType == TDatumType.POLYGON || dstType == TDatumType.MULTIPOLYGON
                  || dstType == TDatumType.LINESTRING);
          break;
        case java.sql.Types.OTHER:
          // NOTE: I ignore subtypes (geography vs geopetry vs none) here just because
          // it makes no difference for OmniSciDB at the moment
          Db_vendor_types.GisType gisType =
                  vendor_types.find_gis_type(otherdb_conn, srcColumns, i);
          if (gisType.srid != dstScale) {
            match = false;
            break;
          }
          switch (dstType) {
            case POINT:
              match = gisType.type.equalsIgnoreCase("POINT");
              break;
            case LINESTRING:
              match = gisType.type.equalsIgnoreCase("LINESTRING");
              break;
            case POLYGON:
              match = gisType.type.equalsIgnoreCase("POLYGON");
              break;
            case MULTIPOLYGON:
              match = gisType.type.equalsIgnoreCase("MULTIPOLYGON");
              break;
            default:
              LOGGER.error("Column type " + JDBCType.valueOf(srcType).getName()
                      + " not Supported");
              exit(1);
          }
          break;
        default:
          LOGGER.error("Column type " + JDBCType.valueOf(srcType).getName()
                  + " not Supported");
          exit(1);
      }
      if (!match) {
        LOGGER.error("Source and destination types for column "
                + srcColumns.getColumnName(i)
                + " do not match. Please make sure that type, precision and scale are exactly the same");
        exit(1);
      }
    }
  }

Here is the caller graph for this function:

Member Data Documentation

OmniSci.Client com.mapd.utility.SQLImporter.client = null

protected

Definition at line 309 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.createMapDConnection().

CommandLine com.mapd.utility.SQLImporter.cmd = null

private

Definition at line 310 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.checkMapDTable(), com.mapd.utility.SQLImporter.createMapDConnection(), com.mapd.utility.SQLImporter.createMapDTable(), com.mapd.utility.SQLImporter.doWork(), com.mapd.utility.SQLImporter.executeQuery(), com.mapd.utility.SQLImporter.run_init(), and com.mapd.utility.SQLImporter.verifyColumnSignaturesMatch().

DateTimeUtils com.mapd.utility.SQLImporter.dateTimeUtils

private

Initial value:

= (milliseconds) -> {
    return milliseconds / 1000;
  }

Definition at line 312 of file SQLImporter.java.

final Logger com.mapd.utility.SQLImporter.LOGGER = LoggerFactory.getLogger(SQLImporter.class)

staticpackage

Definition at line 311 of file SQLImporter.java.

String com.mapd.utility.SQLImporter.session = null

protected

Definition at line 308 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.createMapDConnection(), com.mapd.utility.SQLImporter.executeMapDCommand(), com.mapd.utility.SQLImporter.executeQuery(), com.mapd.utility.SQLImporter.getColumnInfo(), omnisci.thrift.OmniSci.disconnect_args.read(), omnisci.thrift.OmniSci.switch_database_args.read(), omnisci.thrift.OmniSci.clone_session_args.read(), omnisci.thrift.OmniSci.get_server_status_args.read(), omnisci.thrift.OmniSci.get_status_args.read(), omnisci.thrift.OmniSci.get_hardware_info_args.read(), omnisci.thrift.OmniSci.get_tables_args.read(), omnisci.thrift.OmniSci.get_physical_tables_args.read(), omnisci.thrift.OmniSci.get_views_args.read(), omnisci.thrift.OmniSci.get_tables_meta_args.read(), omnisci.thrift.OmniSci.get_table_details_args.read(), omnisci.thrift.OmniSci.get_internal_table_details_args.read(), omnisci.thrift.OmniSci.get_users_args.read(), omnisci.thrift.OmniSci.get_databases_args.read(), omnisci.thrift.OmniSci.start_heap_profile_args.read(), omnisci.thrift.OmniSci.stop_heap_profile_args.read(), omnisci.thrift.OmniSci.get_heap_profile_args.read(), omnisci.thrift.OmniSci.get_memory_args.read(), omnisci.thrift.OmniSci.clear_cpu_memory_args.read(), omnisci.thrift.OmniSci.clear_gpu_memory_args.read(), omnisci.thrift.OmniSci.set_table_epoch_args.read(), omnisci.thrift.OmniSci.set_table_epoch_by_name_args.read(), omnisci.thrift.OmniSci.get_table_epoch_args.read(), omnisci.thrift.OmniSci.get_table_epoch_by_name_args.read(), omnisci.thrift.OmniSci.get_table_epochs_args.read(), omnisci.thrift.OmniSci.set_table_epochs_args.read(), omnisci.thrift.OmniSci.get_session_info_args.read(), omnisci.thrift.OmniSci.sql_execute_args.read(), omnisci.thrift.OmniSci.sql_execute_df_args.read(), omnisci.thrift.OmniSci.sql_execute_gdf_args.read(), omnisci.thrift.OmniSci.deallocate_df_args.read(), omnisci.thrift.OmniSci.sql_validate_args.read(), omnisci.thrift.OmniSci.get_completion_hints_args.read(), omnisci.thrift.OmniSci.set_execution_mode_args.read(), omnisci.thrift.OmniSci.render_vega_args.read(), omnisci.thrift.OmniSci.get_result_row_for_pixel_args.read(), omnisci.thrift.OmniSci.get_dashboard_args.read(), omnisci.thrift.OmniSci.get_dashboards_args.read(), omnisci.thrift.OmniSci.create_dashboard_args.read(), omnisci.thrift.OmniSci.replace_dashboard_args.read(), omnisci.thrift.OmniSci.delete_dashboard_args.read(), omnisci.thrift.OmniSci.share_dashboards_args.read(), omnisci.thrift.OmniSci.delete_dashboards_args.read(), omnisci.thrift.OmniSci.share_dashboard_args.read(), omnisci.thrift.OmniSci.unshare_dashboard_args.read(), omnisci.thrift.OmniSci.unshare_dashboards_args.read(), omnisci.thrift.OmniSci.get_dashboard_grantees_args.read(), omnisci.thrift.OmniSci.get_link_view_args.read(), omnisci.thrift.OmniSci.create_link_args.read(), omnisci.thrift.OmniSci.load_table_binary_args.read(), omnisci.thrift.OmniSci.load_table_binary_columnar_args.read(), omnisci.thrift.OmniSci.load_table_binary_arrow_args.read(), omnisci.thrift.OmniSci.load_table_args.read(), omnisci.thrift.OmniSci.detect_column_types_args.read(), omnisci.thrift.OmniSci.create_table_args.read(), omnisci.thrift.OmniSci.import_table_args.read(), omnisci.thrift.OmniSci.import_geo_table_args.read(), omnisci.thrift.OmniSci.import_table_status_args.read(), omnisci.thrift.OmniSci.get_first_geo_file_in_archive_args.read(), omnisci.thrift.OmniSci.get_all_files_in_archive_args.read(), omnisci.thrift.OmniSci.get_layers_in_geo_file_args.read(), omnisci.thrift.OmniSci.query_get_outer_fragment_count_args.read(), omnisci.thrift.OmniSci.check_table_consistency_args.read(), omnisci.thrift.OmniSci.start_render_query_args.read(), omnisci.thrift.OmniSci.insert_data_args.read(), omnisci.thrift.OmniSci.checkpoint_args.read(), omnisci.thrift.OmniSci.get_roles_args.read(), omnisci.thrift.OmniSci.get_db_objects_for_grantee_args.read(), omnisci.thrift.OmniSci.get_db_object_privs_args.read(), omnisci.thrift.OmniSci.get_all_roles_for_user_args.read(), omnisci.thrift.OmniSci.has_role_args.read(), omnisci.thrift.OmniSci.has_object_privilege_args.read(), omnisci.thrift.OmniSci.set_license_key_args.read(), omnisci.thrift.OmniSci.get_license_claims_args.read(), omnisci.thrift.OmniSci.get_device_parameters_args.read(), omnisci.thrift.OmniSci.register_runtime_extension_functions_args.read(), com.mapd.utility.SQLImporter.tableExists(), omnisci.thrift.OmniSci.disconnect_args.write(), omnisci.thrift.OmniSci.switch_database_args.write(), omnisci.thrift.OmniSci.clone_session_args.write(), omnisci.thrift.OmniSci.get_server_status_args.write(), omnisci.thrift.OmniSci.get_status_args.write(), omnisci.thrift.OmniSci.get_hardware_info_args.write(), omnisci.thrift.OmniSci.get_tables_args.write(), omnisci.thrift.OmniSci.get_physical_tables_args.write(), omnisci.thrift.OmniSci.get_views_args.write(), omnisci.thrift.OmniSci.get_tables_meta_args.write(), omnisci.thrift.OmniSci.get_table_details_args.write(), omnisci.thrift.OmniSci.get_internal_table_details_args.write(), omnisci.thrift.OmniSci.get_users_args.write(), omnisci.thrift.OmniSci.get_databases_args.write(), omnisci.thrift.OmniSci.start_heap_profile_args.write(), omnisci.thrift.OmniSci.stop_heap_profile_args.write(), omnisci.thrift.OmniSci.get_heap_profile_args.write(), omnisci.thrift.OmniSci.get_memory_args.write(), omnisci.thrift.OmniSci.clear_cpu_memory_args.write(), omnisci.thrift.OmniSci.clear_gpu_memory_args.write(), omnisci.thrift.OmniSci.set_table_epoch_args.write(), omnisci.thrift.OmniSci.set_table_epoch_by_name_args.write(), omnisci.thrift.OmniSci.get_table_epoch_args.write(), omnisci.thrift.OmniSci.get_table_epoch_by_name_args.write(), omnisci.thrift.OmniSci.get_table_epochs_args.write(), omnisci.thrift.OmniSci.set_table_epochs_args.write(), omnisci.thrift.OmniSci.get_session_info_args.write(), omnisci.thrift.OmniSci.sql_execute_args.write(), omnisci.thrift.OmniSci.sql_execute_df_args.write(), omnisci.thrift.OmniSci.sql_execute_gdf_args.write(), omnisci.thrift.OmniSci.deallocate_df_args.write(), omnisci.thrift.OmniSci.sql_validate_args.write(), omnisci.thrift.OmniSci.get_completion_hints_args.write(), omnisci.thrift.OmniSci.set_execution_mode_args.write(), omnisci.thrift.OmniSci.render_vega_args.write(), omnisci.thrift.OmniSci.get_result_row_for_pixel_args.write(), omnisci.thrift.OmniSci.get_dashboard_args.write(), omnisci.thrift.OmniSci.get_dashboards_args.write(), omnisci.thrift.OmniSci.create_dashboard_args.write(), omnisci.thrift.OmniSci.replace_dashboard_args.write(), omnisci.thrift.OmniSci.delete_dashboard_args.write(), omnisci.thrift.OmniSci.share_dashboards_args.write(), omnisci.thrift.OmniSci.delete_dashboards_args.write(), omnisci.thrift.OmniSci.share_dashboard_args.write(), omnisci.thrift.OmniSci.unshare_dashboard_args.write(), omnisci.thrift.OmniSci.unshare_dashboards_args.write(), omnisci.thrift.OmniSci.get_dashboard_grantees_args.write(), omnisci.thrift.OmniSci.get_link_view_args.write(), omnisci.thrift.OmniSci.create_link_args.write(), omnisci.thrift.OmniSci.load_table_binary_args.write(), omnisci.thrift.OmniSci.load_table_binary_columnar_args.write(), omnisci.thrift.OmniSci.load_table_binary_arrow_args.write(), omnisci.thrift.OmniSci.load_table_args.write(), omnisci.thrift.OmniSci.detect_column_types_args.write(), omnisci.thrift.OmniSci.create_table_args.write(), omnisci.thrift.OmniSci.import_table_args.write(), omnisci.thrift.OmniSci.import_geo_table_args.write(), omnisci.thrift.OmniSci.import_table_status_args.write(), omnisci.thrift.OmniSci.get_first_geo_file_in_archive_args.write(), omnisci.thrift.OmniSci.get_all_files_in_archive_args.write(), omnisci.thrift.OmniSci.get_layers_in_geo_file_args.write(), omnisci.thrift.OmniSci.query_get_outer_fragment_count_args.write(), omnisci.thrift.OmniSci.check_table_consistency_args.write(), omnisci.thrift.OmniSci.start_render_query_args.write(), omnisci.thrift.OmniSci.insert_data_args.write(), omnisci.thrift.OmniSci.checkpoint_args.write(), omnisci.thrift.OmniSci.get_roles_args.write(), omnisci.thrift.OmniSci.get_db_objects_for_grantee_args.write(), omnisci.thrift.OmniSci.get_db_object_privs_args.write(), omnisci.thrift.OmniSci.get_all_roles_for_user_args.write(), omnisci.thrift.OmniSci.has_role_args.write(), omnisci.thrift.OmniSci.has_object_privilege_args.write(), omnisci.thrift.OmniSci.set_license_key_args.write(), omnisci.thrift.OmniSci.get_license_claims_args.write(), omnisci.thrift.OmniSci.get_device_parameters_args.write(), and omnisci.thrift.OmniSci.register_runtime_extension_functions_args.write().

Db_vendor_types com.mapd.utility.SQLImporter.vendor_types = null

package

Definition at line 316 of file SQLImporter.java.

Referenced by com.mapd.utility.SQLImporter.executeQuery().

---

The documentation for this class was generated from the following file:

• /home/jenkins-slave/workspace/core-os-doxygen/java/utility/src/main/java/com/mapd/utility/SQLImporter.java