16 package com.mapd.logrunner;
18 import com.mapd.thrift.server.MapD;
19 import com.mapd.thrift.server.TColumn;
20 import com.mapd.thrift.server.TColumnData;
21 import com.mapd.thrift.server.TColumnType;
22 import com.mapd.thrift.server.TDBInfo;
23 import com.mapd.thrift.server.TDatum;
24 import com.mapd.thrift.server.TExecuteMode;
25 import com.mapd.thrift.server.TMapDException;
26 import com.mapd.thrift.server.TPixel;
27 import com.mapd.thrift.server.TQueryResult;
28 import com.mapd.thrift.server.TRenderResult;
29 import com.mapd.thrift.server.TRow;
30 import com.mapd.thrift.server.TRowSet;
31 import com.mapd.thrift.server.TTableDetails;
33 import org.apache.thrift.TException;
34 import org.apache.thrift.protocol.TJSONProtocol;
35 import org.apache.thrift.protocol.TProtocol;
36 import org.apache.thrift.transport.THttpClient;
37 import org.apache.thrift.transport.TTransport;
38 import org.apache.thrift.transport.TTransportException;
39 import org.slf4j.Logger;
40 import org.slf4j.LoggerFactory;
42 import java.io.BufferedReader;
43 import java.io.FileNotFoundException;
44 import java.io.FileOutputStream;
45 import java.io.FileReader;
46 import java.io.IOException;
47 import java.util.ArrayList;
48 import java.util.HashMap;
49 import java.util.List;
51 import java.util.concurrent.ArrayBlockingQueue;
52 import java.util.concurrent.ExecutorService;
53 import java.util.concurrent.Executors;
54 import java.util.concurrent.ThreadPoolExecutor;
55 import java.util.concurrent.TimeUnit;
56 import java.util.logging.Level;
63 final static Logger
logger = LoggerFactory.getLogger(LogRunner.class);
66 private HashMap<Integer, String>
json;
71 logger.info(
"Hello, World");
76 }
catch (TTransportException ex) {
77 logger.error(ex.toString());
83 sqlquery =
new HashMap<Integer, String>();
85 json =
new HashMap<Integer, String>();
89 logger.info(
"In doWork here");
91 int numberThreads = 3;
101 logger.info(
"got session");
104 ExecutorService executor =
new ThreadPoolExecutor(numberThreads,
107 TimeUnit.MILLISECONDS,
108 new ArrayBlockingQueue<Runnable>(15),
109 new ThreadPoolExecutor.CallerRunsPolicy());
112 BufferedReader in =
new BufferedReader(
new FileReader(
args[2]));
115 while ((str = in.readLine()) != null) {
116 Runnable worker =
new myThread(str, client, session);
121 logger.info(
"############loop complete");
124 }
catch (IOException e) {
125 logger.error(
"IOException " + e.getMessage());
130 TTransport transport = null;
133 transport =
new THttpClient(
"http://" + hostname +
":" + port);
138 TProtocol protocol =
new TJSONProtocol(transport);
141 return new MapD.Client(protocol);
145 throws TTransportException, TMapDException, TException {
146 String session = client.connect(
"mapd",
"HyperInteractive",
"mapd");
147 logger.info(
"Connected session is " + session);
152 throws TMapDException, TException {
154 logger.info(
"Trying to disconnect session " + session);
155 client.disconnect(session);
158 private void theRest(MapD.Client client,
String session)
throws TException {
160 List<TDBInfo> dbs = client.get_databases(session);
162 for (TDBInfo db : dbs) {
163 logger.info(
"db is " + db.toString());
167 List<String>
tables = client.get_tables(session);
169 for (
String tab : tables) {
170 logger.info(
"Tables is " + tab);
174 logger.info(
"Version " + client.get_version());
177 TTableDetails table_details = client.get_table_details(session,
"flights");
178 for (TColumnType col : table_details.row_desc) {
179 logger.info(
"col name :" + col.col_name);
180 logger.info(
"\tcol encoding :" + col.col_type.encoding);
181 logger.info(
"\tcol is_array :" + col.col_type.is_array);
182 logger.info(
"\tcol nullable :" + col.col_type.nullable);
187 logger.info(
" -- before query -- ");
189 TQueryResult sql_execute = client.sql_execute(session,
190 "Select uniquecarrier,flightnum from flights LIMIT 3;",
199 logger.info(
" -- after query -- ");
201 logger.info(
"TQueryResult execution time is " + sql_execute.getExecution_time_ms());
202 logger.info(
"TQueryResult is " + sql_execute.toString());
203 logger.info(
"TQueryResult getFieldValue is "
204 + sql_execute.getFieldValue(TQueryResult._Fields.ROW_SET));
206 TRowSet row_set = sql_execute.getRow_set();
207 Object fieldValue = sql_execute.getFieldValue(TQueryResult._Fields.ROW_SET);
209 logger.info(
"fieldValue " + fieldValue);
211 logger.info(
"TRowSet is " + row_set.toString());
213 logger.info(
"Get rows size " + row_set.getRowsSize());
214 logger.info(
"Get col size " + row_set.getRowsSize());
216 List<TRow> rows = row_set.getRows();
218 for (TRow row : rows) {
219 List<TDatum> cols = row.getCols();
221 for (TDatum dat : cols) {
222 logger.info(
"ROW " + count +
" " + dat.getFieldValue(TDatum._Fields.VAL));
228 List<TColumn> columns = row_set.getColumns();
230 logger.info(
"columns " + columns);
232 for (TColumn col : columns) {
233 TColumnData data = col.getData();
235 logger.info(
"COL " + count +
" " + data.toString());
254 int logStart = str.indexOf(
']');
255 if (logStart != -1) {
256 String det = str.substring(logStart + 1).trim();
257 String header = str.substring(0, logStart).trim();
259 String[] headDet = header.split(
" .");
262 if (headDet.length != 4 || headDet[0].equals(
"Log")) {
265 Integer pid = Integer.valueOf(headDet[2]);
267 if (det.contains(
"sql_execute :")) {
268 logger.info(
"det " + det);
269 String sl[] = det.split(
":query_str:");
270 logger.info(
"run query " + sl[1]);
272 client.sql_execute(
session, sl[1],
true, null, -1, -1);
273 }
catch (TMapDException ex1) {
274 logger.error(
"Failed to execute " + sl[1] +
" exception " + ex1.toString());
275 }
catch (TException ex) {
276 logger.error(
"Failed to execute " + sl[1] +
" exception " + ex.toString());
285 if (det.contains(
"get_result_row_for_pixel :")) {
286 logger.info(
"det " + det);
287 String ss[] = det.split(
":");
288 String sl[] = det.split(
":table_col_names:");
289 logger.info(
"run get_result_for_pixel " + sl[1]);
290 Map<String, List<String>> tcn =
new HashMap<String, List<String>>();
292 String tn[] = sl[1].split(
":");
293 for (
int i = 0; i < tn.length; i++) {
294 String name[] = tn[i].split(
",");
295 List<String> col =
new ArrayList<String>();
296 for (
int j = 1; j < name.length; j++) {
299 tcn.put(name[0], col);
302 client.get_result_row_for_pixel(
session,
303 Integer.parseInt(ss[3]),
304 new TPixel(
Integer.parseInt(ss[5]), Integer.parseInt(ss[7])),
307 Integer.parseInt(ss[11]),
309 }
catch (TMapDException ex1) {
310 logger.error(
"Failed to execute get_result_row_for_pixel exception "
312 }
catch (TException ex) {
313 logger.error(
"Failed to execute get_result_row_for_pixel exception "
319 if (det.contains(
"render_vega :")) {
320 logger.info(
"det " + det);
321 String ss[] = det.split(
":");
322 String sl[] = det.split(
":vega_json:");
323 json.put(pid, det.substring(det.indexOf(
"render_vega :") + 13, det.length()));
324 logger.info(
"JSON = " + sl[1]);
325 logger.info(
"widget = " + Integer.parseInt(ss[3]));
326 logger.info(
"compressionLevel = " + Integer.parseInt(ss[5]));
327 logger.info(
"run render_vega");
329 logger.info(
"In render: setting gpu mode as we were in CPU mode");
333 client.set_execution_mode(
session, TExecuteMode.GPU);
334 }
catch (TException ex) {
335 logger.error(
"Failed to set_execution_mode exception " + ex.toString());
339 TRenderResult fred = client.render_vega(
session,
340 Integer.parseInt(ss[3]),
346 FileOutputStream fos;
348 fos =
new FileOutputStream(
"/tmp/png.png");
350 fred.image.position(0);
351 byte[] tgxImageDataByte =
new byte[fred.image.limit()];
352 fred.image.get(tgxImageDataByte);
353 fos.write(tgxImageDataByte);
355 }
catch (FileNotFoundException ex) {
356 logger.error(
"Failed to create file exception " + ex.toString());
357 }
catch (IOException ex) {
358 logger.error(
"Failed to create file exception " + ex.toString());
362 }
catch (TException ex) {
363 logger.error(
"Failed to execute render_vega exception " + ex.toString());
368 if (det.contains(
"User mapd sets CPU mode")) {
369 logger.info(
"Set cpu mode");
373 client.set_execution_mode(
session, TExecuteMode.CPU);
374 }
catch (TException ex) {
375 logger.error(
"Failed to set_execution_mode exception " + ex.toString());
380 if (det.contains(
"User mapd sets GPU mode")) {
381 logger.info(
"Set gpu mode");
385 client.set_execution_mode(
session, TExecuteMode.GPU);
386 }
catch (TException ex) {
388 "Failed to execute set_execution_mode exception " + ex.toString());
void theRest(MapD.Client client, String session)
MapD.Client getClient(String hostname, int port)
HashMap< Integer, String > sqlquery
void doWork(String[] args)
void closeSession(MapD.Client client, String session)
myThread(String str1, MapD.Client client1, String session1)
HashMap< Integer, String > originalSql
static final Logger logger
HashMap< Integer, String > json
static void main(String[] args)
String getSession(MapD.Client client)