OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
LogRunner.java
Go to the documentation of this file.
1 /*
2  * Copyright 2015 The Apache Software Foundation.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.mapd.logrunner;
17 
18 import org.apache.thrift.TException;
19 import org.apache.thrift.protocol.TJSONProtocol;
20 import org.apache.thrift.protocol.TProtocol;
21 import org.apache.thrift.transport.THttpClient;
22 import org.apache.thrift.transport.TTransport;
23 import org.apache.thrift.transport.TTransportException;
24 import org.slf4j.Logger;
25 import org.slf4j.LoggerFactory;
26 
27 import java.io.BufferedReader;
28 import java.io.FileNotFoundException;
29 import java.io.FileOutputStream;
30 import java.io.FileReader;
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.HashMap;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.concurrent.ArrayBlockingQueue;
37 import java.util.concurrent.ExecutorService;
38 import java.util.concurrent.Executors;
39 import java.util.concurrent.ThreadPoolExecutor;
40 import java.util.concurrent.TimeUnit;
41 import java.util.logging.Level;
42 
43 import ai.heavy.thrift.server.Heavy;
44 import ai.heavy.thrift.server.TColumn;
45 import ai.heavy.thrift.server.TColumnData;
46 import ai.heavy.thrift.server.TColumnType;
47 import ai.heavy.thrift.server.TDBException;
48 import ai.heavy.thrift.server.TDBInfo;
49 import ai.heavy.thrift.server.TDatum;
50 import ai.heavy.thrift.server.TExecuteMode;
51 import ai.heavy.thrift.server.TPixel;
52 import ai.heavy.thrift.server.TQueryResult;
53 import ai.heavy.thrift.server.TRenderResult;
54 import ai.heavy.thrift.server.TRow;
55 import ai.heavy.thrift.server.TRowSet;
56 import ai.heavy.thrift.server.TTableDetails;
57 
62 public class LogRunner {
63  final static Logger logger = LoggerFactory.getLogger(LogRunner.class);
64  private HashMap<Integer, String> sqlquery;
65  private HashMap<Integer, String> originalSql;
66  private HashMap<Integer, String> json;
67  private boolean gpuMode = false;
68  private boolean cpuMode = false;
69 
70  public static void main(String[] args) throws TException {
71  logger.info("Hello, World");
72 
73  LogRunner x = new LogRunner();
74  try {
75  x.doWork(args);
76  } catch (TTransportException ex) {
77  logger.error(ex.toString());
78  ex.printStackTrace();
79  }
80  }
81 
82  public LogRunner() {
83  sqlquery = new HashMap<Integer, String>();
84  originalSql = new HashMap<Integer, String>();
85  json = new HashMap<Integer, String>();
86  }
87 
88  void doWork(String[] args) throws TTransportException, TException {
89  logger.info("In doWork here");
90 
91  int numberThreads = 3;
92  // Runnable[] worker = new Runnable[numberThreads];
93  //
94  // for (int i = 0; i < numberThreads; i++){
95  //
96  Heavy.Client client = getClient(args[0], Integer.valueOf(args[1]));
97  String session = getSession(client);
98  // worker[i] = new myThread(client, session);
99  // }
100 
101  logger.info("got session");
102  try {
103  // ExecutorService executor = Executors.newFixedThreadPool(6);
104  ExecutorService executor = new ThreadPoolExecutor(numberThreads,
105  numberThreads,
106  0L,
107  TimeUnit.MILLISECONDS,
108  new ArrayBlockingQueue<Runnable>(15),
109  new ThreadPoolExecutor.CallerRunsPolicy());
110  while (true) {
111  // BufferedReader in = new BufferedReader(new
112  // FileReader("/data/logfiles/log1"));
113  BufferedReader in = new BufferedReader(new FileReader(args[2]));
114  String str;
115  int current = 0;
116  while ((str = in.readLine()) != null) {
117  Runnable worker = new myThread(str, client, session);
118  // executor.execute(worker);
119  worker.run();
120  }
121  in.close();
122  logger.info("############loop complete");
123  }
124  // executor.shutdown();
125  } catch (IOException e) {
126  logger.error("IOException " + e.getMessage());
127  }
128  }
129 
130  private Heavy.Client getClient(String hostname, int port) throws TTransportException {
131  TTransport transport = null;
132 
133  // transport = new TSocket("localhost", 6274);
134  transport = new THttpClient("http://" + hostname + ":" + port);
135 
136  transport.open();
137 
138  // TProtocol protocol = new TBinaryProtocol(transport);
139  TProtocol protocol = new TJSONProtocol(transport);
140  // TProtocol protocol = new TProtocol(transport);
141 
142  return new Heavy.Client(protocol);
143  }
144 
145  private String getSession(Heavy.Client client)
146  throws TTransportException, TDBException, TException {
147  String session = client.connect("mapd", "HyperInteractive", "mapd");
148  logger.info("Connected session is " + session);
149  return session;
150  }
151 
152  private void closeSession(Heavy.Client client, String session)
153  throws TDBException, TException {
154  // Now disconnect
155  logger.info("Trying to disconnect session " + session);
156  client.disconnect(session);
157  }
158 
159  private void theRest(Heavy.Client client, String session) throws TException {
160  // lets fetch databases from mapd
161  List<TDBInfo> dbs = client.get_databases(session);
162 
163  for (TDBInfo db : dbs) {
164  logger.info("db is " + db.toString());
165  }
166 
167  // lets fetch tables from mapd
168  List<String> tables = client.get_tables(session);
169 
170  for (String tab : tables) {
171  logger.info("Tables is " + tab);
172  }
173 
174  // lets get the version
175  logger.info("Version " + client.get_version());
176 
177  // get table_details
178  TTableDetails table_details = client.get_table_details(session, "flights");
179  for (TColumnType col : table_details.row_desc) {
180  logger.info("col name :" + col.col_name);
181  logger.info("\tcol encoding :" + col.col_type.encoding);
182  logger.info("\tcol is_array :" + col.col_type.is_array);
183  logger.info("\tcol nullable :" + col.col_type.nullable);
184  logger.info("\tcol type :" + col.col_type.type);
185  }
186 
187  // client.set_execution_mode(session, TExecuteMode.CPU);
188  logger.info(" -- before query -- ");
189 
190  TQueryResult sql_execute = client.sql_execute(session,
191  "Select uniquecarrier,flightnum from flights LIMIT 3;",
192  true,
193  null,
194  -1,
195  -1);
196  // client.send_sql_execute(session, "Select BRAND from ACV ;", true);
197  // logger.info(" -- before query recv -- ");
198  // TQueryResult sql_execute = client.recv_sql_execute();
199 
200  logger.info(" -- after query -- ");
201 
202  logger.info("TQueryResult execution time is " + sql_execute.getExecution_time_ms());
203  logger.info("TQueryResult is " + sql_execute.toString());
204  logger.info("TQueryResult getFieldValue is "
205  + sql_execute.getFieldValue(TQueryResult._Fields.ROW_SET));
206 
207  TRowSet row_set = sql_execute.getRow_set();
208  Object fieldValue = sql_execute.getFieldValue(TQueryResult._Fields.ROW_SET);
209 
210  logger.info("fieldValue " + fieldValue);
211 
212  logger.info("TRowSet is " + row_set.toString());
213 
214  logger.info("Get rows size " + row_set.getRowsSize());
215  logger.info("Get col size " + row_set.getRowsSize());
216 
217  List<TRow> rows = row_set.getRows();
218  int count = 1;
219  for (TRow row : rows) {
220  List<TDatum> cols = row.getCols();
221  if (cols != null) {
222  for (TDatum dat : cols) {
223  logger.info("ROW " + count + " " + dat.getFieldValue(TDatum._Fields.VAL));
224  }
225  count++;
226  }
227  }
228 
229  List<TColumn> columns = row_set.getColumns();
230 
231  logger.info("columns " + columns);
232  count = 1;
233  for (TColumn col : columns) {
234  TColumnData data = col.getData();
235  if (data != null) {
236  logger.info("COL " + count + " " + data.toString());
237  }
238  count++;
239  }
240  }
241 
242  public class myThread implements Runnable {
243  private String str;
244  private Heavy.Client client;
245  private String session;
246 
247  myThread(String str1, Heavy.Client client1, String session1) {
248  str = str1;
249  client = client1;
250  session = session1;
251  }
252 
253  @Override
254  public void run() {
255  int logStart = str.indexOf(']');
256  if (logStart != -1) {
257  String det = str.substring(logStart + 1).trim();
258  String header = str.substring(0, logStart).trim();
259 
260  String[] headDet = header.split(" .");
261  // logger.info("header "+ header + " count " + headDet.length + " detail " + det
262  // );
263  if (headDet.length != 4 || headDet[0].equals("Log")) {
264  return;
265  }
266  Integer pid = Integer.valueOf(headDet[2]);
267 
268  if (det.contains("sql_execute :")) {
269  logger.info("det " + det);
270  String sl[] = det.split(":query_str:");
271  logger.info("run query " + sl[1]);
272  try {
273  client.sql_execute(session, sl[1], true, null, -1, -1);
274  } catch (TDBException ex1) {
275  logger.error(
276  "Failed to execute " + sl[1] + " exception " + ex1.getError_msg());
277  } catch (TException ex) {
278  logger.error("Failed to execute " + sl[1] + " exception " + ex.toString());
279  }
280  return;
281  }
282 
283  // get_result_row_for_pixel
284  // :5pFFQUCKs17GLHOqI7ykK09U8mX7GnLF:widget_id:3:pixel.x:396:pixel.y:53:column_format:1
285  // :PixelRadius:2:table_col_names::points,dest,conv_4326_900913_x(dest_lon) as
286  // x,conv_4326_900913_y(dest_lat) as y,arrdelay as size
287  if (det.contains("get_result_row_for_pixel :")) {
288  logger.info("det " + det);
289  String ss[] = det.split(":");
290  String sl[] = det.split(":table_col_names:");
291  logger.info("run get_result_for_pixel " + sl[1]);
292  Map<String, List<String>> tcn = new HashMap<String, List<String>>();
293 
294  String tn[] = sl[1].split(":");
295  for (int i = 0; i < tn.length; i++) {
296  String name[] = tn[i].split(",");
297  List<String> col = new ArrayList<String>();
298  for (int j = 1; j < name.length; j++) {
299  col.add(name[j]);
300  }
301  tcn.put(name[0], col);
302  }
303  try {
304  client.get_result_row_for_pixel(session,
305  Integer.parseInt(ss[3]),
306  new TPixel(Integer.parseInt(ss[5]), Integer.parseInt(ss[7])),
307  tcn,
308  Boolean.TRUE,
309  Integer.parseInt(ss[11]),
310  null);
311  } catch (TDBException ex1) {
312  logger.error("Failed to execute get_result_row_for_pixel exception "
313  + ex1.getError_msg());
314  } catch (TException ex) {
315  logger.error("Failed to execute get_result_row_for_pixel exception "
316  + ex.toString());
317  }
318  return;
319  }
320 
321  if (det.contains("render_vega :")) {
322  logger.info("det " + det);
323  String ss[] = det.split(":");
324  String sl[] = det.split(":vega_json:");
325  json.put(pid, det.substring(det.indexOf("render_vega :") + 13, det.length()));
326  logger.info("JSON = " + sl[1]);
327  logger.info("widget = " + Integer.parseInt(ss[3]));
328  logger.info("compressionLevel = " + Integer.parseInt(ss[5]));
329  logger.info("run render_vega");
330  if (cpuMode) {
331  logger.info("In render: setting gpu mode as we were in CPU mode");
332  gpuMode = true;
333  cpuMode = false;
334  try {
335  client.set_execution_mode(session, TExecuteMode.GPU);
336  } catch (TException ex) {
337  logger.error("Failed to set_execution_mode exception " + ex.toString());
338  }
339  }
340  try {
341  TRenderResult fred = client.render_vega(session,
342  Integer.parseInt(ss[3]),
343  sl[1],
344  Integer.parseInt(ss[5]),
345  null);
346  if (false) {
347  try {
348  FileOutputStream fos;
349 
350  fos = new FileOutputStream("/tmp/png.png");
351 
352  fred.image.position(0);
353  byte[] tgxImageDataByte = new byte[fred.image.limit()];
354  fred.image.get(tgxImageDataByte);
355  fos.write(tgxImageDataByte);
356  fos.close();
357  } catch (FileNotFoundException ex) {
358  logger.error("Failed to create file exception " + ex.toString());
359  } catch (IOException ex) {
360  logger.error("Failed to create file exception " + ex.toString());
361  }
362  }
363 
364  } catch (TException ex) {
365  logger.error("Failed to execute render_vega exception " + ex.toString());
366  }
367  return;
368  }
369 
370  if (det.contains("User mapd sets CPU mode")) {
371  logger.info("Set cpu mode");
372  cpuMode = true;
373  gpuMode = false;
374  try {
375  client.set_execution_mode(session, TExecuteMode.CPU);
376  } catch (TException ex) {
377  logger.error("Failed to set_execution_mode exception " + ex.toString());
378  }
379  return;
380  }
381 
382  if (det.contains("User mapd sets GPU mode")) {
383  logger.info("Set gpu mode");
384  gpuMode = true;
385  cpuMode = false;
386  try {
387  client.set_execution_mode(session, TExecuteMode.GPU);
388  } catch (TException ex) {
389  logger.error(
390  "Failed to execute set_execution_mode exception " + ex.toString());
391  }
392  return;
393  }
394  }
395  }
396  }
397 }
myThread(String str1, Heavy.Client client1, String session1)
Definition: LogRunner.java:247
HashMap< Integer, String > sqlquery
Definition: LogRunner.java:64
void doWork(String[] args)
Definition: LogRunner.java:88
void theRest(Heavy.Client client, String session)
Definition: LogRunner.java:159
HashMap< Integer, String > originalSql
Definition: LogRunner.java:65
static final Logger logger
Definition: LogRunner.java:63
HashMap< Integer, String > json
Definition: LogRunner.java:66
String getSession(Heavy.Client client)
Definition: LogRunner.java:145
string name
Definition: setup.in.py:72
static void main(String[] args)
Definition: LogRunner.java:70
void closeSession(Heavy.Client client, String session)
Definition: LogRunner.java:152
Heavy.Client getClient(String hostname, int port)
Definition: LogRunner.java:130