OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Classes | Functions
analyze_benchmark Namespace Reference

Classes

class  BenchmarkLoader
 
class  BenchAnalyzer
 
class  PrettyPrint
 

Functions

def compute_speedup
 
def main
 

Function Documentation

def analyze_benchmark.compute_speedup (   x,
  y 
)

Definition at line 136 of file analyze_benchmark.py.

Referenced by analyze_benchmark.BenchAnalyzer.findAnomaliesRatio(), and analyze_benchmark.PrettyPrint.printAttribute().

136 
137 def compute_speedup(x, y):
138  result = []
139  zipped = list(zip(x, y))
140  for q in zipped:
141  result.append(q[0] / q[1])
142  return result
143 
analyze_benchmark.compute_speedup
def compute_speedup
Definition: analyze_benchmark.py:136

+ Here is the caller graph for this function:

def analyze_benchmark.main (   argv)

Definition at line 250 of file analyze_benchmark.py.

250 
251 def main(argv):
252  try:
253  opts, args = getopt.getopt(
254  argv,
255  "hs:r:e:a:p",
256  [
257  "help",
258  "sample=",
259  "reference=",
260  "epsilon=",
261  "attribute=",
262  "print",
263  ],
264  )
265  except getopt.GetOptError:
266  print(
267  "python3 analyze-benchmark.py -s <sample dir> -r <reference dir> -e <epsilon> -a <attribute> -p"
268  )
269  sys.exit(2)
270 
271  dir_artifact_sample = ""
272  dir_artifact_ref = ""
273  epsilon = 0.05
274  query_attribute = (
275  "query_exec_trimmed_avg"
276  ) # default attribute to use for benchmark comparison
277 
278  to_print = False # printing all the results, disabled by default
279 
280  for opt, arg in opts:
281  if opt in ("-h", "--help"):
282  print(
283  """
284  -s/--sample:\t\t\t directory of the results for the benchmarked sample branch
285  -r/--reference:\t\t\t directory of the results for the benchmarked reference branch
286  -e/--epsilon:\t\t\t ratio tolerance for reporting results outside this range
287  -a/--attribute:\t\t\t attribute to be used for benchmark comparison (default: query_total_avg)
288  -p/--print:\t\t\t\t print all the results
289  """
290  )
291  sys.exit()
292  else:
293  if opt in ("-s", "--sample"):
294  dir_artifact_sample = arg
295  assert os.path.isdir(dir_artifact_sample)
296  elif opt in ("-r", "--reference"):
297  dir_artifact_ref = arg
298  assert os.path.isdir(dir_artifact_ref)
299  elif opt in ("-e", "--epsilon"):
300  epsilon = float(arg)
301  elif opt in ("-a", "--attribute"):
302  query_attribute = arg
303  elif opt in ("-p", "--print"):
304  to_print = True
305 
306  assert dir_artifact_ref != ""
307  assert dir_artifact_sample != ""
308  assert epsilon <= 1
309 
310  GPU_list_ref = listdir(dir_artifact_ref)
311  GPU_list_sample = listdir(dir_artifact_sample)
312 
313  for gpu in GPU_list_ref:
314  dir_name_ref = dir_artifact_ref + "/" + gpu + "/Benchmarks"
315  filename_list_ref = listdir(dir_name_ref)
316  dir_name_ref += "/"
317 
318  refBench = BenchmarkLoader(dir_name_ref, filename_list_ref)
319 
320  if gpu in GPU_list_sample:
321  dir_name_sample = dir_artifact_sample + "/" + gpu + "/Benchmarks"
322  filename_list_sample = listdir(dir_name_sample)
323  dir_name_sample += "/"
324 
325  sampleBench = BenchmarkLoader(
326  dir_name_sample, filename_list_sample
327  )
328  first_header = True
329  for index in range(len(filename_list_ref)):
330  refBench.load(filename_list_ref[index])
331  if filename_list_ref[index] in filename_list_sample:
332  sampleBench.load(filename_list_ref[index])
333  if first_header:
334  print(
335  "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
336  )
337  print("++++ " + sampleBench.getGpuName())
338  print(
339  "++++ reference("
340  + refBench.getFrontAttribute("run_label")
341  + "): "
342  + refBench.getFrontAttribute("run_version")
343  )
344  print(
345  "++++ sample("
346  + sampleBench.getFrontAttribute("run_label")
347  + "): "
348  + sampleBench.getFrontAttribute("run_version")
349  )
350  print(
351  "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
352  )
353  first_header = False
354 
355  analyzer = BenchAnalyzer(
356  refBench, sampleBench, query_attribute
357  )
358  analyzer.findAnomaliesRatio(epsilon)
359  if to_print:
360  printer = PrettyPrint(
361  refBench, sampleBench, query_attribute
362  )
363  printer.printAttribute()
364  else:
365  print(
366  "No sample results for table "
367  + refBench.getRunTableName()
368  + " were found."
369  )
370  print(
371  "======================================================================="
372  )
373 
374  else:
375  print(
376  "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
377  )
378  print("++++ No sample results for GPU " + gpu + " were found.")
379  print(
380  "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
381  )
382 
analyze_benchmark.PrettyPrint
Definition: analyze_benchmark.py:144
analyze_benchmark.BenchmarkLoader
Definition: analyze_benchmark.py:10
analyze_benchmark.BenchAnalyzer
Definition: analyze_benchmark.py:67
analyze_benchmark.main
def main
Definition: analyze_benchmark.py:250