OmniSciDB  343343d194
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
analyze-benchmark Namespace Reference

Classes

class  BenchmarkLoader
 
class  BenchAnalyzer
 
class  PrettyPrint
 

Functions

def compute_speedup
 
def main
 

Function Documentation

def analyze-benchmark.compute_speedup (   x,
  y 
)

Definition at line 130 of file analyze-benchmark.py.

Referenced by analyze-benchmark.BenchAnalyzer.findAnomaliesRatio(), and analyze-benchmark.PrettyPrint.printAttribute().

131 def compute_speedup(x, y):
132  result = []
133  zipped = list(zip(x, y))
134  for q in zipped:
135  result.append(q[0] / q[1])
136  return result
137 

+ Here is the caller graph for this function:

def analyze-benchmark.main (   argv)

Definition at line 220 of file analyze-benchmark.py.

221 def main(argv):
222  try:
223  opts, args = getopt.getopt(
224  argv,
225  "hs:r:e:a:p",
226  [
227  "help",
228  "sample=",
229  "reference=",
230  "epsilon=",
231  "attribute=",
232  "print",
233  ],
234  )
235  except getopt.GetOptError:
236  print(
237  "python3 analyze-benchmark.py -s <sample dir> -r <reference dir> -e <epsilon> -a <attribute> -p"
238  )
239  sys.exit(2)
240 
241  dir_artifact_sample = ""
242  dir_artifact_ref = ""
243  epsilon = 0.05
244  query_attribute = (
245  "query_total_avg"
246  ) # default attribute to use for benchmark comparison
247 
248  to_print = False # printing all the results, disabled by default
249 
250  for opt, arg in opts:
251  if opt in ("-h", "--help"):
252  print(
253  """
254  -s/--sample:\t\t\t directory of the results for the benchmarked sample branch
255  -r/--reference:\t\t\t directory of the results for the benchmarked reference branch
256  -e/--epsilon:\t\t\t ratio tolerance for reporting results outside this range
257  -a/--attribute:\t\t\t attribute to be used for benchmark comparison (default: query_total_avg)
258  -p/--print:\t\t\t\t print all the results
259  """
260  )
261  sys.exit()
262  else:
263  if opt in ("-s", "--sample"):
264  dir_artifact_sample = arg
265  assert os.path.isdir(dir_artifact_sample)
266  elif opt in ("-r", "--reference"):
267  dir_artifact_ref = arg
268  assert os.path.isdir(dir_artifact_ref)
269  elif opt in ("-e", "--epsilon"):
270  epsilon = float(arg)
271  elif opt in ("-a", "--attribute"):
272  query_attribute = arg
273  elif opt in ("-p", "--print"):
274  to_print = True
275 
276  assert dir_artifact_ref is not ""
277  assert dir_artifact_sample is not ""
278  assert epsilon <= 1
279 
280  GPU_list_ref = listdir(dir_artifact_ref)
281  GPU_list_sample = listdir(dir_artifact_sample)
282 
283  for gpu in GPU_list_ref:
284  dir_name_ref = dir_artifact_ref + "/" + gpu + "/Benchmarks"
285  filename_list_ref = listdir(dir_name_ref)
286  dir_name_ref += "/"
287 
288  refBench = BenchmarkLoader(dir_name_ref, filename_list_ref)
289 
290  if gpu in GPU_list_sample:
291  dir_name_sample = dir_artifact_sample + "/" + gpu + "/Benchmarks"
292  filename_list_sample = listdir(dir_name_sample)
293  dir_name_sample += "/"
294 
295  sampleBench = BenchmarkLoader(
296  dir_name_sample, filename_list_sample
297  )
298  first_header = True
299  for index in range(len(filename_list_ref)):
300  refBench.load(filename_list_ref[index])
301  if filename_list_ref[index] in filename_list_sample:
302  sampleBench.load(filename_list_ref[index])
303  if first_header:
304  print(
305  "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
306  )
307  print("++++ " + sampleBench.getGpuName())
308  print(
309  "++++ reference("
310  + refBench.getFrontAttribute("run_label")
311  + "): "
312  + refBench.getFrontAttribute("run_version")
313  )
314  print(
315  "++++ sample("
316  + sampleBench.getFrontAttribute("run_label")
317  + "): "
318  + sampleBench.getFrontAttribute("run_version")
319  )
320  print(
321  "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
322  )
323  first_header = False
324 
325  analyzer = BenchAnalyzer(
326  refBench, sampleBench, query_attribute
327  )
328  analyzer.findAnomaliesRatio(epsilon)
329  if to_print:
330  printer = PrettyPrint(
331  refBench, sampleBench, query_attribute
332  )
333  printer.printAttribute()
334  else:
335  print(
336  "No sample results for table "
337  + refBench.getRunTableName()
338  + " were found."
339  )
340  print(
341  "======================================================================="
342  )
343 
344  else:
345  print(
346  "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
347  )
348  print("++++ No sample results for GPU " + gpu + " were found.")
349  print(
350  "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
351  )
352