135 lines · python
1#!/usr/bin/env python2from __future__ import print_function3 4import re, string, sys, os, time5 6DEBUG = 07testDirName = "llvm-test"8test = ["compile", "llc", "jit", "cbe"]9exectime = [10 "llc-time",11 "jit-time",12 "cbe-time",13]14comptime = ["llc", "jit-comptime", "compile"]15 16(tp, exp) = ("compileTime_", "executeTime_")17 18 19def parse(file):20 f = open(file, "r")21 d = f.read()22 23 # Cleanup weird stuff24 d = re.sub(r",\d+:\d", "", d)25 26 r = re.findall(r"TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n", d)27 28 test = {}29 fname = ""30 for t in r:31 if DEBUG:32 print(t)33 if t[0] == "PASS" or t[0] == "FAIL":34 tmp = t[2].split(testDirName)35 36 if DEBUG:37 print(tmp)38 39 if len(tmp) == 2:40 fname = tmp[1].strip("\r\n")41 else:42 fname = tmp[0].strip("\r\n")43 44 if fname not in test:45 test[fname] = {}46 47 for k in test:48 test[fname][k] = "NA"49 test[fname][t[1]] = t[0]50 if DEBUG:51 print(test[fname][t[1]])52 else:53 try:54 n = t[0].split("RESULT-")[1]55 56 if DEBUG:57 print(n)58 59 if n == "llc" or n == "jit-comptime" or n == "compile":60 test[fname][tp + n] = float(t[2].split(" ")[2])61 if DEBUG:62 print(test[fname][tp + n])63 64 elif n.endswith("-time"):65 test[fname][exp + n] = float(t[2].strip("\r\n"))66 if DEBUG:67 print(test[fname][exp + n])68 69 else:70 print("ERROR!")71 sys.exit(1)72 73 except:74 continue75 76 return test77 78 79# Diff results and look for regressions.80def diffResults(d_old, d_new):81 82 for t in sorted(d_old.keys()):83 if DEBUG:84 print(t)85 86 if t in d_new:87 88 # Check if the test passed or failed.89 for x in test:90 if x in d_old[t]:91 if x in d_new[t]:92 if d_old[t][x] == "PASS":93 if d_new[t][x] != "PASS":94 print(t + " *** REGRESSION (" + x + ")\n")95 else:96 if d_new[t][x] == "PASS":97 print(t + " * NEW PASS (" + x + ")\n")98 99 else:100 print(t + "*** REGRESSION (" + x + ")\n")101 102 # For execution time, if there is no result, its a fail.103 for x in exectime:104 if tp + x in d_old[t]:105 if tp + x not in d_new[t]:106 print(t + " *** REGRESSION (" + tp + x + ")\n")107 108 else:109 if tp + x in d_new[t]:110 print(t + " * NEW PASS (" + tp + x + ")\n")111 112 for x in comptime:113 if exp + x in d_old[t]:114 if exp + x not in d_new[t]:115 print(t + " *** REGRESSION (" + exp + x + ")\n")116 117 else:118 if exp + x in d_new[t]:119 print(t + " * NEW PASS (" + exp + x + ")\n")120 121 else:122 print(t + ": Removed from test-suite.\n")123 124 125# Main126if len(sys.argv) < 3:127 print("Usage:", sys.argv[0], "<old log> <new log>")128 sys.exit(-1)129 130d_old = parse(sys.argv[1])131d_new = parse(sys.argv[2])132 133 134diffResults(d_old, d_new)135