178 lines · python
1#!/usr/bin/env python2 3from __future__ import print_function4import re, string, sys, os, time, math5 6DEBUG = 07 8(tp, exp) = ("compile", "exec")9 10 11def parse(file):12 f = open(file, "r")13 d = f.read()14 15 # Cleanup weird stuff16 d = re.sub(r",\d+:\d", "", d)17 18 r = re.findall(r"TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n", d)19 20 test = {}21 fname = ""22 for t in r:23 if DEBUG:24 print(t)25 26 if t[0] == "PASS" or t[0] == "FAIL":27 tmp = t[2].split("llvm-test/")28 29 if DEBUG:30 print(tmp)31 32 if len(tmp) == 2:33 fname = tmp[1].strip("\r\n")34 else:35 fname = tmp[0].strip("\r\n")36 37 if fname not in test:38 test[fname] = {}39 40 test[fname][t[1] + " state"] = t[0]41 test[fname][t[1] + " time"] = float("nan")42 else:43 try:44 n = t[0].split("RESULT-")[1]45 46 if DEBUG:47 print("n == ", n)48 49 if n == "compile-success":50 test[fname]["compile time"] = float(51 t[2].split("program")[1].strip("\r\n")52 )53 54 elif n == "exec-success":55 test[fname]["exec time"] = float(56 t[2].split("program")[1].strip("\r\n")57 )58 if DEBUG:59 print(test[fname][string.replace(n, "-success", "")])60 61 else:62 # print "ERROR!"63 sys.exit(1)64 65 except:66 continue67 68 return test69 70 71# Diff results and look for regressions.72def diffResults(d_old, d_new):73 regressions = {}74 passes = {}75 removed = ""76 77 for x in ["compile state", "compile time", "exec state", "exec time"]:78 regressions[x] = ""79 passes[x] = ""80 81 for t in sorted(d_old.keys()):82 if t in d_new:83 84 # Check if the test passed or failed.85 for x in ["compile state", "compile time", "exec state", "exec time"]:86 87 if x not in d_old[t] and x not in d_new[t]:88 continue89 90 if x in d_old[t]:91 if x in d_new[t]:92 93 if d_old[t][x] == "PASS":94 if d_new[t][x] != "PASS":95 regressions[x] += t + "\n"96 else:97 if d_new[t][x] == "PASS":98 passes[x] += t + "\n"99 100 else:101 regressions[x] += t + "\n"102 103 if x == "compile state" or x == "exec state":104 continue105 106 # For execution time, if there is no result it's a fail.107 if x not in d_old[t] and x not in d_new[t]:108 continue109 elif x not in d_new[t]:110 regressions[x] += t + "\n"111 elif x not in d_old[t]:112 passes[x] += t + "\n"113 114 if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):115 continue116 117 elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]):118 passes[x] += t + "\n"119 120 elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):121 regressions[x] += t + ": NaN%\n"122 123 if (124 d_new[t][x] > d_old[t][x]125 and d_old[t][x] > 0.0126 and (d_new[t][x] - d_old[t][x]) / d_old[t][x] > 0.05127 ):128 regressions[x] += (129 t130 + ": "131 + "{0:.1f}".format(132 100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]133 )134 + "%\n"135 )136 137 else:138 removed += t + "\n"139 140 if len(regressions["compile state"]) != 0:141 print("REGRESSION: Compilation Failed")142 print(regressions["compile state"])143 144 if len(regressions["exec state"]) != 0:145 print("REGRESSION: Execution Failed")146 print(regressions["exec state"])147 148 if len(regressions["compile time"]) != 0:149 print("REGRESSION: Compilation Time")150 print(regressions["compile time"])151 152 if len(regressions["exec time"]) != 0:153 print("REGRESSION: Execution Time")154 print(regressions["exec time"])155 156 if len(passes["compile state"]) != 0:157 print("NEW PASSES: Compilation")158 print(passes["compile state"])159 160 if len(passes["exec state"]) != 0:161 print("NEW PASSES: Execution")162 print(passes["exec state"])163 164 if len(removed) != 0:165 print("REMOVED TESTS")166 print(removed)167 168 169# Main170if len(sys.argv) < 3:171 print("Usage:", sys.argv[0], "<old log> <new log>")172 sys.exit(-1)173 174d_old = parse(sys.argv[1])175d_new = parse(sys.argv[2])176 177diffResults(d_old, d_new)178