126 lines · python
1#!/usr/bin/env python32import argparse3import csv4import re5import sys6import os7from statistics import geometric_mean8 9TIMING_LOG_RE = re.compile(r"(.*)/(.*).tmp(.*)")10 11 12def main():13 parser = argparse.ArgumentParser(14 description="BOLT NFC stat parser",15 formatter_class=argparse.ArgumentDefaultsHelpFormatter,16 )17 parser.add_argument(18 "input", nargs="+", help="timing.log files produced by llvm-bolt-wrapper"19 )20 parser.add_argument(21 "--check_longer_than",22 default=2,23 type=float,24 help="Only warn on tests longer than X seconds for at least one side",25 )26 parser.add_argument(27 "--threshold_single",28 default=10,29 type=float,30 help="Threshold for a single test result swing, abs percent",31 ),32 parser.add_argument(33 "--threshold_agg",34 default=5,35 type=float,36 help="Threshold for geomean test results swing, abs percent",37 ),38 parser.add_argument("--verbose", "-v", action="store_true")39 args = parser.parse_args()40 41 def fmt_delta(value, exc_threshold, above_bound=True):42 formatted_value = format(value, "+.2%")43 if not above_bound:44 formatted_value += "?"45 elif exc_threshold and sys.stdout.isatty(): # terminal supports colors46 return f"\033[1m{formatted_value}\033[0m"47 return formatted_value48 49 # Ratios for geomean computation50 time_ratios = []51 mem_ratios = []52 # Whether any test exceeds the single test threshold (mem or time)53 threshold_single = False54 # Whether geomean exceeds aggregate test threshold (mem or time)55 threshold_agg = False56 57 if args.verbose:58 print(f"# Individual test threshold: +-{args.threshold_single}%")59 print(f"# Aggregate (geomean) test threshold: +-{args.threshold_agg}%")60 print(61 f"# Checking time swings for tests with runtime >"62 f"{args.check_longer_than}s - otherwise marked as ?"63 )64 print("Test/binary BOLT_wall_time BOLT_max_rss")65 66 for input_file in args.input:67 input_dir = os.path.dirname(input_file)68 with open(input_file) as timing_file:69 timing_reader = csv.reader(timing_file, delimiter=";")70 for row in timing_reader:71 test_name = row[0]72 m = TIMING_LOG_RE.match(row[0])73 if m:74 test_name = f"{input_dir}/{m.groups()[1]}/{m.groups()[2]}"75 else:76 # Prepend input dir to unparsed test name77 test_name = input_dir + "#" + test_name78 time_a, time_b = float(row[1]), float(row[3])79 mem_a, mem_b = int(row[2]), int(row[4])80 # Check if time is above bound for at least one side81 time_above_bound = any(82 [x > args.check_longer_than for x in [time_a, time_b]]83 )84 # Compute B/A ratios (for % delta and geomean)85 time_ratio = time_b / time_a if time_a else float('nan')86 mem_ratio = mem_b / mem_a if mem_a else float('nan')87 # Keep ratios for geomean88 if time_above_bound and time_ratio > 0: # must be >0 for gmean89 time_ratios += [time_ratio]90 mem_ratios += [mem_ratio]91 # Deltas: (B/A)-1 = (B-A)/A92 time_delta = time_ratio - 193 mem_delta = mem_ratio - 194 # Check individual test results vs single test threshold95 time_exc = (96 100.0 * abs(time_delta) > args.threshold_single and time_above_bound97 )98 mem_exc = 100.0 * abs(mem_delta) > args.threshold_single99 if time_exc or mem_exc:100 threshold_single = True101 # Print deltas with formatting in verbose mode102 if args.verbose or time_exc or mem_exc:103 print(104 test_name,105 fmt_delta(time_delta, time_exc, time_above_bound),106 fmt_delta(mem_delta, mem_exc),107 )108 109 time_gmean_delta = geometric_mean(time_ratios) - 1110 mem_gmean_delta = geometric_mean(mem_ratios) - 1111 time_agg_threshold = 100.0 * abs(time_gmean_delta) > args.threshold_agg112 mem_agg_threshold = 100.0 * abs(mem_gmean_delta) > args.threshold_agg113 if time_agg_threshold or mem_agg_threshold:114 threshold_agg = True115 if time_agg_threshold or mem_agg_threshold or args.verbose:116 print(117 "Geomean",118 fmt_delta(time_gmean_delta, time_agg_threshold),119 fmt_delta(mem_gmean_delta, mem_agg_threshold),120 )121 exit(threshold_single or threshold_agg)122 123 124if __name__ == "__main__":125 main()126