84 lines · python
1# ===----------------------------------------------------------------------===##2#3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4# See https://llvm.org/LICENSE.txt for license information.5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6#7# ===----------------------------------------------------------------------===##8 9# REQUIRES: enable-spec-benchmarks10 11# RUN: mkdir -p %{temp}12# RUN: echo "%{cxx}" > %{temp}/cxx.subs13# RUN: echo "%{compile_flags}" > %{temp}/compile_flags.subs14# RUN: echo "%{flags}" > %{temp}/flags.subs15# RUN: echo "%{link_flags}" > %{temp}/link_flags.subs16# RUN: echo "%{spec_dir}" > %{temp}/spec_dir.subs17# RUN: %{python} %s %{temp}18# END.19 20import json21import pathlib22import sys23 24test_dir = pathlib.Path(sys.argv[1])25cxx = (test_dir / 'cxx.subs').open().read().strip()26compile_flags = (test_dir / 'compile_flags.subs').open().read().strip()27flags = (test_dir / 'flags.subs').open().read().strip()28link_flags = (test_dir / 'link_flags.subs').open().read().strip()29spec_dir = pathlib.Path((test_dir / 'spec_dir.subs').open().read().strip())30 31# Setup the configuration file32test_dir.mkdir(parents=True, exist_ok=True)33spec_config = test_dir / 'spec-config.cfg'34spec_config.write_text(f"""35default:36 ignore_errors = 137 iterations = 138 label = spec-stdlib39 log_line_width = 409640 makeflags = --jobs=841 mean_anyway = 142 output_format = csv43 preenv = 044 reportable = 045 tune = base46 copies = 147 threads = 148 CC = cc -O3 -std=c18 -Wno-implicit-function-declaration49 CXX = {cxx} {compile_flags} {flags} {link_flags} -Wno-error50 CC_VERSION_OPTION = --version51 CXX_VERSION_OPTION = --version52 EXTRA_PORTABILITY = -DSPEC_NO_CXX17_SPECIAL_MATH_FUNCTIONS # because libc++ doesn't implement the special math functions yet53""")54 55# Build the list of benchmarks. We take all intrate and fprate benchmarks that contain C++ and56# discard the ones that contain Fortran, since this test suite isn't set up to build Fortran code.57spec_benchmarks = set()58no_fortran = set()59with open(spec_dir / 'benchspec' / 'CPU' / 'intrate_any_cpp.bset', 'r') as f:60 spec_benchmarks.update(json.load(f)['benchmarks'])61with open(spec_dir / 'benchspec' / 'CPU' / 'fprate_any_cpp.bset', 'r') as f:62 spec_benchmarks.update(json.load(f)['benchmarks'])63with open(spec_dir / 'benchspec' / 'CPU' / 'no_fortran.bset', 'r') as f:64 no_fortran.update(json.load(f)['benchmarks'])65spec_benchmarks &= no_fortran66 67for benchmark in spec_benchmarks:68 print(f'#--- {benchmark}.sh.test')69 print(f'RUN: rm -rf %{{temp}}') # clean up any previous (potentially incomplete) run70 print(f'RUN: mkdir %{{temp}}')71 print(f'RUN: cp {spec_config} %{{temp}}/spec-config.cfg')72 print(f'RUN: %{{spec_dir}}/bin/runcpu --config %{{temp}}/spec-config.cfg --size train --output-root %{{temp}} --rebuild {benchmark}')73 print(f'RUN: rm -rf %{{temp}}/benchspec') # remove the temporary directory, which can become quite large74 75 # The `runcpu` command above doesn't fail even if the benchmark fails to run. To determine failure, parse the CSV76 # results and ensure there are no compilation errors or runtime errors in the status row. Also print the logs and77 # fail if there are no CSV files at all, which implies a SPEC error.78 print(f'RUN: %{{libcxx-dir}}/utils/parse-spec-results --extract "Base Status" --keep-failed %{{temp}}/result/*.train.csv > %{{temp}}/status || ! cat %{{temp}}/result/*.log')79 print(f'RUN: ! grep -E "CE|RE" %{{temp}}/status || ! cat %{{temp}}/result/*.log')80 81 # If there were no errors, parse the results into LNT-compatible format and print them.82 print(f'RUN: %{{libcxx-dir}}/utils/parse-spec-results %{{temp}}/result/*.train.csv --output-format=lnt > %{{temp}}/results.lnt')83 print(f'RUN: cat %{{temp}}/results.lnt')84