117 lines · yaml
1# This file defines a workflow that runs the libc++ benchmarks when a comment is added to the PR.2#3# The comment is of the form:4#5# /libcxx-bot benchmark <path-to-benchmarks-to-run>6#7# That will cause the specified benchmarks to be run on the PR and on the pull-request target, and8# their results to be compared.9 10name: Benchmark libc++11 12permissions:13 contents: read14 15on:16 issue_comment:17 types:18 - created19 - edited20 21env:22 CC: clang-2223 CXX: clang++-2224 25jobs:26 run-benchmarks:27 permissions:28 pull-requests: write29 30 if: >-31 github.event.issue.pull_request &&32 contains(github.event.comment.body, '/libcxx-bot benchmark')33 34 runs-on: llvm-premerge-libcxx-next-runners # TODO: This should run on a dedicated set of machines35 steps:36 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.037 with:38 python-version: '3.14'39 40 - name: Extract information from the PR41 id: vars42 env:43 COMMENT_BODY: ${{ github.event.comment.body }}44 run: |45 python3 -m venv .venv46 source .venv/bin/activate47 python -m pip install pygithub48 49 cat <<EOF | python >> ${GITHUB_OUTPUT}50 import github51 repo = github.Github("${{ github.token }}").get_repo("${{ github.repository }}")52 pr = repo.get_pull(${{ github.event.issue.number }})53 print(f"pr_base={pr.base.sha}")54 print(f"pr_head={pr.head.sha}")55 EOF56 BENCHMARKS=$(echo "$COMMENT_BODY" | sed -nE 's/\/libcxx-bot benchmark (.+)/\1/p')57 echo "benchmarks=${BENCHMARKS}" >> ${GITHUB_OUTPUT}58 59 - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.060 with:61 ref: ${{ steps.vars.outputs.pr_head }}62 fetch-depth: 063 fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main64 path: repo # Avoid nuking the workspace, where we have the Python virtualenv65 66 - name: Run baseline67 env:68 BENCHMARKS: ${{ steps.vars.outputs.benchmarks }}69 run: |70 source .venv/bin/activate && cd repo71 python -m pip install -r libcxx/utils/requirements.txt72 baseline_commit=$(git merge-base ${{ steps.vars.outputs.pr_base }} ${{ steps.vars.outputs.pr_head }})73 ./libcxx/utils/test-at-commit --commit ${baseline_commit} -B build/baseline -- -sv -j1 --param optimization=speed "$BENCHMARKS"74 ./libcxx/utils/consolidate-benchmarks build/baseline | tee baseline.lnt75 76 - name: Run candidate77 env:78 BENCHMARKS: ${{ steps.vars.outputs.benchmarks }}79 run: |80 source .venv/bin/activate && cd repo81 ./libcxx/utils/test-at-commit --commit ${{ steps.vars.outputs.pr_head }} -B build/candidate -- -sv -j1 --param optimization=speed "$BENCHMARKS"82 ./libcxx/utils/consolidate-benchmarks build/candidate | tee candidate.lnt83 84 - name: Compare baseline and candidate runs85 run: |86 source .venv/bin/activate && cd repo87 ./libcxx/utils/compare-benchmarks baseline.lnt candidate.lnt | tee results.txt88 89 - name: Update comment with results90 run: |91 source .venv/bin/activate && cd repo92 cat <<EOF | python93 import github94 repo = github.Github("${{ github.token }}").get_repo("${{ github.repository }}")95 pr = repo.get_pull(${{ github.event.issue.number }})96 comment = pr.get_issue_comment(${{ github.event.comment.id }})97 with open('results.txt', 'r') as f:98 benchmark_results = f.read()99 100 new_comment_text = f"""101 {comment.body}102 103 <details>104 <summary>105 Benchmark results:106 </summary>107 108 \`\`\`109 {benchmark_results}110 \`\`\`111 112 </details>113 """114 115 comment.edit(new_comment_text)116 EOF117