153 lines · plain
1#===-----------------------------------------------------------------------===//2#3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4# See https://llvm.org/LICENSE.txt for license information.5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6#7#===----------------------------------------------------------------------===//8#9# This is a Nix recipe for collecting reproducers for benchmarking purposes in a10# reproducible way. It works by injecting a linker wrapper that embeds a11# reproducer tarball into a non-allocated section of every linked object, which12# generally causes them to be smuggled out of the build tree in a section of the13# final binaries. In principle, this technique should let us collect reproducers14# from any project packaged by Nix without project-specific knowledge, but as15# you can see below, many interesting ones need a few hacks.16#17# If you have Nix installed, you can build the reproducers with the following18# command:19#20# TMPDIR=/var/tmp nix-build -j6 --log-format bar collect.nix21#22# This will result in building several large projects including Chromium and23# Firefox, which will take some time, and it will also build most of the24# dependencies for non-native targets. Eventually you will get a result25# directory containing all the reproducers.26#27# The following projects have been tested successfully:28# - chrome (native only, cross builds fail building the qtbase dependency)29# - firefox (all targets)30# - linux-kernel (all targets, requires patched nixpkgs)31# - ladybird (native only, same problem as chromium)32# - llvm (all targets)33 34{35 nixpkgs ? fetchTarball "https://github.com/NixOS/nixpkgs/archive/992f916556fcfaa94451ebc7fc6e396134bbf5b1.tar.gz",36 system ? builtins.currentSystem,37}:38let39 reproducerPkgs =40 crossSystem:41 let42 pkgsCross = import nixpkgs { inherit crossSystem; };43 # Wraps the given stdenv and lld package into a variant that collects44 # the reproducer and builds with debug info.45 reproducerCollectingStdenv =46 stdenv: lld:47 let48 bintools = stdenv.cc.bintools.override {49 extraBuildCommands = ''50 wrap ${stdenv.cc.targetPrefix}nix-wrap-lld ${pkgsCross.path}/pkgs/build-support/bintools-wrapper/ld-wrapper.sh ${lld}/bin/ld.lld51 export lz4=${pkgsCross.lib.getBin pkgsCross.buildPackages.lz4}/bin/lz452 substituteAll ${./ld-wrapper.sh} $out/bin/${stdenv.cc.targetPrefix}ld53 chmod +x $out/bin/${stdenv.cc.targetPrefix}ld54 substituteAll ${./ld-wrapper.sh} $out/bin/${stdenv.cc.targetPrefix}ld.lld55 chmod +x $out/bin/${stdenv.cc.targetPrefix}ld.lld56 '';57 };58 in59 pkgsCross.withCFlags [ "-g1" ] (60 stdenv.override (old: {61 allowedRequisites = null;62 cc = stdenv.cc.override { inherit bintools; };63 })64 );65 withReproducerCollectingStdenv =66 pkg:67 pkg.override {68 stdenv = reproducerCollectingStdenv pkgsCross.stdenv pkgsCross.buildPackages.lld;69 };70 withReproducerCollectingClangStdenv =71 pkg:72 pkg.override {73 clangStdenv = reproducerCollectingStdenv pkgsCross.clangStdenv pkgsCross.buildPackages.lld;74 };75 in76 {77 # For benchmarking the linker we want to disable LTO as otherwise we would78 # just be benchmarking the LLVM optimizer. Also, we generally want the79 # package to use the regular stdenv in order to simplify wrapping it.80 # Firefox normally uses the rustc stdenv but uses the regular one if81 # LTO is disabled so we kill two birds with one stone by disabling it.82 # Chromium uses the rustc stdenv unconditionally so we need the stuff83 # below to make sure that it finds our wrapped stdenv.84 chrome =85 (pkgsCross.chromium.override {86 newScope =87 extra:88 pkgsCross.newScope (89 extra90 // {91 pkgsBuildBuild = {92 pkg-config = pkgsCross.pkgsBuildBuild.pkg-config;93 rustc = {94 llvmPackages = rec {95 stdenv = reproducerCollectingStdenv pkgsCross.pkgsBuildBuild.rustc.llvmPackages.stdenv pkgsCross.pkgsBuildBuild.rustc.llvmPackages.lld;96 bintools = stdenv.cc.bintools;97 };98 };99 };100 }101 );102 pkgs = {103 rustc = {104 llvmPackages = {105 stdenv = reproducerCollectingStdenv pkgsCross.rustc.llvmPackages.stdenv pkgsCross.rustc.llvmPackages.lld;106 };107 };108 };109 }).browser.overrideAttrs110 (old: {111 configurePhase = old.configurePhase + ''112 echo use_thin_lto = false >> out/Release/args.gn113 echo is_cfi = false >> out/Release/args.gn114 '';115 });116 firefox = (withReproducerCollectingStdenv pkgsCross.firefox-unwrapped).override {117 ltoSupport = false;118 pgoSupport = false;119 };120 # Doesn't work as-is because the kernel derivation calls the linker121 # directly instead of the wrapper. See:122 # https://github.com/NixOS/nixpkgs/blob/d3fdff1631946f3e51318317375d638dae3d6aa2/pkgs/os-specific/linux/kernel/common-flags.nix#L12123 linux-kernel = (withReproducerCollectingStdenv pkgsCross.linux_latest).dev;124 ladybird = withReproducerCollectingStdenv pkgsCross.ladybird;125 llvm = withReproducerCollectingStdenv pkgsCross.llvm;126 webkitgtk = withReproducerCollectingClangStdenv pkgsCross.webkitgtk;127 hello = withReproducerCollectingStdenv pkgsCross.hello;128 };129 targets = {130 x86_64 = reproducerPkgs { config = "x86_64-unknown-linux-gnu"; };131 aarch64 = reproducerPkgs { config = "aarch64-unknown-linux-gnu"; };132 riscv64 = reproducerPkgs { config = "riscv64-unknown-linux-gnu"; };133 };134 pkgs = import nixpkgs { };135in136pkgs.runCommand "lld-speed-test" { } ''137 extract_reproducer() {138 ${pkgs.coreutils}/bin/mkdir -p $out/$2139 ${pkgs.llvm}/bin/llvm-objcopy -O binary --only-section=.lld_repro --set-section-flags .lld_repro=alloc $1 - | ${pkgs.gnutar}/bin/tar x -I ${pkgs.lib.getBin pkgs.buildPackages.lz4}/bin/lz4 --strip-components=1 -C $out/$2140 }141 142 extract_reproducer ${targets.aarch64.hello}/bin/hello hello-arm64143 extract_reproducer ${targets.x86_64.hello}/bin/hello hello-x64144 extract_reproducer ${targets.aarch64.chrome}/libexec/chromium/chromium chrome145 extract_reproducer ${targets.aarch64.ladybird}/lib/liblagom-web.so ladybird146 extract_reproducer ${targets.aarch64.firefox}/lib/firefox/libxul.so firefox-arm64147 extract_reproducer ${targets.x86_64.firefox}/lib/firefox/libxul.so firefox-x64148 extract_reproducer ${targets.riscv64.firefox}/lib/firefox/libxul.so firefox-riscv64149 extract_reproducer ${pkgs.lib.getLib targets.aarch64.llvm}/lib/libLLVM.so llvm-arm64150 extract_reproducer ${pkgs.lib.getLib targets.x86_64.llvm}/lib/libLLVM.so llvm-x64151 extract_reproducer ${pkgs.lib.getLib targets.riscv64.llvm}/lib/libLLVM.so llvm-riscv6152''153