From 4333a9d5a83e5a90981f2d8272db6b3125aa10b6 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 4 Aug 2025 23:50:02 +0300 Subject: [PATCH] ci: Collect code coverage in tests This adds the necessary infrastructure to collect code coverage in CI, which could be useful to look at munually or track consistently via something like codecov. Co-authored-by: Jade Lovelace --- .github/workflows/ci.yml | 30 +++- ci/gha/tests/default.nix | 176 +++++++++++++++++---- ci/gha/tests/wrapper.nix | 16 ++ nix-meson-build-support/common/meson.build | 8 + tests/functional/flakes/run.sh | 2 + tests/functional/shell.sh | 2 + 6 files changed, 200 insertions(+), 34 deletions(-) create mode 100644 ci/gha/tests/wrapper.nix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7e2782d8..1745988da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,18 +29,21 @@ jobs: - scenario: on ubuntu runs-on: ubuntu-24.04 os: linux - sanitizers: false + instrumented: false primary: true + stdenv: stdenv - scenario: on macos runs-on: macos-14 os: darwin - sanitizers: false + instrumented: false primary: true - - scenario: on ubuntu (with sanitizers) + stdenv: stdenv + - scenario: on ubuntu (with sanitizers / coverage) runs-on: ubuntu-24.04 os: linux - sanitizers: true + instrumented: true primary: false + stdenv: clangStdenv name: tests ${{ matrix.scenario }} runs-on: ${{ matrix.runs-on }} timeout-minutes: 60 @@ -63,13 +66,28 @@ jobs: if: matrix.os == 'linux' - name: Run component tests run: | - nix build --file ci/gha/tests componentTests -L \ - --arg withSanitizers ${{ matrix.sanitizers }} + nix build --file ci/gha/tests/wrapper.nix componentTests -L \ + --arg withInstrumentation ${{ matrix.instrumented }} \ + --argstr stdenv "${{ matrix.stdenv }}" - name: Run flake checks and prepare the installer tarball run: | ci/gha/tests/build-checks ci/gha/tests/prepare-installer-for-github-actions if: ${{ matrix.primary }} + - name: Collect code coverage + run: | + nix build --file ci/gha/tests/wrapper.nix codeCoverage.coverageReports -L \ + --arg withInstrumentation ${{ matrix.instrumented }} \ + --argstr stdenv "${{ matrix.stdenv }}" \ + --out-link coverage-reports + cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY + if: ${{ matrix.instrumented }} + - name: Upload coverage reports + uses: actions/upload-artifact@v4 + with: + name: coverage-reports + path: coverage-reports/ + if: ${{ matrix.instrumented }} - name: Upload installer tarball uses: actions/upload-artifact@v4 with: diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index ce44d7cf7..d2bee699b 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -5,15 +5,78 @@ getStdenv ? p: p.stdenv, componentTestsPrefix ? "", withSanitizers ? false, + withCoverage ? false, + ... }: let inherit (pkgs) lib; hydraJobs = nixFlake.hydraJobs; packages' = nixFlake.packages.${system}; + stdenv = (getStdenv pkgs); + + enableSanitizersLayer = finalAttrs: prevAttrs: { + mesonFlags = + (prevAttrs.mesonFlags or [ ]) + ++ [ + # Run all tests with UBSAN enabled. Running both with ubsan and + # without doesn't seem to have much immediate benefit for doubling + # the GHA CI workaround. + # + # TODO: Work toward enabling "address,undefined" if it seems feasible. + # This would maybe require dropping Boost coroutines and ignoring intentional + # memory leaks with detect_leaks=0. + (lib.mesonOption "b_sanitize" "undefined") + ] + ++ (lib.optionals stdenv.cc.isClang [ + # https://www.github.com/mesonbuild/meson/issues/764 + (lib.mesonBool "b_lundef" false) + ]); + }; + + collectCoverageLayer = finalAttrs: prevAttrs: { + env = + let + # https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#the-code-coverage-workflow + coverageFlags = [ + "-fprofile-instr-generate" + "-fcoverage-mapping" + ]; + in + { + CFLAGS = toString coverageFlags; + CXXFLAGS = toString coverageFlags; + }; + + # Done in a pre-configure hook, because $NIX_BUILD_TOP needs to be substituted. + preConfigure = + prevAttrs.preConfigure or "" + + '' + mappingFlag=" -fcoverage-prefix-map=$NIX_BUILD_TOP/${finalAttrs.src.name}=${finalAttrs.src}" + CFLAGS+="$mappingFlag" + CXXFLAGS+="$mappingFlag" + ''; + }; + + componentOverrides = + (lib.optional withSanitizers enableSanitizersLayer) + ++ (lib.optional withCoverage collectCoverageLayer); in -{ +rec { + nixComponents = + (nixFlake.lib.makeComponents { + inherit pkgs; + inherit getStdenv; + }).overrideScope + ( + final: prev: { + nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; }; + + mesonComponentOverrides = lib.composeManyExtensions componentOverrides; + } + ); + /** Top-level tests for the flake outputs, as they would be built by hydra. These tests generally can't be overridden to run with sanitizers. @@ -52,33 +115,6 @@ in }; componentTests = - let - nixComponents = - (nixFlake.lib.makeComponents { - inherit pkgs; - inherit getStdenv; - }).overrideScope - ( - final: prev: { - nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; }; - - mesonComponentOverrides = finalAttrs: prevAttrs: { - mesonFlags = - (prevAttrs.mesonFlags or [ ]) - ++ lib.optionals withSanitizers [ - # Run all tests with UBSAN enabled. Running both with ubsan and - # without doesn't seem to have much immediate benefit for doubling - # the GHA CI workaround. - # - # TODO: Work toward enabling "address,undefined" if it seems feasible. - # This would maybe require dropping Boost coroutines and ignoring intentional - # memory leaks with detect_leaks=0. - (lib.mesonOption "b_sanitize" "undefined") - ]; - }; - } - ); - in (lib.concatMapAttrs ( pkgName: pkg: lib.concatMapAttrs (testName: test: { @@ -88,4 +124,88 @@ in // lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) { "${componentTestsPrefix}nix-functional-tests" = nixComponents.nix-functional-tests; }; + + codeCoverage = + let + componentsTestsToProfile = + (builtins.mapAttrs (n: v: nixComponents.${n}.tests.run) { + "nix-util-tests" = { }; + "nix-store-tests" = { }; + "nix-fetchers-tests" = { }; + "nix-expr-tests" = { }; + "nix-flake-tests" = { }; + }) + // { + inherit (nixComponents) nix-functional-tests; + }; + + coverageProfileDrvs = lib.mapAttrs ( + n: v: + v.overrideAttrs ( + finalAttrs: prevAttrs: { + outputs = (prevAttrs.outputs or [ "out" ]) ++ [ "profraw" ]; + env = { + LLVM_PROFILE_FILE = "${placeholder "profraw"}/%m"; + }; + } + ) + ) componentsTestsToProfile; + + coverageProfiles = lib.mapAttrsToList (n: v: lib.getOutput "profraw" v) coverageProfileDrvs; + + mergedProfdata = + pkgs.runCommand "merged-profdata" + { + __structuredAttrs = true; + nativeBuildInputs = [ pkgs.llvmPackages.libllvm ]; + inherit coverageProfiles; + } + '' + rawProfiles=() + for dir in "''\${coverageProfiles[@]}"; do + rawProfiles+=($dir/*) + done + llvm-profdata merge -sparse -output $out "''\${rawProfiles[@]}" + ''; + + coverageReports = + let + nixComponentDrvs = lib.filter (lib.isDerivation) (lib.attrValues nixComponents); + in + pkgs.runCommand "code-coverage-report" + { + nativeBuildInputs = [ + pkgs.llvmPackages.libllvm + ]; + __structuredAttrs = true; + nixComponents = nixComponentDrvs; + } + '' + # ${toString (lib.map (v: v.src) nixComponentDrvs)} + + binaryFiles=() + for dir in "''\${nixComponents[@]}"; do + readarray -t filesInDir < <(find "$dir" -type f -executable) + binaryFiles+=("''\${filesInDir[@]}") + done + + arguments=$(concatStringsSep " -object " binaryFiles) + llvm-cov show $arguments -instr-profile ${mergedProfdata} -output-dir $out -format=html + + { + echo "# Code coverage summary (generated via \`llvm-cov\`):" + echo + echo '```' + llvm-cov report $arguments -instr-profile ${mergedProfdata} -format=text -use-color=false + echo '```' + echo + } >> $out/index.txt + + ''; + in + assert withCoverage; + assert stdenv.cc.isClang; + { + inherit coverageProfileDrvs mergedProfdata coverageReports; + }; } diff --git a/ci/gha/tests/wrapper.nix b/ci/gha/tests/wrapper.nix new file mode 100644 index 000000000..dc280ebbb --- /dev/null +++ b/ci/gha/tests/wrapper.nix @@ -0,0 +1,16 @@ +{ + nixFlake ? builtins.getFlake ("git+file://" + toString ../../..), + system ? builtins.currentSystem, + pkgs ? nixFlake.inputs.nixpkgs.legacyPackages.${system}, + stdenv ? "stdenv", + componentTestsPrefix ? "", + withInstrumentation ? false, +}@args: +import ./. ( + args + // { + getStdenv = p: p.${stdenv}; + withSanitizers = withInstrumentation; + withCoverage = withInstrumentation; + } +) diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index bb57ca941..fd686f140 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -32,3 +32,11 @@ do_pch = cxx.get_id() == 'clang' if cxx.get_id() == 'clang' add_project_arguments('-fpch-instantiate-templates', language : 'cpp') endif + +# Clang gets grumpy about missing libasan symbols if -shared-libasan is not +# passed when building shared libs, at least on Linux +if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option( + 'b_sanitize', +)) + add_project_link_arguments('-shared-libasan', language : 'cpp') +endif diff --git a/tests/functional/flakes/run.sh b/tests/functional/flakes/run.sh index c92ddca2b..0a2947825 100755 --- a/tests/functional/flakes/run.sh +++ b/tests/functional/flakes/run.sh @@ -41,11 +41,13 @@ nix run -f shell-hello.nix env > $TEST_ROOT/actual-env # - we unset TMPDIR on macOS if it contains /var/folders. bad. https://github.com/NixOS/nix/issues/7731 # - _ is set by bash and is expected to differ because it contains the original command # - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +# - __LLVM_PROFILE_RT_INIT_ONCE - implementation detail of LLVM source code coverage collection sed -i \ -e 's/PATH=.*/PATH=.../' \ -e 's/_=.*/_=.../' \ -e '/^TMPDIR=\/var\/folders\/.*/d' \ -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + -e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \ $TEST_ROOT/expected-env $TEST_ROOT/actual-env sort $TEST_ROOT/expected-env | uniq > $TEST_ROOT/expected-env.sorted # nix run appears to clear _. I don't understand why. Is this ok? diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index 51032ff1b..9769c90d1 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -34,11 +34,13 @@ nix shell -f shell-hello.nix hello -c env > "$TEST_ROOT/actual-env" # - we unset TMPDIR on macOS if it contains /var/folders # - _ is set by bash and is expectedf to differ because it contains the original command # - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +# - __LLVM_PROFILE_RT_INIT_ONCE - implementation detail of LLVM source code coverage collection sed -i \ -e 's/PATH=.*/PATH=.../' \ -e 's/_=.*/_=.../' \ -e '/^TMPDIR=\/var\/folders\/.*/d' \ -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + -e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \ "$TEST_ROOT/expected-env" "$TEST_ROOT/actual-env" sort "$TEST_ROOT/expected-env" > "$TEST_ROOT/expected-env.sorted" sort "$TEST_ROOT/actual-env" > "$TEST_ROOT/actual-env.sorted"