diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7e2782d8..1745988da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,18 +29,21 @@ jobs: - scenario: on ubuntu runs-on: ubuntu-24.04 os: linux - sanitizers: false + instrumented: false primary: true + stdenv: stdenv - scenario: on macos runs-on: macos-14 os: darwin - sanitizers: false + instrumented: false primary: true - - scenario: on ubuntu (with sanitizers) + stdenv: stdenv + - scenario: on ubuntu (with sanitizers / coverage) runs-on: ubuntu-24.04 os: linux - sanitizers: true + instrumented: true primary: false + stdenv: clangStdenv name: tests ${{ matrix.scenario }} runs-on: ${{ matrix.runs-on }} timeout-minutes: 60 @@ -63,13 +66,28 @@ jobs: if: matrix.os == 'linux' - name: Run component tests run: | - nix build --file ci/gha/tests componentTests -L \ - --arg withSanitizers ${{ matrix.sanitizers }} + nix build --file ci/gha/tests/wrapper.nix componentTests -L \ + --arg withInstrumentation ${{ matrix.instrumented }} \ + --argstr stdenv "${{ matrix.stdenv }}" - name: Run flake checks and prepare the installer tarball run: | ci/gha/tests/build-checks ci/gha/tests/prepare-installer-for-github-actions if: ${{ matrix.primary }} + - name: Collect code coverage + run: | + nix build --file ci/gha/tests/wrapper.nix codeCoverage.coverageReports -L \ + --arg withInstrumentation ${{ matrix.instrumented }} \ + --argstr stdenv "${{ matrix.stdenv }}" \ + --out-link coverage-reports + cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY + if: ${{ matrix.instrumented }} + - name: Upload coverage reports + uses: actions/upload-artifact@v4 + with: + name: coverage-reports + path: coverage-reports/ + if: ${{ matrix.instrumented }} - name: Upload installer tarball uses: actions/upload-artifact@v4 with: diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index ce44d7cf7..d2bee699b 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -5,15 +5,78 @@ getStdenv ? p: p.stdenv, componentTestsPrefix ? "", withSanitizers ? false, + withCoverage ? false, + ... }: let inherit (pkgs) lib; hydraJobs = nixFlake.hydraJobs; packages' = nixFlake.packages.${system}; + stdenv = (getStdenv pkgs); + + enableSanitizersLayer = finalAttrs: prevAttrs: { + mesonFlags = + (prevAttrs.mesonFlags or [ ]) + ++ [ + # Run all tests with UBSAN enabled. Running both with ubsan and + # without doesn't seem to have much immediate benefit for doubling + # the GHA CI workaround. + # + # TODO: Work toward enabling "address,undefined" if it seems feasible. + # This would maybe require dropping Boost coroutines and ignoring intentional + # memory leaks with detect_leaks=0. + (lib.mesonOption "b_sanitize" "undefined") + ] + ++ (lib.optionals stdenv.cc.isClang [ + # https://www.github.com/mesonbuild/meson/issues/764 + (lib.mesonBool "b_lundef" false) + ]); + }; + + collectCoverageLayer = finalAttrs: prevAttrs: { + env = + let + # https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#the-code-coverage-workflow + coverageFlags = [ + "-fprofile-instr-generate" + "-fcoverage-mapping" + ]; + in + { + CFLAGS = toString coverageFlags; + CXXFLAGS = toString coverageFlags; + }; + + # Done in a pre-configure hook, because $NIX_BUILD_TOP needs to be substituted. + preConfigure = + prevAttrs.preConfigure or "" + + '' + mappingFlag=" -fcoverage-prefix-map=$NIX_BUILD_TOP/${finalAttrs.src.name}=${finalAttrs.src}" + CFLAGS+="$mappingFlag" + CXXFLAGS+="$mappingFlag" + ''; + }; + + componentOverrides = + (lib.optional withSanitizers enableSanitizersLayer) + ++ (lib.optional withCoverage collectCoverageLayer); in -{ +rec { + nixComponents = + (nixFlake.lib.makeComponents { + inherit pkgs; + inherit getStdenv; + }).overrideScope + ( + final: prev: { + nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; }; + + mesonComponentOverrides = lib.composeManyExtensions componentOverrides; + } + ); + /** Top-level tests for the flake outputs, as they would be built by hydra. These tests generally can't be overridden to run with sanitizers. @@ -52,33 +115,6 @@ in }; componentTests = - let - nixComponents = - (nixFlake.lib.makeComponents { - inherit pkgs; - inherit getStdenv; - }).overrideScope - ( - final: prev: { - nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; }; - - mesonComponentOverrides = finalAttrs: prevAttrs: { - mesonFlags = - (prevAttrs.mesonFlags or [ ]) - ++ lib.optionals withSanitizers [ - # Run all tests with UBSAN enabled. Running both with ubsan and - # without doesn't seem to have much immediate benefit for doubling - # the GHA CI workaround. - # - # TODO: Work toward enabling "address,undefined" if it seems feasible. - # This would maybe require dropping Boost coroutines and ignoring intentional - # memory leaks with detect_leaks=0. - (lib.mesonOption "b_sanitize" "undefined") - ]; - }; - } - ); - in (lib.concatMapAttrs ( pkgName: pkg: lib.concatMapAttrs (testName: test: { @@ -88,4 +124,88 @@ in // lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) { "${componentTestsPrefix}nix-functional-tests" = nixComponents.nix-functional-tests; }; + + codeCoverage = + let + componentsTestsToProfile = + (builtins.mapAttrs (n: v: nixComponents.${n}.tests.run) { + "nix-util-tests" = { }; + "nix-store-tests" = { }; + "nix-fetchers-tests" = { }; + "nix-expr-tests" = { }; + "nix-flake-tests" = { }; + }) + // { + inherit (nixComponents) nix-functional-tests; + }; + + coverageProfileDrvs = lib.mapAttrs ( + n: v: + v.overrideAttrs ( + finalAttrs: prevAttrs: { + outputs = (prevAttrs.outputs or [ "out" ]) ++ [ "profraw" ]; + env = { + LLVM_PROFILE_FILE = "${placeholder "profraw"}/%m"; + }; + } + ) + ) componentsTestsToProfile; + + coverageProfiles = lib.mapAttrsToList (n: v: lib.getOutput "profraw" v) coverageProfileDrvs; + + mergedProfdata = + pkgs.runCommand "merged-profdata" + { + __structuredAttrs = true; + nativeBuildInputs = [ pkgs.llvmPackages.libllvm ]; + inherit coverageProfiles; + } + '' + rawProfiles=() + for dir in "''\${coverageProfiles[@]}"; do + rawProfiles+=($dir/*) + done + llvm-profdata merge -sparse -output $out "''\${rawProfiles[@]}" + ''; + + coverageReports = + let + nixComponentDrvs = lib.filter (lib.isDerivation) (lib.attrValues nixComponents); + in + pkgs.runCommand "code-coverage-report" + { + nativeBuildInputs = [ + pkgs.llvmPackages.libllvm + ]; + __structuredAttrs = true; + nixComponents = nixComponentDrvs; + } + '' + # ${toString (lib.map (v: v.src) nixComponentDrvs)} + + binaryFiles=() + for dir in "''\${nixComponents[@]}"; do + readarray -t filesInDir < <(find "$dir" -type f -executable) + binaryFiles+=("''\${filesInDir[@]}") + done + + arguments=$(concatStringsSep " -object " binaryFiles) + llvm-cov show $arguments -instr-profile ${mergedProfdata} -output-dir $out -format=html + + { + echo "# Code coverage summary (generated via \`llvm-cov\`):" + echo + echo '```' + llvm-cov report $arguments -instr-profile ${mergedProfdata} -format=text -use-color=false + echo '```' + echo + } >> $out/index.txt + + ''; + in + assert withCoverage; + assert stdenv.cc.isClang; + { + inherit coverageProfileDrvs mergedProfdata coverageReports; + }; } diff --git a/ci/gha/tests/wrapper.nix b/ci/gha/tests/wrapper.nix new file mode 100644 index 000000000..dc280ebbb --- /dev/null +++ b/ci/gha/tests/wrapper.nix @@ -0,0 +1,16 @@ +{ + nixFlake ? builtins.getFlake ("git+file://" + toString ../../..), + system ? builtins.currentSystem, + pkgs ? nixFlake.inputs.nixpkgs.legacyPackages.${system}, + stdenv ? "stdenv", + componentTestsPrefix ? "", + withInstrumentation ? false, +}@args: +import ./. ( + args + // { + getStdenv = p: p.${stdenv}; + withSanitizers = withInstrumentation; + withCoverage = withInstrumentation; + } +) diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index bb57ca941..fd686f140 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -32,3 +32,11 @@ do_pch = cxx.get_id() == 'clang' if cxx.get_id() == 'clang' add_project_arguments('-fpch-instantiate-templates', language : 'cpp') endif + +# Clang gets grumpy about missing libasan symbols if -shared-libasan is not +# passed when building shared libs, at least on Linux +if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option( + 'b_sanitize', +)) + add_project_link_arguments('-shared-libasan', language : 'cpp') +endif diff --git a/tests/functional/flakes/run.sh b/tests/functional/flakes/run.sh index c92ddca2b..0a2947825 100755 --- a/tests/functional/flakes/run.sh +++ b/tests/functional/flakes/run.sh @@ -41,11 +41,13 @@ nix run -f shell-hello.nix env > $TEST_ROOT/actual-env # - we unset TMPDIR on macOS if it contains /var/folders. bad. https://github.com/NixOS/nix/issues/7731 # - _ is set by bash and is expected to differ because it contains the original command # - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +# - __LLVM_PROFILE_RT_INIT_ONCE - implementation detail of LLVM source code coverage collection sed -i \ -e 's/PATH=.*/PATH=.../' \ -e 's/_=.*/_=.../' \ -e '/^TMPDIR=\/var\/folders\/.*/d' \ -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + -e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \ $TEST_ROOT/expected-env $TEST_ROOT/actual-env sort $TEST_ROOT/expected-env | uniq > $TEST_ROOT/expected-env.sorted # nix run appears to clear _. I don't understand why. Is this ok? diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index 51032ff1b..9769c90d1 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -34,11 +34,13 @@ nix shell -f shell-hello.nix hello -c env > "$TEST_ROOT/actual-env" # - we unset TMPDIR on macOS if it contains /var/folders # - _ is set by bash and is expectedf to differ because it contains the original command # - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +# - __LLVM_PROFILE_RT_INIT_ONCE - implementation detail of LLVM source code coverage collection sed -i \ -e 's/PATH=.*/PATH=.../' \ -e 's/_=.*/_=.../' \ -e '/^TMPDIR=\/var\/folders\/.*/d' \ -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + -e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \ "$TEST_ROOT/expected-env" "$TEST_ROOT/actual-env" sort "$TEST_ROOT/expected-env" > "$TEST_ROOT/expected-env.sorted" sort "$TEST_ROOT/actual-env" > "$TEST_ROOT/actual-env.sorted"