Switched to lcov for coverage collection, greatly simplified coverage.py

Since we already have fairly complicated scriptts, I figured it wouldn't
be too hard to use the gcov tools and directly parse their output. Boy
was I wrong.

The gcov intermediary format is a bit of a mess. In version 5.4, a
text-based intermediary format is written to a single .gcov file per
executable. This changed sometime before version 7.5, when it started
writing separate .gcov files per .o files. And in version 9 this
intermediary format has been entirely replaced with an incompatible json
format!

Ironically, this means the internal-only .gcda/.gcno binary format has
actually been more stable than the intermediary format.

Also there's no way to avoid temporary .gcov files generated in the
project root, which risks messing with how test.py runs parallel tests.
Fortunately this looks like it will be fixed in gcov version 9.

---

Ended up switching to lcov, which was the right way to go. lcov handles
all of the gcov parsing, provides an easily parsable output, and even
provides a set of higher-level commands to manage coverage collection
from different runs.

Since this is all provided by lcov, was able to simplify coverage.py
quite a bit. Now it just parses the .info files output by lcov.

scripts/test.py

@@ -21,7 +21,6 @@ import errno
 import signal
 
 TESTDIR = 'tests'
-RESULTDIR = 'results' # only used for coverage
 RULES = """
 define FLATTEN
 %(path)s%%$(subst /,.,$(target)): $(target)
@@ -35,22 +34,27 @@ $(foreach target,$(SRC),$(eval $(FLATTEN)))
 %(path)s.test: %(path)s.test.o $(foreach t,$(subst /,.,$(OBJ)),%(path)s.$t)
     $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
 """
-COVERAGE_TEST_RULES = """
+COVERAGE_RULES = """
 %(path)s.test: override CFLAGS += -fprofile-arcs -ftest-coverage
 
-# delete lingering coverage info during build
-%(path)s.test: | %(path)s.test.clean
-.PHONY: %(path)s.test.clean
-%(path)s.test.clean:
+# delete lingering coverage
+%(path)s.test: | %(path)s.info.clean
+.PHONY: %(path)s.clean
+%(path)s.clean:
     rm -f %(path)s*.gcda
 
-override TEST_GCDAS += %(path)s*.gcda
-"""
-COVERAGE_RESULT_RULES = """
-# dependencies defined in test makefiles
-.PHONY: %(results)s/coverage.gcov
-%(results)s/coverage.gcov: $(patsubst %%,%%.gcov,$(wildcard $(TEST_GCDAS)))
-    ./scripts/coverage.py -s $^ --filter="$(SRC)" --merge=$@
+# accumulate coverage info
+.PHONY: %(path)s.info
+%(path)s.info:
+    $(strip $(LCOV) -c \\
+        $(addprefix -d ,$(wildcard %(path)s*.gcda)) \\
+        --rc 'geninfo_adjust_src_path=$(shell pwd)' \\
+        -o $@)
+    $(LCOV) -e $@ $(addprefix /,$(SRC)) -o $@
+
+.PHONY: %(path)s.cumul.info
+%(path)s.cumul.info: %(path)s.info
+    $(LCOV) -a $< $(addprefix -a ,$(wildcard $@)) -o $@
 """
 GLOBALS = """
 //////////////// AUTOGENERATED TEST ////////////////
@@ -539,8 +543,7 @@ class TestSuite:
 
             # add coverage hooks?
             if args.get('coverage', False):
-                mk.write(COVERAGE_TEST_RULES.replace(4*' ', '\t') % dict(
-                    results=args['results'],
+                mk.write(COVERAGE_RULES.replace(4*' ', '\t') % dict(
                     path=self.path))
                 mk.write('\n')
 
@@ -749,40 +752,14 @@ def main(**args):
                 failed += 1
 
     if args.get('coverage', False):
-        # mkdir -p resultdir
-        os.makedirs(args['results'], exist_ok=True)
-
         # collect coverage info
-        hits, branches = 0, 0
-
-        with open(args['results'] + '/coverage.mk', 'w') as mk:
-            mk.write(COVERAGE_RESULT_RULES.replace(4*' ', '\t') % dict(
-                results=args['results']))
-
         cmd = (['make', '-f', 'Makefile'] +
             list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
-            ['-f', args['results'] + '/coverage.mk',
-                args['results'] + '/coverage.gcov'])
-        mpty, spty = pty.openpty()
+            [re.sub('\.test$', '.cumul.info', target) for target in targets])
         if args.get('verbose', False):
             print(' '.join(shlex.quote(c) for c in cmd))
-        proc = sp.Popen(cmd, stdout=spty)
-        os.close(spty)
-        mpty = os.fdopen(mpty, 'r', 1)
-        while True:
-            try:
-                line = mpty.readline()
-            except OSError as e:
-                if e.errno == errno.EIO:
-                    break
-                raise
-            if args.get('verbose', False):
-                sys.stdout.write(line)
-            # get coverage status
-            m = re.match('^TOTALS +([0-9]+)/([0-9]+)', line)
-            if m:
-                hits = int(m.group(1))
-                branches = int(m.group(2))
+        proc = sp.Popen(cmd,
+            stdout=sp.DEVNULL if not args.get('verbose', False) else None)
         proc.wait()
         if proc.returncode != 0:
             sys.exit(-3)
@@ -803,9 +780,6 @@ def main(**args):
         100*(passed/total if total else 1.0)))
     print('tests failed %d/%d (%.2f%%)' % (failed, total,
         100*(failed/total if total else 1.0)))
-    if args.get('coverage', False):
-        print('coverage %d/%d (%.2f%%)' % (hits, branches,
-            100*(hits/branches if branches else 1.0)))
     return 1 if failed > 0 else 0
 
 if __name__ == "__main__":
@@ -818,9 +792,6 @@ if __name__ == "__main__":
             directory of tests, a specific file, a suite by name, and even a \
             specific test case by adding brackets. For example \
             \"test_dirs[0]\" or \"{0}/test_dirs.toml[0]\".".format(TESTDIR))
-    parser.add_argument('--results', default=RESULTDIR,
-        help="Directory to store results. Created implicitly. Only used in \
-            this script for coverage information if --coverage is provided.")
     parser.add_argument('-D', action='append', default=[],
         help="Overriding parameter definitions.")
     parser.add_argument('-v', '--verbose', action='store_true',
@@ -848,8 +819,8 @@ if __name__ == "__main__":
     parser.add_argument('--disk',
         help="Specify a file to use for persistent/reentrant tests.")
     parser.add_argument('--coverage', action='store_true',
-        help="Collect coverage information across tests. This is stored in \
-            the results directory. Coverage is not reset between runs \
-            allowing multiple test runs to contribute to coverage \
-            information.")
+        help="Collect coverage information during testing. This uses lcov/gcov \
+            to accumulate coverage information into *.info files. Note \
+            coverage is not reset between runs, allowing multiple runs to \
+            contribute to coverage.")
     sys.exit(main(**vars(parser.parse_args())))