Add a script to measure coverage per module.

This makes it possible to increase coverage threshold to "each module
has 100% branch coverage from its own tests".

This implementation supports `make maxi_cov` and `tox -e maxi_cov`.
It is also enabled in CI.

.github/workflows/tests.yml

@@ -9,8 +9,8 @@ on:
       - main
 
 jobs:
-  main:
-    name: Run code quality checks
+  coverage:
+    name: Run test coverage checks
     runs-on: ubuntu-latest
     steps:
     - name: Check out repository
@@ -23,6 +23,21 @@ jobs:
       run: pip install tox
     - name: Run tests with coverage
       run: tox -e coverage
+    - name: Run tests with per-module coverage
+      run: tox -e maxi_cov
+
+  quality:
+    name: Run code quality checks
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v3
+    - name: Install Python 3.x
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Install tox
+      run: pip install tox
     - name: Check code formatting
       run: tox -e black
     - name: Check code style
@@ -34,7 +49,9 @@ jobs:
 
   matrix:
     name: Run tests on Python ${{ matrix.python }}
-    needs: main
+    needs:
+      - coverage
+      - quality
     runs-on: ubuntu-latest
     strategy:
       matrix:

Makefile

@@ -1,4 +1,4 @@
-.PHONY: default style test coverage build clean
+.PHONY: default style test coverage maxi_cov build clean
 
 export PYTHONASYNCIODEBUG=1
 export PYTHONPATH=src
@@ -20,6 +20,11 @@ coverage:
 	coverage html
 	coverage report --show-missing --fail-under=100
 
+maxi_cov:
+	python tests/maxi_cov.py
+	coverage html
+	coverage report --show-missing --fail-under=100
+
 build:
 	python setup.py build_ext --inplace
 

setup.cfg

@@ -21,7 +21,9 @@ lines_after_imports = 2
 [coverage:run]
 branch = True
 omit =
-    */__main__.py
+    # */websockets matches src/websockets and .tox/**/site-packages/websockets
+    */websockets/__main__.py
+    tests/maxi_cov.py
 
 [coverage:paths]
 source =

tests/maxi_cov.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+
+"""Measure coverage of each module by its test module."""
+
+import glob
+import os.path
+import subprocess
+import sys
+
+
+UNMAPPED_SRC_FILES = ["websockets/version.py"]
+UNMAPPED_TEST_FILES = ["tests/test_exports.py"]
+
+IGNORED_FILES = [
+    # */websockets matches src/websockets and .tox/**/site-packages/websockets.
+    # There are no tests for the __main__ module.
+    "*/websockets/__main__.py",
+    # This approach isn't applicable to the test suite of the legacy
+    # implementation, due to the huge test_client_server test module.
+    "*/websockets/legacy/*",
+    "tests/legacy/*",
+    # Test utilities don't fit anywhere because they are shared.
+    "tests/extensions/utils.py",
+    "tests/utils.py",
+    # There is no point measure the coverage of this script.
+    "tests/maxi_cov.py",
+]
+
+
+def check_environment():
+    """Check that prerequisites for running this script are met."""
+    try:
+        import websockets  # noqa: F401
+    except ImportError:
+        print("failed to import websockets; is src on PYTHONPATH?")
+        return False
+    try:
+        import coverage  # noqa: F401
+    except ImportError:
+        print("failed to locate Coverage.py; is it installed?")
+        return False
+    return True
+
+
+def get_mapping(src_dir="src"):
+    """Return a dict mapping each source file to its test file."""
+
+    # List source and test files.
+
+    src_files = glob.glob(
+        os.path.join(src_dir, "websockets/**/*.py"),
+        recursive=True,
+    )
+
+    test_files = glob.glob(
+        "tests/**/*.py",
+        recursive=True,
+    )
+
+    src_files = [
+        os.path.relpath(src_file, src_dir)
+        for src_file in sorted(src_files)
+        if os.path.basename(src_file) != "__init__.py"
+        and os.path.basename(src_file) != "__main__.py"
+        and "legacy" not in os.path.dirname(src_file)
+    ]
+    test_files = [
+        test_file
+        for test_file in sorted(test_files)
+        if os.path.basename(test_file) != "__init__.py"
+        and os.path.basename(test_file).startswith("test_")
+        and "legacy" not in os.path.dirname(test_file)
+    ]
+
+    # Map source files to test files.
+
+    mapping = {}
+    unmapped_test_files = []
+
+    for test_file in test_files:
+        dir_name, file_name = os.path.split(test_file)
+        assert dir_name.startswith("tests")
+        assert file_name.startswith("test_")
+        src_file = os.path.join(
+            "websockets" + dir_name[len("tests") :],
+            file_name[len("test_") :],
+        )
+        if src_file in src_files:
+            mapping[src_file] = test_file
+        else:
+            unmapped_test_files.append(test_file)
+
+    unmapped_src_files = list(set(src_files) - set(mapping))
+
+    # Ensure that all files are mapped.
+
+    assert unmapped_src_files == UNMAPPED_SRC_FILES
+    assert unmapped_test_files == UNMAPPED_TEST_FILES
+
+    return mapping
+
+
+def run_coverage(mapping, src_dir="src"):
+    # Initialize a new coverage measurement session. The --source option
+    # includes all files in the report, even if they're never imported.
+    print("\nInitializing session\n", flush=True)
+    subprocess.run(
+        [
+            sys.executable,
+            "-m",
+            "coverage",
+            "run",
+            "--source",
+            ",".join([os.path.join(src_dir, "websockets"), "tests"]),
+            "--omit",
+            ",".join(IGNORED_FILES),
+            "-m",
+            "unittest",
+        ]
+        + UNMAPPED_TEST_FILES,
+        check=True,
+    )
+    # Append coverage of each source module by the corresponding test module.
+    for src_file, test_file in mapping.items():
+        print(f"\nTesting {src_file} with {test_file}\n", flush=True)
+        subprocess.run(
+            [
+                sys.executable,
+                "-m",
+                "coverage",
+                "run",
+                "--append",
+                "--include",
+                ",".join([os.path.join(src_dir, src_file), test_file]),
+                "-m",
+                "unittest",
+                test_file,
+            ],
+            check=True,
+        )
+
+
+if __name__ == "__main__":
+    if not check_environment():
+        sys.exit(1)
+    src_dir = sys.argv[1] if len(sys.argv) == 2 else "src"
+    mapping = get_mapping(src_dir)
+    run_coverage(mapping, src_dir)

tox.ini

@@ -21,6 +21,12 @@ commands =
     python -m coverage report --show-missing --fail-under=100
 deps = coverage
 
+[testenv:maxi_cov]
+commands =
+    python tests/maxi_cov.py {envsitepackagesdir}
+    python -m coverage report --show-missing --fail-under=100
+deps = coverage
+
 [testenv:black]
 commands = black --check src tests
 deps = black