platform-ci/platform_ci/test_reports.py

"""Judge already-produced JUnit XML reports of a native test run.

This module only reads declared report files with the standard-library XML parser. It starts no process and never
reruns tests. Which reports are declared, and that each declared report exists and is nonempty, is the packaging
module's responsibility, so zero reports is a valid input here.
Like the legacy Maven gate, zero tests or all-skipped tests are accepted; only failures, errors and malformed
reports fail, and DTDs and entity declarations are never parsed.
"""

import logging
from collections.abc import Sequence
from dataclasses import dataclass, field
from pathlib import Path
from xml.etree import ElementTree
from xml.parsers import expat

from platform_ci.safe_xml import ForbiddenDeclarationError, parse_without_declarations
from platform_ci.workspace import GIT_METADATA_DIRECTORY_NAME, PLATFORM_CHECKOUT_DIRECTORY_NAME, PipelineError

LOG = logging.getLogger(__name__)


class TestReportError(PipelineError):
   """Raised when a declared report path is outside the workspace or is not a regular file."""

   __test__ = False


@dataclass(frozen=True)
class JUnitSummary:
   """Totals over the declared reports plus one message per problem; `passed` is true only without problems."""

   reports: int = 0
   tests: int = 0
   skipped: int = 0
   failures: int = 0
   errors: int = 0
   problems: tuple[str, ...] = field(default_factory=tuple)

   @property
   def executed(self) -> int:
      """Tests that were not skipped."""
      return self.tests - self.skipped

   @property
   def passed(self) -> bool:
      """Whether every report parsed and none recorded a failure or an error."""
      return not self.problems


def check_junit_reports(workspace_directory: Path, report_paths: Sequence[Path]) -> JUnitSummary:
   """Parse each declared JUnit report and total the results; report paths are relative to, or inside, the workspace.

   Every path must resolve, through symbolic links, to a regular file inside the workspace and outside Git metadata
   and the platform checkout; otherwise `TestReportError` is raised. A malformed report, a DOCTYPE or ENTITY
   declaration, a root other than `testsuite`/`testsuites`, a bad count, or skipped exceeding tests is a problem,
   as is any failed or errored count or any `testcase` with a `failure` or `error` child.
   """
   workspace_root = workspace_directory.resolve()
   totals = [0, 0, 0, 0]
   problems: list[str] = []
   resolved_reports = [_resolve_report(workspace_root, path) for path in report_paths]
   for relative_path, path in resolved_reports:
      try:
         tests, skipped, failures, errors = _parse_report(path)
      except (ForbiddenDeclarationError, expat.ExpatError, ValueError, OSError) as error:
         problems.append(f"{relative_path}: {_describe(error)}")
         continue
      for index, value in enumerate((tests, skipped, failures, errors)):
         totals[index] += value
      if skipped > tests:
         problems.append(f"{relative_path}: skipped={skipped} exceeds tests={tests}")
      if failures or errors:
         problems.append(f"{relative_path}: failures={failures} errors={errors}")
   summary = JUnitSummary(len(resolved_reports), totals[0], totals[1], totals[2], totals[3], tuple(problems))
   LOG.info("junit: %d reports, %d tests, %d skipped, %d executed", summary.reports, summary.tests, summary.skipped, summary.executed)
   return summary


def _resolve_report(workspace_root: Path, report_path: Path) -> tuple[str, Path]:
   """Return the workspace-relative name and resolved path of a report, proving it is a real file inside the workspace."""
   resolved = (workspace_root / report_path).resolve()
   if not resolved.is_relative_to(workspace_root):
      raise TestReportError(f"report {report_path} resolves outside the workspace")
   relative = resolved.relative_to(workspace_root)
   if GIT_METADATA_DIRECTORY_NAME in relative.parts or PLATFORM_CHECKOUT_DIRECTORY_NAME in relative.parts[:1]:
      raise TestReportError(f"report {report_path} resolves into Git metadata or the platform checkout")
   if not resolved.is_file():
      raise TestReportError(f"report {report_path} is not a regular file")
   return relative.as_posix(), resolved


def _parse_report(path: Path) -> tuple[int, int, int, int]:
   """Return the tests, skipped, failures and errors of one report, counting failing test cases as well as attributes."""
   root = parse_without_declarations(path.read_bytes())
   if root.tag == "testsuite":
      suites = [root]
   elif root.tag == "testsuites":
      suites = list(root.findall("testsuite"))
   else:
      raise ValueError("root element is not testsuite or testsuites")
   tests = sum(_count(suite, "tests") for suite in suites)
   skipped = sum(_count(suite, "skipped") for suite in suites)
   failures = sum(_count(suite, "failures") for suite in suites)
   errors = sum(_count(suite, "errors") for suite in suites)
   failing_cases = sum(1 for case in root.iter("testcase") if case.find("failure") is not None)
   erroring_cases = sum(1 for case in root.iter("testcase") if case.find("error") is not None)
   return tests, skipped, max(failures, failing_cases), max(errors, erroring_cases)


def _count(suite: ElementTree.Element, attribute: str) -> int:
   """Return a nonnegative integer attribute, with an absent attribute counting as zero."""
   text = suite.get(attribute)
   if text is None:
      return 0
   if not text.isascii() or not text.isdigit():
      raise ValueError(f"{attribute} is not a nonnegative integer")
   return int(text)


def _describe(error: Exception) -> str:
   """Return a short problem description that never echoes report content."""
   if isinstance(error, ForbiddenDeclarationError):
      return "DTD/entity declarations are not allowed"
   if isinstance(error, expat.ExpatError):
      return "malformed XML"
   if isinstance(error, OSError):
      return "cannot be read"
   return str(error)