platform-ci/tests/test_test_reports.py

"""JUnit report judgment: legacy Maven allow-empty behavior, strict XML handling, workspace containment."""

import tempfile
import unittest
from pathlib import Path
from typing import override

from platform_ci.test_reports import JUnitSummary, TestReportError, check_junit_reports

PASSING = '<testsuite name="a" tests="3" skipped="1" failures="0" errors="0"><testcase name="x"/><testcase name="y"><skipped/></testcase></testsuite>'


class JUnitReportTest(unittest.TestCase):
   """Reports are parsed as data; only failures, errors and malformed reports fail."""

   @override
   def setUp(self) -> None:
      """Create an empty workspace."""
      directory = tempfile.TemporaryDirectory()
      self.addCleanup(directory.cleanup)
      self.workspace = Path(directory.name).resolve()

   def _report(self, content: str | bytes, name: str = "target/surefire-reports/TEST-a.xml") -> Path:
      path = self.workspace / name
      path.parent.mkdir(parents=True, exist_ok=True)
      if isinstance(content, str):
         content = content.encode("utf-8")
      path.write_bytes(content)
      return Path(name)

   def _check(self, *contents: str | bytes) -> JUnitSummary:
      return check_junit_reports(self.workspace, [self._report(content, f"r/TEST-{index}.xml") for index, content in enumerate(contents)])

   def test_passing_report_is_summarized(self) -> None:
      """Counts are totaled and skipped tests are not executed ones."""
      summary = self._check(PASSING, '<testsuite tests="2" failures="0" errors="0"/>')
      self.assertTrue(summary.passed)
      self.assertEqual((summary.reports, summary.tests, summary.skipped, summary.executed, summary.failures, summary.errors), (2, 5, 1, 4, 0, 0))

   def test_zero_reports_zero_tests_and_all_skipped_are_allowed(self) -> None:
      """The old Maven --allow-empty behavior: no new zero-report or all-skipped gate."""
      self.assertTrue(check_junit_reports(self.workspace, []).passed)
      self.assertTrue(self._check('<testsuite tests="0"/>').passed)
      all_skipped = self._check('<testsuite tests="2" skipped="2"><testcase name="a"><skipped/></testcase></testsuite>')
      self.assertTrue(all_skipped.passed)
      self.assertEqual(all_skipped.executed, 0)

   def test_failed_or_errored_counts_fail(self) -> None:
      """A nonzero failures or errors attribute fails, and every report is still totaled."""
      for attributes, expected in (('failures="1"', (1, 0)), ('errors="2"', (0, 2))):
         with self.subTest(attributes):
            summary = self._check(f'<testsuite tests="3" {attributes}/>')
            self.assertFalse(summary.passed)
            self.assertEqual((summary.failures, summary.errors), expected)

   def test_failing_or_erroring_testcases_fail_even_when_counts_claim_success(self) -> None:
      """Child elements are authoritative over a summary that says zero."""
      for child, expected in (("failure", (1, 0)), ("error", (0, 1))):
         with self.subTest(child):
            summary = self._check(f'<testsuite tests="1" failures="0" errors="0"><testcase name="t"><{child} message="m"/></testcase></testsuite>')
            self.assertFalse(summary.passed)
            self.assertEqual((summary.failures, summary.errors), expected)

   def test_testsuites_root_sums_its_suites(self) -> None:
      """An aggregate root counts each direct suite once."""
      summary = self._check('<testsuites><testsuite tests="2" skipped="1"/><testsuite tests="3" failures="1"/></testsuites>')
      self.assertEqual((summary.tests, summary.skipped, summary.failures), (5, 1, 1))
      self.assertFalse(summary.passed)

   def test_one_bad_report_fails_but_the_others_are_still_counted(self) -> None:
      """Problems name the workspace-relative report and never include report content."""
      summary = self._check(PASSING, '<testsuite tests="1" failures="1"/>')
      self.assertEqual(summary.tests, 4)
      self.assertEqual(len(summary.problems), 1)
      self.assertTrue(summary.problems[0].startswith("r/TEST-1.xml:"))

   def test_malformed_reports_fail(self) -> None:
      """Broken XML, wrong roots, empty files, bad counts and skipped above tests are all problems."""
      cases = {
         "truncated": "<testsuite tests='1'>",
         "empty": "",
         "not xml": "tests passed",
         "wrong root": '<report tests="1"/>',
         "negative count": '<testsuite tests="-1"/>',
         "word count": '<testsuite tests="many"/>',
         "skipped above tests": '<testsuite tests="1" skipped="2"/>',
         "invalid bytes": b"\xff\xfe\x00<testsuite",
      }
      for description, content in cases.items():
         with self.subTest(description):
            summary = self._check(content)
            self.assertFalse(summary.passed)
            self.assertEqual(len(summary.problems), 1)

   def test_doctype_and_entities_are_rejected_without_expansion(self) -> None:
      """DTD and entity declarations fail, including an entity bomb, and nothing is expanded or fetched."""
      bomb = '<!DOCTYPE testsuite [<!ENTITY a "aaaaaaaaaa"><!ENTITY b "&a;&a;&a;&a;&a;&a;&a;&a;">]><testsuite tests="0" name="&b;"/>'
      external = '<!DOCTYPE testsuite SYSTEM "http://127.0.0.1:9/evil.dtd"><testsuite tests="0"/>'
      entity_only = '<!DOCTYPE testsuite [<!ENTITY x SYSTEM "file:///etc/passwd">]><testsuite tests="0"/>'
      internal = '<!DOCTYPE testsuite [<!ENTITY x "value">]><testsuite tests="0" name="&x;"/>'
      cases: dict[str, str | bytes] = {
         "bomb": bomb,
         "external dtd": external,
         "external entity": entity_only,
         "internal entity": internal,
         "utf-16 bomb": bomb.encode("utf-16"),
         "utf-16 external entity": entity_only.encode("utf-16"),
         "utf-16 internal entity": internal.encode("utf-16"),
         "utf-8 with declaration": b'<?xml version="1.0" encoding="UTF-8"?>' + external.encode("utf-8"),
      }
      for description, content in cases.items():
         with self.subTest(description):
            summary = self._check(content)
            self.assertFalse(summary.passed)
            self.assertIn("not allowed", summary.problems[0])

   def test_utf16_report_without_declarations_is_parsed_once(self) -> None:
      """A BOM-marked UTF-16 report parses normally, and nested text and cases are counted once."""
      summary = self._check(PASSING.encode("utf-16"))
      self.assertTrue(summary.passed)
      self.assertEqual((summary.tests, summary.skipped), (3, 1))

   def test_report_outside_the_workspace_is_rejected_including_through_a_symlink(self) -> None:
      """An absolute path, parent traversal and a link to an outside file all raise."""
      with tempfile.TemporaryDirectory() as outside_name:
         outside = Path(outside_name) / "TEST-out.xml"
         outside.write_text(PASSING, encoding="utf-8")
         with self.assertRaises(TestReportError):
            check_junit_reports(self.workspace, [outside])
         link = self.workspace / "link.xml"
         try:
            link.symlink_to(outside)
         except OSError:
            self.skipTest("symbolic links are not available")
         with self.assertRaises(TestReportError):
            check_junit_reports(self.workspace, [Path("link.xml")])
      with self.assertRaises(TestReportError):
         check_junit_reports(self.workspace, [Path("../elsewhere.xml")])

   def test_git_metadata_platform_checkout_missing_files_and_directories_are_rejected(self) -> None:
      """Only real report files in the checkout are read; a missing declared report is packaging's concern, not a pass here."""
      git_report = self._report(PASSING, ".git/TEST-a.xml")
      platform_report = self._report(PASSING, ".platform-ci/TEST-a.xml")
      (self.workspace / "dir").mkdir()
      for path in (git_report, platform_report, Path("missing.xml"), Path("dir")):
         with self.subTest(str(path)), self.assertRaises(TestReportError):
            check_junit_reports(self.workspace, [path])

   def test_parser_runs_no_tools(self) -> None:
      """The module imports no process-starting facility, so tests can never be rerun from here."""
      source = (Path(__file__).resolve().parent.parent / "platform_ci" / "test_reports.py").read_text(encoding="utf-8")
      for forbidden in ("subprocess", "os.system", "socket", "urllib"):
         self.assertNotIn(f"import {forbidden}", source)