ci: run the adapter tests and fuzz campaigns on every push #55
No files matched your search
@@ -6,7 +6,6 @@ on:
|
||||
jobs:
|
||||
windows:
|
||||
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/windows.yaml@v1
|
||||
secrets: inherit
|
||||
with:
|
||||
commands: |
|
||||
dotnet restore MarkdownViewr.Windows.slnx -p:Platform=x64 --locked-mode
|
||||
@@ -16,8 +15,22 @@ jobs:
|
||||
$env:Platform = 'x64'
|
||||
dotnet format MarkdownViewr.Windows.slnx --verify-no-changes --severity warn --no-restore
|
||||
Remove-Item Env:Platform
|
||||
# The host runner keeps its workspace between runs; the coverage gate must see this run's report only.
|
||||
if (Test-Path TestResults) { Remove-Item TestResults -Recurse -Force }
|
||||
dotnet test tests/MarkdownViewr.Windows.Core.Tests --no-build --settings coverage.runsettings --collect:"XPlat Code Coverage" --logger trx --results-directory TestResults
|
||||
pwsh -NoProfile -File ./build/Assert-Coverage.ps1 -ResultsDirectory TestResults -Packages MarkdownViewr.Windows.Core
|
||||
# Windows adapters (files, watcher, registry) against the real operating system. The Desktop
|
||||
# category drives the app with mouse and keyboard and needs an interactive session.
|
||||
dotnet test tests/MarkdownViewr.Windows.UiTests --no-build --filter "Category!=Desktop" --logger trx --results-directory TestResults
|
||||
# The runs above include the seeded parser campaigns at 500 iterations. Main and tags repeat
|
||||
# them at 10000; a filter that matches no test fails, so a renamed category cannot pass unseen.
|
||||
if ($env:GITHUB_REF -eq 'refs/heads/main' -or $env:GITHUB_REF -like 'refs/tags/*') {
|
||||
$env:K22_FUZZ_ITERATIONS = '10000'
|
||||
Write-Output "Parser campaigns at $env:K22_FUZZ_ITERATIONS iterations for $env:GITHUB_REF"
|
||||
dotnet test tests/MarkdownViewr.Windows.Core.Tests --no-build --filter "Category=Fuzz" -- RunConfiguration.TreatNoTestsAsError=true
|
||||
dotnet test tests/MarkdownViewr.Windows.UiTests --no-build --filter "Category=Fuzz" -- RunConfiguration.TreatNoTestsAsError=true
|
||||
Remove-Item Env:K22_FUZZ_ITERATIONS
|
||||
}
|
||||
# PowerShell build scripts: analyzer and Pester boundary tests with pinned modules saved to the job's temp directory.
|
||||
$modules = Join-Path ($env:RUNNER_TEMP ?? [IO.Path]::GetTempPath()) "psmodules-$env:GITHUB_RUN_ID"
|
||||
New-Item -ItemType Directory -Force -Path $modules | Out-Null
|
||||
|
||||
@@ -13,12 +13,13 @@ Use Windows and the .NET 10 SDK selected by `global.json`:
|
||||
```powershell
|
||||
dotnet restore MarkdownViewr.Windows.slnx -p:Platform=x64 --locked-mode
|
||||
dotnet build MarkdownViewr.Windows.slnx -p:Platform=x64 -warnaserror
|
||||
dotnet test tests/MarkdownViewr.Windows.Core.Tests --settings coverage.runsettings --collect:'XPlat Code Coverage' --results-directory TestResults
|
||||
./build/Assert-Coverage.ps1 -ResultsDirectory TestResults
|
||||
dotnet test tests/MarkdownViewr.Windows.Core.Tests --no-build --settings coverage.runsettings --collect:'XPlat Code Coverage' --results-directory TestResults # TestResults must start empty
|
||||
pwsh -NoProfile -File ./build/Assert-Coverage.ps1 -ResultsDirectory TestResults -Packages MarkdownViewr.Windows.Core
|
||||
dotnet test tests/MarkdownViewr.Windows.UiTests --no-build --filter "Category!=Desktop"
|
||||
dotnet format MarkdownViewr.Windows.slnx --verify-no-changes --severity warn --no-restore
|
||||
```
|
||||
|
||||
Desktop tests require an explicitly available interactive desktop. Run `dotnet test tests/MarkdownViewr.Windows.UiTests` only when input will not interfere with other work. Preserve 100% measured core coverage; use xUnit and deterministic injected time/filesystem boundaries. Follow `Method_Scenario_Expectation` test names.
|
||||
Desktop tests require an explicitly available interactive desktop. Run `dotnet test tests/MarkdownViewr.Windows.UiTests --no-build --filter "Category=Desktop"` only when input will not interfere with other work. Preserve 100% measured core coverage; use xUnit and deterministic injected time/filesystem boundaries. Follow `Method_Scenario_Expectation` test names.
|
||||
|
||||
## Style and Contributions
|
||||
|
||||
|
||||
@@ -9,12 +9,16 @@ its Windows desktop reference architecture.
|
||||
```powershell
|
||||
dotnet restore MarkdownViewr.Windows.slnx -p:Platform=x64 --locked-mode
|
||||
dotnet build MarkdownViewr.Windows.slnx -p:Platform=x64 -warnaserror
|
||||
dotnet test MarkdownViewr.Windows.slnx -p:Platform=x64 --no-build --filter "FullyQualifiedName!~UiTests" --settings coverage.runsettings --collect:"XPlat Code Coverage" --results-directory TestResults
|
||||
./build/Assert-Coverage.ps1 -ResultsDirectory TestResults
|
||||
dotnet test tests/MarkdownViewr.Windows.UiTests --no-build # launches the app on the desktop
|
||||
dotnet test tests/MarkdownViewr.Windows.Core.Tests --no-build --settings coverage.runsettings --collect:"XPlat Code Coverage" --results-directory TestResults # TestResults must start empty
|
||||
pwsh -NoProfile -File ./build/Assert-Coverage.ps1 -ResultsDirectory TestResults -Packages MarkdownViewr.Windows.Core
|
||||
dotnet test tests/MarkdownViewr.Windows.UiTests --no-build --filter "Category!=Desktop" # Windows adapters; no window, no input
|
||||
dotnet test tests/MarkdownViewr.Windows.UiTests --no-build --filter "Category=Desktop" # drives the app with mouse and keyboard
|
||||
dotnet format MarkdownViewr.Windows.slnx --verify-no-changes --severity warn --no-restore
|
||||
```
|
||||
|
||||
Build through the solution. Passing `-p:Platform=x64` to one test project selects a different
|
||||
output folder, so a following `--no-build` test run would use stale binaries.
|
||||
|
||||
## Rules
|
||||
|
||||
- GitHub Flow: topic branch `<type>/<desc>` → PR to `main` → squash merge. Conventional
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ This Product follows the [KofTwentyTwo standards](https://github.com/KofTwentyTw
|
||||
- Preserve original MIT notices and dependency credits.
|
||||
- Disclose substantial AI assistance and include a `Co-Authored-By:` trailer naming the model.
|
||||
|
||||
Required CI covers build/coverage/style, reader and installed-update acceptance, parser fuzzing, tooling tests, dependency/secret/workflow scans, CodeQL, PR title, and DCO.
|
||||
CI runs on every push and covers the build, formatting, core tests with the coverage gate, Windows adapter tests, parser fuzzing, and tooling tests. It also verifies npm registry signatures, and the NuGet restore reports known vulnerable packages. Desktop and installed-update acceptance, the secret and workflow scans, and CodeQL are not automated at present; [DEVELOPERS.md](DEVELOPERS.md#what-runs-where) lists what to run by hand. Local hooks check the commit title, the sign-off, and secrets.
|
||||
|
||||
Under the single-maintainer policy, `review / automated` also requires a genuine current-commit acknowledgement from the configured human maintainer. An agent cannot provide that acknowledgement. See [repository setup](docs/repository-setup.md).
|
||||
|
||||
|
||||
+21
-7
@@ -28,26 +28,40 @@ To isolate preview data, set `MARKDOWNVIEWR_WINDOWS_DATA_DIR` for that process t
|
||||
|
||||
## Tests and quality gates
|
||||
|
||||
After the solution build above:
|
||||
|
||||
```powershell
|
||||
dotnet test tests/MarkdownViewr.Windows.Core.Tests --settings coverage.runsettings --collect:"XPlat Code Coverage" --results-directory TestResults
|
||||
./build/Assert-Coverage.ps1 -ResultsDirectory TestResults
|
||||
if (Test-Path TestResults) { Remove-Item TestResults -Recurse -Force }
|
||||
dotnet test tests/MarkdownViewr.Windows.Core.Tests --no-build --settings coverage.runsettings --collect:"XPlat Code Coverage" --results-directory TestResults
|
||||
pwsh -NoProfile -File ./build/Assert-Coverage.ps1 -ResultsDirectory TestResults -Packages MarkdownViewr.Windows.Core
|
||||
dotnet test tests/MarkdownViewr.Windows.UiTests --no-build --filter "Category!=Desktop"
|
||||
dotnet format MarkdownViewr.Windows.slnx --verify-no-changes --severity warn --no-restore
|
||||
npm ci --ignore-scripts
|
||||
npm run lint
|
||||
npm test
|
||||
```
|
||||
|
||||
Core tests use xUnit and Coverlet; retain 100% measured core line coverage. Name tests `Method_Scenario_Expectation` and add meaningful regression coverage for fixes. Deterministic tests inject time and filesystem boundaries.
|
||||
Core tests use xUnit and Coverlet; retain 100% measured core line coverage. The coverage gate reads every report under the results directory, so start from an empty one, and run the gate as its own process: called in-process, a failure does not stop the calling script. Name tests `Method_Scenario_Expectation` and add meaningful regression coverage for fixes. Deterministic tests inject time and filesystem boundaries.
|
||||
|
||||
Native desktop tests use FlaUI/UIA3 and require an available interactive desktop:
|
||||
The `UiTests` assembly holds two kinds of test. Adapter tests exercise the Windows file, watcher, and registry code against the real operating system without opening a window. Tests in the `Desktop` category use FlaUI/UIA3 to drive the app with the mouse and keyboard and need an interactive desktop:
|
||||
|
||||
```powershell
|
||||
dotnet test tests/MarkdownViewr.Windows.UiTests
|
||||
dotnet test tests/MarkdownViewr.Windows.UiTests --no-build --filter "Category=Desktop"
|
||||
```
|
||||
|
||||
Reserve the desktop before running physical-input tests. Hosted CI also exercises published output and actual stable/dev installation, update/restart, registration, and coexistence. See [installed-update acceptance](docs/installed-updates.md). Synthetic unsigned fixtures do not establish production signing.
|
||||
Reserve the desktop before running them. Build through the solution: passing `-p:Platform=x64` to one test project selects a different output folder, and a following `--no-build` run would test stale binaries.
|
||||
|
||||
For PowerShell, use PSScriptAnalyzer 1.25.0 with `PSScriptAnalyzerSettings.psd1`, then Pester 5.7.1 against `build/*.Tests.ps1`. Reproduce the exact sequential analyzer partitions and negative fixtures in [scripts.yml](.github/workflows/scripts.yml); engine exceptions and rule findings are failures.
|
||||
For PowerShell, use PSScriptAnalyzer 1.25.0 with `PSScriptAnalyzerSettings.psd1`, then Pester 5.7.1 against `build/*.Tests.ps1`. Reproduce the exact sequential analyzer partitions in [build.yaml](.gitea/workflows/build.yaml); engine exceptions and rule findings are failures.
|
||||
|
||||
### What runs where
|
||||
|
||||
The repository is hosted on Gitea, with GitHub as a mirror. [build.yaml](.gitea/workflows/build.yaml) runs on every push to any branch: locked restore, zero-warning build, formatting, core tests with the coverage gate, the adapter tests, the seeded parser campaigns, script analysis with Pester, and the JavaScript lint and tests. A pull request shows the result for its head commit.
|
||||
|
||||
The workflows under `.github/` belong to the mirror, where GitHub Actions is currently switched off. Until it is switched on again, these checks have no automated run and are run by hand before a release:
|
||||
|
||||
- the `Desktop` tests, against a local build and against published output;
|
||||
- [installed-update acceptance](docs/installed-updates.md): actual stable and development installation, update and restart, registration, and coexistence. Its synthetic unsigned fixtures do not establish production signing;
|
||||
- secret and workflow scans, and CodeQL.
|
||||
|
||||
Run the [seeded parser campaigns](docs/security/fuzzing.md) when changing parser or activation boundaries. Never use private documents as a corpus.
|
||||
|
||||
|
||||
+135
-22
@@ -12,25 +12,42 @@ BeforeAll `
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Writes a synthetic report so tests can exercise release-gate exit codes.
|
||||
Writes a synthetic report for one package: one line per entry of -Hits, in one class.
|
||||
#>
|
||||
function Write-CoverageReport
|
||||
{
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $Directory,
|
||||
[Parameter(Mandatory)] [ValidateRange(0, 1)] [double] $Rate
|
||||
[string] $Package = 'Probe',
|
||||
[int[]] $Hits = @(1, 1),
|
||||
# What the report claims in its summary attribute; the gate must not believe it.
|
||||
[string] $StatedRate = '1'
|
||||
)
|
||||
|
||||
$rateText = $Rate.ToString([Globalization.CultureInfo]::InvariantCulture)
|
||||
$secondLineHits = if ($Rate -eq 1) { 1 } else { 0 }
|
||||
$number = 0
|
||||
$lines = foreach ($count in $Hits) { $number++; "<line number=`"$number`" hits=`"$count`" />" }
|
||||
$report = @"
|
||||
<coverage><packages><package name="Probe" line-rate="$rateText"><classes>
|
||||
<class filename="Probe.cs"><lines><line number="1" hits="1" /><line number="2" hits="$secondLineHits" /></lines></class>
|
||||
<coverage line-rate="$StatedRate"><packages><package name="$Package" line-rate="$StatedRate"><classes>
|
||||
<class filename="$Package.cs"><lines>$($lines -join '')</lines></class>
|
||||
</classes></package></packages></coverage>
|
||||
"@
|
||||
New-Item -ItemType Directory -Force -Path $Directory | Out-Null
|
||||
Set-Content -LiteralPath (Join-Path $Directory 'coverage.cobertura.xml') -Value $report
|
||||
}
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Runs the gate as CI does, in its own process, and returns its exit code and combined output.
|
||||
#>
|
||||
function Invoke-CoverageGate
|
||||
{
|
||||
[CmdletBinding()]
|
||||
param([Parameter(ValueFromRemainingArguments)] [string[]] $Arguments)
|
||||
|
||||
$output = (& pwsh -NoProfile -File $script:CoverageGate -ResultsDirectory $script:ReportsDirectory @Arguments 2>&1) -join "`n"
|
||||
[pscustomobject]@{ ExitCode = $LASTEXITCODE; Output = $output }
|
||||
}
|
||||
}
|
||||
|
||||
Describe 'Required coverage gate' `
|
||||
@@ -43,32 +60,128 @@ Describe 'Required coverage gate' `
|
||||
|
||||
It 'rejects a run that produced no report' `
|
||||
{
|
||||
$output = (& pwsh -NoProfile -File $script:CoverageGate -ResultsDirectory $script:ReportsDirectory -Packages Probe 2>&1) -join "`n"
|
||||
$LASTEXITCODE | Should -Be 1
|
||||
$output | Should -Match 'No coverage.cobertura.xml'
|
||||
$result = Invoke-CoverageGate -Packages Probe
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match 'No coverage.cobertura.xml'
|
||||
}
|
||||
|
||||
It 'rejects an absent required package' `
|
||||
{
|
||||
Write-CoverageReport -Directory $script:ReportsDirectory -Rate 1
|
||||
$output = (& pwsh -NoProfile -File $script:CoverageGate -ResultsDirectory $script:ReportsDirectory -Packages Missing 2>&1) -join "`n"
|
||||
$LASTEXITCODE | Should -Be 1
|
||||
$output | Should -Match 'Missing.*not found'
|
||||
Write-CoverageReport -Directory $script:ReportsDirectory
|
||||
$result = Invoke-CoverageGate -Packages Missing
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match "'Missing' not found"
|
||||
}
|
||||
|
||||
It 'rejects coverage below the required percentage' `
|
||||
It 'does not take a longer or differently cased package name for the required one' `
|
||||
{
|
||||
Write-CoverageReport -Directory $script:ReportsDirectory -Rate 0.5
|
||||
$output = (& pwsh -NoProfile -File $script:CoverageGate -ResultsDirectory $script:ReportsDirectory -Packages Probe 2>&1) -join "`n"
|
||||
$LASTEXITCODE | Should -Be 1
|
||||
$output | Should -Match 'Coverage gate FAILED'
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'longer') -Package 'ProbeExtra'
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'cased') -Package 'probe'
|
||||
$result = Invoke-CoverageGate -Packages Probe
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match "'Probe' not found"
|
||||
}
|
||||
|
||||
It 'rejects coverage below the required percentage and names the uncovered lines' `
|
||||
{
|
||||
Write-CoverageReport -Directory $script:ReportsDirectory -Hits 1, 0, 1, 0
|
||||
$result = Invoke-CoverageGate -Packages Probe
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match 'Probe line coverage: 50% \(2 of 4 lines'
|
||||
$result.Output | Should -Match 'uncovered: Probe\.cs lines 2, 4(\s|$)'
|
||||
$result.Output | Should -Match 'Coverage gate FAILED'
|
||||
}
|
||||
|
||||
It 'accepts full coverage of the required package' `
|
||||
{
|
||||
Write-CoverageReport -Directory $script:ReportsDirectory -Rate 1
|
||||
$output = (& pwsh -NoProfile -File $script:CoverageGate -ResultsDirectory $script:ReportsDirectory -Packages Probe 2>&1) -join "`n"
|
||||
$LASTEXITCODE | Should -Be 0
|
||||
$output | Should -Match 'Coverage gate passed'
|
||||
Write-CoverageReport -Directory $script:ReportsDirectory
|
||||
$result = Invoke-CoverageGate -Packages Probe
|
||||
$result.ExitCode | Should -Be 0
|
||||
$result.Output | Should -Match 'Probe line coverage: 100% \(2 of 2 lines'
|
||||
$result.Output | Should -Match 'Coverage gate passed'
|
||||
}
|
||||
|
||||
It 'counts the lines itself instead of believing the stated rate <StatedRate>' -ForEach @(
|
||||
@{ StatedRate = '1' }
|
||||
@{ StatedRate = '1.5' }
|
||||
@{ StatedRate = 'NaN' }
|
||||
@{ StatedRate = '0,5' }
|
||||
) `
|
||||
{
|
||||
Write-CoverageReport -Directory $script:ReportsDirectory -Hits 1, 0 -StatedRate $StatedRate
|
||||
$result = Invoke-CoverageGate -Packages Probe
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match 'Probe line coverage: 50% \(1 of 2 lines'
|
||||
}
|
||||
|
||||
It 'never shows a package just under the bar as having reached it' `
|
||||
{
|
||||
Write-CoverageReport -Directory $script:ReportsDirectory -Hits (@(1) * 19999 + 0)
|
||||
$result = Invoke-CoverageGate -Packages Probe
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match 'Probe line coverage: 99\.99% \(19999 of 20000 lines'
|
||||
}
|
||||
|
||||
It 'rejects a package no report measured a line of' `
|
||||
{
|
||||
Write-CoverageReport -Directory $script:ReportsDirectory -Hits @()
|
||||
$result = Invoke-CoverageGate -Packages Probe
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match 'no measured lines'
|
||||
}
|
||||
|
||||
It 'ignores an empty report when another test project measured the package' `
|
||||
{
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'first') -Hits @()
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'second') -Hits 1, 0
|
||||
$result = Invoke-CoverageGate -Packages Probe
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match 'Probe line coverage: 50% \(1 of 2 lines'
|
||||
}
|
||||
|
||||
It 'takes the test project that covers the package best' `
|
||||
{
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'first') -Hits 1, 0
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'second') -Hits 1, 1
|
||||
$result = Invoke-CoverageGate
|
||||
$result.ExitCode | Should -Be 0
|
||||
$result.Output | Should -Match 'Probe line coverage: 100% \(2 of 2 lines.*second'
|
||||
}
|
||||
|
||||
It 'lists uncovered lines only from the report it used' `
|
||||
{
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'first') -Hits 0, 0, 0, 1
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'second') -Hits 1, 1, 0, 1
|
||||
$result = Invoke-CoverageGate -Packages Probe
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match 'Probe line coverage: 75% \(3 of 4 lines.*second'
|
||||
$result.Output | Should -Match 'uncovered: Probe\.cs lines 3(\s|$)'
|
||||
$result.Output | Should -Not -Match 'lines 1, 2, 3'
|
||||
}
|
||||
|
||||
It 'gates every package named in one comma-separated argument' `
|
||||
{
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'first') -Package 'First'
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'second') -Package 'Second' -Hits 1, 0
|
||||
$result = Invoke-CoverageGate -Packages 'First,Second'
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match 'First line coverage: 100%'
|
||||
$result.Output | Should -Match 'Coverage gate FAILED: Second'
|
||||
}
|
||||
|
||||
It 'gates every measured package when none is named' `
|
||||
{
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'first') -Package 'First'
|
||||
Write-CoverageReport -Directory (Join-Path $script:ReportsDirectory 'second') -Package 'Second' -Hits 1, 0
|
||||
$result = Invoke-CoverageGate
|
||||
$result.ExitCode | Should -Be 1
|
||||
$result.Output | Should -Match 'Coverage gate FAILED: Second'
|
||||
}
|
||||
|
||||
It 'holds the package to a lower bar when one is given' `
|
||||
{
|
||||
Write-CoverageReport -Directory $script:ReportsDirectory -Hits 1, 0
|
||||
(Invoke-CoverageGate -MinimumPercent 50).ExitCode | Should -Be 0
|
||||
(Invoke-CoverageGate -MinimumPercent 51).ExitCode | Should -Be 1
|
||||
}
|
||||
}
|
||||
+62
-26
@@ -1,21 +1,32 @@
|
||||
#Requires -Version 7.4
|
||||
# Copyright (c) 2026 James Maes
|
||||
# SPDX-License-Identifier: MIT
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Fails when any named assembly is below the required line coverage.
|
||||
|
||||
.DESCRIPTION
|
||||
Reads every coverage.cobertura.xml under -ResultsDirectory (one per test project
|
||||
run) and, for each package, takes the best line rate any report measured. A package
|
||||
can be split across test projects, and the report that exercises it most is
|
||||
authoritative. Without -Packages, every package the reports measured is gated:
|
||||
coverage.runsettings already limits measurement to the assemblies held to the bar.
|
||||
Exits 1 on any shortfall, or when a package named in -Packages is missing.
|
||||
run) and, for each package, takes the report that covers it best. A package can be
|
||||
split across test projects, and the report that exercises it most is authoritative.
|
||||
Without -Packages, every package the reports measured is gated: coverage.runsettings
|
||||
already limits measurement to the assemblies held to the bar.
|
||||
|
||||
The rate is counted from the lines each report lists, not read from its summary
|
||||
attributes, so a report cannot state a rate its lines do not support.
|
||||
|
||||
Exits 1 on any shortfall, when a package named in -Packages is missing, or when no
|
||||
report measured a single line of it.
|
||||
|
||||
Every report under the directory counts, including one left by an earlier run, so
|
||||
collect into an empty directory. Run the script as its own process: called
|
||||
in-process, its exit code does not stop the caller.
|
||||
|
||||
.EXAMPLE
|
||||
./build/Assert-Coverage.ps1 -ResultsDirectory TestResults
|
||||
pwsh -NoProfile -File ./build/Assert-Coverage.ps1 -ResultsDirectory TestResults
|
||||
|
||||
.EXAMPLE
|
||||
./build/Assert-Coverage.ps1 -ResultsDirectory TestResults -Packages MarkdownViewr.Windows.Core
|
||||
pwsh -NoProfile -File ./build/Assert-Coverage.ps1 -ResultsDirectory TestResults -Packages MarkdownViewr.Windows.Core
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
@@ -37,45 +48,70 @@ if ($files.Count -eq 0)
|
||||
exit 1
|
||||
}
|
||||
|
||||
$reports = foreach ($file in $files) { [xml](Get-Content -LiteralPath $file.FullName -Raw) }
|
||||
# One entry per package per report: where it was measured, how many lines, and how many of them ran.
|
||||
$measured = foreach ($file in $files)
|
||||
{
|
||||
$document = [xml](Get-Content -LiteralPath $file.FullName -Raw)
|
||||
foreach ($package in $document.SelectNodes('/coverage/packages/package'))
|
||||
{
|
||||
$lines = $package.SelectNodes('classes/class/lines/line')
|
||||
$covered = 0
|
||||
foreach ($line in $lines)
|
||||
{
|
||||
if ([int]::Parse($line.GetAttribute('hits'), [Globalization.CultureInfo]::InvariantCulture) -gt 0) { $covered++ }
|
||||
}
|
||||
[pscustomobject]@{
|
||||
Name = $package.GetAttribute('name')
|
||||
Report = $file.FullName
|
||||
Lines = $lines.Count
|
||||
Covered = $covered
|
||||
Package = $package
|
||||
}
|
||||
}
|
||||
}
|
||||
$measured = @($measured)
|
||||
if ($Packages.Count -eq 0)
|
||||
{
|
||||
$Packages = @($reports | ForEach-Object { @($_.coverage.packages.package) } | ForEach-Object { $_.name } | Where-Object { $_ } | Sort-Object -Unique)
|
||||
$Packages = @($measured | ForEach-Object { $_.Name } | Where-Object { $_ } | Sort-Object -Unique)
|
||||
if ($Packages.Count -eq 0)
|
||||
{
|
||||
Write-Error 'The coverage reports measured no packages; check the Include filter in coverage.runsettings.' -ErrorAction Continue
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
$required = $MinimumPercent / 100
|
||||
$failed = $false
|
||||
|
||||
foreach ($name in $Packages)
|
||||
{
|
||||
$rates = foreach ($doc in $reports)
|
||||
{
|
||||
$package = @($doc.coverage.packages.package) | Where-Object { $_.name -eq $name }
|
||||
if ($package) { [double]::Parse($package.'line-rate', [Globalization.CultureInfo]::InvariantCulture) }
|
||||
}
|
||||
if (-not $rates)
|
||||
$found = @($measured | Where-Object { $_.Name -ceq $name })
|
||||
if ($found.Count -eq 0)
|
||||
{
|
||||
Write-Error "Package '$name' not found in any coverage report." -ErrorAction Continue
|
||||
$failed = $true
|
||||
continue
|
||||
}
|
||||
|
||||
$best = ($rates | Measure-Object -Maximum).Maximum
|
||||
Write-Output ("{0} line coverage: {1}% (required {2}%)" -f $name, [math]::Round($best * 100, 2), $MinimumPercent)
|
||||
if ($best -lt $required)
|
||||
# A package with no measured lines proves nothing.
|
||||
$counted = @($found | Where-Object { $_.Lines -gt 0 })
|
||||
if ($counted.Count -eq 0)
|
||||
{
|
||||
foreach ($doc in $reports)
|
||||
Write-Error "Package '$name' has no measured lines in any coverage report." -ErrorAction Continue
|
||||
$failed = $true
|
||||
continue
|
||||
}
|
||||
|
||||
# Best covered share first; between equal shares, the report that measured more lines.
|
||||
$best = $counted | Sort-Object -Property @{ Expression = { $_.Covered / $_.Lines }; Descending = $true }, @{ Expression = { $_.Lines }; Descending = $true } | Select-Object -First 1
|
||||
$percent = 100 * $best.Covered / $best.Lines
|
||||
# Rounded down, so a package just under the bar is never shown as having reached it.
|
||||
$shown = [math]::Floor($percent * 100) / 100
|
||||
Write-Output ("{0} line coverage: {1}% ({2} of {3} lines, required {4}%), from {5}" -f $name, $shown, $best.Covered, $best.Lines, $MinimumPercent, $best.Report)
|
||||
if ($percent -lt $MinimumPercent)
|
||||
{
|
||||
foreach ($class in $best.Package.SelectNodes('classes/class'))
|
||||
{
|
||||
$package = @($doc.coverage.packages.package) | Where-Object { $_.name -eq $name }
|
||||
foreach ($class in @($package.classes.class))
|
||||
{
|
||||
$missed = @($class.lines.line) | Where-Object { $_.hits -eq '0' } | ForEach-Object { $_.number }
|
||||
if ($missed) { Write-Output " uncovered: $($class.filename) lines $($missed -join ', ')" }
|
||||
}
|
||||
$missed = @($class.SelectNodes('lines/line[@hits="0"]') | ForEach-Object { $_.GetAttribute('number') })
|
||||
if ($missed.Count -gt 0) { Write-Output " uncovered: $($class.GetAttribute('filename')) lines $($missed -join ', ')" }
|
||||
}
|
||||
Write-Error "Coverage gate FAILED: $name is below $MinimumPercent%." -ErrorAction Continue
|
||||
$failed = $true
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Coverage contract for the 100% gate (K22-TEST-10): every hand-written line of the
|
||||
UI-free core and the command line counts. There is deliberately no Include filter:
|
||||
coverlet measures the projects the unit tests reference whose sources are in this
|
||||
repository (Core and the CLI), never test assemblies or NuGet packages, and an Include
|
||||
filter would silently drop the CLI, whose assembly name (the app id) contains dashes.
|
||||
UI-free core counts. There is deliberately no Include filter: coverlet measures the
|
||||
projects the unit tests reference whose sources are in this repository, never test
|
||||
assemblies or NuGet packages, and the gate names the package it requires.
|
||||
Excluded, with intent:
|
||||
- GeneratedCodeAttribute: source-generated JSON contexts and MVVM Toolkit members.
|
||||
- ExcludeFromCodeCoverageAttribute: thin adapters a test cannot reach, each with a
|
||||
|
||||
@@ -5,6 +5,13 @@ Releases follow the
|
||||
Pushing a protected `v*` tag on `main` runs [`.github/workflows/release.yml`](../.github/workflows/release.yml),
|
||||
which calls the shared `release-velopack.yml` workflow in KofTwentyTwo/standards.
|
||||
|
||||
## Current state
|
||||
|
||||
The repository moved to Gitea, and GitHub is now a mirror with GitHub Actions switched
|
||||
off. The release workflow this document describes exists only on GitHub, so until a
|
||||
signing release job exists on Gitea, pushing a `v*` tag builds and tests the tag on
|
||||
Gitea and publishes nothing. Do not tag expecting a release.
|
||||
|
||||
## Workflow baseline
|
||||
|
||||
The signing release builder is pinned to standards v1.1.1 (`96cbdf208a15c9c524676bd15ff9e323228a952b`); other shared gates retain v0.1.2. The release caller checks main ancestry and reruns build, coverage, formatting, UI tests, security scans and CodeQL before publication.
|
||||
|
||||
@@ -14,10 +14,8 @@ dotnet tool install vpk --version 1.2.161 --tool-path artifacts/smoke-tools
|
||||
|
||||
The harness publishes the actual starter app four times with locked dependencies,
|
||||
self-contained .NET and Windows App SDK settings matching the release builder.
|
||||
Core and CLI declare the Windows runtime identifiers so their committed locks
|
||||
support runtime-specific publishing. Their single-file analyzer setting also locks
|
||||
the SDK's ILLink tool dependency used when publishing the CLI as one executable.
|
||||
The harness publishes and runs each CLI version when that head exists.
|
||||
Core declares the Windows runtime identifiers so its committed lock supports
|
||||
runtime-specific publishing.
|
||||
Keep `VelopackApp.Build().Run()` directly in
|
||||
the app's entry point: the packer's startup check cannot recognize an indirect
|
||||
AppKit wrapper call. Do not disable that pack-time check.
|
||||
@@ -79,8 +77,8 @@ checks do not establish the corruption's cause.
|
||||
|
||||
## Evidence and cleanup
|
||||
|
||||
CI requires `installed-updates` and retains JSON results, process/installer logs
|
||||
and TRX UI reports. Local evidence stays under a fresh `artifacts/installed-smoke-*`
|
||||
The harness has no automated run at present; run it by hand before a release. It
|
||||
writes JSON results, process/installer logs and TRX UI reports. Local evidence stays under a fresh `artifacts/installed-smoke-*`
|
||||
directory. Existing output directories are refused so each attempt tests fresh
|
||||
packages and results. Packer children receive a private `VELOPACK_TEMP` directory
|
||||
named for the unique test package ID, isolating staging from other app builds.
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
"tests/MarkdownViewr.Windows.UiTests/ActivationFuzzTests.cs"
|
||||
],
|
||||
"command": "dotnet test tests/MarkdownViewr.Windows.Core.Tests -c Release -p:Platform=x64 --no-build --no-restore --filter Category=Fuzz",
|
||||
"workflow": ".github/workflows/fuzz.yml",
|
||||
"workflow": ".gitea/workflows/build.yaml",
|
||||
"evidence": "docs/security/fuzzing.md"
|
||||
}
|
||||
}
|
||||
@@ -35,13 +35,12 @@ dotnet test tests/MarkdownViewr.Windows.UiTests -c Release -p:Platform=x64 --fil
|
||||
$env:K22_FUZZ_ITERATIONS = '10000'
|
||||
```
|
||||
|
||||
Each target uses seeds 22023 and 99173. PR/main campaigns run 500 iterations per
|
||||
seed/target (5000 generated property evaluations total); weekly and every release
|
||||
reuse run 10000 (100000 total). `fuzz.yml` runs on PRs, `main`, weekly and manual
|
||||
requests; release.yml calls it and requires success before publication. The
|
||||
`fuzzing` check is declared in the repository protection plan; apply and verify
|
||||
that live protection after its first hosted run. The job has a 15-minute bound;
|
||||
iteration configuration outside 1–100000 fails explicitly.
|
||||
Each target uses seeds 22023 and 99173. Every push runs 500 iterations per
|
||||
seed/target (5000 generated property evaluations total) as part of the ordinary
|
||||
test runs in `.gitea/workflows/build.yaml`. Pushes to `main` and to a tag repeat
|
||||
the campaigns at 10000 (100000 total), and that step fails if its filter matches
|
||||
no test. There is no scheduled campaign. Iteration configuration outside 1–100000
|
||||
fails explicitly.
|
||||
|
||||
Failures retain seed, target, zero-based iteration and JSON-escaped synthetic
|
||||
input in TRX/CI logs. To replay the sequence, use the same checkout/runtime/seed
|
||||
|
||||
@@ -6,8 +6,6 @@
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<RuntimeIdentifiers>win-x86;win-x64;win-arm64</RuntimeIdentifiers>
|
||||
<!-- Include the SDK tool dependency in the lock used by single-file CLI publishing. -->
|
||||
<EnableSingleFileAnalyzer>true</EnableSingleFileAnalyzer>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -30,12 +30,6 @@
|
||||
"resolved": "3.0.296",
|
||||
"contentHash": "1Yp/6OK/Ev66hE2YLe9UvIgtJrKOX/xvoGDe0UbdtuG2ZxUoDoyONdEn+HWCDuy02Mu8SmglHEd/9SGPrM1qYg=="
|
||||
},
|
||||
"Microsoft.NET.ILLink.Tasks": {
|
||||
"type": "Direct",
|
||||
"requested": "[10.0.14, )",
|
||||
"resolved": "10.0.14",
|
||||
"contentHash": "YlTamXppEQls6DYWt7RdUuwWDb0lyw/vXCaGeQpLCpsLmfwGuCkOz/vIGgMTV7ECsJcavkCOkxZXv9s8rdAHkQ=="
|
||||
},
|
||||
"AngleSharp": {
|
||||
"type": "Transitive",
|
||||
"resolved": "1.7.2",
|
||||
|
||||
@@ -20,6 +20,7 @@ namespace MarkdownViewr.Windows.UiTests;
|
||||
/// UI tests share one desktop, so they never run in parallel.
|
||||
/// </summary>
|
||||
[Collection("Desktop")]
|
||||
[Trait("Category", "Desktop")]
|
||||
public partial class AppTests
|
||||
{
|
||||
private readonly ITestOutputHelper _output;
|
||||
|
||||
Reference in new issue
Block a user