Files
qqq-frontend-next/scripts/acceptance-gate.mjs
T
james.maes d0818d1a61 feat(mobile): tablet project, 44px touch targets and a phone coverage gate (#708)
Adds an iPad (WebKit, touch) acceptance project that runs the @mobile and
@tablet tests, sizes controls to 44 x 44 CSS px on coarse pointers, and makes
the gate require a passing touch-project test for every required row unless
the row records why it is desktop-only.
2026-09-25 17:05:03 -05:00

106 lines
5.3 KiB
JavaScript

/*
* Copyright 2026 QRun.IO, Inc.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
*/
// Fails unless every required matrix row has at least one acceptance test and every test for it
// passed on the first attempt in every configured browser. Unknown IDs, skips and flakes fail.
// --partial checks only the rows exercised by a filtered run and never reports acceptance.
import { readdirSync, readFileSync, writeFileSync } from 'node:fs'
const partial = process.argv.includes('--partial')
const matrixFiles = readdirSync('tests/acceptance/matrix').filter((name) => name.endsWith('.json')).sort()
const matrix = { rows: matrixFiles.flatMap((name) => {
const area = JSON.parse(readFileSync(`tests/acceptance/matrix/${name}`, 'utf8'))
return area.rows.map((row) => {
if (!area.prefixes.some((prefix) => row.id.startsWith(prefix + '-'))) throw new Error(`${row.id} does not belong to ${name}`)
return { ...row, area: area.area }
})
}) }
const duplicates = matrix.rows.map((row) => row.id).filter((id, index, ids) => ids.indexOf(id) !== index)
if (duplicates.length) throw new Error(`Duplicate matrix IDs: ${duplicates.join(', ')}`)
const report = JSON.parse(readFileSync('test-results/acceptance/report.json', 'utf8'))
const rows = new Map(matrix.rows.map((row) => [row.id, row]))
const results = new Map()
const problems = []
const byProject = Object.fromEntries((report.config?.projects ?? []).map((project) => [project.name, { passed: 0, failed: 0, skipped: 0, flaky: 0 }]))
function walk(suite, titles = []) {
for (const child of suite.suites ?? []) walk(child, [...titles, child.title])
for (const spec of suite.specs ?? []) {
const ids = [...`${titles.join(' ')} ${spec.title}`.matchAll(/\[([A-Z]{3}-\d{3})\]/g)].map((match) => match[1])
if (ids.length === 0) problems.push(`Test without a matrix ID: ${spec.file} ${spec.title}`)
for (const test of spec.tests) {
const attempts = test.results ?? []
const outcome = test.status === 'skipped' ? 'skipped'
: attempts.length === 1 && attempts[0].status === 'passed' ? 'passed'
: attempts.some((attempt) => attempt.status === 'passed') ? 'flaky' : 'failed'
byProject[test.projectName] ??= { passed: 0, failed: 0, skipped: 0, flaky: 0 }
byProject[test.projectName][outcome]++
for (const id of ids) {
if (!rows.has(id)) problems.push(`Unknown matrix ID ${id} in ${spec.file}`)
const list = results.get(id) ?? []
list.push({ title: spec.title, project: test.projectName, outcome })
results.set(id, list)
}
}
}
}
for (const suite of report.suites ?? []) walk(suite)
const summary = { passed: 0, failed: 0, missing: 0, excluded: 0 }
for (const row of matrix.rows) {
if (row.required === false) {
summary.excluded++
if (!row.approval) problems.push(`${row.id} is excluded without a recorded approval`)
continue
}
const list = results.get(row.id)
if (!list) {
if (partial) continue
summary.missing++
problems.push(`${row.id} ${row.feature}: no acceptance test`)
continue
}
const bad = list.filter(({ outcome }) => outcome !== 'passed')
if (bad.length) {
summary.failed++
for (const { title, project, outcome } of bad) problems.push(`${row.id} ${outcome} [${project}] ${title}`)
} else summary.passed++
}
// Phone coverage (QRun-IO/qqq#708): when a touch project ran (mobile: phone, tablet: iPad), every
// required row passes there too, unless the row names why it is not a touch scenario (`desktopOnly`).
const touchProjects = ['mobile', 'tablet'].filter((project) => byProject[project])
const phone = { covered: 0, desktopOnly: 0, uncovered: 0 }
if (touchProjects.length) for (const row of matrix.rows) {
if (row.required === false) continue
if (row.desktopOnly !== undefined) {
if (typeof row.desktopOnly !== 'string' || !row.desktopOnly.trim()) problems.push(`${row.id} desktopOnly needs the reason it is not a touch scenario`)
phone.desktopOnly++
continue
}
const list = results.get(row.id) ?? []
if (partial && !list.length) continue
const missing = touchProjects.filter((project) => !list.some((result) => result.project === project && result.outcome === 'passed'))
if (missing.length) {
phone.uncovered++
problems.push(`${row.id} ${row.feature}: no passing @mobile test in ${missing.join(', ')}`)
} else phone.covered++
}
if (touchProjects.length) summary.phone = phone
// Every configured project must run something (the phone project runs only @mobile tests).
if (!partial) for (const [project, counts] of Object.entries(byProject)) {
if (Object.values(counts).every((count) => count === 0)) problems.push(`Project ${project} ran no tests`)
}
const status = { partial, generatedAt: new Date().toISOString(), summary, byProject, problems,
rows: matrix.rows.map((row) => ({ id: row.id, results: results.get(row.id) ?? [] })) }
writeFileSync('test-results/acceptance/gate.json', JSON.stringify(status, null, 2) + '\n')
console.log(`Acceptance gate${partial ? ' (partial run, not an acceptance result)' : ''}:`, JSON.stringify(summary), JSON.stringify(byProject))
for (const problem of problems) console.log(` FAIL ${problem}`)
process.exit(problems.length ? 1 : 0)