detect: confine header detection to avoid deleting adjacent comments (#30) #41

Merged
KofTwentyTwo merged 1 commits from feature/GH-30-detect-overmatch into develop 2026-06-09 15:17:56 +00:00
3 changed files with 205 additions and 56 deletions

No files matched your search

+77
View File
@@ -76,6 +76,22 @@ func phpFileType() model.FileType {
}
}
// pyFileType returns a "# " line-comment Python file type. It is used to exercise the
// issue #30 regression: a line-comment header followed by a blank line then a foreign
// doc comment must not absorb (and therefore must not delete) that doc comment.
func pyFileType() model.FileType {
return model.FileType{
Name: "Python",
Extensions: []string{".py"},
CommentStyle: model.CommentStyle{Block: false, LinePrefix: "# "},
PreserveFirst: []model.PreserveRule{
{Kind: model.PreserveBOM, Before: false},
{Kind: model.PreserveShebang, Before: false},
{Kind: model.PreserveCodingPragma, Before: false},
},
}
}
// agplConfig is the canonical apply config: AGPL, reuse+notice, an explicit year so
// the rendered header is deterministic and independent of the wall clock.
func agplConfig() model.Config {
@@ -413,6 +429,67 @@ func TestApplyFile(t *testing.T) {
})
}
// lookupLicense fetches a curated license by id or fails the test. Local to these
// tests to avoid depending on a sibling package's test helper.
func lookupLicense(t *testing.T, id string) model.License {
t.Helper()
lic, ok := spdx.Lookup(id)
require.True(t, ok, "license %q must be in the curated set", id)
return lic
}
// TestApplyFilePreservesDocCommentBelowBlankLine is the issue #30 end-to-end safety
// guard. A line-comment file carrying a managed header, a blank line, then an unrelated
// file doc comment must keep that doc comment intact across a relicense: the detected
// span covers only the header, so the replace swaps only the header bytes.
func TestApplyFilePreservesDocCommentBelowBlankLine(t *testing.T) {
ft := pyFileType()
// Step 1: insert an MIT header into a file whose body opens with a doc comment.
mitFunc := func(f model.FileType) (string, error) {
return render.Header(render.HeaderInput{
License: lookupLicense(t, "MIT"),
Holder: "Acme Corp",
Year: "2024",
Style: model.StyleReuse,
FileType: f,
})
}
body := "# This module wires the dashboard widgets together.\n" +
"# It is intentionally documented at the top of the file.\n" +
"\n" +
"def main():\n pass\n"
withHeader, _, action, err := ApplyFile([]byte(body), ft, mitFunc)
require.NoError(t, err)
require.Equal(t, "insert", action)
require.Contains(t, string(withHeader), "SPDX-License-Identifier: MIT")
require.Contains(t, string(withHeader), "wires the dashboard widgets",
"the foreign doc comment must be present after insert")
// Step 2: relicense to Apache-2.0. This replaces the detected header span.
apacheFunc := func(f model.FileType) (string, error) {
return render.Header(render.HeaderInput{
License: lookupLicense(t, "Apache-2.0"),
Holder: "Acme Corp",
Year: "2026",
Style: model.StyleReuse,
FileType: f,
})
}
relicensed, _, action2, err := ApplyFile(withHeader, ft, apacheFunc)
require.NoError(t, err)
require.Equal(t, "replace", action2)
got := string(relicensed)
assert.Contains(t, got, "SPDX-License-Identifier: Apache-2.0", "new license written")
assert.NotContains(t, got, "SPDX-License-Identifier: MIT", "old license id removed")
// The safety-critical assertion: the foreign doc comment survives the relicense.
assert.Contains(t, got, "# This module wires the dashboard widgets together.",
"the doc comment below the blank line must survive a relicense")
assert.Contains(t, got, "# It is intentionally documented at the top of the file.")
assert.Contains(t, got, "def main():")
}
func TestApplyFileIdempotentAfterPreserveFirstConstructs(t *testing.T) {
cases := []struct {
name string
+53 -46
View File
@@ -89,11 +89,12 @@ var curatedFingerprints = []struct {
// reports presence, the byte span, the matched SPDX id / holder / year if any, and
// whether the match came via the sentinel.
//
// The span (StartByte inclusive, EndByte exclusive) covers exactly the leading
// comment block(s) recognized as a license header, including any blank lines between
// adjacent comment blocks but excluding the preserve-first prefixes and the file
// content that follows. When no header is found, Present is false and the offsets
// are zero.
// The span (StartByte inclusive, EndByte exclusive) covers exactly the contiguous
// leading comment recognized as a license header, plus its trailing blank-line
// separator, but excluding the preserve-first prefixes and the file content that
// follows. A blank line inside the leading region ends it, so a foreign doc comment
// below a blank separator is never absorbed. When no header is found, Present is false
// and the offsets are zero.
func Detect(content []byte, ft model.FileType) (model.DetectedHeader, error) {
// Uncommentable types (JSON and friends) can never carry a header; report absence
// rather than risk matching incidental text.
@@ -200,8 +201,8 @@ func FingerprintLicense(headerText string) (string, bool) {
// leadingCommentRegion finds the contiguous comment block(s) at offset start and
// returns their byte span plus the extracted comment text (delimiters stripped,
// joined by newlines). For block-comment types it spans the first block comment; for
// line-comment types it spans the run of consecutive comment lines (blank lines
// between comment lines are included so a header split by a blank line is one region).
// line-comment types it spans the run of CONTIGUOUS comment lines (a truly-blank line
// ends the run -- see lineCommentRegion for the issue #30 safety rationale).
//
// The bool is false when there is no comment at start, in which case no header is
// possible. Leading blank lines before the comment are tolerated and included in the
@@ -283,10 +284,22 @@ func blockCommentRegion(content []byte, start int, cs model.CommentStyle) (int,
return openOffset, end, stripBlockInner(inner), true
}
// lineCommentRegion spans the run of consecutive line comments (each beginning with
// the LinePrefix, ignoring leading whitespace) starting at start, tolerating blank
// lines between comment lines, and returns the joined comment text with prefixes
// stripped. The run ends at the first non-blank, non-comment line.
// lineCommentRegion spans the run of CONTIGUOUS line comments (each beginning with
// the LinePrefix, ignoring leading whitespace) starting at start, and returns the
// joined comment text with prefixes stripped. The run ends at the first line that is
// neither a comment line nor empty -- and, critically, a truly-blank line ALSO ends
// the run.
//
// WHY a blank line ends the run (issue #30, safety-critical): a blank line between a
// license header and a following file doc comment is a hard boundary. Earlier this
// function tolerated blank lines between comment lines, so "header / blank / doc
// comment" collapsed into one detected span; apply then deleted the doc comment on
// relicense/year-refresh. Our own rendered headers never contain a truly-blank
// internal line -- the renderer emits separator lines as a bare prefix (e.g. "#" or
// "//"), which is still a comment line and stays in-region -- so confining the region
// to contiguous comment lines is byte-idempotent for headers we write while refusing
// to absorb a foreign comment that a blank line has separated from the header. When in
// doubt we under-detect (leave the foreign comment alone) rather than over-detect.
func lineCommentRegion(content []byte, start int, cs model.CommentStyle) (int, int, string, bool) {
prefix := strings.TrimRight(cs.LinePrefix, " \t")
if prefix == "" {
@@ -296,7 +309,6 @@ func lineCommentRegion(content []byte, start int, cs model.CommentStyle) (int, i
pos := start
end := start
var lines []string
pendingBlank := 0
sawComment := false
for pos < len(content) {
@@ -305,26 +317,16 @@ func lineCommentRegion(content []byte, start int, cs model.CommentStyle) (int, i
body := strings.TrimRight(raw, "\r\n")
trimmed := strings.TrimLeft(body, " \t")
switch {
case strings.HasPrefix(trimmed, prefix):
// A comment line: flush any tolerated blank lines into the region, then record.
for i := 0; i < pendingBlank; i++ {
lines = append(lines, "")
}
pendingBlank = 0
text := strings.TrimPrefix(trimmed, prefix)
text = strings.TrimPrefix(text, " ")
lines = append(lines, text)
end = lineEnd
sawComment = true
case strings.TrimSpace(body) == "":
// Blank line: tolerate it only between comment lines, do not extend the span yet.
pendingBlank++
default:
// Non-comment content ends the leading comment run.
pos = len(content)
continue
if !strings.HasPrefix(trimmed, prefix) {
// Anything that is not a comment line -- including a truly-blank line --
// ends the contiguous leading comment run.
break
}
text := strings.TrimPrefix(trimmed, prefix)
text = strings.TrimPrefix(text, " ")
lines = append(lines, text)
end = lineEnd
sawComment = true
pos = lineEnd
}
@@ -336,24 +338,29 @@ func lineCommentRegion(content []byte, start int, cs model.CommentStyle) (int, i
// spdxIdentifierTag extracts the SPDX id from an "SPDX-License-Identifier:" line, if
// present. This is the universal REUSE tag and a definitive license signal.
//
// SAFETY (issue #30): the tag must sit at the START of a comment line (leading
// whitespace allowed), not buried mid-prose. commentText is the leading comment with
// per-line prefixes already stripped and lines joined by "\n", so requiring the tag to
// begin a line means it is an actual REUSE tag rather than a sentence that merely
// mentions the tag (e.g. a doc comment explaining the convention). Matching the tag
// anywhere previously let such prose qualify a foreign comment as a managed header.
func spdxIdentifierTag(commentText string) (string, bool) {
const tag = "spdx-license-identifier:"
lower := strings.ToLower(commentText)
idx := strings.Index(lower, tag)
if idx < 0 {
return "", false
for _, line := range strings.Split(commentText, "\n") {
trimmed := strings.TrimLeft(line, " \t")
if !strings.HasPrefix(strings.ToLower(trimmed), tag) {
continue
}
id := strings.TrimSpace(trimmed[len(tag):])
if id == "" {
// The tag is on its own line but carries no value; not a usable signal.
continue
}
// A multi-license expression is reported as-is; the first token is the primary id.
return id, true
}
rest := commentText[idx+len(tag):]
if nl := strings.IndexAny(rest, "\r\n"); nl >= 0 {
rest = rest[:nl]
}
id := strings.TrimSpace(rest)
if id == "" {
return "", false
}
// A multi-license expression is reported as-is; the first token is the primary id.
// (id is already non-empty here, so it always has at least one field.)
return id, true
return "", false
}
// matchStandardHeader checks the comment text against every curated SPDX
+75 -10
View File
@@ -198,6 +198,18 @@ For more information, please refer to <http://unlicense.org/>`
ft: lineFT(),
wantPresent: false,
},
{
// SAFETY (issue #30): a comment that merely MENTIONS the SPDX tag mid-prose
// (not on its own comment line) is documentation about the tag, not a managed
// header. It must NOT be treated as a license header, or apply would rewrite a
// real doc comment.
name: "SPDX tag mentioned mid-prose is not a managed header",
content: "// To mark a file's license, add an SPDX-License-Identifier: MIT line\n" +
"// near the top of the file. This helper documents that convention.\n" +
"\npackage widget\n",
ft: lineFT(),
wantPresent: false,
},
}
for _, tc := range tests {
@@ -233,6 +245,30 @@ func TestDetectSpanCoversCommentRegion(t *testing.T) {
assert.Equal(t, "package main\n", string(content[got.EndByte:]))
}
// TestDetectExcludesDocCommentBelowBlankLine is the issue #30 regression guard at the
// Detect level: a managed header, a truly-blank line, then an unrelated file doc
// comment must yield a span that covers ONLY the header (through the blank separator),
// never the doc comment below. Absorbing the doc comment would make apply delete it on
// relicense/year-refresh.
func TestDetectExcludesDocCommentBelowBlankLine(t *testing.T) {
docComment := "// Package widget renders widgets for the dashboard.\n"
content := []byte("// " + Sentinel + "\n" +
"// SPDX-License-Identifier: MIT\n" +
"\n" +
docComment +
"// It exposes a single Render entry point.\n" +
"\npackage widget\n")
got, err := Detect(content, lineFT())
require.NoError(t, err)
require.True(t, got.Present)
assert.Equal(t, 0, got.StartByte)
// The span ends at the blank separator; the doc comment is wholly outside it.
rest := string(content[got.EndByte:])
assert.Equal(t, docComment+"// It exposes a single Render entry point.\n\npackage widget\n", rest)
assert.NotContains(t, string(content[got.StartByte:got.EndByte]), "renders widgets")
}
// TestDetectPreserveBoundaryError covers Detect's preserve-boundary error guard via
// the preserveBoundaryFn seam. The real PreserveBoundary never errors, so the guard
// is otherwise unreachable; injecting an error confirms Detect propagates it and
@@ -541,7 +577,29 @@ func TestLeadingCommentRegionLine(t *testing.T) {
cs := model.CommentStyle{Block: false, LinePrefix: "// "}
ft := model.FileType{CommentStyle: cs}
t.Run("run of comment lines with tolerated blank between", func(t *testing.T) {
// SAFETY (issue #30): a truly-blank line ENDS the leading comment run. WHY: a
// blank line between a license header and a following file doc comment is a hard
// boundary; absorbing the doc comment into the header span made apply delete it on
// relicense/year-refresh. Our own rendered headers never contain a truly-blank
// internal line (separator lines are emitted as a bare prefix, e.g. "#"/"//"), so
// stopping at the blank under-detects nothing we wrote and protects foreign
// comments below the header.
t.Run("truly blank line ends the region (doc comment below is excluded)", func(t *testing.T) {
content := []byte("// header part A\n\n// unrelated doc comment\ncode\n")
s, e, text, ok := leadingCommentRegion(content, 0, ft)
require.True(t, ok)
assert.Equal(t, 0, s)
// The blank separator is folded into the header span (it round-trips with our
// own rendered trailing blank), and the doc comment below it is wholly outside.
assert.Equal(t, "// unrelated doc comment\ncode\n", string(content[e:]))
assert.Contains(t, text, "part A")
assert.NotContains(t, text, "unrelated doc comment")
})
// A bare-prefix line ("//" with no body) is still a comment line, not a blank, so
// it does NOT end the run. This is exactly the shape our renderer emits for the
// separator between REUSE tags and the notice block, so it must stay in-region.
t.Run("bare prefix separator line stays in the region", func(t *testing.T) {
content := []byte("// line one\n//\n// line three\nbody\n")
s, e, text, ok := leadingCommentRegion(content, 0, ft)
require.True(t, ok)
@@ -551,15 +609,6 @@ func TestLeadingCommentRegionLine(t *testing.T) {
assert.Contains(t, text, "line three")
})
t.Run("blank line then comment is still one region", func(t *testing.T) {
content := []byte("// header part A\n\n// header part B\ncode\n")
_, e, text, ok := leadingCommentRegion(content, 0, ft)
require.True(t, ok)
assert.Equal(t, "code\n", string(content[e:]))
assert.Contains(t, text, "part A")
assert.Contains(t, text, "part B")
})
t.Run("non-comment line ends the run", func(t *testing.T) {
content := []byte("// header\ncode();\n// trailing comment\n")
_, e, _, ok := leadingCommentRegion(content, 0, ft)
@@ -620,6 +669,22 @@ func TestSpdxIdentifierTag(t *testing.T) {
wantID: "(MIT OR Apache-2.0)",
wantOK: true,
},
{
// SAFETY (issue #30): the tag must sit at the START of a comment line. A
// mention buried mid-prose is documentation about the tag, not a managed
// header, and must not qualify.
name: "tag mentioned mid-prose does not qualify",
text: "add an SPDX-License-Identifier: MIT line near the top",
wantOK: false,
},
{
// Leading whitespace before the tag on its own line is allowed (comment
// bodies may be indented); it still counts as own-line.
name: "tag on its own line with leading whitespace qualifies",
text: "Copyright 2026 Acme\n SPDX-License-Identifier: MIT\n",
wantID: "MIT",
wantOK: true,
},
{
name: "no tag present",
text: "just a normal comment",