detect: confine header detection to avoid deleting adjacent comments (#30) #41
No files matched your search
@@ -76,6 +76,22 @@ func phpFileType() model.FileType {
|
||||
}
|
||||
}
|
||||
|
||||
// pyFileType returns a "# " line-comment Python file type. It is used to exercise the
|
||||
// issue #30 regression: a line-comment header followed by a blank line then a foreign
|
||||
// doc comment must not absorb (and therefore must not delete) that doc comment.
|
||||
func pyFileType() model.FileType {
|
||||
return model.FileType{
|
||||
Name: "Python",
|
||||
Extensions: []string{".py"},
|
||||
CommentStyle: model.CommentStyle{Block: false, LinePrefix: "# "},
|
||||
PreserveFirst: []model.PreserveRule{
|
||||
{Kind: model.PreserveBOM, Before: false},
|
||||
{Kind: model.PreserveShebang, Before: false},
|
||||
{Kind: model.PreserveCodingPragma, Before: false},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// agplConfig is the canonical apply config: AGPL, reuse+notice, an explicit year so
|
||||
// the rendered header is deterministic and independent of the wall clock.
|
||||
func agplConfig() model.Config {
|
||||
@@ -413,6 +429,67 @@ func TestApplyFile(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// lookupLicense fetches a curated license by id or fails the test. Local to these
|
||||
// tests to avoid depending on a sibling package's test helper.
|
||||
func lookupLicense(t *testing.T, id string) model.License {
|
||||
t.Helper()
|
||||
lic, ok := spdx.Lookup(id)
|
||||
require.True(t, ok, "license %q must be in the curated set", id)
|
||||
return lic
|
||||
}
|
||||
|
||||
// TestApplyFilePreservesDocCommentBelowBlankLine is the issue #30 end-to-end safety
|
||||
// guard. A line-comment file carrying a managed header, a blank line, then an unrelated
|
||||
// file doc comment must keep that doc comment intact across a relicense: the detected
|
||||
// span covers only the header, so the replace swaps only the header bytes.
|
||||
func TestApplyFilePreservesDocCommentBelowBlankLine(t *testing.T) {
|
||||
ft := pyFileType()
|
||||
|
||||
// Step 1: insert an MIT header into a file whose body opens with a doc comment.
|
||||
mitFunc := func(f model.FileType) (string, error) {
|
||||
return render.Header(render.HeaderInput{
|
||||
License: lookupLicense(t, "MIT"),
|
||||
Holder: "Acme Corp",
|
||||
Year: "2024",
|
||||
Style: model.StyleReuse,
|
||||
FileType: f,
|
||||
})
|
||||
}
|
||||
body := "# This module wires the dashboard widgets together.\n" +
|
||||
"# It is intentionally documented at the top of the file.\n" +
|
||||
"\n" +
|
||||
"def main():\n pass\n"
|
||||
withHeader, _, action, err := ApplyFile([]byte(body), ft, mitFunc)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "insert", action)
|
||||
require.Contains(t, string(withHeader), "SPDX-License-Identifier: MIT")
|
||||
require.Contains(t, string(withHeader), "wires the dashboard widgets",
|
||||
"the foreign doc comment must be present after insert")
|
||||
|
||||
// Step 2: relicense to Apache-2.0. This replaces the detected header span.
|
||||
apacheFunc := func(f model.FileType) (string, error) {
|
||||
return render.Header(render.HeaderInput{
|
||||
License: lookupLicense(t, "Apache-2.0"),
|
||||
Holder: "Acme Corp",
|
||||
Year: "2026",
|
||||
Style: model.StyleReuse,
|
||||
FileType: f,
|
||||
})
|
||||
}
|
||||
relicensed, _, action2, err := ApplyFile(withHeader, ft, apacheFunc)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "replace", action2)
|
||||
|
||||
got := string(relicensed)
|
||||
assert.Contains(t, got, "SPDX-License-Identifier: Apache-2.0", "new license written")
|
||||
assert.NotContains(t, got, "SPDX-License-Identifier: MIT", "old license id removed")
|
||||
// The safety-critical assertion: the foreign doc comment survives the relicense.
|
||||
assert.Contains(t, got, "# This module wires the dashboard widgets together.",
|
||||
"the doc comment below the blank line must survive a relicense")
|
||||
assert.Contains(t, got, "# It is intentionally documented at the top of the file.")
|
||||
assert.Contains(t, got, "def main():")
|
||||
}
|
||||
|
||||
func TestApplyFileIdempotentAfterPreserveFirstConstructs(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
|
||||
+53
-46
@@ -89,11 +89,12 @@ var curatedFingerprints = []struct {
|
||||
// reports presence, the byte span, the matched SPDX id / holder / year if any, and
|
||||
// whether the match came via the sentinel.
|
||||
//
|
||||
// The span (StartByte inclusive, EndByte exclusive) covers exactly the leading
|
||||
// comment block(s) recognized as a license header, including any blank lines between
|
||||
// adjacent comment blocks but excluding the preserve-first prefixes and the file
|
||||
// content that follows. When no header is found, Present is false and the offsets
|
||||
// are zero.
|
||||
// The span (StartByte inclusive, EndByte exclusive) covers exactly the contiguous
|
||||
// leading comment recognized as a license header, plus its trailing blank-line
|
||||
// separator, but excluding the preserve-first prefixes and the file content that
|
||||
// follows. A blank line inside the leading region ends it, so a foreign doc comment
|
||||
// below a blank separator is never absorbed. When no header is found, Present is false
|
||||
// and the offsets are zero.
|
||||
func Detect(content []byte, ft model.FileType) (model.DetectedHeader, error) {
|
||||
// Uncommentable types (JSON and friends) can never carry a header; report absence
|
||||
// rather than risk matching incidental text.
|
||||
@@ -200,8 +201,8 @@ func FingerprintLicense(headerText string) (string, bool) {
|
||||
// leadingCommentRegion finds the contiguous comment block(s) at offset start and
|
||||
// returns their byte span plus the extracted comment text (delimiters stripped,
|
||||
// joined by newlines). For block-comment types it spans the first block comment; for
|
||||
// line-comment types it spans the run of consecutive comment lines (blank lines
|
||||
// between comment lines are included so a header split by a blank line is one region).
|
||||
// line-comment types it spans the run of CONTIGUOUS comment lines (a truly-blank line
|
||||
// ends the run -- see lineCommentRegion for the issue #30 safety rationale).
|
||||
//
|
||||
// The bool is false when there is no comment at start, in which case no header is
|
||||
// possible. Leading blank lines before the comment are tolerated and included in the
|
||||
@@ -283,10 +284,22 @@ func blockCommentRegion(content []byte, start int, cs model.CommentStyle) (int,
|
||||
return openOffset, end, stripBlockInner(inner), true
|
||||
}
|
||||
|
||||
// lineCommentRegion spans the run of consecutive line comments (each beginning with
|
||||
// the LinePrefix, ignoring leading whitespace) starting at start, tolerating blank
|
||||
// lines between comment lines, and returns the joined comment text with prefixes
|
||||
// stripped. The run ends at the first non-blank, non-comment line.
|
||||
// lineCommentRegion spans the run of CONTIGUOUS line comments (each beginning with
|
||||
// the LinePrefix, ignoring leading whitespace) starting at start, and returns the
|
||||
// joined comment text with prefixes stripped. The run ends at the first line that is
|
||||
// neither a comment line nor empty -- and, critically, a truly-blank line ALSO ends
|
||||
// the run.
|
||||
//
|
||||
// WHY a blank line ends the run (issue #30, safety-critical): a blank line between a
|
||||
// license header and a following file doc comment is a hard boundary. Earlier this
|
||||
// function tolerated blank lines between comment lines, so "header / blank / doc
|
||||
// comment" collapsed into one detected span; apply then deleted the doc comment on
|
||||
// relicense/year-refresh. Our own rendered headers never contain a truly-blank
|
||||
// internal line -- the renderer emits separator lines as a bare prefix (e.g. "#" or
|
||||
// "//"), which is still a comment line and stays in-region -- so confining the region
|
||||
// to contiguous comment lines is byte-idempotent for headers we write while refusing
|
||||
// to absorb a foreign comment that a blank line has separated from the header. When in
|
||||
// doubt we under-detect (leave the foreign comment alone) rather than over-detect.
|
||||
func lineCommentRegion(content []byte, start int, cs model.CommentStyle) (int, int, string, bool) {
|
||||
prefix := strings.TrimRight(cs.LinePrefix, " \t")
|
||||
if prefix == "" {
|
||||
@@ -296,7 +309,6 @@ func lineCommentRegion(content []byte, start int, cs model.CommentStyle) (int, i
|
||||
pos := start
|
||||
end := start
|
||||
var lines []string
|
||||
pendingBlank := 0
|
||||
sawComment := false
|
||||
|
||||
for pos < len(content) {
|
||||
@@ -305,26 +317,16 @@ func lineCommentRegion(content []byte, start int, cs model.CommentStyle) (int, i
|
||||
body := strings.TrimRight(raw, "\r\n")
|
||||
trimmed := strings.TrimLeft(body, " \t")
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(trimmed, prefix):
|
||||
// A comment line: flush any tolerated blank lines into the region, then record.
|
||||
for i := 0; i < pendingBlank; i++ {
|
||||
lines = append(lines, "")
|
||||
}
|
||||
pendingBlank = 0
|
||||
text := strings.TrimPrefix(trimmed, prefix)
|
||||
text = strings.TrimPrefix(text, " ")
|
||||
lines = append(lines, text)
|
||||
end = lineEnd
|
||||
sawComment = true
|
||||
case strings.TrimSpace(body) == "":
|
||||
// Blank line: tolerate it only between comment lines, do not extend the span yet.
|
||||
pendingBlank++
|
||||
default:
|
||||
// Non-comment content ends the leading comment run.
|
||||
pos = len(content)
|
||||
continue
|
||||
if !strings.HasPrefix(trimmed, prefix) {
|
||||
// Anything that is not a comment line -- including a truly-blank line --
|
||||
// ends the contiguous leading comment run.
|
||||
break
|
||||
}
|
||||
text := strings.TrimPrefix(trimmed, prefix)
|
||||
text = strings.TrimPrefix(text, " ")
|
||||
lines = append(lines, text)
|
||||
end = lineEnd
|
||||
sawComment = true
|
||||
pos = lineEnd
|
||||
}
|
||||
|
||||
@@ -336,24 +338,29 @@ func lineCommentRegion(content []byte, start int, cs model.CommentStyle) (int, i
|
||||
|
||||
// spdxIdentifierTag extracts the SPDX id from an "SPDX-License-Identifier:" line, if
|
||||
// present. This is the universal REUSE tag and a definitive license signal.
|
||||
//
|
||||
// SAFETY (issue #30): the tag must sit at the START of a comment line (leading
|
||||
// whitespace allowed), not buried mid-prose. commentText is the leading comment with
|
||||
// per-line prefixes already stripped and lines joined by "\n", so requiring the tag to
|
||||
// begin a line means it is an actual REUSE tag rather than a sentence that merely
|
||||
// mentions the tag (e.g. a doc comment explaining the convention). Matching the tag
|
||||
// anywhere previously let such prose qualify a foreign comment as a managed header.
|
||||
func spdxIdentifierTag(commentText string) (string, bool) {
|
||||
const tag = "spdx-license-identifier:"
|
||||
lower := strings.ToLower(commentText)
|
||||
idx := strings.Index(lower, tag)
|
||||
if idx < 0 {
|
||||
return "", false
|
||||
for _, line := range strings.Split(commentText, "\n") {
|
||||
trimmed := strings.TrimLeft(line, " \t")
|
||||
if !strings.HasPrefix(strings.ToLower(trimmed), tag) {
|
||||
continue
|
||||
}
|
||||
id := strings.TrimSpace(trimmed[len(tag):])
|
||||
if id == "" {
|
||||
// The tag is on its own line but carries no value; not a usable signal.
|
||||
continue
|
||||
}
|
||||
// A multi-license expression is reported as-is; the first token is the primary id.
|
||||
return id, true
|
||||
}
|
||||
rest := commentText[idx+len(tag):]
|
||||
if nl := strings.IndexAny(rest, "\r\n"); nl >= 0 {
|
||||
rest = rest[:nl]
|
||||
}
|
||||
id := strings.TrimSpace(rest)
|
||||
if id == "" {
|
||||
return "", false
|
||||
}
|
||||
// A multi-license expression is reported as-is; the first token is the primary id.
|
||||
// (id is already non-empty here, so it always has at least one field.)
|
||||
return id, true
|
||||
return "", false
|
||||
}
|
||||
|
||||
// matchStandardHeader checks the comment text against every curated SPDX
|
||||
|
||||
@@ -198,6 +198,18 @@ For more information, please refer to <http://unlicense.org/>`
|
||||
ft: lineFT(),
|
||||
wantPresent: false,
|
||||
},
|
||||
{
|
||||
// SAFETY (issue #30): a comment that merely MENTIONS the SPDX tag mid-prose
|
||||
// (not on its own comment line) is documentation about the tag, not a managed
|
||||
// header. It must NOT be treated as a license header, or apply would rewrite a
|
||||
// real doc comment.
|
||||
name: "SPDX tag mentioned mid-prose is not a managed header",
|
||||
content: "// To mark a file's license, add an SPDX-License-Identifier: MIT line\n" +
|
||||
"// near the top of the file. This helper documents that convention.\n" +
|
||||
"\npackage widget\n",
|
||||
ft: lineFT(),
|
||||
wantPresent: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
@@ -233,6 +245,30 @@ func TestDetectSpanCoversCommentRegion(t *testing.T) {
|
||||
assert.Equal(t, "package main\n", string(content[got.EndByte:]))
|
||||
}
|
||||
|
||||
// TestDetectExcludesDocCommentBelowBlankLine is the issue #30 regression guard at the
|
||||
// Detect level: a managed header, a truly-blank line, then an unrelated file doc
|
||||
// comment must yield a span that covers ONLY the header (through the blank separator),
|
||||
// never the doc comment below. Absorbing the doc comment would make apply delete it on
|
||||
// relicense/year-refresh.
|
||||
func TestDetectExcludesDocCommentBelowBlankLine(t *testing.T) {
|
||||
docComment := "// Package widget renders widgets for the dashboard.\n"
|
||||
content := []byte("// " + Sentinel + "\n" +
|
||||
"// SPDX-License-Identifier: MIT\n" +
|
||||
"\n" +
|
||||
docComment +
|
||||
"// It exposes a single Render entry point.\n" +
|
||||
"\npackage widget\n")
|
||||
|
||||
got, err := Detect(content, lineFT())
|
||||
require.NoError(t, err)
|
||||
require.True(t, got.Present)
|
||||
assert.Equal(t, 0, got.StartByte)
|
||||
// The span ends at the blank separator; the doc comment is wholly outside it.
|
||||
rest := string(content[got.EndByte:])
|
||||
assert.Equal(t, docComment+"// It exposes a single Render entry point.\n\npackage widget\n", rest)
|
||||
assert.NotContains(t, string(content[got.StartByte:got.EndByte]), "renders widgets")
|
||||
}
|
||||
|
||||
// TestDetectPreserveBoundaryError covers Detect's preserve-boundary error guard via
|
||||
// the preserveBoundaryFn seam. The real PreserveBoundary never errors, so the guard
|
||||
// is otherwise unreachable; injecting an error confirms Detect propagates it and
|
||||
@@ -541,7 +577,29 @@ func TestLeadingCommentRegionLine(t *testing.T) {
|
||||
cs := model.CommentStyle{Block: false, LinePrefix: "// "}
|
||||
ft := model.FileType{CommentStyle: cs}
|
||||
|
||||
t.Run("run of comment lines with tolerated blank between", func(t *testing.T) {
|
||||
// SAFETY (issue #30): a truly-blank line ENDS the leading comment run. WHY: a
|
||||
// blank line between a license header and a following file doc comment is a hard
|
||||
// boundary; absorbing the doc comment into the header span made apply delete it on
|
||||
// relicense/year-refresh. Our own rendered headers never contain a truly-blank
|
||||
// internal line (separator lines are emitted as a bare prefix, e.g. "#"/"//"), so
|
||||
// stopping at the blank under-detects nothing we wrote and protects foreign
|
||||
// comments below the header.
|
||||
t.Run("truly blank line ends the region (doc comment below is excluded)", func(t *testing.T) {
|
||||
content := []byte("// header part A\n\n// unrelated doc comment\ncode\n")
|
||||
s, e, text, ok := leadingCommentRegion(content, 0, ft)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, 0, s)
|
||||
// The blank separator is folded into the header span (it round-trips with our
|
||||
// own rendered trailing blank), and the doc comment below it is wholly outside.
|
||||
assert.Equal(t, "// unrelated doc comment\ncode\n", string(content[e:]))
|
||||
assert.Contains(t, text, "part A")
|
||||
assert.NotContains(t, text, "unrelated doc comment")
|
||||
})
|
||||
|
||||
// A bare-prefix line ("//" with no body) is still a comment line, not a blank, so
|
||||
// it does NOT end the run. This is exactly the shape our renderer emits for the
|
||||
// separator between REUSE tags and the notice block, so it must stay in-region.
|
||||
t.Run("bare prefix separator line stays in the region", func(t *testing.T) {
|
||||
content := []byte("// line one\n//\n// line three\nbody\n")
|
||||
s, e, text, ok := leadingCommentRegion(content, 0, ft)
|
||||
require.True(t, ok)
|
||||
@@ -551,15 +609,6 @@ func TestLeadingCommentRegionLine(t *testing.T) {
|
||||
assert.Contains(t, text, "line three")
|
||||
})
|
||||
|
||||
t.Run("blank line then comment is still one region", func(t *testing.T) {
|
||||
content := []byte("// header part A\n\n// header part B\ncode\n")
|
||||
_, e, text, ok := leadingCommentRegion(content, 0, ft)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "code\n", string(content[e:]))
|
||||
assert.Contains(t, text, "part A")
|
||||
assert.Contains(t, text, "part B")
|
||||
})
|
||||
|
||||
t.Run("non-comment line ends the run", func(t *testing.T) {
|
||||
content := []byte("// header\ncode();\n// trailing comment\n")
|
||||
_, e, _, ok := leadingCommentRegion(content, 0, ft)
|
||||
@@ -620,6 +669,22 @@ func TestSpdxIdentifierTag(t *testing.T) {
|
||||
wantID: "(MIT OR Apache-2.0)",
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
// SAFETY (issue #30): the tag must sit at the START of a comment line. A
|
||||
// mention buried mid-prose is documentation about the tag, not a managed
|
||||
// header, and must not qualify.
|
||||
name: "tag mentioned mid-prose does not qualify",
|
||||
text: "add an SPDX-License-Identifier: MIT line near the top",
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
// Leading whitespace before the tag on its own line is allowed (comment
|
||||
// bodies may be indented); it still counts as own-line.
|
||||
name: "tag on its own line with leading whitespace qualifies",
|
||||
text: "Copyright 2026 Acme\n SPDX-License-Identifier: MIT\n",
|
||||
wantID: "MIT",
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "no tag present",
|
||||
text: "just a normal comment",
|
||||
|
||||
Reference in new issue
Block a user