diff --git a/README.md b/README.md index 154feb3..eea832e 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ On launch, gclo shows a brief branded splash, then the **Quick Sync** connect ca 8. **Cancel** at any time: in-flight repos stop cleanly, unstarted repos are marked Canceled, and the summary still appears. After a run, **Retry failed** re-runs exactly the failed rows, and **Open folder** opens the target folder in Explorer. -**When something goes wrong:** failures are isolated per repository — one broken repo never stops the rest, and a failed row shows its error message inline. Failed clones are cleaned up, and a clone whose checkout never finished (a crash, a kill, a cleanup that could not delete the folder) is flagged inside its `.git`, so a partial working tree is never mistaken for an up-to-date repo on the next run — gclo completes the checkout instead. Repositories with Windows-impossible paths are the deliberate exception: gclo checks every path in the incoming tree against Windows file-system rules *before* checkout — invalid characters, reserved device names like `CON`, trailing spaces or dots, names that differ only by case — and when it finds any, the repo is marked Failed with a per-path list of reasons, but the fully fetched `.git` is **kept** (nothing was ever checked out). Recovery happens in place, with no re-download: click the row's **Resolve…** link to rename each offending path (safe names are pre-suggested) or skip it, and gclo materializes the working tree right there; the CLI does the same automatically with [`--sanitize-paths`](docs/CLI.md#windows-invalid-paths-and---sanitize-paths). When WSL is installed with git in its default distribution, the same dialog offers **Clone in WSL instead**: the repository is cloned unchanged into `~/gclo//` inside WSL (reachable from Explorer as `\wsl.localhost\\home\...`), the token travels only through the environment, and the Windows copy is left as it is. Long paths beyond the classic 260-character limit are handled automatically (`core.longpaths`). +**When something goes wrong:** failures are isolated per repository — one broken repo never stops the rest, and a failed row shows its error message inline. Failed clones are cleaned up, and a clone whose checkout never finished (a crash, a kill, a cleanup that could not delete the folder) is flagged inside its `.git`, so a partial working tree is never mistaken for an up-to-date repo on the next run — gclo completes the checkout instead. Repositories with Windows-impossible paths are the deliberate exception: gclo checks every path in the incoming tree against Windows file-system rules *before* checkout — invalid characters, reserved device names like `CON`, trailing spaces or dots, names that differ only by case — and when it finds any, the repo is marked Failed with a per-path list of reasons, but the fully fetched `.git` is **kept** (nothing was ever checked out). Recovery happens in place, with no re-download: click the row's **Resolve…** link to rename each offending path (safe names are pre-suggested) or skip it, and gclo materializes the working tree right there; the CLI does the same automatically with [`--sanitize-paths`](docs/CLI.md#windows-invalid-paths-and---sanitize-paths). This applies to a repository gclo could not check out when it cloned it. A repository that was already checked out normally is never rewritten: when an update brings it such a path, it fails and nothing in it is changed, and **Resolve…** (or `--sanitize-paths`) then says how to get a fresh clone with the renames instead of touching it. When WSL is installed with git in its default distribution, the same dialog offers **Clone in WSL instead**: the repository is cloned unchanged into `~/gclo//` inside WSL (reachable from Explorer as `\wsl.localhost\\home\...`), the token travels only through the environment, and the Windows copy is left as it is. Long paths beyond the classic 260-character limit are handled automatically (`core.longpaths`). **Settings** (File > Settings…) let you set a default GitHub token (stored in the Windows Credential Manager), the default target folder (with a folder picker), the default parallelism, and the theme (System, Light, or Dark); an Advanced section toggles the startup splash and its duration. diff --git a/docs/CLI.md b/docs/CLI.md index 7f8639f..c32d146 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -158,12 +158,31 @@ With `--sanitize-paths`, gclo checks such a repository out anyway: upstream tip has not moved. A later sync that brings *new* invalid paths not covered by the stored mapping fails with those paths listed — sync again with `--sanitize-paths` (or use the GUI's **Resolve…** dialog) and the new renames - and skips are merged into the stored mapping. Use `--include ` to - target just that repository. + and skips are merged into the stored mapping; a newer choice for a path + replaces the stored one. Use `--include ` to target just that + repository. +- A repository that was already checked out normally is never rewritten this + way. When an update brings such a repository its first invalid path, it + fails with those paths listed; with `--sanitize-paths` the failure says + that the repository is checked out normally, and nothing in it is changed: + its working tree may hold work, and writing it by hand could destroy that. + To get it with the renames, move or delete its folder and sync again (the + fresh clone is then sanitized), or use the desktop app's **Clone in WSL**. + The same holds when the paths of a clone that was never checked out, or of + an empty one, are resolved and files were put into its folder by hand: a file + that is already where the repository has one, and is not what an unfinished + checkout would have left there, is not replaced by the renaming. Only the working tree is renamed — the repository's history and index still hold -the original paths, so `git status` reports the renamed and skipped files as -local changes. Treat a sanitized repository as a read-only mirror. +the original paths, so `git status` reports renamed and skipped paths as local +changes. Treat a sanitized repository as a read-only mirror: later syncs +rewrite its working tree, so edits made there are overwritten. A commit made +there is not dropped: while origin has nothing new, a sync leaves the tree as +it is, and once origin has moved on as well the sync fails for that +repository, as it does for any clone with local commits. The branch is only +ever moved to the tip of the branch it tracks: forward, or, when origin's +history was rewritten, off a commit that this clone's record of earlier +fetches shows was origin's own. #### `--json` @@ -181,7 +200,8 @@ run ends (also on cancellation): - `type` is always `"summary"`. - `failures[].invalidPaths` is the **full** list of offending paths when the failure was Windows path validation (the message itself only quotes the - first three); `null` for any other failure. + first three), also when `--sanitize-paths` then could not fix the repository + (`error` says why); `null` for any other failure. - `sanitized[]` lists the repositories `--sanitize-paths` checked out with renames and skips; they are counted under `cloned`, but `skippedPaths` are not on disk. diff --git a/docs/security/threat-model.md b/docs/security/threat-model.md index 4400533..78e4f9f 100644 --- a/docs/security/threat-model.md +++ b/docs/security/threat-model.md @@ -63,7 +63,7 @@ flowchart LR | Credential Manager | out/in | Tokens by `gclo[:]:account:` target | Windows user session | `CredentialManagerVault`; scope derived from the data root | | `settings.json`, `accounts.json` (with `accounts.json.lock`) | out/in | Non-secret settings and account metadata | File ACL (user profile) | Written beside the file and put in place in one step; sanitized on load; an accounts file whose content is not valid is moved aside (`.corrupt-*`), one that cannot be read is left in place; the previous accounts file is kept as `.bak`. Changes to the accounts are serialized across gclo processes by the lock file: a process of the same user that holds it open can delay account changes (they fail after a few seconds, nothing is written), not read or alter them | | Activity logs | out | Actions and errors; token *lengths* only | File ACL | `IActivityLog` callers never pass tokens | -| `.git/gclo-recovery.json`, `gclo.checkoutpending` config key | out/in | Path-recovery plan; checkout-pending marker | File ACL | Written atomically; read only for the repository it belongs to | +| `.git/gclo-recovery.json`, `gclo.checkoutpending` and `gclo.recoverymanaged` config keys | out/in | Path-recovery plan; checkout-pending marker; "gclo wrote this working tree" marker | File ACL | Written atomically; read only for the repository it belongs to (the markers from the repository's own configuration, never the user's or the machine's). A repository marked as written by gclo has its working tree rewritten on sync; one that is not, and was checked out by git, is never rewritten | | Command line (CLI) | in | Options, org, target, filters | n/a | `OptionReader` / `RepoFilterSpec`; parallelism bounded 1..64; token options refused on argv | | `GCLO_DATA_DIR`, `GCLO_UITEST_FIXTURE` env | in | Data root override; offline test seam | n/a | Fixture only answers for the exact fixture token (every other token, and the question whether a folder is a repository, go to the real services); ignored by an installed build; logged when enabled; never set in production (#145) | | `wsl.exe -e sh -s` | out/in | POSIX script on stdin, org/repo/url as positional args, `GCLO_GIT_TOKEN` via `WSLENV` | PAT via env | Nothing is re-parsed by wsl.exe's command line; token never on argv or disk; UTF-16 wsl.exe messages decoded | diff --git a/gclo.Cli.Tests/SyncCommandTests.cs b/gclo.Cli.Tests/SyncCommandTests.cs index 22fe283..e1c8415 100644 --- a/gclo.Cli.Tests/SyncCommandTests.cs +++ b/gclo.Cli.Tests/SyncCommandTests.cs @@ -468,7 +468,9 @@ public sealed class SyncCommandTests : IDisposable Assert.Contains("\"sanitized\":[{\"repo\":\"fixable\",\"renamed\":1,\"skipped\":1,\"skippedPaths\":[\"dup\"]}]", json); Assert.Contains("\"repo\":\"stuck\"", json); Assert.Contains("path sanitization failed: disk full", json); - Assert.Contains("\"invalidPaths\":null", json); // a non-path failure has no list + // The paths that made it fail in the first place are still what is wrong with it. + Assert.Contains("\"path\":\"bad:name.txt\"", json); + Assert.DoesNotContain("\"invalidPaths\":null", json); } @@ -720,6 +722,27 @@ public sealed class SyncCommandTests : IDisposable + [Fact] + public async Task SanitizePaths_RecoveryRefused_StillNamesThePathsInTheDocument() + { + using var console = new ConsoleCapture(); + var lister = new FakeRepoLister { Result = [Repo("a")] }; + var git = new FakeGit + { + OnClone = (_, _, _, _, _) => Task.FromException(InvalidPaths()), + OnApplyRecovery = (_, _, _) => Task.FromException(new InvalidOperationException("This repository is checked out normally.")), + }; + + int code = await Run(lister, git, "--org", "acme", "--target", Target, "--token-env", EnvVar, "--sanitize-paths", "--json"); + + Assert.Equal(1, code); + Assert.Contains("path sanitization failed: This repository is checked out normally.", console.Out); + Assert.Contains("\"path\":\"bad:name.txt\"", console.Out); + Assert.DoesNotContain("\"invalidPaths\":null", console.Out); + } + + + [Fact] public async Task InvalidPathFailure_WithoutSanitize_PrintsOffendingPaths() { diff --git a/gclo.Cli/SyncCommand.cs b/gclo.Cli/SyncCommand.cs index 569fd34..c1fd604 100644 --- a/gclo.Cli/SyncCommand.cs +++ b/gclo.Cli/SyncCommand.cs @@ -670,9 +670,12 @@ internal static class SyncCommand } catch(Exception ex) { + // The paths that made the repository fail are still what is wrong with it, + // whatever stopped the sanitization (for one, a clone that was checked out + // normally is never rewritten): the report keeps them. progress.Report(new RepoProgress( pending.Repo, SyncStatus.Failed, $"path sanitization failed: {ex.Message}", - InvalidPaths: (ex as InvalidRepositoryPathsException)?.Paths)); + InvalidPaths: (ex as InvalidRepositoryPathsException)?.Paths ?? pending.Paths)); return false; } } diff --git a/gclo.Engine.Tests/RecoveryLocalWorkTests.cs b/gclo.Engine.Tests/RecoveryLocalWorkTests.cs new file mode 100644 index 0000000..cba458b --- /dev/null +++ b/gclo.Engine.Tests/RecoveryLocalWorkTests.cs @@ -0,0 +1,1438 @@ +/* + * Copyright (c) 2026 James Maes (KofTwentyTwo) + * SPDX-License-Identifier: MIT + */ + +using System.Text; +using System.Text.Json.Nodes; +using LibGit2Sharp; +using static gclo.Engine.Tests.GitTestHelpers; + + +namespace gclo.Engine.Tests; + + +/// +/// Path recovery writes a working tree by hand, so it is only done to a tree that is +/// gclo's to write, never moves a branch off a commit made in the repository, and +/// finishes what an interrupted apply left (#147). Real repositories on the local +/// file system, like . +/// +public sealed class RecoveryLocalWorkTests : IDisposable +{ + private const string Token = "test-token"; + + private const string PendingKey = "gclo.checkoutpending"; + + private const string ManagedKey = "gclo.recoverymanaged"; + + private readonly string _root = + Path.Combine(Path.GetTempPath(), "gclo-tests", Guid.NewGuid().ToString("N")); + + private readonly LibGit2GitClient _client = new(); + + + + public RecoveryLocalWorkTests() + { + Directory.CreateDirectory(_root); + } + + + + public void Dispose() => TryDeleteDirectory(_root); + + + + // ---------------------------------------------------------------- a clone that was checked out is never rewritten + + [Theory] + [InlineData("clean")] + [InlineData("uncommitted edit")] + [InlineData("untracked file in the way")] + [InlineData("local commit")] + public async Task OrdinaryClone_IsRefused_WhateverItsState_AndNothingInItChanges(string state) + { + (string source, string target) = await CloneOrdinaryAsync(("readme.txt", "v1"), ("work.txt", "work")); + switch(state) + { + case "uncommitted edit": + File.WriteAllText(Path.Combine(target, "work.txt"), "my local edit"); + break; + case "untracked file in the way": + File.WriteAllText(Path.Combine(target, "bad_file.txt"), "mine, never committed"); + break; + case "local commit": + CommitInWorkingTree(target, "mine.txt", "my work"); + break; + default: + break; + } + string before = HeadSha(target); + Dictionary filesBefore = ReadAllFiles(target); + AppendForgedCommit(source, ("bad:file.txt", "bad"), ("readme.txt", "v2")); + await PullExpectingInvalidPathsAsync(target); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None)); + + Assert.Contains("is checked out normally", ex.Message); + Assert.Contains("move or delete the folder", ex.Message); + Assert.Contains(target, ex.Message); + Assert.Equal(before, HeadSha(target)); + Assert.Equal(filesBefore, ReadAllFiles(target)); + Assert.False(File.Exists(RecoveryFilePath(target))); + Assert.False(IsFlagSet(target, ManagedKey), "a refused apply leaves the repository an ordinary clone"); + Assert.False(IsFlagSet(target, PendingKey)); + + // And it stays an ordinary clone to the next pull, which reports the same paths. + await PullExpectingInvalidPathsAsync(target); + Assert.Equal(filesBefore, ReadAllFiles(target)); + } + + + + [Fact] + public async Task OrdinaryClone_WithAnUnreadableRecoveryFileInIt_IsNotTurnedIntoAManagedRepository() + { + (_, string target) = await CloneOrdinaryAsync(("readme.txt", "v1"), ("work.txt", "work")); + File.WriteAllText(RecoveryFilePath(target), "{ this is not a recovery file"); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + Assert.Contains("is corrupt", ex.Message); + Assert.False(IsFlagSet(target, ManagedKey), "a file that could not be read proves nothing about the repository"); + + // The message says to delete it. After that the clone is an ordinary clone again, + // and a pull does what a pull does: it leaves local edits alone. + File.Delete(RecoveryFilePath(target)); + File.WriteAllText(Path.Combine(target, "work.txt"), "my local edit"); + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal("my local edit", File.ReadAllText(Path.Combine(target, "work.txt"))); + Assert.False(IsFlagSet(target, ManagedKey)); + } + + + + [Fact] + public async Task EmptyClone_WhoseOriginGetsItsFirstCommitWithAnInvalidPath_CanBeResolved() + { + string source = Path.Combine(_root, "empty-source"); + Repository.Init(source); + string target = Path.Combine(_root, "empty-clone"); + await _client.CloneAsync(source, target, Token, null, CancellationToken.None); + string tip = FirstForgedCommit(source, ("bad:file.txt", "bad"), ("keep.txt", "keep")); + await PullExpectingInvalidPathsAsync(target); + + // Nothing was ever checked out here, so the tree is gclo's to write. + await _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None); + + Assert.Equal(tip, HeadSha(target)); + Assert.Equal("bad", File.ReadAllText(Path.Combine(target, "bad_file.txt"))); + Assert.Equal("keep", File.ReadAllText(Path.Combine(target, "keep.txt"))); + // The branch tracks origin, so later pulls follow it. + using(var repo = new Repository(target)) + { + Assert.Equal("refs/remotes/origin/" + repo.Head.FriendlyName, repo.Head.TrackedBranch?.CanonicalName); + } + string next = AppendForgedCommit(source, ("more.txt", "m")); + await _client.PullAsync(target, Token, CancellationToken.None); + Assert.Equal(next, HeadSha(target)); + Assert.Equal("m", File.ReadAllText(Path.Combine(target, "more.txt"))); + } + + + + // ---------------------------------------------------------------- a managed repository: its branch + + [Fact] + public async Task Managed_FollowsOrigin_WhenItsCommitIsContainedInOriginsTip() + { + (string source, string target) = await CreateManagedAsync("managed-follow"); + string tip = AppendForgedCommit(source, ("other.txt", "o")); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(tip, HeadSha(target)); + Assert.Equal(tip, MaterializedTip(target)); + Assert.Equal("o", File.ReadAllText(Path.Combine(target, "other.txt"))); + } + + + + [Fact] + public async Task Managed_WithALocalCommitAndOriginMovedOn_IsNotMovedOffIt() + { + (string source, string target) = await CreateManagedAsync("managed-diverged"); + string local = CommitInWorkingTree(target, "mine.txt", "my work"); + AppendForgedCommit(source, ("other.txt", "o")); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("have diverged", ex.Message); + Assert.Equal(local, HeadSha(target)); + Assert.Equal("my work", File.ReadAllText(Path.Combine(target, "mine.txt"))); + } + + + + [Fact] + public async Task Managed_WithALocalCommitAndOriginUnchanged_SyncsWithoutMovingTheBranch() + { + (_, string target) = await CreateManagedAsync("managed-ahead"); + string local = CommitInWorkingTree(target, "mine.txt", "my work"); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(local, HeadSha(target)); + Assert.Equal("my work", File.ReadAllText(Path.Combine(target, "mine.txt"))); + Assert.Equal("bad", File.ReadAllText(Path.Combine(target, "bad_file.txt"))); + } + + + + [Fact] + public async Task Managed_ALocalCommitThatWasSyncedOnce_IsStillNotDropped_WhenOriginMovesOn() + { + // A sync with the local commit in place goes through without complaint. That + // must not turn it into "origin's commit": what counts is whether origin's + // branch ever pointed at it. + (string source, string target) = await CreateManagedAsync("managed-synced-local-commit"); + string local = CommitInWorkingTree(target, "mine.txt", "my work"); + await _client.PullAsync(target, Token, CancellationToken.None); + Assert.Equal(local, HeadSha(target)); + AppendForgedCommit(source, ("other.txt", "from a colleague")); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("have diverged", ex.Message); + Assert.Equal(local, HeadSha(target)); + Assert.Equal("my work", File.ReadAllText(Path.Combine(target, "mine.txt"))); + } + + + + [Fact] + public async Task Managed_WhoseOriginWasRewritten_FollowsIt_WhenNobodyCommittedThere() + { + (string source, string target) = await CreateManagedAsync("managed-rewritten"); + // Upstream replaces its history: a new root commit, unrelated to the old one. + string rewritten = RewriteHistory(source, ("bad:file.txt", "bad again"), ("fresh.txt", "fresh")); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(rewritten, HeadSha(target)); + Assert.Equal("fresh", File.ReadAllText(Path.Combine(target, "fresh.txt"))); + Assert.Equal("bad again", File.ReadAllText(Path.Combine(target, "bad_file.txt"))); + Assert.False(File.Exists(Path.Combine(target, "keep.txt"))); + } + + + + [Fact] + public async Task Managed_OriginRewrittenAfterAnInterruptedPull_IsStillFollowed() + { + (string source, string target) = await CreateManagedAsync("managed-interrupted-then-rewritten"); + string moved = AppendForgedCommit(source, ("other.txt", "o")); + await InterruptAsync(target, "keep.txt", () => _client.PullAsync(target, Token, CancellationToken.None)); + // The branch moved before the files were written; nobody committed anything. + Assert.Equal(moved, HeadSha(target)); + Assert.Null(MaterializedTip(target)); + string rewritten = RewriteHistory(source, ("bad:file.txt", "bad again"), ("fresh.txt", "fresh")); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(rewritten, HeadSha(target)); + Assert.Equal(rewritten, MaterializedTip(target)); + Assert.Equal("fresh", File.ReadAllText(Path.Combine(target, "fresh.txt"))); + Assert.False(File.Exists(Path.Combine(target, "other.txt")), "what the interrupted pull wrote and origin no longer has is gone"); + Assert.False(File.Exists(Path.Combine(target, "keep.txt"))); + } + + + + [Fact] + public async Task Managed_AnUncoveredInvalidPath_FailsBeforeTheBranchMoves() + { + (string source, string target) = await CreateManagedAsync("managed-uncovered"); + string before = HeadSha(target); + AppendForgedCommit(source, ("worse:new.txt", "w")); + + await PullExpectingInvalidPathsAsync(target); + + // Nothing was delivered, so nothing moved: the branch still names the commit on disk. + Assert.Equal(before, HeadSha(target)); + Assert.Equal(before, MaterializedTip(target)); + + // Resolving the new path then delivers the commit. + await _client.ApplyRecoveryAsync(target, Recovery(renames: [("worse:new.txt", "worse_new.txt")]), CancellationToken.None); + Assert.False(string.Equals(before, HeadSha(target), StringComparison.Ordinal), "the branch moved to origin's commit"); + Assert.Equal("w", File.ReadAllText(Path.Combine(target, "worse_new.txt"))); + } + + + + // ---------------------------------------------------------------- a managed repository: failures + + [Fact] + public async Task Managed_WhatWouldFailHalfWay_IsFoundBeforeAnythingChanges_AFileWhereAFolderIsNeeded() + { + (string source, string target) = await CreateManagedAsync("managed-file-for-folder"); + string before = HeadSha(target); + File.WriteAllText(Path.Combine(target, "docs"), "a file of the user's"); + AppendForgedCommit(source, ("docs/a.txt", "a")); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("'docs' is a file in this folder", ex.Message); + Assert.Equal(before, HeadSha(target)); + Assert.Equal(before, MaterializedTip(target)); + Assert.Equal("a file of the user's", File.ReadAllText(Path.Combine(target, "docs"))); + } + + + + [Fact] + public async Task Managed_WhatWouldFailHalfWay_IsFoundBeforeAnythingChanges_ALinkOnTheWay() + { + (string source, string target) = await CreateManagedAsync("managed-link"); + string before = HeadSha(target); + string elsewhere = Path.Combine(_root, "elsewhere"); + Directory.CreateDirectory(elsewhere); + string link = Path.Combine(target, "vendor"); + WorkingTreeTests.CreateDirectoryLink(link, elsewhere); + try + { + AppendForgedCommit(source, ("vendor/lib.txt", "lib")); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("is a link to another location", ex.Message); + Assert.Equal(before, HeadSha(target)); + Assert.Equal(before, MaterializedTip(target)); + Assert.Empty(Directory.EnumerateFileSystemEntries(elsewhere)); + } + finally + { + WorkingTreeTests.RemoveLink(link); + } + } + + + + [Fact] + public async Task Managed_APullInterruptedHalfWay_IsCompletedByTheNext_AndWhatLeftTheTreeIsStillRemoved() + { + (string source, string target) = await CreateManagedAsync("managed-interrupted-pull"); + ReplaceEntry(source, remove: "keep.txt", add: ("zz-new.txt", "new")); + string tip = AppendForgedCommit(source, ("aa-new.txt", "new")); + // "bad_file.txt" is held open: the pull fails after the branch moved and + // "keep.txt" was removed. + await InterruptAsync(target, "bad_file.txt", () => _client.PullAsync(target, Token, CancellationToken.None)); + Assert.Null(MaterializedTip(target)); + Assert.False(IsFlagSet(target, PendingKey), "the never-checked-out marker is not for a repository that has a tree"); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(tip, HeadSha(target)); + Assert.Equal(tip, MaterializedTip(target)); + Assert.Equal("new", File.ReadAllText(Path.Combine(target, "aa-new.txt"))); + Assert.Equal("new", File.ReadAllText(Path.Combine(target, "zz-new.txt"))); + Assert.False(File.Exists(Path.Combine(target, "keep.txt"))); + } + + + + [Fact] + public async Task Managed_ReResolvingAtTheSameCommit_ThatFailsHalfWay_IsCompletedByTheNextPull() + { + (_, string target) = await CreateManagedAsync("managed-interrupted-resolve"); + string tip = HeadSha(target); + // A different name for the same path; the apply fails after the old name was + // removed, before everything is written. + await InterruptAsync( + target, "keep.txt", + () => _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "renamed.txt")]), CancellationToken.None)); + Assert.Null(MaterializedTip(target)); + + // The tip did not move, but the tree is unfinished: "already at the tip" does not apply. + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(tip, MaterializedTip(target)); + Assert.Equal("bad", File.ReadAllText(Path.Combine(target, "renamed.txt"))); + Assert.Equal("keep", File.ReadAllText(Path.Combine(target, "keep.txt"))); + Assert.False(File.Exists(Path.Combine(target, "bad_file.txt"))); + } + + + + // ---------------------------------------------------------------- a managed repository: its recovery file + + [Fact] + public async Task Managed_WhoseRecoveryFileWasDeleted_AsTheMessageForACorruptOneAdvises_CanBeResolvedAgain() + { + (string source, string target) = await CreateManagedAsync("managed-file-deleted"); + File.Delete(RecoveryFilePath(target)); + + // It is still known to be recovery-managed, so the pull reports the invalid + // path again instead of treating the renamed tree as an ordinary clone... + await PullExpectingInvalidPathsAsync(target); + // ...and resolving rebuilds the file instead of refusing a "checked-out clone". + await _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None); + + Assert.Equal(HeadSha(target), MaterializedTip(target)); + string tip = AppendForgedCommit(source, ("other.txt", "o")); + await _client.PullAsync(target, Token, CancellationToken.None); + Assert.Equal(tip, HeadSha(target)); + } + + + + [Fact] + public async Task Managed_ByTheReleasedBuild_WhichWroteNoIndexAndNoFlag_KeepsUpdatingAfterTheUpgrade() + { + (string source, string target) = await CreateManagedAsync("managed-released-build"); + // What 1.1.0 leaves: the recovery file with its manifest and tip, but no index + // and no managed flag. + File.Delete(Path.Combine(target, ".git", "index")); + UnsetFlag(target, ManagedKey); + string tip = AppendForgedCommit(source, ("other.txt", "o")); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(tip, HeadSha(target)); + Assert.Equal("o", File.ReadAllText(Path.Combine(target, "other.txt"))); + Assert.True(IsFlagSet(target, ManagedKey), "once its file has been read, the repository is known to be managed"); + } + + + + [Fact] + public async Task Managed_ByTheReleasedBuild_IsFlaggedOnAPullThatHasNothingToDo_SoLosingTheFileLaterIsNotADeadEnd() + { + (_, string target) = await CreateManagedAsync("managed-released-build-idle"); + UnsetFlag(target, ManagedKey); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.True(IsFlagSet(target, ManagedKey)); + } + + + + [Fact] + public async Task Managed_ARecoveryFileFromABuildThatRecordedLess_IsCompleted() + { + (string source, string target) = await CreateManagedAsync("managed-old-file"); + // Files written before the manifest existed have neither list nor tip. + JsonNode document = JsonNode.Parse(File.ReadAllText(RecoveryFilePath(target)))!; + document.AsObject().Remove("MaterializedPaths"); + document.AsObject().Remove("MaterializedTip"); + File.WriteAllText(RecoveryFilePath(target), document.ToJsonString()); + string tip = AppendForgedCommit(source, ("other.txt", "o")); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(tip, HeadSha(target)); + Assert.Equal(tip, MaterializedTip(target)); + Assert.Equal("o", File.ReadAllText(Path.Combine(target, "other.txt"))); + } + + + + [Fact] + public async Task Managed_WithATwinThatDiffersOnlyByCaseSkipped_KeepsUpdating() + { + // Skipping one of two names that differ only by case leaves a tree git itself + // reads as changed; that must not stop later updates. + string source = CreateForgedRepo(_root, ("readme.md", "lower"), ("README.md", "upper"), ("keep.txt", "keep")); + string target = await CloneExpectingInvalidPathsAsync(source, "managed-twins"); + using(var repo = new Repository(source)) + { + Assert.NotNull(repo.Head.Tip["README.md"]); + } + await _client.ApplyRecoveryAsync(target, Recovery(skips: ["README.md"]), CancellationToken.None); + Assert.Equal("lower", File.ReadAllText(Path.Combine(target, "readme.md"))); + string tip = AppendForgedCommit(source, ("other.txt", "o")); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(tip, HeadSha(target)); + Assert.Equal("o", File.ReadAllText(Path.Combine(target, "other.txt"))); + Assert.Equal("lower", File.ReadAllText(Path.Combine(target, "readme.md"))); + } + + + + // ---------------------------------------------------------------- what git sees afterwards + + [Fact] + public async Task AfterAnApply_TheIndexDescribesTheCommit_SoOnlyRenamedAndSkippedPathsDiffer() + { + string source = CreateForgedRepo( + _root, ("bad:file.txt", "bad"), ("skip:me.txt", "skipped"), ("keep.txt", "keep"), ("src/app.cs", "code")); + string target = await CloneExpectingInvalidPathsAsync(source, "index"); + + await _client.ApplyRecoveryAsync( + target, Recovery(renames: [("bad:file.txt", "bad_file.txt")], skips: ["skip:me.txt"]), CancellationToken.None); + + using var repo = new Repository(target); + Assert.Equal( + new[] { "bad:file.txt", "keep.txt", "skip:me.txt", "src/app.cs" }, + repo.Index.Select(e => e.Path).Order(StringComparer.Ordinal), + StringComparer.Ordinal); + RepositoryStatus status = repo.RetrieveStatus(new StatusOptions { IncludeUntracked = true }); + // The files that kept their names are clean; nothing reads as "everything deleted". + Assert.Equal(FileStatus.Unaltered, repo.RetrieveStatus("keep.txt")); + Assert.Equal(FileStatus.Unaltered, repo.RetrieveStatus("src/app.cs")); + Assert.Equal( + new[] { "bad:file.txt", "skip:me.txt" }, + status.Missing.Select(e => e.FilePath).Order(StringComparer.Ordinal), + StringComparer.Ordinal); + Assert.Equal("bad_file.txt", Assert.Single(status.Untracked).FilePath); + } + + + + [Fact] + public async Task AttributeFiles_ThatChangeUpstream_ApplyToFilesWrittenBeforeThem_NotTheStaleOneOnDisk() + { + string source = CreateForgedRepo( + _root, + ("!early.txt", "one\ntwo\n"), + (".gitattributes", "*.txt text eol=crlf\n"), + ("bad:file.txt", "one\ntwo\n")); + string target = await CloneExpectingInvalidPathsAsync(source, "attributes-change"); + UseCrLfByDefault(target); + await _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None); + Assert.Equal("one\r\ntwo\r\n", File.ReadAllText(Path.Combine(target, "!early.txt"))); + // Upstream switches the line endings; the old attributes file is still on disk + // when "!early.txt", which sorts before it, is rewritten. + ReplaceEntry(source, remove: ".gitattributes", add: (".gitattributes", "*.txt text eol=lf\n")); + AppendForgedCommit(source, ("!early.txt", "one\ntwo\nthree\n")); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal("one\ntwo\nthree\n", File.ReadAllText(Path.Combine(target, "!early.txt"))); + Assert.Equal("one\ntwo\n", File.ReadAllText(Path.Combine(target, "bad_file.txt"))); + } + + + + // ---------------------------------------------------------------- the newer choice wins + + [Fact] + public async Task AStoredSkip_IsReplacedByALaterRename_OfTheSamePath() + { + string source = CreateForgedRepo(_root, ("bad:file.txt", "bad"), ("keep.txt", "keep")); + string target = await CloneExpectingInvalidPathsAsync(source, "skip-then-rename"); + await _client.ApplyRecoveryAsync(target, Recovery(skips: ["bad:file.txt"]), CancellationToken.None); + Assert.False(File.Exists(Path.Combine(target, "bad_file.txt"))); + + await _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None); + + Assert.Equal("bad", File.ReadAllText(Path.Combine(target, "bad_file.txt"))); + Assert.Contains("bad_file.txt", File.ReadAllText(RecoveryFilePath(target))); + } + + + + [Fact] + public async Task AStoredRename_IsReplacedByALaterSkip_OfTheSamePath() + { + (_, string target) = await CreateManagedAsync("rename-then-skip"); + Assert.True(File.Exists(Path.Combine(target, "bad_file.txt"))); + + await _client.ApplyRecoveryAsync(target, Recovery(skips: ["bad:file.txt"]), CancellationToken.None); + + Assert.False(File.Exists(Path.Combine(target, "bad_file.txt")), "the file written under the old choice is removed"); + Assert.Equal("keep", File.ReadAllText(Path.Combine(target, "keep.txt"))); + Assert.DoesNotContain("bad_file.txt", File.ReadAllText(RecoveryFilePath(target))); + } + + + + // ---------------------------------------------------------------- an empty repository is only empty while nothing is in it + + [Fact] + public async Task EmptyClone_WithAFileOfTheUsersWhereTheRepositoryHasOne_IsRefused_AndTheFileIsKept() + { + string source = Path.Combine(_root, "empty-source-2"); + Repository.Init(source); + string target = Path.Combine(_root, "empty-clone-2"); + await _client.CloneAsync(source, target, Token, null, CancellationToken.None); + File.WriteAllText(Path.Combine(target, "keep.txt"), "mine, put here by hand"); + FirstForgedCommit(source, ("bad:file.txt", "bad"), ("keep.txt", "origin's")); + await PullExpectingInvalidPathsAsync(target); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None)); + + Assert.Contains("'keep.txt' is already in this folder", ex.Message); + Assert.Equal("mine, put here by hand", File.ReadAllText(Path.Combine(target, "keep.txt"))); + Assert.False(File.Exists(Path.Combine(target, "bad_file.txt"))); + Assert.False(IsFlagSet(target, ManagedKey)); + Assert.False(File.Exists(RecoveryFilePath(target))); + + // With the file out of the way the empty clone resolves as before. + File.Delete(Path.Combine(target, "keep.txt")); + await _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None); + Assert.Equal("origin's", File.ReadAllText(Path.Combine(target, "keep.txt"))); + } + + + + [Fact] + public async Task ABranchWithoutCommitsInACheckedOutClone_IsStillACheckedOutClone_AndIsRefused() + { + // What "git checkout --orphan topic" leaves: no commit on the branch, the index + // and the working tree of the branch it was started from. + (string source, string target) = await CloneOrdinaryAsync(("readme.txt", "v1"), ("work.txt", "work")); + using(var repo = new Repository(target)) + { + repo.Refs.UpdateTarget("HEAD", "refs/heads/topic"); + } + File.WriteAllText(Path.Combine(target, "work.txt"), "my local edit"); + using(var repo = new Repository(source)) + { + Commit onTopic = NewRootCommit(repo, [("bad:file.txt", "bad"), ("work.txt", "origin's")]); + repo.Refs.Add("refs/heads/topic", onTopic.Id); + } + Dictionary filesBefore = ReadAllFiles(target); + + // The pull does not take "no commit yet" for "nothing checked out"... + InvalidOperationException pull = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + Assert.Contains("has no commits here yet, but files are staged", pull.Message); + + // ...and neither does resolving. + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None)); + + Assert.Contains("is checked out normally", ex.Message); + Assert.Equal(filesBefore, ReadAllFiles(target)); + Assert.False(IsFlagSet(target, ManagedKey)); + } + + + + [Fact] + public async Task ABranchWithoutCommitsInACheckedOutClone_IsNotCheckedOutOver_ByAPull_EvenWithValidPaths() + { + (string source, string target) = await CloneOrdinaryAsync(("readme.txt", "v1"), ("work.txt", "work")); + using(var repo = new Repository(target)) + { + repo.Refs.UpdateTarget("HEAD", "refs/heads/topic"); + } + File.WriteAllText(Path.Combine(target, "work.txt"), "my local edit"); + using(var repo = new Repository(source)) + { + Commit onTopic = NewRootCommit(repo, [("work.txt", "origin's")]); + repo.Refs.Add("refs/heads/topic", onTopic.Id); + } + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("gclo does not check out over them", ex.Message); + Assert.Equal("my local edit", File.ReadAllText(Path.Combine(target, "work.txt"))); + } + + + + // ---------------------------------------------------------------- what counts as origin's commit + + [Fact] + public async Task Managed_ACommitThatWasPushed_IsNotTakenForOneOfOrigins_WhenOriginIsRewrittenWithoutIt() + { + (string source, string target) = await CreateManagedAsync("managed-pushed"); + string mine = CommitInWorkingTree(target, "mine.txt", "my work"); + // What "git push" records in this clone: the remote-tracking branch now points + // at the pushed commit, and its reflog says a push put it there. + using(var repo = new Repository(target)) + { + repo.Refs.UpdateTarget(repo.Refs[repo.Head.TrackedBranch.CanonicalName], new ObjectId(mine), "update by push"); + } + // A colleague force-pushes a history that does not have the commit. + RewriteHistory(source, ("bad:file.txt", "bad again"), ("keep.txt", "keep"), ("fresh.txt", "fresh")); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("have diverged", ex.Message); + Assert.Equal(mine, HeadSha(target)); + Assert.Equal("my work", File.ReadAllText(Path.Combine(target, "mine.txt"))); + } + + + + [Fact] + public async Task Managed_ABranchThatTracksAnotherLocalBranch_IsNeverMovedOffItsCommits() + { + (_, string target) = await CreateManagedAsync("managed-local-upstream"); + string start = HeadSha(target); + string mine = CommitInWorkingTree(target, "mine.txt", "my work"); + using(var repo = new Repository(target)) + { + // "I committed on the wrong branch": the commit goes to a branch that tracks + // the local one, which is put back and later moves on without it. + string main = repo.Head.FriendlyName; + Branch feature = repo.CreateBranch("feature", repo.Head.Tip); + repo.Branches.Update(feature, b => + { + b.Remote = "."; + b.UpstreamBranch = "refs/heads/" + main; + }); + Signature signature = MakeSignature(); + Commit startCommit = repo.Lookup(start); + Commit elsewhere = repo.ObjectDatabase.CreateCommit( + signature, signature, "main moves on", startCommit.Tree, [startCommit], prettifyMessage: false); + repo.Refs.UpdateTarget(repo.Refs["refs/heads/" + main], elsewhere.Id); + repo.Refs.UpdateTarget("HEAD", "refs/heads/feature"); + } + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("have diverged", ex.Message); + Assert.Equal(mine, HeadSha(target)); + } + + + + [Fact] + public async Task Managed_OriginRewritten_IsFollowed_GoingByAnEarlierFetch_NotOnlyByTheLatest() + { + (string source, string target) = await CreateManagedAsync("managed-rewritten-after-two"); + AppendForgedCommit(source, ("second.txt", "2")); + await _client.PullAsync(target, Token, CancellationToken.None); // the branch is where a fetch put it + string rewritten = RewriteHistory(source, ("bad:file.txt", "bad again"), ("keep.txt", "keep")); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(rewritten, HeadSha(target)); + Assert.False(File.Exists(Path.Combine(target, "second.txt"))); + } + + + + [Fact] + public async Task Managed_OriginRewritten_IsFollowed_WhenTheOlderRecordsOfWhatWasFetchedHaveExpired() + { + (string source, string target) = await CreateManagedAsync("managed-expired-reflog"); + DeleteReflogOfTheTrackedBranch(target); + string rewritten = RewriteHistory(source, ("bad:file.txt", "bad again"), ("keep.txt", "keep")); + + // The fetch itself records where the branch was before it moved it. + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(rewritten, HeadSha(target)); + } + + + + [Fact] + public async Task Managed_WithNoRecordOfWhatWasFetched_StillFastForwards_AndRefusesARewriteWithoutClaimingLocalCommits() + { + (string source, string target) = await CreateManagedAsync("managed-no-reflog"); + using(var repo = new Repository(target)) + { + repo.Config.Set("core.logAllRefUpdates", false, ConfigurationLevel.Local); + } + DeleteReflogOfTheTrackedBranch(target); + + string next = AppendForgedCommit(source, ("more.txt", "m")); + await _client.PullAsync(target, Token, CancellationToken.None); + Assert.Equal(next, HeadSha(target)); // contained in origin's tip: no record is needed + + RewriteHistory(source, ("bad:file.txt", "bad again"), ("keep.txt", "keep")); + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("have diverged", ex.Message); + Assert.Contains("the record of what was fetched is gone", ex.Message); + Assert.DoesNotContain("has local commits", ex.Message); + Assert.Equal(next, HeadSha(target)); + } + + + + [Fact] + public async Task NeverCheckedOut_WithACommitMadeByHand_IsNotMovedOffIt_ByAPull() + { + string source = CreateForgedRepo(_root, ("bad:file.txt", "bad"), ("keep.txt", "keep")); + string target = await CloneExpectingInvalidPathsAsync(source, "pending-with-commit"); + string mine; + using(var repo = new Repository(target)) + { + Signature signature = MakeSignature(); + Commit commit = repo.ObjectDatabase.CreateCommit( + signature, signature, "made by hand", repo.Head.Tip.Tree, [repo.Head.Tip], prettifyMessage: false); + repo.Refs.UpdateTarget(repo.Refs.Head.ResolveToDirectReference(), commit.Id); + mine = commit.Sha; + } + + await PullExpectingInvalidPathsAsync(target); + + Assert.Equal(mine, HeadSha(target)); + } + + + + // ---------------------------------------------------------------- a pull that has nothing to write, writes nothing + + [Fact] + public async Task Managed_ACommitMadeOnTop_WithOriginUnchanged_LeavesEveryOtherFileAsItIs() + { + (_, string target) = await CreateManagedAsync("managed-commit-and-edit"); + File.WriteAllText(Path.Combine(target, "keep.txt"), "edited, not committed"); + string mine = CommitInWorkingTree(target, "mine.txt", "my work"); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(mine, HeadSha(target)); + Assert.Equal("edited, not committed", File.ReadAllText(Path.Combine(target, "keep.txt"))); + Assert.Equal("bad", File.ReadAllText(Path.Combine(target, "bad_file.txt"))); + Assert.Equal("my work", File.ReadAllText(Path.Combine(target, "mine.txt"))); + } + + + + [Fact] + public async Task Managed_WhoseBranchWasMovedBackByHand_IsBroughtToOriginsTipAgain() + { + (string source, string target) = await CreateManagedAsync("managed-moved-back"); + string first = HeadSha(target); + string tip = AppendForgedCommit(source, ("second.txt", "2")); + await _client.PullAsync(target, Token, CancellationToken.None); + using(var repo = new Repository(target)) + { + repo.Refs.UpdateTarget(repo.Refs.Head.ResolveToDirectReference(), new ObjectId(first)); // "git reset --soft HEAD~1" + } + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(tip, HeadSha(target)); + Assert.Equal(tip, MaterializedTip(target)); + } + + + + [Fact] + public async Task Managed_ARecordedTipThatIsNotACommit_IsNotTrusted_AndTheTreeIsWrittenAgain() + { + (_, string target) = await CreateManagedAsync("managed-odd-tip"); + File.WriteAllText(Path.Combine(target, "keep.txt"), "changed on disk"); + string file = RecoveryFilePath(target); + JsonNode document = JsonNode.Parse(File.ReadAllText(file))!; + document["MaterializedTip"] = "not a commit id"; + File.WriteAllText(file, document.ToJsonString()); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal("keep", File.ReadAllText(Path.Combine(target, "keep.txt"))); + Assert.Equal(HeadSha(target), MaterializedTip(target)); + } + + + + // ---------------------------------------------------------------- found before anything changes + + [Fact] + public async Task Managed_AnotherGitProcessAtWork_IsFoundBeforeAnythingChanges() + { + (string source, string target) = await CreateManagedAsync("managed-index-lock"); + string before = HeadSha(target); + string tip = AppendForgedCommit(source, ("more.txt", "m")); + string lockFile = Path.Combine(target, ".git", "index.lock"); + File.WriteAllText(lockFile, ""); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("its index is locked", ex.Message); + Assert.Equal(before, HeadSha(target)); + Assert.Equal(before, MaterializedTip(target)); + + File.Delete(lockFile); + await _client.PullAsync(target, Token, CancellationToken.None); + Assert.Equal(tip, HeadSha(target)); + } + + + + [Fact] + public async Task Managed_AFolderWithSomeonesFilesWhereTheRepositoryNowHasAFile_IsFoundBeforeAnythingChanges() + { + (string source, string target) = await CreateManagedAsync("managed-folder-in-the-way"); + string before = HeadSha(target); + Directory.CreateDirectory(Path.Combine(target, "notes", "deeper")); + File.WriteAllText(Path.Combine(target, "notes", "deeper", "mine.txt"), "mine"); + AppendForgedCommit(source, ("notes", "origin's file")); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("'notes' is a folder with files in it ('notes/deeper/mine.txt' is one)", ex.Message); + Assert.Equal(before, HeadSha(target)); + Assert.Equal(before, MaterializedTip(target)); + Assert.Equal("mine", File.ReadAllText(Path.Combine(target, "notes", "deeper", "mine.txt"))); + } + + + + [Fact] + public async Task Managed_AFolderOfItsOwnThatUpstreamTurnedIntoAFile_IsReplaced() + { + (string source, string target) = await CreateManagedAsync("managed-own-folder-to-file"); + AppendForgedCommit(source, ("notes/a.txt", "a"), ("notes/deeper/b.txt", "b")); + await _client.PullAsync(target, Token, CancellationToken.None); + using(var repo = new Repository(source)) + { + Commit head = repo.Head.Tip; + TreeDefinition definition = TreeDefinition.From(head.Tree).Remove("notes"); + definition.Add("notes", NewBlob(repo, "now a file"), Mode.NonExecutableFile); + Signature signature = MakeSignature(); + Commit commit = repo.ObjectDatabase.CreateCommit( + signature, signature, "folder becomes file", repo.ObjectDatabase.CreateTree(definition), [head], prettifyMessage: false); + repo.Refs.UpdateTarget(repo.Refs.Head.ResolveToDirectReference(), commit.Id); + } + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal("now a file", File.ReadAllText(Path.Combine(target, "notes"))); + } + + + + [Fact] + public async Task Managed_ALinkOnTheWayToAFileThatLeftTheTree_IsFoundBeforeAnythingChanges() + { + (string source, string target) = await CreateManagedAsync("managed-link-to-stale"); + AppendForgedCommit(source, ("sub/old.txt", "old")); + await _client.PullAsync(target, Token, CancellationToken.None); + string before = HeadSha(target); + // The folder gclo wrote into is replaced by a link to a folder elsewhere that + // happens to have a file of the same name. + string elsewhere = Path.Combine(_root, "elsewhere-stale"); + Directory.CreateDirectory(elsewhere); + File.WriteAllText(Path.Combine(elsewhere, "old.txt"), "someone else's"); + Directory.Delete(Path.Combine(target, "sub"), recursive: true); + string link = Path.Combine(target, "sub"); + WorkingTreeTests.CreateDirectoryLink(link, elsewhere); + try + { + ReplaceEntry(source, remove: "sub/old.txt", add: ("other.txt", "o")); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains("is a link to another location", ex.Message); + Assert.Equal(before, HeadSha(target)); + Assert.Equal(before, MaterializedTip(target)); + Assert.Equal("someone else's", File.ReadAllText(Path.Combine(elsewhere, "old.txt"))); + } + finally + { + WorkingTreeTests.RemoveLink(link); + } + } + + + + [Theory] + [InlineData("docs", "docs/a.txt")] + [InlineData("Docs", "docs/a.txt")] + public async Task Managed_AFileOfItsOwnThatUpstreamTurnedIntoAFolder_IsNotTakenForSomeoneElses(string file, string inFolder) + { + if(!string.Equals(file, "docs", StringComparison.Ordinal) && !OperatingSystem.IsWindows()) + { + return; // only a file system that ignores case sees 'Docs' in the way of 'docs' + } + (string source, string target) = await CreateManagedAsync("managed-own-file-to-folder-" + file); + AppendForgedCommit(source, (file, "a file")); + await _client.PullAsync(target, Token, CancellationToken.None); + ReplaceEntry(source, remove: file, add: (inFolder, "in a folder")); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal("in a folder", File.ReadAllText(Path.Combine(target, "docs", "a.txt"))); + } + + + + [Fact] + public async Task Managed_AFileThatLeftTheTree_WhereAFolderOfTheUsersNowSits_LeavesTheFolder_AndTheSyncSucceeds() + { + (string source, string target) = await CreateManagedAsync("managed-stale-is-a-folder"); + AppendForgedCommit(source, ("notes", "a file")); + await _client.PullAsync(target, Token, CancellationToken.None); + File.Delete(Path.Combine(target, "notes")); + Directory.CreateDirectory(Path.Combine(target, "notes")); + File.WriteAllText(Path.Combine(target, "notes", "mine.txt"), "mine"); + ReplaceEntry(source, remove: "notes", add: ("other.txt", "o")); + string tip = HeadSha(source); + + await _client.PullAsync(target, Token, CancellationToken.None); + await _client.PullAsync(target, Token, CancellationToken.None); // and it stays fine + + Assert.Equal(tip, HeadSha(target)); + Assert.Equal(tip, MaterializedTip(target)); + Assert.Equal("mine", File.ReadAllText(Path.Combine(target, "notes", "mine.txt"))); + Assert.Equal("o", File.ReadAllText(Path.Combine(target, "other.txt"))); + } + + + + // ---------------------------------------------------------------- what an interrupted apply had and was about to have + + [Fact] + public async Task Managed_AFileThatCouldNotBeRemovedWhenThePullWasInterrupted_IsRemovedByTheNext() + { + (string source, string target) = await CreateManagedAsync("managed-stale-held"); + string tip = HeadSha(source); + ReplaceEntry(source, remove: "keep.txt", add: ("zz-new.txt", "new")); + tip = HeadSha(source); + // "keep.txt" left the tree and is held open: removing it is what fails. + await InterruptAsync(target, "keep.txt", () => _client.PullAsync(target, Token, CancellationToken.None)); + Assert.Null(MaterializedTip(target)); + Assert.True(File.Exists(Path.Combine(target, "keep.txt"))); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(tip, MaterializedTip(target)); + Assert.False(File.Exists(Path.Combine(target, "keep.txt")), "it is still known to be gclo's, and it left the tree"); + } + + + + [Fact] + public async Task Managed_AFileWrittenBeforeThePullWasInterrupted_IsRemovedByTheNext_WhenItHasLeftTheTreeByThen() + { + (string source, string target) = await CreateManagedAsync("managed-new-then-gone"); + AppendForgedCommit(source, ("aa-new.txt", "new"), ("keep.txt", "changed")); + // "aa-new.txt" is written, then "keep.txt" (held open) stops the pull. + await InterruptAsync(target, "keep.txt", () => _client.PullAsync(target, Token, CancellationToken.None)); + Assert.True(File.Exists(Path.Combine(target, "aa-new.txt"))); + ReplaceEntry(source, remove: "aa-new.txt", add: ("other.txt", "o")); + + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.False(File.Exists(Path.Combine(target, "aa-new.txt")), "written by the interrupted pull, so gclo's to remove"); + Assert.Equal("changed", File.ReadAllText(Path.Combine(target, "keep.txt"))); + Assert.Equal("o", File.ReadAllText(Path.Combine(target, "other.txt"))); + } + + + + // ---------------------------------------------------------------- written by the released build, recovery file lost + + [Fact] + public async Task ResolvedByTheReleasedBuild_WhoseRecoveryFileWasLost_CanBeResolvedAgain_AsTheMessageAdvises() + { + (string source, string target) = await CreateManagedAsync("released-build-file-lost"); + // What 1.1.0 leaves (no index, no managed flag), with its recovery file gone + // before this build ever read it. + File.Delete(Path.Combine(target, ".git", "index")); + UnsetFlag(target, ManagedKey); + File.Delete(RecoveryFilePath(target)); + + // The sync reports the paths again, which is what offers Resolve in the app + // and what --sanitize-paths acts on... + await PullExpectingInvalidPathsAsync(target); + // ...and resolving rebuilds the file. + await _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None); + + Assert.Equal(HeadSha(target), MaterializedTip(target)); + Assert.True(IsFlagSet(target, ManagedKey)); + string tip = AppendForgedCommit(source, ("other.txt", "o")); + await _client.PullAsync(target, Token, CancellationToken.None); + Assert.Equal(tip, HeadSha(target)); + } + + + + [Fact] + public async Task ARefusedApply_LeavesTheRepositorysConfigurationAsItWas() + { + (string source, string target) = await CloneOrdinaryAsync(("readme.txt", "v1")); + AppendForgedCommit(source, ("bad:file.txt", "bad")); + await PullExpectingInvalidPathsAsync(target); + using(var repo = new Repository(target)) + { + repo.Config.Unset("core.longpaths", ConfigurationLevel.Local); // the pull set it; the refused apply must not + } + string config = Path.Combine(target, ".git", "config"); + string before = File.ReadAllText(config); + + await Assert.ThrowsAsync( + () => _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None)); + + Assert.Equal(before, File.ReadAllText(config)); + } + + + + // ---------------------------------------------------------------- what the released build can leave behind + + /// + /// Puts a managed repository into the state a pull of the released build leaves + /// when it fails after its first step: the branch is at origin's new commit, the + /// tree and the recovery file are still at the previous one, and there is neither + /// an index nor the managed flag. + /// + private static void LeaveAsAFailedPullOfTheReleasedBuild(string target) + { + using var repo = new Repository(target); + Commands.Fetch(repo, "origin", [], null, null); + repo.Refs.UpdateTarget(repo.Refs.Head.ResolveToDirectReference(), repo.Head.TrackedBranch.Tip.Id); + repo.Config.Unset(ManagedKey, ConfigurationLevel.Local); + File.Delete(Path.Combine(repo.Info.Path, "index")); + } + + + + [Fact] + public async Task AfterAPullOfTheReleasedBuildThatWasInterrupted_TheNextPullWritesTheTree() + { + (string source, string target) = await CreateManagedAsync("released-build-interrupted"); + string first = HeadSha(target); + string tip = AppendForgedCommit(source, ("new.txt", "new"), ("keep.txt", "changed")); + LeaveAsAFailedPullOfTheReleasedBuild(target); + Assert.Equal(tip, HeadSha(target)); + Assert.Equal(first, MaterializedTip(target)); + + // The branch being ahead of what was written is origin's doing here, not a + // commit made in the folder: the tree still has to be written. + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal(tip, MaterializedTip(target)); + Assert.Equal("new", File.ReadAllText(Path.Combine(target, "new.txt"))); + Assert.Equal("changed", File.ReadAllText(Path.Combine(target, "keep.txt"))); + } + + + + [Fact] + public async Task AfterAPullOfTheReleasedBuildThatFailedOnANewInvalidPath_TheNextPullReportsItAgain() + { + (string source, string target) = await CreateManagedAsync("released-build-new-invalid"); + AppendForgedCommit(source, ("another:bad.txt", "bad too")); + LeaveAsAFailedPullOfTheReleasedBuild(target); + + // Reporting nothing would take the Resolve link away and leave the tree a commit behind. + InvalidRepositoryPathsException ex = await Assert.ThrowsAsync( + () => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Contains(ex.Paths, p => string.Equals(p.RepoPath, "another:bad.txt", StringComparison.Ordinal)); + } + + + + // ---------------------------------------------------------------- a tree that only looks like one gclo wrote + + [Fact] + public async Task AClonedRepositoryWhoseIndexWasDeleted_KeepsItsEditedFile_WhenResolved() + { + (string source, string target) = await CloneOrdinaryAsync(("readme.txt", "v1"), ("work.txt", "work")); + File.WriteAllText(Path.Combine(target, "work.txt"), "my local edit"); + File.WriteAllText(Path.Combine(target, "untracked.txt"), "mine"); + File.Delete(Path.Combine(target, ".git", "index")); + AppendForgedCommit(source, ("bad:file.txt", "bad"), ("readme.txt", "v2")); + await PullExpectingInvalidPathsAsync(target); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None)); + + Assert.Contains("'work.txt' is already in this folder", ex.Message); + Assert.Equal("my local edit", File.ReadAllText(Path.Combine(target, "work.txt"))); + Assert.Equal("mine", File.ReadAllText(Path.Combine(target, "untracked.txt"))); + Assert.Equal("v1", File.ReadAllText(Path.Combine(target, "readme.txt"))); + Assert.False(IsFlagSet(target, ManagedKey)); + } + + + + [Fact] + public async Task AClonedRepositoryWhoseIndexWasDeleted_AndThatHasNothingToRename_StaysAnOrdinaryClone() + { + (_, string target) = await CloneOrdinaryAsync(("readme.txt", "v1"), ("work.txt", "work")); + File.Delete(Path.Combine(target, ".git", "index")); + + await _client.PullAsync(target, Token, CancellationToken.None); // nothing new on origin + + Assert.False(IsFlagSet(target, ManagedKey), "a tree with nothing to rename is not one path recovery wrote"); + Assert.False(File.Exists(RecoveryFilePath(target))); + + // And it is pulled like any clone: with an edit in it, and nothing new on + // origin, the sync goes through and the edit is still there. + File.WriteAllText(Path.Combine(target, "work.txt"), "my local edit"); + await _client.PullAsync(target, Token, CancellationToken.None); + + Assert.Equal("my local edit", File.ReadAllText(Path.Combine(target, "work.txt"))); + Assert.False(IsFlagSet(target, ManagedKey)); + Assert.False(File.Exists(RecoveryFilePath(target))); + } + + + + [Fact] + public async Task ARepositoryClonedByGitWithoutACheckout_IsNotTakenForOneGcloWrote() + { + string source = CreateForgedRepo(_root, ("readme.txt", "v1")); + string target = Path.Combine(_root, "cloned-by-git"); + Repository.Clone(source, target, new CloneOptions { Checkout = false }); + File.WriteAllText(Path.Combine(target, "readme.txt"), "put here by hand"); + AppendForgedCommit(source, ("bad:file.txt", "bad")); + await PullExpectingInvalidPathsAsync(target); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None)); + + Assert.Contains("is checked out normally", ex.Message); + Assert.Equal("put here by hand", File.ReadAllText(Path.Combine(target, "readme.txt"))); + } + + + + [Fact] + public async Task NeverCheckedOut_WithAFileOfTheUsersWhereTheRepositoryHasOne_IsRefused_AndTheFileIsKept() + { + string source = CreateForgedRepo(_root, ("bad:file.txt", "bad"), ("keep.txt", "keep")); + string target = await CloneExpectingInvalidPathsAsync(source, "pending-with-a-file"); + File.WriteAllText(Path.Combine(target, "keep.txt"), "mine, put here by hand"); + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None)); + + Assert.Contains("'keep.txt' is already in this folder", ex.Message); + Assert.Equal("mine, put here by hand", File.ReadAllText(Path.Combine(target, "keep.txt"))); + Assert.False(File.Exists(Path.Combine(target, "bad_file.txt"))); + } + + + + [Fact] + public async Task NeverCheckedOut_WithAFileAnUnfinishedCheckoutLeft_IsResolved() + { + // What a checkout that was cut short leaves: a file with exactly the commit's content. + string source = CreateForgedRepo(_root, ("bad:file.txt", "bad"), ("keep.txt", "keep")); + string target = await CloneExpectingInvalidPathsAsync(source, "pending-with-its-own-file"); + File.WriteAllText(Path.Combine(target, "keep.txt"), "keep"); + + await _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None); + + Assert.Equal("keep", File.ReadAllText(Path.Combine(target, "keep.txt"))); + Assert.Equal("bad", File.ReadAllText(Path.Combine(target, "bad_file.txt"))); + } + + + + [Fact] + public async Task Managed_ACommitPutOnTheTrackingBranchByHand_IsNotTakenForOneOfOrigins() + { + (string source, string target) = await CreateManagedAsync("managed-hand-updated-ref"); + string mine = CommitInWorkingTree(target, "mine.txt", "my work"); + using(var repo = new Repository(target)) + { + // Not a fetch: whatever wrote this entry, it does not say origin had the commit. + repo.Refs.UpdateTarget(repo.Refs[repo.Head.TrackedBranch.CanonicalName], new ObjectId(mine), ""); + } + RewriteHistory(source, ("bad:file.txt", "bad again"), ("keep.txt", "keep")); + + await Assert.ThrowsAsync(() => _client.PullAsync(target, Token, CancellationToken.None)); + + Assert.Equal(mine, HeadSha(target)); + } + + + + // ---------------------------------------------------------------- helpers + + /// Removes what this clone remembers of where origin's branch was, as expiry does. + private static void DeleteReflogOfTheTrackedBranch(string workdir) + { + using var repo = new Repository(workdir); + string log = Path.Combine(repo.Info.Path, "logs", repo.Head.TrackedBranch.CanonicalName.Replace('/', Path.DirectorySeparatorChar)); + File.Delete(log); + } + + + + /// A forged source with valid paths, and an ordinary (fully checked-out) clone of it. + private async Task<(string Source, string Target)> CloneOrdinaryAsync(params (string Name, string Content)[] entries) + { + string source = CreateForgedRepo(_root, entries); + string target = Path.Combine(_root, "clone-" + Guid.NewGuid().ToString("N")[..8]); + await _client.CloneAsync(source, target, Token, null, CancellationToken.None); + return (source, target); + } + + + + /// A source with one invalid path and "keep.txt", and a clone of it resolved by renaming that path. + private async Task<(string Source, string Target)> CreateManagedAsync(string name) + { + string source = CreateForgedRepo(_root, ("bad:file.txt", "bad"), ("keep.txt", "keep")); + string target = await CloneExpectingInvalidPathsAsync(source, name); + await _client.ApplyRecoveryAsync(target, Recovery(renames: [("bad:file.txt", "bad_file.txt")]), CancellationToken.None); + return (source, target); + } + + + + private async Task CloneExpectingInvalidPathsAsync(string source, string name) + { + string target = Path.Combine(_root, name); + await Assert.ThrowsAsync( + () => _client.CloneAsync(source, target, Token, null, CancellationToken.None)); + return target; + } + + + + private Task PullExpectingInvalidPathsAsync(string target) + => Assert.ThrowsAsync(() => _client.PullAsync(target, Token, CancellationToken.None)); + + + + /// + /// Runs while is held open + /// by a reader that lets others read it but not replace it: everything that is + /// checked before the tree is rewritten passes, and the rewrite itself fails when + /// it reaches that file, after the point of no return. + /// + private static async Task InterruptAsync(string target, string fileName, Func operation) + { + using var held = new FileStream(Path.Combine(target, fileName), FileMode.Open, FileAccess.Read, FileShare.Read); + await Assert.ThrowsAnyAsync(operation); + } + + + + private static PathRecovery Recovery((string From, string To)[]? renames = null, string[]? skips = null) + => new( + (renames ?? []).ToDictionary(r => r.From, r => r.To, StringComparer.Ordinal), + new HashSet(skips ?? [], StringComparer.Ordinal)); + + + + /// Adds a commit that removes one entry and writes (or overwrites) another. + private static void ReplaceEntry(string workdir, string remove, (string Name, string Content) add) + { + using var repo = new Repository(workdir); + Commit head = repo.Head.Tip; + TreeDefinition definition = TreeDefinition.From(head.Tree).Remove(remove); + definition.Add(add.Name, NewBlob(repo, add.Content), Mode.NonExecutableFile); + Signature signature = MakeSignature(); + Commit commit = repo.ObjectDatabase.CreateCommit( + signature, signature, "forged change", repo.ObjectDatabase.CreateTree(definition), [head], prettifyMessage: false); + repo.Refs.UpdateTarget(repo.Refs.Head.ResolveToDirectReference(), commit.Id); + } + + + + /// The first commit of a repository that has none yet; returns its SHA. + private static string FirstForgedCommit(string workdir, params (string Name, string Content)[] entries) + { + using var repo = new Repository(workdir); + Commit commit = NewRootCommit(repo, entries); + repo.Refs.Add(repo.Refs.Head.TargetIdentifier, commit.Id); + return commit.Sha; + } + + + + /// Replaces the branch's history with a single new root commit, as a force-push does; returns its SHA. + private static string RewriteHistory(string workdir, params (string Name, string Content)[] entries) + { + using var repo = new Repository(workdir); + Commit commit = NewRootCommit(repo, entries); + repo.Refs.UpdateTarget(repo.Refs.Head.ResolveToDirectReference(), commit.Id); + return commit.Sha; + } + + + + private static Commit NewRootCommit(Repository repo, (string Name, string Content)[] entries) + { + var definition = new TreeDefinition(); + foreach((string name, string content) in entries) + { + definition.Add(name, NewBlob(repo, content), Mode.NonExecutableFile); + } + Signature signature = MakeSignature(); + return repo.ObjectDatabase.CreateCommit( + signature, signature, "forged root", repo.ObjectDatabase.CreateTree(definition), [], prettifyMessage: false); + } + + + + private static Blob NewBlob(Repository repo, string content) + => repo.ObjectDatabase.CreateBlob(new MemoryStream(Encoding.UTF8.GetBytes(content))); + + + + /// An ordinary commit made in the working tree, the way a person would. + private static string CommitInWorkingTree(string workdir, string fileName, string content) + { + using var repo = new Repository(workdir); + File.WriteAllText(Path.Combine(workdir, fileName), content); + Commands.Stage(repo, fileName); + Signature signature = MakeSignature(); + return repo.Commit("local work", signature, signature).Sha; + } + + + + /// Every file of the working tree (not of .git) with its content. + private static Dictionary ReadAllFiles(string workdir) + => Directory.EnumerateFiles(workdir, "*", SearchOption.AllDirectories) + .Select(f => Path.GetRelativePath(workdir, f)) + .Where(f => !f.StartsWith(".git" + Path.DirectorySeparatorChar, StringComparison.Ordinal)) + .ToDictionary(f => f, f => File.ReadAllText(Path.Combine(workdir, f)), StringComparer.Ordinal); + + + + /// Without attributes, this setting writes text files with CRLF line endings. + private static void UseCrLfByDefault(string repoPath) + { + using var repo = new Repository(repoPath); + repo.Config.Set("core.autocrlf", "true", ConfigurationLevel.Local); + } + + + + private static bool IsFlagSet(string repoPath, string key) + { + using var repo = new Repository(repoPath); + return repo.Config.Get(key, ConfigurationLevel.Local)?.Value == true; + } + + + + private static void UnsetFlag(string repoPath, string key) + { + using var repo = new Repository(repoPath); + repo.Config.Unset(key, ConfigurationLevel.Local); + } + + + + private static string RecoveryFilePath(string repoPath) + { + using var repo = new Repository(repoPath); + return Path.Combine(repo.Info.Path, "gclo-recovery.json"); + } + + + + /// The commit the recovery file says is on disk; null while an apply is unfinished. + private static string? MaterializedTip(string repoPath) + => JsonNode.Parse(File.ReadAllText(RecoveryFilePath(repoPath)))!["MaterializedTip"]?.GetValue(); +} diff --git a/gclo.Engine.Tests/WorkingTreeTests.cs b/gclo.Engine.Tests/WorkingTreeTests.cs index fd141b7..4ea2a44 100644 --- a/gclo.Engine.Tests/WorkingTreeTests.cs +++ b/gclo.Engine.Tests/WorkingTreeTests.cs @@ -329,6 +329,77 @@ public sealed class WorkingTreeTests : IDisposable + [Fact] + public void Create_WhereAFolderWithoutFilesSits_ReplacesItWithTheFile() + { + // Upstream turned a folder into a file; its files left with the old tree, and + // empty folders are nobody's. + Directory.CreateDirectory(Path.Combine(_workdir, "docs", "empty", "deeper")); + + using(FileStream file = WorkingTree.Create(_workdir, "docs")) + { + file.Write("now a file"u8); + } + + Assert.Equal("now a file", File.ReadAllText(Path.Combine(_workdir, "docs"))); + } + + + + [Fact] + public void Create_WhereAFolderHoldingOnlyALinkSits_ReplacesIt_AndLeavesTheLinksTargetAlone() + { + File.WriteAllText(Path.Combine(_outside, "kept.txt"), "kept"); + Directory.CreateDirectory(Path.Combine(_workdir, "docs", "inner")); + LinkDirectory(Path.Combine(_workdir, "docs", "inner", "linked"), _outside); + + using(FileStream file = WorkingTree.Create(_workdir, "docs")) + { + file.Write("now a file"u8); + } + + Assert.Equal("now a file", File.ReadAllText(Path.Combine(_workdir, "docs"))); + Assert.Equal("kept", File.ReadAllText(Path.Combine(_outside, "kept.txt"))); + } + + + + [Fact] + public void Verify_ChecksWhatCreateChecks_WithoutTouchingTheDisk() + { + var verified = new HashSet(StringComparer.Ordinal); + Directory.CreateDirectory(Path.Combine(_workdir, "src")); + LinkDirectory(Path.Combine(_workdir, "vendor"), _outside); + + WorkingTree.Verify(_workdir, "src/app/main.cs", verified); + WorkingTree.Verify(_workdir, "readme.txt"); + Assert.Throws(() => WorkingTree.Verify(_workdir, "../outside/x.txt", verified)); + Assert.Throws(() => WorkingTree.Verify(_workdir, ".git/config", verified)); + Assert.Throws(() => WorkingTree.Verify(_workdir, "vendor/lib.txt", verified)); + + Assert.False(Directory.Exists(Path.Combine(_workdir, "src", "app")), "nothing is created by a check"); + Assert.False(File.Exists(Path.Combine(_workdir, "readme.txt"))); + Assert.Contains(Path.Combine(_workdir, "src"), verified); + Assert.Empty(Directory.EnumerateFileSystemEntries(_outside)); + } + + + + [Fact] + public void Create_WhereAFolderWithFilesSits_IsRefused_AndTheFilesSurvive() + { + string mine = Path.Combine(_workdir, "docs", "deeper", "mine.txt"); + Directory.CreateDirectory(Path.GetDirectoryName(mine)!); + File.WriteAllText(mine, "not gclo's to remove"); + + InvalidOperationException ex = Assert.Throws(() => WorkingTree.Create(_workdir, "docs")); + + Assert.Contains("is a folder with files in it", ex.Message); + Assert.Equal("not gclo's to remove", File.ReadAllText(mine)); + } + + + [Fact] public void Create_OutsideOrInGit_IsRefused_BeforeAnythingIsTouched() { @@ -378,6 +449,74 @@ public sealed class WorkingTreeTests : IDisposable + [Fact] + public void Delete_WhereAFolderNowSits_LeavesIt_WithEverythingInIt() + { + Directory.CreateDirectory(Path.Combine(_workdir, "notes", "empty")); + File.WriteAllText(Path.Combine(_workdir, "notes", "mine.txt"), "mine"); + + WorkingTree.Delete(_workdir, "notes"); + + Assert.Equal("mine", File.ReadAllText(Path.Combine(_workdir, "notes", "mine.txt"))); + Assert.True(Directory.Exists(Path.Combine(_workdir, "notes", "empty"))); + } + + + + [Fact] + public void Create_WhereAFolderWithFilesSits_IsRefused_BeforeAnythingInItIsTouched() + { + Directory.CreateDirectory(Path.Combine(_workdir, "docs", "aaa-empty")); + Directory.CreateDirectory(Path.Combine(_workdir, "docs", "zzz")); + File.WriteAllText(Path.Combine(_workdir, "docs", "zzz", "mine.txt"), "mine"); + LinkDirectory(Path.Combine(_workdir, "docs", "bbb-link"), _outside); + + Assert.Throws(() => WorkingTree.Create(_workdir, "docs").Dispose()); + + // Not even the empty folder or the link that come before the file is found. + Assert.True(Directory.Exists(Path.Combine(_workdir, "docs", "aaa-empty"))); + Assert.True(Directory.Exists(Path.Combine(_workdir, "docs", "bbb-link"))); + Assert.Equal("mine", File.ReadAllText(Path.Combine(_workdir, "docs", "zzz", "mine.txt"))); + } + + + + [Fact] + public void Create_WhereAReadOnlyEmptyFolderSits_ReplacesIt() + { + var folder = new DirectoryInfo(Path.Combine(_workdir, "docs")); + folder.Create(); + folder.Attributes |= FileAttributes.ReadOnly; + + using(FileStream file = WorkingTree.Create(_workdir, "docs")) + { + file.Write("now a file"u8); + } + + Assert.Equal("now a file", File.ReadAllText(Path.Combine(_workdir, "docs"))); + } + + + + [Fact] + public void FilesUnder_ListsTheFilesOfARealFolder_NotThoseBehindALink() + { + Directory.CreateDirectory(Path.Combine(_workdir, "docs", "deeper")); + File.WriteAllText(Path.Combine(_workdir, "docs", "a.txt"), "a"); + File.WriteAllText(Path.Combine(_workdir, "docs", "deeper", "b.txt"), "b"); + File.WriteAllText(Path.Combine(_outside, "outside.txt"), "o"); + LinkDirectory(Path.Combine(_workdir, "docs", "linked"), _outside); + LinkDirectory(Path.Combine(_workdir, "whole-link"), _outside); + File.WriteAllText(Path.Combine(_workdir, "plain.txt"), "p"); + + Assert.Equal(["docs/a.txt", "docs/deeper/b.txt"], WorkingTree.FilesUnder(_workdir, "docs").Order(StringComparer.Ordinal), StringComparer.Ordinal); + Assert.Empty(WorkingTree.FilesUnder(_workdir, "whole-link")); + Assert.Empty(WorkingTree.FilesUnder(_workdir, "plain.txt")); + Assert.Empty(WorkingTree.FilesUnder(_workdir, "not/there")); + } + + + [Fact] public void Delete_MissingFile_IsFine() { diff --git a/gclo.Engine/IGitClient.cs b/gclo.Engine/IGitClient.cs index cd00349..4a687d2 100644 --- a/gclo.Engine/IGitClient.cs +++ b/gclo.Engine/IGitClient.cs @@ -41,15 +41,24 @@ public interface IGitClient /// - /// Materializes the working tree of a fetched-but-never-checked-out repository - /// (one whose clone failed with ) - /// by writing HEAD's blobs to disk with 's renames - /// and skips applied. Any previously persisted recovery is merged in first, - /// with winning on conflicts, so a repository that - /// gains new invalid paths can be fixed without restating earlier choices. - /// Throws when the effective - /// (post-mapping) path set is still invalid or collides; on success the merged - /// recovery is persisted so later pulls re-apply it to new commits. + /// Writes the working tree of a repository whose paths Windows cannot create, with + /// 's renames and skips applied. It applies to a + /// repository that was fetched but never checked out (its clone failed with + /// , or it was empty) and to one an + /// earlier call already wrote this way, for which it also brings in origin's new + /// commits. Any previously persisted recovery is merged in first, the newer choice + /// for a path winning, so a repository that gains new invalid paths can be fixed + /// without restating earlier choices. /// + /// + /// The tree is written by hand, with force, so this refuses + /// () a clone that was checked out + /// normally: that tree may hold someone's work. It moves a branch only to the + /// tip of the branch it tracks, and off the current commit only when that is + /// contained in that tip or was itself once origin's tip. Throws + /// when the effective path set is + /// still invalid or collides, before anything is changed; on success the merged + /// recovery is persisted so later pulls re-apply it to new commits. + /// Task ApplyRecoveryAsync(string path, PathRecovery recovery, CancellationToken cancellationToken); } diff --git a/gclo.Engine/LibGit2GitClient.cs b/gclo.Engine/LibGit2GitClient.cs index da5e047..adac830 100644 --- a/gclo.Engine/LibGit2GitClient.cs +++ b/gclo.Engine/LibGit2GitClient.cs @@ -47,11 +47,25 @@ public sealed class LibGit2GitClient : IGitClient /// private const string CheckoutPendingKey = "gclo.checkoutpending"; + /// + /// Local git config flag meaning "gclo writes this repository's working tree + /// itself, through a path recovery". It outlives the recovery file, so a + /// repository whose file was deleted (as the message for a corrupt one advises) + /// is still known to be recovery-managed and can be resolved again, instead of + /// being taken for a clone someone checked out (#147). Set only once a recovery + /// has been read or written successfully, never because a file merely exists. + /// + private const string RecoveryManagedKey = "gclo.recoverymanaged"; + /// File name (under .git) of the persisted . private const string RecoveryFileName = "gclo-recovery.json"; private static readonly JsonSerializerOptions s_recoveryJsonOptions = new() { WriteIndented = true }; + /// No renames, no skips: every entry of a tree under its own path. + private static readonly PathRecovery s_noRecovery = new( + new Dictionary(StringComparer.Ordinal), new HashSet(StringComparer.Ordinal)); + /// /// Name of the remote that exists only for the duration of one fetch, when a /// folder's origin names the listed repository but spells its URL differently @@ -262,22 +276,16 @@ public sealed class LibGit2GitClient : IGitClient // recovery). Both take a dedicated path — their working tree is materialized // from the tip, never by a merge checkout that trusts the existing tree. // Every other repository takes the normal fetch + fast-forward pull below. - string recoveryFile = GetRecoveryFilePath(repo); - if(File.Exists(recoveryFile)) + if(File.Exists(GetRecoveryFilePath(repo)) || IsRecoveryManaged(repo) + || (!IsCheckoutPending(repo) && WasWrittenWithoutAnIndex(repo) && HasPathsWindowsCannotCreate(repo))) { - // A recovery-managed repo stays recovery-managed: re-materialize the new - // tip through the stored mapping. Freshly-invalid paths the mapping does - // not cover surface as a typed failure listing the effective paths. - AdvanceHeadToTrackedTip(repo); - RecoveryDocument stored = LoadRecoveryDocument(recoveryFile, WorkingDirectoryOf(repo)); - if(stored.MaterializedTip is not null && string.Equals(stored.MaterializedTip, repo.Head.Tip?.Sha, StringComparison.Ordinal)) - { - // The tree on disk already reflects this tip: nothing to rewrite. A - // recovery-managed repo used to re-materialize every blob on every - // sync, touching gigabytes and every mtime for zero change (#31). - return; - } - ApplyRecoveryCore(repo, ToRecovery(stored), ct); + // A recovery-managed repo stays recovery-managed: the new tip is written + // through the stored mapping, or nothing is touched when the tree on disk + // already reflects it (a managed repo used to re-materialize every blob on + // every sync, touching gigabytes and every mtime for zero change, #31). + // Freshly-invalid paths the mapping does not cover surface as a typed + // failure listing the effective paths. + ApplyRecoveryCore(repo, incoming: null, ct); return; } @@ -613,6 +621,20 @@ public sealed class LibGit2GitClient : IGitClient return; // the remote is still empty; nothing to pull } + /////////////////////////////////////////////////////////////////////////////// + // No commit on the branch does not mean nothing was checked out: a branch // + // started with "checkout --orphan" keeps the index and the working tree of // + // the branch it was started from. Checking out over that, with force, would // + // replace someone's files. // + /////////////////////////////////////////////////////////////////////////////// + if(repo.Index.Count > 0) + { + throw new InvalidOperationException( + $"'{branchName}' has no commits here yet, but files are staged in this folder (a branch started with " + + "'git checkout --orphan'?). gclo does not check out over them; fetched, but nothing was checked out. " + + "Commit or remove them, or check out another branch."); + } + if(OperatingSystem.IsWindows()) { IReadOnlyList invalidPaths = WindowsPathValidator.Validate(remoteBranch.Tip.Tree); @@ -662,48 +684,109 @@ public sealed class LibGit2GitClient : IGitClient /// - /// Materializes HEAD's tree into the working directory with the recovery's renames - /// and skips applied, after validating that the effective path set is actually - /// creatable on Windows. Any recovery persisted from an earlier run is merged in - /// first (the incoming recovery wins on conflicts), so a recovery-managed repo - /// that gains new invalid paths can be fixed incrementally. Persists the merged - /// recovery so later pulls re-apply it, then clears the pending-checkout marker. + /// Materializes a commit's tree into the working directory with renames and skips + /// applied, after validating that the effective path set is actually creatable on + /// Windows. is what the user just chose (it wins over + /// stored choices path by path), or null when a pull re-applies what is stored. /// - private static void ApplyRecoveryCore(Repository repo, PathRecovery recovery, CancellationToken ct) + /// + /// + /// This writes a working tree by hand, with force. It is therefore only ever done + /// to a tree that is gclo's to write (#147): + /// + /// + /// A repository whose clone was fetched but never checked out (the pending + /// marker), or that is empty: nothing was ever written there. + /// A recovery-managed repository: an earlier apply wrote its tree, and the + /// documentation says to treat it as a read-only mirror. + /// + /// + /// A clone that was checked out normally is refused, whatever its state. Proving + /// that such a clone holds nothing of the user's is not possible in general + /// (untracked files, files git was told to ignore changes in, an apply interrupted + /// half-way and edited since), and a wrong guess destroys work. The way to get such + /// a repository with renamed paths is a fresh clone. + /// + /// + /// Commits are protected in every case: the branch only moves where + /// allows. And an apply that is interrupted is completed by + /// the next pull: before the first destructive step the recovery file is saved + /// without a materialized tip, listing everything that was and will be on disk. + /// + /// + private static void ApplyRecoveryCore(Repository repo, PathRecovery? incoming, CancellationToken ct) { + string workdir = WorkingDirectoryOf(repo); + string recoveryFile = GetRecoveryFilePath(repo); + RecoveryDocument? stored = File.Exists(recoveryFile) ? LoadRecoveryDocument(recoveryFile, workdir) : null; + // "Managed" by gclo's own word (its file, its flag), or only by what the + // repository looks like; the second is trusted less (see below). + bool managedByRecord = stored is not null || IsRecoveryManaged(repo); + bool managed = managedByRecord || WasWrittenWithoutAnIndex(repo); + + /////////////////////////////////////////////////////////////////////////////// + // An empty repository counts as never checked out only while git has staged // + // nothing in it either: a branch started with "checkout --orphan" has no // + // commit yet, and a working tree full of someone's files. // + /////////////////////////////////////////////////////////////////////////////// + bool emptyAndUntouched = repo.Info.IsHeadUnborn && repo.Index.Count == 0; + if(!managed && !IsCheckoutPending(repo) && !emptyAndUntouched) + { + throw new InvalidOperationException( + "This repository is checked out normally. Resolving its paths would replace the working tree with one that gclo " + + "writes itself, which could overwrite work in it, so gclo does not do that to a clone that was checked out. " + + $"To get it with renamed paths, move or delete the folder '{Path.TrimEndingDirectorySeparator(workdir)}' and sync again" + + (OperatingSystem.IsWindows() ? ", or use Clone in WSL in the desktop app." : ".")); + } + + // Only now: a refused apply leaves the repository's configuration as it was too. if(OperatingSystem.IsWindows()) { repo.Config.Set("core.longpaths", true, ConfigurationLevel.Local); } - string workdir = WorkingDirectoryOf(repo); - string recoveryFile = GetRecoveryFilePath(repo); - RecoveryDocument? previous = File.Exists(recoveryFile) ? LoadRecoveryDocument(recoveryFile, workdir) : null; - if(previous is not null) + PathRecovery recovery = MergeRecoveries(stored is null ? s_noRecovery : ToRecovery(stored), incoming ?? s_noRecovery); + Commit tip = TargetOf(repo); + if(incoming is null && IsAlreadyOnDisk(repo, stored, tip)) { - recovery = MergeRecoveries(stored: ToRecovery(previous), incoming: recovery); + // (The file was read without complaint, so the repository is known to be + // managed from here on even if the file is lost.) + repo.Config.Set(RecoveryManagedKey, true, ConfigurationLevel.Local); + return; } - Commit tip = repo.Head.Tip - ?? throw new InvalidOperationException("Repository has no commits; nothing to materialize."); - List<(string OriginalPath, string EffectivePath, Blob Blob)> entries = CollectEffectiveEntries(tip.Tree, recovery); // Validate BEFORE writing anything: a bad mapping (still-invalid segment, or // two originals landing on one destination) must fail cleanly with the - // effective paths, leaving the working tree untouched. + // effective paths, leaving the working tree and the branch untouched. IReadOnlyList stillInvalid = WindowsPathValidator.ValidatePaths(entries.Select(e => e.EffectivePath)); if(stillInvalid.Count > 0) { throw new InvalidRepositoryPathsException(stillInvalid); } + HashSet? ours = stored?.MaterializedPaths is null ? null : new HashSet(stored.MaterializedPaths, StringComparer.Ordinal); + var effectiveNow = new HashSet(entries.Select(e => e.EffectivePath), StringComparer.Ordinal); + RefuseWhatCannotBeWritten(repo, workdir, entries, ours, effectiveNow, replaceOnlyWhatTheCommitHas: !managedByRecord); + + ///////////////////////////////////////////////////////////////////////////// + // The point of no return. Record first that this tree is being rewritten, // + // and everything that was and will be on disk by gclo's doing; then move the // + // branch, describe the commit in the index (as after any checkout: without // + // it git reports every file as deleted, and attribute lookups can already // + // see the new tree), remove what left the tree, and write the files. // + ///////////////////////////////////////////////////////////////////////////// + SaveRecovery(repo, recovery, (ours ?? []).Union(effectiveNow, StringComparer.Ordinal), materializedTip: null); + repo.Config.Set(RecoveryManagedKey, true, ConfigurationLevel.Local); + MoveHeadTo(repo, tip); + repo.Index.Replace(tip.Tree); + repo.Index.Write(); // A real checkout removes what left the tree; this materialization must too, - // or a recovery-managed working tree accumulates every file upstream ever - // deleted or renamed (#31). The previous apply's manifest says what it wrote. - var effectiveNow = new HashSet(entries.Select(e => e.EffectivePath), StringComparer.Ordinal); - foreach(string stale in previous?.MaterializedPaths ?? []) + // or the working tree accumulates every file upstream ever deleted or renamed + // (#31). + foreach(string stale in ours ?? []) { if(!effectiveNow.Contains(stale)) { @@ -712,8 +795,10 @@ public sealed class LibGit2GitClient : IGitClient } // Parent folders are shared by many files; each is checked for a link once. + // Attribute files go first: they decide how the files after them are written + // (line endings), and a stale one on disk would otherwise outrank the index. var verifiedFolders = new HashSet(StringComparer.Ordinal); - foreach((string? originalPath, string? effectivePath, Blob? blob) in entries) + foreach((string? originalPath, string? effectivePath, Blob? blob) in entries.OrderBy(e => IsAttributesFile(e.OriginalPath) ? 0 : 1)) { ct.ThrowIfCancellationRequested(); @@ -725,25 +810,366 @@ public sealed class LibGit2GitClient : IGitClient content.CopyTo(file); } - SaveRecovery(repo, recovery, entries.Select(e => e.EffectivePath), tip.Sha); + SaveRecovery(repo, recovery, effectiveNow, tip.Sha); repo.Config.Set(CheckoutPendingKey, false, ConfigurationLevel.Local); } /// - /// Overlays onto : renames - /// union with the incoming value winning on a shared original path; skips union. + /// The repository's own word that gclo wrote its working tree. Read from the + /// repository's configuration only: a value set for the user or the machine says + /// nothing about this folder. + /// + private static bool IsRecoveryManaged(Repository repo) + => repo.Config.Get(RecoveryManagedKey, ConfigurationLevel.Local)?.Value == true; + + + + /// + /// True for a repository that gclo cloned (it carries gclo's checkout marker, + /// whatever its value), has commits, and has no index: git never checked it out, + /// so whatever tree is there was written by gclo. That is how a repository looks + /// that the released build resolved (it wrote files and no index), also after its + /// recovery file was lost. A clone made by git, with or without a checkout, has + /// no such marker and is not taken for one. The inference is still only an + /// inference (an index can be deleted by hand), so a tree recognised this way is + /// rewritten only where its files are what the current commit has + /// (). + /// + private static bool WasWrittenWithoutAnIndex(Repository repo) + => !repo.Info.IsHeadUnborn + && repo.Config.Get(CheckoutPendingKey, ConfigurationLevel.Local) is not null + && !File.Exists(Path.Combine(repo.Info.Path, "index")); + + + + /// + /// True when the current commit holds a path that cannot be created here. Only + /// then can a tree without an index be one that path recovery wrote: a repository + /// with nothing to rename is an ordinary clone whatever became of its index, and + /// takes the ordinary pull, which leaves local changes alone and marks nothing. + /// + private static bool HasPathsWindowsCannotCreate(Repository repo) + => OperatingSystem.IsWindows() && WindowsPathValidator.Validate(repo.Head.Tip.Tree).Count > 0; + + + + /// + /// True when a pull has nothing to write: the branch already is at the commit to + /// write, and what gclo last wrote is that commit or one it builds on. In the + /// second case the commits on top were made here and origin brought nothing; + /// rewriting the tree to what was committed would throw away everything else in it. + /// + private static bool IsAlreadyOnDisk(Repository repo, RecoveryDocument? stored, Commit tip) + { + if(stored?.MaterializedTip is not { } written + || !string.Equals(repo.Head.Tip?.Sha, tip.Sha, StringComparison.Ordinal)) + { + return false; // an apply is unfinished, or the branch still has to move + } + + if(string.Equals(written, tip.Sha, StringComparison.Ordinal)) + { + return true; + } + + /////////////////////////////////////////////////////////////////////////////// + // The branch is ahead of what was written. That is "commits made here" only // + // when origin has nothing beyond what was written. If origin does, the // + // branch was moved to origin's commit and the tree was never written for it: // + // the released build moved the branch first, so a pull of its that failed // + // half-way (a new invalid path, a cancel, a locked file) leaves exactly that. // + /////////////////////////////////////////////////////////////////////////////// + Commit? writtenCommit = ObjectId.TryParse(written, out ObjectId writtenId) ? repo.Lookup(writtenId) : null; + if(writtenCommit is null) + { + return false; + } + + Commit? upstream = repo.Info.IsHeadDetached ? null : repo.Head.TrackedBranch?.Tip; + bool originHasNothingNew = upstream is null + || string.Equals(repo.ObjectDatabase.FindMergeBase(upstream, writtenCommit)?.Sha, upstream.Sha, StringComparison.Ordinal); + return originHasNothingNew + && string.Equals(repo.ObjectDatabase.FindMergeBase(writtenCommit, tip)?.Sha, written, StringComparison.Ordinal); + } + + + + private static bool IsAttributesFile(string repoPath) + => repoPath.EndsWith(".gitattributes", StringComparison.Ordinal) + && (repoPath.Length == ".gitattributes".Length || repoPath[^(".gitattributes".Length + 1)] == '/'); + + + + /// + /// The commit an apply writes. gclo moves a branch only to what origin has, and only + /// when that drops no commit (#147): + /// + /// An empty repository (unborn HEAD): origin's tip for that branch, its first + /// checkout. + /// Origin has nothing the current commit lacks (no upstream, a detached HEAD, + /// the same commit, or commits made here on top of origin's): the current + /// commit. + /// The current commit is contained in origin's tip: origin's tip. + /// The histories diverged: origin's tip only when the current commit is one + /// origin's branch itself once pointed at (). Then + /// nobody committed here and origin rewrote its history. Otherwise a refusal. + /// + /// + private static Commit TargetOf(Repository repo) + { + Commit? head = repo.Head.Tip; + if(head is null) + { + const string Prefix = "refs/heads/"; + string targetRef = repo.Refs.Head.TargetIdentifier; + Commit? first = targetRef.StartsWith(Prefix, StringComparison.Ordinal) + ? repo.Branches["origin/" + targetRef[Prefix.Length..]]?.Tip + : null; + return first ?? throw new InvalidOperationException( + "This repository has no commits, and neither has its branch on origin; there is nothing to materialize."); + } + + Branch? tracked = repo.Info.IsHeadDetached ? null : repo.Head.TrackedBranch; + Commit? upstream = tracked?.Tip; + if(upstream is null) + { + return head; + } + + string? mergeBase = repo.ObjectDatabase.FindMergeBase(head, upstream)?.Sha; + if(string.Equals(mergeBase, upstream.Sha, StringComparison.Ordinal)) + { + return head; + } + + if(string.Equals(mergeBase, head.Sha, StringComparison.Ordinal) || WasOnceTheTipOf(repo, tracked!, head)) + { + return upstream; + } + + throw new InvalidOperationException(tracked!.IsRemote + ? $"'{repo.Head.FriendlyName}' and its branch on origin have diverged, and nothing here shows that the commits " + + "in this folder came from origin: either commits were made here, or origin's history was rewritten and the " + + "record of what was fetched is gone. gclo only fast-forwards and never merges. " + + "Push or reset commits made here, or delete the folder to re-clone." + : $"'{repo.Head.FriendlyName}' and the local branch it tracks ('{tracked.FriendlyName}') have diverged. " + + "gclo only fast-forwards and never merges. Merge or reset by hand, or check out a branch that tracks origin."); + } + + + + /// + /// True when origin's branch itself pointed at at some + /// time, going by the reflog of the remote-tracking branch : + /// every clone, fetch, and pull that moves the branch adds an entry saying where + /// it went, and only those entries count. Other things write there too (a push + /// records the commit that was made here, a hand-made update records anything), + /// and they do not. Where the branch was before the oldest entry that is left + /// (older ones expire) counts as well. A branch that tracks another local branch + /// has no such record of origin, and nothing is taken from its reflog. + /// + private static bool WasOnceTheTipOf(Repository repo, Branch tracked, Commit commit) + { + if(!tracked.IsRemote) + { + return false; + } + + ReflogEntry? oldest = null; + foreach(ReflogEntry entry in repo.Refs.Log(tracked.CanonicalName)) // newest first + { + if(entry.To == commit.Id && IsWrittenByAFetch(entry)) + { + return true; + } + oldest = entry; + } + return oldest is not null && oldest.From == commit.Id; + } + + + + /// True for a reflog entry whose message says a clone, a fetch, or a pull wrote it. + private static bool IsWrittenByAFetch(ReflogEntry entry) + { + string message = entry.Message ?? ""; + return message.StartsWith("fetch", StringComparison.Ordinal) + || message.StartsWith("pull", StringComparison.Ordinal) + || message.StartsWith("clone", StringComparison.Ordinal); + } + + + + /// Points the current branch at , creating it (with its upstream) for an empty repository. + private static void MoveHeadTo(Repository repo, Commit tip) + { + if(repo.Info.IsHeadUnborn) + { + string targetRef = repo.Refs.Head.TargetIdentifier; + repo.Refs.Add(targetRef, tip.Id); + Branch local = repo.Branches[targetRef["refs/heads/".Length..]]; + repo.Branches.Update(local, b => + { + b.Remote = "origin"; + b.UpstreamBranch = targetRef; + }); + return; + } + + if(!string.Equals(repo.Head.Tip.Sha, tip.Sha, StringComparison.Ordinal)) + { + repo.Refs.UpdateTarget(repo.Refs.Head.ResolveToDirectReference(), tip.Id); + } + } + + + + /// + /// Finds, before anything has changed, what it can of what would make the apply + /// fail half-way or write over something that is not gclo's: another git process at + /// work, a destination that is not a usable path in the working tree, a link on the + /// way to it or to a file that is to be removed, a file sitting where a folder is + /// needed, a folder holding someone's files where a file is needed, and, where + /// gclo has no record of having written the tree, a file at a destination that is + /// not what the current commit has there. Failing here leaves the branch, the + /// index and every file as they were (#147). What only shows while writing (a file + /// held open) still fails later; the next pull then completes the apply. + /// + /// + /// replaceOnlyWhatTheCommitHas is true when nothing records that gclo wrote + /// this tree (a clone that was never checked out, an empty one, a tree recognised + /// only by its shape). A file that is already at a destination is then replaced + /// only when it is, byte for byte, what the current commit has for that path: an + /// unfinished checkout's file, not something a person put there or changed. + /// + private static void RefuseWhatCannotBeWritten( + Repository repo, + string workdir, + List<(string OriginalPath, string EffectivePath, Blob Blob)> entries, + HashSet? ours, + HashSet effectiveNow, + bool replaceOnlyWhatTheCommitHas) + { + if(File.Exists(Path.Combine(repo.Info.Path, "index.lock"))) + { + throw new InvalidOperationException( + "Another git process is working in this repository (its index is locked). Try again when it has finished; " + + $"if no git is running, one that crashed left '{Path.Combine(repo.Info.Path, "index.lock")}' behind, and deleting that file clears it."); + } + + Tree? current = replaceOnlyWhatTheCommitHas ? repo.Head.Tip?.Tree : null; + + // The file system decides whether two names are one file; on Windows it does + // not tell 'Docs' from 'docs', so neither does the question "is this ours". + var oursAsOnDisk = new HashSet( + ours ?? [], OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); + var verifiedFolders = new HashSet(StringComparer.Ordinal); + var checkedFolders = new HashSet(StringComparer.Ordinal); + foreach((string? originalPath, string? effectivePath, _) in entries) + { + WorkingTree.Verify(workdir, effectivePath, verifiedFolders); + + string full = Path.Combine(workdir, effectivePath.Replace('/', Path.DirectorySeparatorChar)); + if(replaceOnlyWhatTheCommitHas && File.Exists(full) && !IsWhatTheCommitHas(full, originalPath, current)) + { + throw new InvalidOperationException( + $"'{effectivePath}' is already in this folder, and it is not a file gclo has a record of writing " + + "or one the current commit has in that form, so it is not gclo's to replace. Move or delete it, then try again; " + + "if that goes for many files, move or delete the whole folder and sync again."); + } + + string? foreign = WorkingTree.FilesUnder(workdir, effectivePath).FirstOrDefault(file => !oursAsOnDisk.Contains(file)); + if(foreign is not null) + { + throw new InvalidOperationException( + $"'{effectivePath}' is a folder with files in it ('{foreign}' is one), where the repository has a file. " + + "Move or delete the folder, then try again."); + } + + for(int slash = effectivePath.IndexOf('/', StringComparison.Ordinal); slash > 0; slash = effectivePath.IndexOf('/', slash + 1)) + { + string folder = effectivePath[..slash]; + if(checkedFolders.Add(folder) + && !oursAsOnDisk.Contains(folder) + && File.Exists(Path.Combine(workdir, folder.Replace('/', Path.DirectorySeparatorChar)))) + { + throw new InvalidOperationException( + $"'{folder}' is a file in this folder, and the repository has a folder of that name. " + + "Move or delete the file, then try again."); + } + } + } + + // What left the tree is removed through real folders only, like what is written. + foreach(string stale in ours ?? []) + { + if(!effectiveNow.Contains(stale)) + { + WorkingTree.Verify(workdir, stale, verifiedFolders); + } + } + } + + + + /// + /// True when the file at holds exactly what a checkout of + /// writes for + /// (line-ending conversion included). False when the commit has no such file. + /// + private static bool IsWhatTheCommitHas(string full, string originalPath, Tree? commitTree) + { + if(commitTree?[originalPath]?.Target is not Blob blob) + { + return false; + } + + using Stream expected = blob.GetContentStream(new FilteringOptions(originalPath)); + using var actual = new FileStream(full, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + byte[] expectedBuffer = new byte[81920]; + byte[] actualBuffer = new byte[81920]; + while(true) + { + int expectedRead = expected.ReadAtLeast(expectedBuffer, expectedBuffer.Length, throwOnEndOfStream: false); + int actualRead = actual.ReadAtLeast(actualBuffer, actualBuffer.Length, throwOnEndOfStream: false); + if(expectedRead != actualRead || !expectedBuffer.AsSpan(0, expectedRead).SequenceEqual(actualBuffer.AsSpan(0, actualRead))) + { + return false; + } + if(expectedRead == 0) + { + return true; + } + } + } + + + + /// + /// Overlays onto . The newer + /// choice for a path wins whatever kind it is: an incoming rename replaces a stored + /// rename or a stored skip of that path, and an incoming skip replaces a stored + /// rename. Everything else is kept from both. /// private static PathRecovery MergeRecoveries(PathRecovery stored, PathRecovery incoming) { - var renames = new Dictionary(stored.SegmentRenames, StringComparer.Ordinal); + var renames = new Dictionary(StringComparer.Ordinal); + foreach((string? originalPath, string? replacement) in stored.SegmentRenames) + { + if(!incoming.SkippedPaths.Contains(originalPath)) + { + renames[originalPath] = replacement; + } + } foreach((string? originalPath, string? replacement) in incoming.SegmentRenames) { renames[originalPath] = replacement; } - var skips = new HashSet(stored.SkippedPaths, StringComparer.Ordinal); + var skips = new HashSet( + stored.SkippedPaths.Where(path => !incoming.SegmentRenames.ContainsKey(path)), StringComparer.Ordinal); skips.UnionWith(incoming.SkippedPaths); return new PathRecovery(renames, skips); @@ -855,25 +1281,30 @@ public sealed class LibGit2GitClient : IGitClient /// - /// Moves the current branch's ref to its upstream tip without touching the working - /// tree. Only used for never-checked-out (marker) and recovery-managed repos, whose - /// local branch can hold no local work — so a plain ref move is safe where the - /// normal pull path would insist on a fast-forward merge. + /// Moves the current branch to where origin's is, without touching the working + /// tree, for a repository whose clone was never checked out. By the rule every + /// branch move here follows (): a commit somebody made in + /// it by hand is not dropped. /// private static void AdvanceHeadToTrackedTip(Repository repo) { - Branch tracked = repo.Head.TrackedBranch; - if(tracked?.Tip is null || string.Equals(repo.Head.Tip?.Sha, tracked.Tip.Sha, StringComparison.Ordinal)) + if(repo.Info.IsHeadUnborn) { return; } - repo.Refs.UpdateTarget(repo.Refs.Head.ResolveToDirectReference(), tracked.Tip.Id); + + Commit target = TargetOf(repo); + if(!string.Equals(repo.Head.Tip.Sha, target.Sha, StringComparison.Ordinal)) + { + repo.Refs.UpdateTarget(repo.Refs.Head.ResolveToDirectReference(), target.Id); + } } + /// The never-checked-out marker, from the repository's own configuration only. private static bool IsCheckoutPending(Repository repo) - => repo.Config.Get(CheckoutPendingKey)?.Value == true; + => repo.Config.Get(CheckoutPendingKey, ConfigurationLevel.Local)?.Value == true; @@ -886,7 +1317,7 @@ public sealed class LibGit2GitClient : IGitClient /// Original path to replacement path map; null when absent from the file. /// Paths omitted from the working tree; null when absent from the file. /// Effective paths the last apply wrote, so the next one can delete what left the tree; null in files from before this field existed. - /// SHA of the commit the last apply materialized; null in older files. + /// SHA of the commit the last apply materialized; null in older files, and while an apply is under way (so a tree left unfinished is never taken for done). private sealed record RecoveryDocument( Dictionary? SegmentRenames, List? SkippedPaths, @@ -896,13 +1327,13 @@ public sealed class LibGit2GitClient : IGitClient private static void SaveRecovery( - Repository repo, PathRecovery recovery, IEnumerable materializedPaths, string tipSha) + Repository repo, PathRecovery recovery, IEnumerable materializedPaths, string? materializedTip) { var document = new RecoveryDocument( new Dictionary(recovery.SegmentRenames, StringComparer.Ordinal), recovery.SkippedPaths.Order(StringComparer.Ordinal).ToList(), materializedPaths.Order(StringComparer.Ordinal).ToList(), - tipSha); + materializedTip); // Atomic replace: a crash mid-write must not leave a truncated document that // fails every later pull of this repository (#31). diff --git a/gclo.Engine/WorkingTree.cs b/gclo.Engine/WorkingTree.cs index f7b991c..1a9d411 100644 --- a/gclo.Engine/WorkingTree.cs +++ b/gclo.Engine/WorkingTree.cs @@ -85,6 +85,17 @@ internal static class WorkingTree + /// + /// Everything checks before it touches the disk, without + /// touching it: the path resolves inside the working directory, and no folder on + /// the way is a link. For finding out up front, before an operation that cannot be + /// undone begins, whether its writes can succeed. + /// + public static void Verify(string workdir, string repoPath, ISet? verifiedFolders = null) + => RefuseLinkOnTheWay(workdir, Resolve(workdir, repoPath), verifiedFolders); + + + /// /// Opens for writing as a new, empty file, creating its /// folders. Whatever file is already there is removed first, read-only or not, so @@ -109,13 +120,20 @@ internal static class WorkingTree /// - /// Deletes (a missing file is fine) and then the folders - /// it leaves empty, stopping at the working directory itself. + /// Deletes the file (a missing one is fine) and then + /// the folders it leaves empty, stopping at the working directory itself. A folder + /// that sits there now is left alone with everything in it: the file gclo wrote + /// is gone already, and whoever put the folder there owns it. /// public static void Delete(string workdir, string repoPath) { string full = Resolve(workdir, repoPath); RefuseLinkOnTheWay(workdir, full, verifiedFolders: null); + var folder = new DirectoryInfo(full); + if(folder.Exists && folder.LinkTarget is null) + { + return; + } RemoveFile(full); string root = Root(workdir); @@ -213,9 +231,67 @@ internal static class WorkingTree /// - /// Removes what is at when it is a file or a link, read-only - /// or not: a link is removed itself (even one whose target is gone, even one to a - /// directory), never what it points at. + /// The files inside the real folder at , as paths + /// relative to the working directory; nothing when no folder is there, or a link + /// is. Links inside it are not entered and not listed. + /// + public static IEnumerable FilesUnder(string workdir, string repoPath) + { + var folder = new DirectoryInfo(Resolve(workdir, repoPath)); + return folder.Exists && folder.LinkTarget is null ? FilesIn(folder, repoPath) : []; + } + + + + private static IEnumerable FilesIn(DirectoryInfo folder, string repoPath) + { + foreach(FileInfo file in folder.EnumerateFiles()) + { + yield return repoPath + "/" + file.Name; + } + + foreach(DirectoryInfo child in folder.EnumerateDirectories()) + { + if(child.LinkTarget is not null) + { + continue; + } + foreach(string inner in FilesIn(child, repoPath + "/" + child.Name)) + { + yield return inner; + } + } + } + + + + /// + /// Removes a folder that holds nothing but empty folders and links, bottom up. + /// Called only for a folder already found to hold no file. + /// + private static void RemoveEmptyFolders(DirectoryInfo folder) + { + foreach(DirectoryInfo child in folder.EnumerateDirectories()) + { + if(child.LinkTarget is not null) + { + child.Delete(); // a link is nobody's content; remove the link itself + continue; + } + RemoveEmptyFolders(child); + } + + folder.Attributes = FileAttributes.Directory; + folder.Delete(); + } + + + + /// + /// Removes what is at when it is a file, a link, or a folder + /// that holds no files, read-only or not: a link is removed itself (even one whose + /// target is gone, even one to a directory), never what it points at. A folder that + /// still holds files is refused. /// private static void RemoveFile(string full) { @@ -226,6 +302,22 @@ internal static class WorkingTree return; } + if(directory.Exists) + { + // A real folder where a file belongs (upstream turned a folder into a file). + // Whatever the tree had in it is gone by now; if files remain they are not + // gclo's to remove, and empty folders are nobody's. Looked at as a whole + // first: a folder that is refused loses nothing, not even an empty subfolder. + if(FilesIn(directory, directory.Name).Any()) + { + throw new InvalidOperationException( + $"'{directory.FullName}' is a folder with files in it, where the repository has a file. " + + "Move or delete the folder, then try again."); + } + RemoveEmptyFolders(directory); + return; + } + var file = new FileInfo(full); bool isLink = file.LinkTarget is not null; if(!file.Exists && !isLink)