diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8e02c9a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: ci + +# Build and run the acceptance suite on every PR and on pushes to main. The Reqnroll scenarios +# drive the real built protostar binary, so this also exercises the CLI end to end. Run on both +# Linux and Windows to cover the OS-specific binary name and install paths. +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 # MinVer needs full history to compute the version. + + - uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2 # v5 + with: + dotnet-version: '10.0.x' + + - name: Build + run: dotnet build -c Release + + - name: Test + run: dotnet test -c Release --no-build --verbosity normal diff --git a/.gitignore b/.gitignore index 859638c..e81e677 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ appsettings.*.local.json # OS .DS_Store Thumbs.db + +# Reqnroll generates code-behind beside each .feature file; it is rebuilt every compile. +*.feature.cs diff --git a/protostar.sln b/protostar.sln index 101b3e0..962f168 100644 --- a/protostar.sln +++ b/protostar.sln @@ -7,6 +7,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Protostar.Cli", "src\Protostar.Cli\Protostar.Cli.csproj", "{B94673CE-1472-46C5-8538-82062FDD51E6}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{0C88DD14-F956-CE84-757C-A364CCF449FC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Protostar.Cli.Acceptance", "test\Protostar.Cli.Acceptance\Protostar.Cli.Acceptance.csproj", "{DC3CF955-4BC2-481C-B640-3FEC51094494}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -29,11 +33,24 @@ Global {B94673CE-1472-46C5-8538-82062FDD51E6}.Release|x64.Build.0 = Release|Any CPU {B94673CE-1472-46C5-8538-82062FDD51E6}.Release|x86.ActiveCfg = Release|Any CPU {B94673CE-1472-46C5-8538-82062FDD51E6}.Release|x86.Build.0 = Release|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|x64.ActiveCfg = Debug|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|x64.Build.0 = Debug|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|x86.ActiveCfg = Debug|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|x86.Build.0 = Debug|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|Any CPU.Build.0 = Release|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|x64.ActiveCfg = Release|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|x64.Build.0 = Release|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|x86.ActiveCfg = Release|Any CPU + {DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution {B94673CE-1472-46C5-8538-82062FDD51E6} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {DC3CF955-4BC2-481C-B640-3FEC51094494} = {0C88DD14-F956-CE84-757C-A364CCF449FC} EndGlobalSection EndGlobal diff --git a/src/Protostar.Cli/Commands/UninstallCommand.cs b/src/Protostar.Cli/Commands/UninstallCommand.cs index de8356e..669765f 100644 --- a/src/Protostar.Cli/Commands/UninstallCommand.cs +++ b/src/Protostar.Cli/Commands/UninstallCommand.cs @@ -13,6 +13,10 @@ public sealed class Settings : CommandSettings [CommandOption("-d|--dir ")] [Description("Install directory to remove from. Defaults to the per-user location.")] public string? Dir { get; init; } + + [CommandOption("--no-modify-path")] + [Description("Do not remove the install directory from PATH.")] + public bool NoModifyPath { get; init; } } protected override int Execute(CommandContext context, Settings settings, CancellationToken cancellation) @@ -39,7 +43,8 @@ protected override int Execute(CommandContext context, Settings settings, Cancel return 1; } - PathManager.RemoveFromPath(dir); + if (!settings.NoModifyPath) + PathManager.RemoveFromPath(dir); AnsiConsole.MarkupLine($"Removed [aqua]protostar[/] from [grey]{Markup.Escape(dir)}[/]."); return 0; } diff --git a/test/Protostar.Cli.Acceptance/Features/DefaultCommand.feature b/test/Protostar.Cli.Acceptance/Features/DefaultCommand.feature new file mode 100644 index 0000000..2e71553 --- /dev/null +++ b/test/Protostar.Cli.Acceptance/Features/DefaultCommand.feature @@ -0,0 +1,10 @@ +Feature: Default command guidance + As a new user + I want a bare protostar invocation to explain itself + So that I know it is working and where to go next + + Scenario: Running with no arguments prints guidance + When I run protostar with no arguments + Then the exit code is 0 + And the output contains "Live, continuous refinement of agent skills." + And the output contains "protostar --help" diff --git a/test/Protostar.Cli.Acceptance/Features/Help.feature b/test/Protostar.Cli.Acceptance/Features/Help.feature new file mode 100644 index 0000000..4c417cc --- /dev/null +++ b/test/Protostar.Cli.Acceptance/Features/Help.feature @@ -0,0 +1,10 @@ +Feature: Help lists available commands + As a user + I want protostar --help to list the commands + So that I can discover what protostar can do + + Scenario: --help lists the install and uninstall commands + When I run protostar with "--help" + Then the exit code is 0 + And the output contains "install" + And the output contains "uninstall" diff --git a/test/Protostar.Cli.Acceptance/Features/Install.feature b/test/Protostar.Cli.Acceptance/Features/Install.feature new file mode 100644 index 0000000..9ef428c --- /dev/null +++ b/test/Protostar.Cli.Acceptance/Features/Install.feature @@ -0,0 +1,19 @@ +Feature: Installing protostar + As a user + I want protostar install to place the binary in a directory + So that I can run it from my PATH + + Scenario: Installing into a clean sandbox directory + Given a clean install sandbox + When I run protostar with "install --dir {installDir} --no-modify-path" + Then the exit code is 0 + And the output contains "Installed" + And a protostar binary exists in the install dir + + Scenario: Re-installing into the same directory succeeds + Given a clean install sandbox + When I run protostar with "install --dir {installDir} --no-modify-path" + And I run protostar with "install --dir {installDir} --no-modify-path" + Then the exit code is 0 + And the output contains "Installed" + And a protostar binary exists in the install dir diff --git a/test/Protostar.Cli.Acceptance/Features/Uninstall.feature b/test/Protostar.Cli.Acceptance/Features/Uninstall.feature new file mode 100644 index 0000000..3ae5870 --- /dev/null +++ b/test/Protostar.Cli.Acceptance/Features/Uninstall.feature @@ -0,0 +1,18 @@ +Feature: Uninstalling protostar + As a user + I want protostar uninstall to remove an installed binary + So that I can cleanly remove the tool + + Scenario: Uninstalling removes the installed binary + Given a clean install sandbox + When I run protostar with "install --dir {installDir} --no-modify-path" + And I run protostar with "uninstall --dir {installDir} --no-modify-path" + Then the exit code is 0 + And the output contains "Removed" + And no protostar binary exists in the install dir + + Scenario: Uninstalling when nothing is installed is a no-op + Given a clean install sandbox + When I run protostar with "uninstall --dir {installDir} --no-modify-path" + Then the exit code is 0 + And the output contains "Nothing to remove" diff --git a/test/Protostar.Cli.Acceptance/Features/Version.feature b/test/Protostar.Cli.Acceptance/Features/Version.feature new file mode 100644 index 0000000..44206b8 --- /dev/null +++ b/test/Protostar.Cli.Acceptance/Features/Version.feature @@ -0,0 +1,9 @@ +Feature: Reporting the version + As a user who installed protostar + I want protostar --version to report a version + So that I can confirm which build I am running + + Scenario: --version prints a semantic version + When I run protostar with "--version" + Then the exit code is 0 + And the output matches "\d+\.\d+\.\d+" diff --git a/test/Protostar.Cli.Acceptance/Protostar.Cli.Acceptance.csproj b/test/Protostar.Cli.Acceptance/Protostar.Cli.Acceptance.csproj new file mode 100644 index 0000000..77c3946 --- /dev/null +++ b/test/Protostar.Cli.Acceptance/Protostar.Cli.Acceptance.csproj @@ -0,0 +1,27 @@ + + + + net10.0 + enable + enable + + false + false + + + + + + + + + + + + + + + + + + diff --git a/test/Protostar.Cli.Acceptance/Steps/CliSteps.cs b/test/Protostar.Cli.Acceptance/Steps/CliSteps.cs new file mode 100644 index 0000000..cb2498d --- /dev/null +++ b/test/Protostar.Cli.Acceptance/Steps/CliSteps.cs @@ -0,0 +1,69 @@ +using CliWrap.Buffered; +using Protostar.Cli.Acceptance.Support; +using Reqnroll; +using Xunit; + +namespace Protostar.Cli.Acceptance.Steps; + +/// +/// Step definitions that drive the real protostar binary as a user would and assert on its +/// stdout/stderr, exit code, and filesystem side effects. One per scenario. +/// +[Binding] +public sealed class CliSteps : IDisposable +{ + private Sandbox? _sandbox; + private BufferedCommandResult? _result; + + private Sandbox Sandbox => _sandbox ??= new Sandbox(); + + private BufferedCommandResult Result => + _result ?? throw new InvalidOperationException("No protostar invocation has run in this scenario yet."); + + // Spectre disables colour when stdout is redirected, so captured output is plain text. We still + // merge stderr in case a future command writes there. + private string Output => Result.StandardOutput + Result.StandardError; + + [Given("a clean install sandbox")] + public void GivenACleanInstallSandbox() => _ = Sandbox; + + [When("I run protostar with no arguments")] + public async Task WhenIRunWithNoArguments() => + _result = await CliRunner.RunAsync([]); + + [When("I run protostar with {string}")] + public async Task WhenIRunWith(string argLine) + { + // Split on spaces, then substitute the {installDir} placeholder as a whole token so a + // sandbox path containing spaces still arrives as a single argument. + var args = argLine + .Split(' ', StringSplitOptions.RemoveEmptyEntries) + .Select(token => token == "{installDir}" ? Sandbox.InstallDir : token) + .ToList(); + _result = await CliRunner.RunAsync(args); + } + + [Then("the exit code is {int}")] + public void ThenTheExitCodeIs(int expected) => + Assert.Equal(expected, Result.ExitCode); + + [Then("the output contains {string}")] + public void ThenTheOutputContains(string text) => + Assert.Contains(text, Output); + + [Then("the output matches {string}")] + public void ThenTheOutputMatches(string pattern) => + Assert.Matches(pattern, Output); + + [Then("a protostar binary exists in the install dir")] + public void ThenABinaryExists() => + Assert.True(File.Exists(Sandbox.InstalledBinary), + $"Expected an installed binary at '{Sandbox.InstalledBinary}'."); + + [Then("no protostar binary exists in the install dir")] + public void ThenNoBinaryExists() => + Assert.False(File.Exists(Sandbox.InstalledBinary), + $"Did not expect a binary at '{Sandbox.InstalledBinary}'."); + + public void Dispose() => _sandbox?.Dispose(); +} diff --git a/test/Protostar.Cli.Acceptance/Support/CliRunner.cs b/test/Protostar.Cli.Acceptance/Support/CliRunner.cs new file mode 100644 index 0000000..8c61ba8 --- /dev/null +++ b/test/Protostar.Cli.Acceptance/Support/CliRunner.cs @@ -0,0 +1,58 @@ +using CliWrap; +using CliWrap.Buffered; + +namespace Protostar.Cli.Acceptance.Support; + +/// +/// Locates the built protostar binary and runs it as a child process, the way a user would. +/// Output capture is buffered; stdout/stderr and the exit code are returned for assertions. +/// +public static class CliRunner +{ + private static readonly Lazy LazyBinary = new(ResolveBinary); + + /// Absolute path to the protostar binary under test. + public static string BinaryPath => LazyBinary.Value; + + public static async Task RunAsync(IEnumerable args) + { + return await CliWrap.Cli.Wrap(BinaryPath) + .WithArguments(args) + // Non-zero exits are expected in some scenarios; assert on them rather than throwing. + .WithValidation(CommandResultValidation.None) + .ExecuteBufferedAsync(); + } + + private static string ResolveBinary() + { + // CI publishes a self-contained binary and points us at it. + var fromEnv = Environment.GetEnvironmentVariable("PROTOSTAR_BIN"); + if (!string.IsNullOrWhiteSpace(fromEnv) && File.Exists(fromEnv)) + return fromEnv; + + // Local/dev: the ProjectReference builds the CLI; find its apphost under src/Protostar.Cli/bin. + var repoRoot = FindRepoRoot(); + var exeName = OperatingSystem.IsWindows() ? "protostar.exe" : "protostar"; + var binDir = Path.Combine(repoRoot, "src", "Protostar.Cli", "bin"); + if (!Directory.Exists(binDir)) + throw new FileNotFoundException( + $"protostar build output not found under '{binDir}'. Build the solution first, or set PROTOSTAR_BIN."); + + var match = Directory.EnumerateFiles(binDir, exeName, SearchOption.AllDirectories) + .OrderByDescending(File.GetLastWriteTimeUtc) + .FirstOrDefault(); + + return match ?? throw new FileNotFoundException( + $"Could not find '{exeName}' under '{binDir}'. Build the solution first, or set PROTOSTAR_BIN."); + } + + private static string FindRepoRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "protostar.sln"))) + dir = dir.Parent; + + return dir?.FullName ?? throw new DirectoryNotFoundException( + "Could not locate the repo root (protostar.sln) from the test output directory."); + } +} diff --git a/test/Protostar.Cli.Acceptance/Support/Sandbox.cs b/test/Protostar.Cli.Acceptance/Support/Sandbox.cs new file mode 100644 index 0000000..3b23a47 --- /dev/null +++ b/test/Protostar.Cli.Acceptance/Support/Sandbox.cs @@ -0,0 +1,39 @@ +namespace Protostar.Cli.Acceptance.Support; + +/// +/// A throwaway directory tree for install/uninstall scenarios. Scenarios install into +/// with --no-modify-path so a test never touches the real +/// machine's filesystem outside this sandbox or its PATH. Disposed at the end of each scenario. +/// +public sealed class Sandbox : IDisposable +{ + public Sandbox() + { + Root = Path.Combine(Path.GetTempPath(), "protostar-acceptance", Guid.NewGuid().ToString("n")); + InstallDir = Path.Combine(Root, "bin"); + Directory.CreateDirectory(Root); + } + + /// Root of the sandbox; removed on dispose. + public string Root { get; } + + /// Directory passed to protostar install --dir. + public string InstallDir { get; } + + /// Where an install is expected to place the binary. + public string InstalledBinary => + Path.Combine(InstallDir, OperatingSystem.IsWindows() ? "protostar.exe" : "protostar"); + + public void Dispose() + { + try + { + if (Directory.Exists(Root)) + Directory.Delete(Root, recursive: true); + } + catch + { + // Best-effort cleanup; a leaked temp dir must never fail a test. + } + } +} diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..2725020 --- /dev/null +++ b/test/README.md @@ -0,0 +1,42 @@ +# Tests + +## Protostar.Cli.Acceptance + +Behaviour-driven acceptance tests for the protostar CLI, written with +[Reqnroll](https://reqnroll.net) (the maintained successor to SpecFlow) and run by xUnit. Each +scenario validates the CLI **the way a user runs it**: it executes the real built binary as a child +process (via [CliWrap](https://github.com/Tyrrrz/CliWrap)) and asserts on stdout/stderr, the exit +code, and filesystem side effects. There is no mocking of the command layer. + +### Layout + +- `Features/*.feature` — Gherkin scenarios in user language (version, default command, help, + install, uninstall). +- `Steps/CliSteps.cs` — step definitions binding the Gherkin to `CliRunner` + `Sandbox`. +- `Support/CliRunner.cs` — locates the protostar binary and runs it. +- `Support/Sandbox.cs` — a throwaway temp directory per scenario, so install/uninstall never touch + the real machine. Install/uninstall scenarios pass `--no-modify-path`, so the suite never edits + your PATH. + +### Running + +From the repo root: + +```bash +dotnet test +``` + +The acceptance project has a `ProjectReference` to `Protostar.Cli`, so the CLI is built first and +its binary is discovered automatically under `src/Protostar.Cli/bin/`. To point the suite at a +specific binary instead (for example a published self-contained build), set `PROTOSTAR_BIN`: + +```bash +PROTOSTAR_BIN=/path/to/protostar dotnet test +``` + +### Extending to harness integrations + +When the CLI gains harness integration (hook install, skill discovery), make every harness path +(config dir, settings file, skills dir) redirectable via an environment variable or flag, then add +a `Support/Harness` fixture beside `Sandbox` that builds a fake harness layout in a temp dir. Use a +Gherkin `Scenario Outline` with one Examples row per harness. See PROT-41 for the full plan.