test: Reqnroll BDD acceptance suite for the CLI (PROT-41)

.github/workflows/ci.yml

@@ -0,0 +1,38 @@
+name: ci
+
+# Build and run the acceptance suite on every PR and on pushes to main. The Reqnroll scenarios
+# drive the real built protostar binary, so this also exercises the CLI end to end. Run on both
+# Linux and Windows to cover the OS-specific binary name and install paths.
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          fetch-depth: 0          # MinVer needs full history to compute the version.
+
+      - uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2 # v5
+        with:
+          dotnet-version: '10.0.x'
+
+      - name: Build
+        run: dotnet build -c Release
+
+      - name: Test
+        run: dotnet test -c Release --no-build --verbosity normal

.gitignore

@@ -25,3 +25,6 @@ appsettings.*.local.json
 # OS
 .DS_Store
 Thumbs.db
+
+# Reqnroll generates code-behind beside each .feature file; it is rebuilt every compile.
+*.feature.cs

protostar.sln

@@ -7,6 +7,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Protostar.Cli", "src\Protostar.Cli\Protostar.Cli.csproj", "{B94673CE-1472-46C5-8538-82062FDD51E6}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{0C88DD14-F956-CE84-757C-A364CCF449FC}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Protostar.Cli.Acceptance", "test\Protostar.Cli.Acceptance\Protostar.Cli.Acceptance.csproj", "{DC3CF955-4BC2-481C-B640-3FEC51094494}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -29,11 +33,24 @@ Global
 		{B94673CE-1472-46C5-8538-82062FDD51E6}.Release|x64.Build.0 = Release|Any CPU
 		{B94673CE-1472-46C5-8538-82062FDD51E6}.Release|x86.ActiveCfg = Release|Any CPU
 		{B94673CE-1472-46C5-8538-82062FDD51E6}.Release|x86.Build.0 = Release|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|x64.Build.0 = Debug|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Debug|x86.Build.0 = Debug|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|x64.ActiveCfg = Release|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|x64.Build.0 = Release|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|x86.ActiveCfg = Release|Any CPU
+		{DC3CF955-4BC2-481C-B640-3FEC51094494}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{B94673CE-1472-46C5-8538-82062FDD51E6} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{DC3CF955-4BC2-481C-B640-3FEC51094494} = {0C88DD14-F956-CE84-757C-A364CCF449FC}
 	EndGlobalSection
 EndGlobal

src/Protostar.Cli/Commands/UninstallCommand.cs

@@ -13,6 +13,10 @@ public sealed class Settings : CommandSettings
         [CommandOption("-d|--dir <DIR>")]
         [Description("Install directory to remove from. Defaults to the per-user location.")]
         public string? Dir { get; init; }
+
+        [CommandOption("--no-modify-path")]
+        [Description("Do not remove the install directory from PATH.")]
+        public bool NoModifyPath { get; init; }
     }
 
     protected override int Execute(CommandContext context, Settings settings, CancellationToken cancellation)
@@ -39,7 +43,8 @@ protected override int Execute(CommandContext context, Settings settings, Cancel
             return 1;
         }
 
-        PathManager.RemoveFromPath(dir);
+        if (!settings.NoModifyPath)
+            PathManager.RemoveFromPath(dir);
         AnsiConsole.MarkupLine($"Removed [aqua]protostar[/] from [grey]{Markup.Escape(dir)}[/].");
         return 0;
     }

test/Protostar.Cli.Acceptance/Features/DefaultCommand.feature

@@ -0,0 +1,10 @@
+Feature: Default command guidance
+  As a new user
+  I want a bare protostar invocation to explain itself
+  So that I know it is working and where to go next
+
+  Scenario: Running with no arguments prints guidance
+    When I run protostar with no arguments
+    Then the exit code is 0
+    And the output contains "Live, continuous refinement of agent skills."
+    And the output contains "protostar --help"

test/Protostar.Cli.Acceptance/Features/Help.feature

@@ -0,0 +1,10 @@
+Feature: Help lists available commands
+  As a user
+  I want protostar --help to list the commands
+  So that I can discover what protostar can do
+
+  Scenario: --help lists the install and uninstall commands
+    When I run protostar with "--help"
+    Then the exit code is 0
+    And the output contains "install"
+    And the output contains "uninstall"

test/Protostar.Cli.Acceptance/Features/Install.feature

@@ -0,0 +1,19 @@
+Feature: Installing protostar
+  As a user
+  I want protostar install to place the binary in a directory
+  So that I can run it from my PATH
+
+  Scenario: Installing into a clean sandbox directory
+    Given a clean install sandbox
+    When I run protostar with "install --dir {installDir} --no-modify-path"
+    Then the exit code is 0
+    And the output contains "Installed"
+    And a protostar binary exists in the install dir
+
+  Scenario: Re-installing into the same directory succeeds
+    Given a clean install sandbox
+    When I run protostar with "install --dir {installDir} --no-modify-path"
+    And I run protostar with "install --dir {installDir} --no-modify-path"
+    Then the exit code is 0
+    And the output contains "Installed"
+    And a protostar binary exists in the install dir

test/Protostar.Cli.Acceptance/Features/Uninstall.feature

@@ -0,0 +1,18 @@
+Feature: Uninstalling protostar
+  As a user
+  I want protostar uninstall to remove an installed binary
+  So that I can cleanly remove the tool
+
+  Scenario: Uninstalling removes the installed binary
+    Given a clean install sandbox
+    When I run protostar with "install --dir {installDir} --no-modify-path"
+    And I run protostar with "uninstall --dir {installDir} --no-modify-path"
+    Then the exit code is 0
+    And the output contains "Removed"
+    And no protostar binary exists in the install dir
+
+  Scenario: Uninstalling when nothing is installed is a no-op
+    Given a clean install sandbox
+    When I run protostar with "uninstall --dir {installDir} --no-modify-path"
+    Then the exit code is 0
+    And the output contains "Nothing to remove"

test/Protostar.Cli.Acceptance/Features/Version.feature

@@ -0,0 +1,9 @@
+Feature: Reporting the version
+  As a user who installed protostar
+  I want protostar --version to report a version
+  So that I can confirm which build I am running
+
+  Scenario: --version prints a semantic version
+    When I run protostar with "--version"
+    Then the exit code is 0
+    And the output matches "\d+\.\d+\.\d+"

test/Protostar.Cli.Acceptance/Protostar.Cli.Acceptance.csproj

@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <!-- Test project: never packed, never shipped. -->
+    <IsPackable>false</IsPackable>
+    <IsPublishable>false</IsPublishable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
+    <!-- Reqnroll.xUnit pins xUnit v2 (xunit.core 2.8.1); use the matching v2 runner. -->
+    <PackageReference Include="Reqnroll.xUnit" Version="3.3.4" />
+    <PackageReference Include="xunit" Version="2.9.3" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
+    <!-- Drives the real built binary as a child process. -->
+    <PackageReference Include="CliWrap" Version="3.10.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <!-- Build the CLI before the tests so its binary is available for the black-box runner. -->
+    <ProjectReference Include="..\..\src\Protostar.Cli\Protostar.Cli.csproj" />
+  </ItemGroup>
+
+</Project>

test/Protostar.Cli.Acceptance/Steps/CliSteps.cs

@@ -0,0 +1,69 @@
+using CliWrap.Buffered;
+using Protostar.Cli.Acceptance.Support;
+using Reqnroll;
+using Xunit;
+
+namespace Protostar.Cli.Acceptance.Steps;
+
+/// <summary>
+/// Step definitions that drive the real protostar binary as a user would and assert on its
+/// stdout/stderr, exit code, and filesystem side effects. One <see cref="Sandbox"/> per scenario.
+/// </summary>
+[Binding]
+public sealed class CliSteps : IDisposable
+{
+    private Sandbox? _sandbox;
+    private BufferedCommandResult? _result;
+
+    private Sandbox Sandbox => _sandbox ??= new Sandbox();
+
+    private BufferedCommandResult Result =>
+        _result ?? throw new InvalidOperationException("No protostar invocation has run in this scenario yet.");
+
+    // Spectre disables colour when stdout is redirected, so captured output is plain text. We still
+    // merge stderr in case a future command writes there.
+    private string Output => Result.StandardOutput + Result.StandardError;
+
+    [Given("a clean install sandbox")]
+    public void GivenACleanInstallSandbox() => _ = Sandbox;
+
+    [When("I run protostar with no arguments")]
+    public async Task WhenIRunWithNoArguments() =>
+        _result = await CliRunner.RunAsync([]);
+
+    [When("I run protostar with {string}")]
+    public async Task WhenIRunWith(string argLine)
+    {
+        // Split on spaces, then substitute the {installDir} placeholder as a whole token so a
+        // sandbox path containing spaces still arrives as a single argument.
+        var args = argLine
+            .Split(' ', StringSplitOptions.RemoveEmptyEntries)
+            .Select(token => token == "{installDir}" ? Sandbox.InstallDir : token)
+            .ToList();
+        _result = await CliRunner.RunAsync(args);
+    }
+
+    [Then("the exit code is {int}")]
+    public void ThenTheExitCodeIs(int expected) =>
+        Assert.Equal(expected, Result.ExitCode);
+
+    [Then("the output contains {string}")]
+    public void ThenTheOutputContains(string text) =>
+        Assert.Contains(text, Output);
+
+    [Then("the output matches {string}")]
+    public void ThenTheOutputMatches(string pattern) =>
+        Assert.Matches(pattern, Output);
+
+    [Then("a protostar binary exists in the install dir")]
+    public void ThenABinaryExists() =>
+        Assert.True(File.Exists(Sandbox.InstalledBinary),
+            $"Expected an installed binary at '{Sandbox.InstalledBinary}'.");
+
+    [Then("no protostar binary exists in the install dir")]
+    public void ThenNoBinaryExists() =>
+        Assert.False(File.Exists(Sandbox.InstalledBinary),
+            $"Did not expect a binary at '{Sandbox.InstalledBinary}'.");
+
+    public void Dispose() => _sandbox?.Dispose();
+}

test/Protostar.Cli.Acceptance/Support/CliRunner.cs

@@ -0,0 +1,58 @@
+using CliWrap;
+using CliWrap.Buffered;
+
+namespace Protostar.Cli.Acceptance.Support;
+
+/// <summary>
+/// Locates the built protostar binary and runs it as a child process, the way a user would.
+/// Output capture is buffered; stdout/stderr and the exit code are returned for assertions.
+/// </summary>
+public static class CliRunner
+{
+    private static readonly Lazy<string> LazyBinary = new(ResolveBinary);
+
+    /// <summary>Absolute path to the protostar binary under test.</summary>
+    public static string BinaryPath => LazyBinary.Value;
+
+    public static async Task<BufferedCommandResult> RunAsync(IEnumerable<string> args)
+    {
+        return await CliWrap.Cli.Wrap(BinaryPath)
+            .WithArguments(args)
+            // Non-zero exits are expected in some scenarios; assert on them rather than throwing.
+            .WithValidation(CommandResultValidation.None)
+            .ExecuteBufferedAsync();
+    }
+
+    private static string ResolveBinary()
+    {
+        // CI publishes a self-contained binary and points us at it.
+        var fromEnv = Environment.GetEnvironmentVariable("PROTOSTAR_BIN");
+        if (!string.IsNullOrWhiteSpace(fromEnv) && File.Exists(fromEnv))
+            return fromEnv;
+
+        // Local/dev: the ProjectReference builds the CLI; find its apphost under src/Protostar.Cli/bin.
+        var repoRoot = FindRepoRoot();
+        var exeName = OperatingSystem.IsWindows() ? "protostar.exe" : "protostar";
+        var binDir = Path.Combine(repoRoot, "src", "Protostar.Cli", "bin");
+        if (!Directory.Exists(binDir))
+            throw new FileNotFoundException(
+                $"protostar build output not found under '{binDir}'. Build the solution first, or set PROTOSTAR_BIN.");
+
+        var match = Directory.EnumerateFiles(binDir, exeName, SearchOption.AllDirectories)
+            .OrderByDescending(File.GetLastWriteTimeUtc)
+            .FirstOrDefault();
+
+        return match ?? throw new FileNotFoundException(
+            $"Could not find '{exeName}' under '{binDir}'. Build the solution first, or set PROTOSTAR_BIN.");
+    }
+
+    private static string FindRepoRoot()
+    {
+        var dir = new DirectoryInfo(AppContext.BaseDirectory);
+        while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "protostar.sln")))
+            dir = dir.Parent;
+
+        return dir?.FullName ?? throw new DirectoryNotFoundException(
+            "Could not locate the repo root (protostar.sln) from the test output directory.");
+    }
+}

test/Protostar.Cli.Acceptance/Support/Sandbox.cs

@@ -0,0 +1,39 @@
+namespace Protostar.Cli.Acceptance.Support;
+
+/// <summary>
+/// A throwaway directory tree for install/uninstall scenarios. Scenarios install into
+/// <see cref="InstallDir"/> with <c>--no-modify-path</c> so a test never touches the real
+/// machine's filesystem outside this sandbox or its PATH. Disposed at the end of each scenario.
+/// </summary>
+public sealed class Sandbox : IDisposable
+{
+    public Sandbox()
+    {
+        Root = Path.Combine(Path.GetTempPath(), "protostar-acceptance", Guid.NewGuid().ToString("n"));
+        InstallDir = Path.Combine(Root, "bin");
+        Directory.CreateDirectory(Root);
+    }
+
+    /// <summary>Root of the sandbox; removed on dispose.</summary>
+    public string Root { get; }
+
+    /// <summary>Directory passed to <c>protostar install --dir</c>.</summary>
+    public string InstallDir { get; }
+
+    /// <summary>Where an install is expected to place the binary.</summary>
+    public string InstalledBinary =>
+        Path.Combine(InstallDir, OperatingSystem.IsWindows() ? "protostar.exe" : "protostar");
+
+    public void Dispose()
+    {
+        try
+        {
+            if (Directory.Exists(Root))
+                Directory.Delete(Root, recursive: true);
+        }
+        catch
+        {
+            // Best-effort cleanup; a leaked temp dir must never fail a test.
+        }
+    }
+}