feat(cli): gonext plugin test — in-process fake host + conformance suite

cli/gonext/cmd/plugin/test.go

@@ -1,23 +1,43 @@
 package plugin
 
 import (
+	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
 
 	"github.com/Singleton-Solution/GoNext/cli/gonext/internal/plugintest"
+	"github.com/Singleton-Solution/GoNext/packages/go/plugins/conformance"
 )
 
-// runTest implements `gonext plugin test [--json] <bundle>`. It returns the
-// process exit code (see [ExitOK], [ExitFail], [ExitUsage]).
+// runTest implements `gonext plugin test [--json] [--suite=conformance]
+// [--record-fixtures=DIR] <bundle>`. It returns the process exit code
+// (see [ExitOK], [ExitFail], [ExitUsage]).
+//
+// The two suites:
+//
+//   - default suite (no --suite): runs the bundle contract checks from
+//     [plugintest.Run] (manifest schema, layout, capability vocabulary,
+//     WASM header). Fast, host-independent, suitable for every save.
+//   - --suite=conformance: also runs the in-memory conformance scenarios
+//     under [conformance.NewSuite] against a fakehost.Host. Slower, more
+//     thorough, suitable for CI and pre-publish.
+//
+// We keep `default` and `conformance` as separate codepaths so a plugin
+// author who is fine-tuning their bundle layout doesn't pay the cost of
+// the behavioural scenarios — and so the marketplace's ingest pipeline
+// stays on the stable `default` shape until conformance graduates.
 func runTest(args []string, stdout, stderr io.Writer) int {
 	fs := flag.NewFlagSet("gonext plugin test", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	fs.Usage = func() {
 		fmt.Fprintln(stderr, testUsage)
 	}
 	jsonOut := fs.Bool("json", false, "emit the report as a single JSON object on stdout")
+	suite := fs.String("suite", "", "named suite to run; only `conformance` is recognised today")
+	recordFixtures := fs.String("record-fixtures", "",
+		"with --suite=conformance: directory to dump one JSON fixture per scenario")
 
 	if err := fs.Parse(args); err != nil {
 		// flag.ContinueOnError already printed the error via fs.SetOutput.
@@ -40,22 +60,48 @@ func runTest(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintln(stderr, testUsage)
 		return ExitUsage
 	}
+	bundle := rest[0]
 
-	report, err := plugintest.Run(rest[0])
+	// --record-fixtures only makes sense in --suite=conformance mode.
+	// Rejecting it loudly elsewhere prevents the silent-no-op trap.
+	if *recordFixtures != "" && *suite != "conformance" {
+		fmt.Fprintln(stderr,
+			"gonext plugin test: --record-fixtures requires --suite=conformance")
+		return ExitUsage
+	}
+
+	switch *suite {
+	case "", "default":
+		return runDefaultSuite(bundle, *jsonOut, stdout, stderr)
+	case "conformance":
+		return runConformanceSuite(bundle, *jsonOut, *recordFixtures, stdout, stderr)
+	default:
+		fmt.Fprintf(stderr, "gonext plugin test: unknown suite %q (want: default | conformance)\n",
+			*suite)
+		return ExitUsage
+	}
+}
+
+// runDefaultSuite runs the bundle-contract checks via plugintest. This
+// is the original `gonext plugin test` behaviour and remains the
+// fastest path.
+func runDefaultSuite(bundle string, jsonOut bool, stdout, stderr io.Writer) int {
+	report, err := plugintest.Run(bundle)
 	if err != nil {
-		// Couldn't even open the bundle — that's a usage-class problem, not
-		// a contract failure. Make it visible regardless of format.
-		if *jsonOut {
-			// Emit an empty-checks report so the marketplace ingestor still
-			// gets a valid JSON document.
+		// Couldn't even open the bundle — that's a usage-class
+		// problem, not a contract failure. Make it visible
+		// regardless of format.
+		if jsonOut {
+			// Emit an empty-checks report so the marketplace
+			// ingestor still gets a valid JSON document.
 			report.Pass = false
 			_ = writeJSON(stdout, report)
 		}
 		fmt.Fprintf(stderr, "gonext plugin test: %s\n", err)
 		return ExitFail
 	}
 
-	if *jsonOut {
+	if jsonOut {
 		if err := writeJSON(stdout, report); err != nil {
 			fmt.Fprintf(stderr, "gonext plugin test: writing JSON: %s\n", err)
 			return ExitFail
@@ -73,6 +119,36 @@ func runTest(args []string, stdout, stderr io.Writer) int {
 	return ExitOK
 }
 
+// runConformanceSuite runs the in-memory conformance scenarios. The
+// suite mounts a fakehost.Host per scenario and asserts on the
+// recorded host-call trace.
+func runConformanceSuite(bundle string, jsonOut bool, recordDir string, stdout, stderr io.Writer) int {
+	s := conformance.NewSuite()
+	s.RecordFixtures = recordDir
+	report, err := s.Run(context.Background(), bundle)
+	if err != nil {
+		fmt.Fprintf(stderr, "gonext plugin test: conformance: %s\n", err)
+		return ExitFail
+	}
+
+	if jsonOut {
+		if err := report.WriteJSON(stdout); err != nil {
+			fmt.Fprintf(stderr, "gonext plugin test: writing JSON: %s\n", err)
+			return ExitFail
+		}
+	} else {
+		if err := report.WriteHuman(stdout); err != nil {
+			fmt.Fprintf(stderr, "gonext plugin test: writing report: %s\n", err)
+			return ExitFail
+		}
+	}
+
+	if !report.Pass {
+		return ExitFail
+	}
+	return ExitOK
+}
+
 // writeJSON emits the report as a pretty-printed JSON document followed by a
 // trailing newline.
 func writeJSON(w io.Writer, r plugintest.Report) error {
@@ -84,23 +160,42 @@ func writeJSON(w io.Writer, r plugintest.Report) error {
 const testUsage = `gonext plugin test — run the plugin contract checks against a bundle
 
 Usage:
-  gonext plugin test [--json] <bundle>
+  gonext plugin test [--json] [--suite=conformance] [--record-fixtures=DIR] <bundle>
 
 Arguments:
-  <bundle>   Path to a plugin bundle. Either a directory containing
-             manifest.json at its root, or a .gnplugin (zip) archive.
+  <bundle>             Path to a plugin bundle. Either a directory
+                       containing manifest.json at its root, or a
+                       .gnplugin (zip) archive.
 
 Flags:
-  --json     Emit the report as a single JSON object on stdout. Without
-             this flag, a human-readable PASS/FAIL/SKIP table is printed.
+  --json               Emit the report as a single JSON object on stdout.
+                       Without this flag, a human-readable PASS/FAIL/SKIP
+                       table is printed.
+
+  --suite=NAME         Choose which suite to run.
+                         (empty | default)  Bundle contract checks only —
+                                            fast, host-independent.
+                         conformance        Adds scenario-based assertions
+                                            against an in-memory fake host
+                                            (declared caps match usage,
+                                            init+teardown idempotent,
+                                            1s synthetic-job budget, etc.).
+
+  --record-fixtures=DIR  Only valid with --suite=conformance. After the run
+                         finishes, dump one JSON fixture per scenario into
+                         DIR for later replay. Existing files are
+                         overwritten.
 
 Exit codes:
   0   all checks passed (skipped checks do not count against pass)
   1   at least one check failed, or the bundle could not be opened
   2   usage error (bad flags or missing argument)
 
-Today the runner performs the checks that don't require the WASM host:
-manifest schema validation, bundle layout, capability vocabulary, and a
-read-only WebAssembly header check. The other contract checks from
-docs/11-testing-ci.md §7.1 are emitted as rows with status "skipped" and
-reason "runtime-not-available" until the host lands.`
+Default suite: manifest schema validation, bundle layout, capability
+vocabulary, and a read-only WebAssembly header check. The remaining
+contract checks from docs/11-testing-ci.md §7.1 are emitted as skipped
+rows until the host lands.
+
+Conformance suite: parses the manifest (v1 OR legacy), runs the built-in
+scenarios from packages/go/plugins/conformance, and (with
+--record-fixtures) writes the recorded fake-host event trace to disk.`

cli/gonext/cmd/plugin/test_test.go

@@ -0,0 +1,185 @@
+package plugin
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// conformanceManifest is a v1-schema manifest that satisfies every
+// built-in conformance scenario.
+const conformanceManifest = `{
+  "$schema": "https://wpc.dev/schemas/plugin-manifest-v1.json",
+  "slug": "gn-seo",
+  "name": "GN SEO",
+  "version": "1.0.0",
+  "abi_version": 1,
+  "license": "MIT",
+  "server": { "wasm": "server/plugin.wasm" },
+  "capabilities": {"posts.read": {}, "posts.write": {}, "kv": {}},
+  "hooks": {
+    "actions": [{"name": "save_post", "handler": "onSave"}]
+  },
+  "jobs": ["gn-seo.recompute"]
+}`
+
+// failingConformanceManifest has hooks but no capabilities — the
+// capabilities.match-usage scenario fails on this.
+const failingConformanceManifest = `{
+  "$schema": "https://wpc.dev/schemas/plugin-manifest-v1.json",
+  "slug": "gn-bad",
+  "name": "Bad",
+  "version": "1.0.0",
+  "abi_version": 1,
+  "license": "MIT",
+  "server": { "wasm": "server/plugin.wasm" },
+  "capabilities": {},
+  "hooks": {
+    "actions": [{"name": "save_post", "handler": "onSave"}]
+  }
+}`
+
+func TestRunTest_Conformance_HappyPath(t *testing.T) {
+	dir := writeBundleDir(t, []byte(conformanceManifest), validHeader)
+	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
+	got := Run([]string{"test", "--suite=conformance", dir}, stdout, stderr)
+	if got != ExitOK {
+		t.Fatalf("exit = %d; want %d. stderr=%q stdout=%q",
+			got, ExitOK, stderr.String(), stdout.String())
+	}
+	out := stdout.String()
+	for _, sub := range []string{"capabilities.declared", "init.idempotent", "PASS"} {
+		if !strings.Contains(out, sub) {
+			t.Errorf("conformance output missing %q; got %q", sub, out)
+		}
+	}
+}
+
+func TestRunTest_Conformance_Failing(t *testing.T) {
+	dir := writeBundleDir(t, []byte(failingConformanceManifest), validHeader)
+	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
+	got := Run([]string{"test", "--suite=conformance", dir}, stdout, stderr)
+	if got != ExitFail {
+		t.Fatalf("exit = %d; want %d. stderr=%q stdout=%q",
+			got, ExitFail, stderr.String(), stdout.String())
+	}
+	if !strings.Contains(stdout.String(), "FAIL") {
+		t.Errorf("expected FAIL row; got %q", stdout.String())
+	}
+}
+
+func TestRunTest_Conformance_JSON(t *testing.T) {
+	dir := writeBundleDir(t, []byte(conformanceManifest), validHeader)
+	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
+	got := Run([]string{"test", "--suite=conformance", "--json", dir}, stdout, stderr)
+	if got != ExitOK {
+		t.Fatalf("exit = %d; want %d. stderr=%q",
+			got, ExitOK, stderr.String())
+	}
+	var doc struct {
+		Bundle  string `json:"bundle"`
+		Suite   string `json:"suite"`
+		Pass    bool   `json:"pass"`
+		Results []struct {
+			Name   string `json:"name"`
+			Status string `json:"status"`
+		} `json:"results"`
+	}
+	if err := json.NewDecoder(stdout).Decode(&doc); err != nil {
+		t.Fatalf("decode JSON: %v\nstdout=%s", err, stdout.String())
+	}
+	if doc.Suite != "conformance" {
+		t.Errorf("suite = %q; want conformance", doc.Suite)
+	}
+	if !doc.Pass {
+		t.Errorf("pass = false; want true. results=%+v", doc.Results)
+	}
+	if len(doc.Results) == 0 {
+		t.Errorf("expected at least one result, got 0")
+	}
+}
+
+func TestRunTest_Conformance_RecordFixtures(t *testing.T) {
+	dir := writeBundleDir(t, []byte(conformanceManifest), validHeader)
+	fixtures := filepath.Join(t.TempDir(), "out")
+	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
+	got := Run([]string{
+		"test", "--suite=conformance",
+		"--record-fixtures=" + fixtures, dir,
+	}, stdout, stderr)
+	if got != ExitOK {
+		t.Fatalf("exit = %d; want %d. stderr=%q", got, ExitOK, stderr.String())
+	}
+	entries, err := os.ReadDir(fixtures)
+	if err != nil {
+		t.Fatalf("read fixtures dir: %v", err)
+	}
+	if len(entries) == 0 {
+		t.Errorf("expected fixtures dumped; got 0 entries")
+	}
+}
+
+func TestRunTest_RecordFixtures_RequiresConformance(t *testing.T) {
+	dir := writeBundleDir(t, []byte(minimalManifest), validHeader)
+	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
+	got := Run([]string{"test", "--record-fixtures=/tmp/x", dir}, stdout, stderr)
+	if got != ExitUsage {
+		t.Fatalf("exit = %d; want ExitUsage. stderr=%q stdout=%q",
+			got, stderr.String(), stdout.String())
+	}
+	if !strings.Contains(stderr.String(), "requires --suite=conformance") {
+		t.Errorf("stderr missing diagnostic; got %q", stderr.String())
+	}
+}
+
+func TestRunTest_UnknownSuite(t *testing.T) {
+	dir := writeBundleDir(t, []byte(minimalManifest), validHeader)
+	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
+	got := Run([]string{"test", "--suite=banana", dir}, stdout, stderr)
+	if got != ExitUsage {
+		t.Fatalf("exit = %d; want ExitUsage. stderr=%q stdout=%q",
+			got, stderr.String(), stdout.String())
+	}
+	if !strings.Contains(stderr.String(), "unknown suite") {
+		t.Errorf("stderr missing diagnostic; got %q", stderr.String())
+	}
+}
+
+func TestRunTest_DefaultSuite_StillWorks(t *testing.T) {
+	// Sanity: passing --suite=default (explicit) routes through the
+	// default path and reports the contract checks, not the
+	// conformance scenarios.
+	dir := writeBundleDir(t, []byte(minimalManifest), validHeader)
+	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
+	got := Run([]string{"test", "--suite=default", dir}, stdout, stderr)
+	if got != ExitOK {
+		t.Fatalf("exit = %d; want %d. stderr=%q", got, ExitOK, stderr.String())
+	}
+	if !strings.Contains(stdout.String(), "manifest.schema") {
+		t.Errorf("default suite missing manifest.schema row; got %q", stdout.String())
+	}
+}
+
+func TestRunTest_Conformance_SEOExample(t *testing.T) {
+	// End-to-end: drive `gonext plugin test --suite=conformance` against
+	// the real seo example bundle. This is the "smoke-test against
+	// examples/plugins/seo" the issue calls for.
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("getwd: %v", err)
+	}
+	bundle := filepath.Clean(filepath.Join(wd,
+		"..", "..", "..", "..", "examples", "plugins", "seo"))
+	if _, err := os.Stat(filepath.Join(bundle, "manifest.json")); err != nil {
+		t.Skipf("seo example not found at %s: %v", bundle, err)
+	}
+	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
+	got := Run([]string{"test", "--suite=conformance", bundle}, stdout, stderr)
+	if got != ExitOK {
+		t.Fatalf("seo conformance: exit = %d; want %d.\nstdout=%s\nstderr=%s",
+			got, ExitOK, stdout.String(), stderr.String())
+	}
+}