feat: regression testing

.github/workflows/regression.yml

@@ -0,0 +1,72 @@
+name: Regression Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  run_all_tests:
+    runs-on: ubuntu-latest
+    #if: "!contains(github.event.pull_request.title, '[NO-REGRESSION-TEST]')"
+    steps:
+      - name: Setup Go environment
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+
+      - name: Setup Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+          components: rust-analyzer
+
+      - name: Setup Python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Checkout pull request code
+        uses: actions/checkout@v4
+        with:
+          path: 'pr_repo'
+          submodules: true
+
+      - name: Install Python dependencies
+        run: |
+          pip install -r ./pr_repo/script/requirements.txt
+          pip install ./pr_repo/pylsp
+
+      - name: Checkout main branch code
+        uses: actions/checkout@v4
+        with:
+          ref: 'main'
+          path: 'main_repo'
+
+      - name: Compile both binaries
+        run: |
+          (cd main_repo && go build -o ../abcoder_old)
+          (cd pr_repo && go build -o ../abcoder_new)
+
+      - name: Run test scripts and generate outputs
+        run: |
+          LANGS="go rust python" OUTDIR=out_old ABCEXE=./abcoder_old ./pr_repo/script/run_all_testdata.sh
+          LANGS="go rust python" OUTDIR=out_new ABCEXE=./abcoder_new ./pr_repo/script/run_all_testdata.sh
+
+      - name: Compare outputs and check for regression
+        id: diff_check
+        run: ./pr_repo/script/diffjson.py out_old out_new
+        continue-on-error: true
+
+      - name: Upload output directories
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: regression-outputs
+          path: |
+            out_old
+            out_new
+          retention-days: 3

lang/lsp/client.go

@@ -17,6 +17,7 @@ package lsp
 import (
 	"bufio"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -65,22 +66,6 @@ func NewLSPClient(repo string, openfile string, wait time.Duration, opts ClientO
 		if err != nil {
 			return nil, err
 		}
-
-		// wait for "textDocument/publishDiagnostics" notification
-		// 	resp := cli.WaitFirstNotify("textDocument/publishDiagnostics")
-		// again:
-		// 	var diagnostics lsp.PublishDiagnosticsParams
-		// 	if err := json.Unmarshal(*resp.Params, &diagnostics); err != nil {
-		// 		logger.Fatalf("Failed to unmarshal diagnostics: %v", err)
-		// 	}
-		// 	if len(diagnostics.Diagnostics) > 0 {
-		// 		// wait again
-		// 		resp = cli.WaitFirstNotify("textDocument/publishDiagnostics")
-		// 		if retry > 0 {
-		// 			retry--
-		// 			goto again
-		// 		}
-		// 	}
 	}
 
 	time.Sleep(wait)
@@ -93,6 +78,19 @@ func (c *LSPClient) Close() error {
 	return c.Conn.Close()
 }
 
+// Extra wrapper around json rpc to
+// 1. implement a transparent, generic cache
+func (cli *LSPClient) Call(ctx context.Context, method string, params, result interface{}, opts ...jsonrpc2.CallOption) error {
+	var raw json.RawMessage
+	if err := cli.Conn.Call(ctx, method, params, &raw); err != nil {
+		return err
+	}
+	if err := json.Unmarshal(raw, result); err != nil {
+		return err
+	}
+	return nil
+}
+
 type initializeParams struct {
 	ProcessID int `json:"processId,omitempty"`
 

lang/lsp/clients_test.go

@@ -160,10 +160,10 @@ Output`
 	// references
 	refRange := Range{
 		Start: Position{
-			Line:      13,
-			Character: 13,
+			Line:      48,
+			Character: 6,
 		},
-	}
+	} // trait $0MyTrait {
 	t.Run("references", func(t *testing.T) {
 		id := Location{
 			URI:   entity_mod_uri,
@@ -173,6 +173,9 @@ Output`
 		if err != nil {
 			t.Fatalf("Find Reference failed: %v", err)
 		}
+		if len(references) != 4 {
+			t.Fatalf("Expected 4 references, got %d\n%+v\n", len(references), references)
+		}
 		if _, err := json.Marshal(references); err != nil {
 			t.Fatalf("Marshal Reference failed: %v", err)
 		}
@@ -198,6 +201,12 @@ Output`
 		if err != nil {
 			t.Fatalf("Semantic Tokens failed: %v", err)
 		}
+		if len(tokens) != 149 {
+			t.Fatalf("Expected 149 semantic tokens, got %d\n%+v", len(tokens), tokens)
+		}
+		if len(tokens) == 0 {
+			t.Fatalf("Semantic Tokens should not be empty")
+		}
 		if _, err := json.Marshal(tokens); err != nil {
 			t.Fatalf("Marshal Semantic Tokens failed: %v", err)
 		}
@@ -230,18 +239,6 @@ Output`
 			if len(definition) != 1 {
 				t.Fatalf("Find Definition should have found entry, but got none at %#v", pos)
 			}
-			// t.Logf("Find Definition %#v ->\n%#v", pos, definition)
-		}
-	})
-
-	// workspaceSymbol
-	t.Run("workspaceSymbol", func(t *testing.T) {
-		symbols, err := rustLSP.WorkspaceSymbols(context.Background(), "add")
-		if err != nil {
-			t.Fatalf("Workspace Symbol failed: %v", err)
-		}
-		if _, err := json.Marshal(symbols); err != nil {
-			t.Fatalf("Marshal Workspace Symbols failed: %v", err)
 		}
 	})
 

lang/lsp/handler.go

@@ -93,9 +93,9 @@ loop:
 
 func (h *lspHandler) Handle(ctx context.Context, conn *jsonrpc2.Conn, req *jsonrpc2.Request) {
 	// This method will be called for both requests and notifications
-	log.Info("handle method: %s\n", req.Method)
+	log.Debug("handle method: %s\n", req.Method)
 	if req.Params != nil {
-		log.Info("param: %s\n", string(*req.Params))
+		log.Debug("param: %s\n", string(*req.Params))
 	}
 	if req.Notif {
 		// This is a notification
@@ -126,6 +126,10 @@ func (h *lspHandler) sendNotify(req *jsonrpc2.Request) {
 
 func (h *lspHandler) handleNotification(ctx context.Context, conn *jsonrpc2.Conn, req *jsonrpc2.Request) {
 	switch req.Method {
+	case "textDocument/publishDiagnostics":
+		// This notification is sent from the server to the client to signal results of validation runs.
+		log.Debug("Received publishDiagnostics notification:\n%s\n", string(*req.Params))
+		return
 	// exit
 	case "exit":
 		log.Info("Received exit notification\n")