Factor out callback server stuff; update README

README.md

@@ -1,18 +1,35 @@
 ## Complement-Crypto
+*EXPERIMENTAL: As of Jan 2024 this repo is under active development currently so things will break constantly.*
 
-Complement for Rust SDK crypto.
+Complement Crypto is an end-to-end test suite for next generation Matrix _clients_, designed to test the full spectrum of E2EE APIs. 
 
-**EXPERIMENTAL: As of Jan 2024 this repo is under active development currently so things will break constantly.**
+### Rationale
 
-### What is it? Why?
-
-Complement-Crypto extends the existing Complement test suite to support full end-to-end testing of the Rust SDK. End-to-end testing is defined at the FFI / JS SDK layer through to a real homeserver, a real sliding sync proxy, real federation, to another rust SDK on FFI / JS SDK.
+Complement-Crypto extends the existing Complement test suite to support full end-to-end testing of the Matrix Rust SDK. End-to-end testing is defined at the FFI / JS SDK layer through to a real homeserver, a real sliding sync proxy, real federation, to another rust SDK on FFI / JS SDK.
 
 Why:
-- To detect "unable to decrypt" failures and add regression tests for them.
+- To detect "unable to decrypt" failures and *add regression tests* for them.
 - To ensure cross-client compatibility (e.g mobile clients work with web clients and vice versa).
 - To enable new kinds of security tests (active attacker tests)
 
+Goals:
+- Must work under Github Actions / Gitlab CI/CD.
+- Must be fast (where fast is no slower than the slowest action in CI, in other words it shouldn't be slowing down existing workflows).
+- Must be able to test next-gen clients Element X and Element-Web R (Rust crypto).
+- Must be able to test Synapse.
+- Must be able to test the full spectrum of E2EE tasks (key backups, x-signing, etc)
+- Should be able to test over real federation.
+- Should be able to manipulate network conditions.
+- Should be able to manipulate program state (e.g restart, sigkill, clear storage).
+- Could test other homeservers than Synapse.
+- Could test other clients than rust SDK backed ones.
+- Could provide benchmarking/performance testing.
+
+Anti-Goals:
+- Esoteric test edge cases e.g Synapse worker race conditions, FFI concurrency control issues. For these, a unit test in the respective project would be more appropriate.
+- UI testing. This is not a goal because it slows down tests, is less portable e.g needs emulators and is usually significantly more flakey than no-UI tests.
+
+
 ### How do I run it?
 
 *See [FAQ.md](FAQ.md) for more information.*

internal/deploy/callback_addon.go

@@ -0,0 +1,50 @@
+package deploy
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/complement-crypto/internal/api"
+	"github.com/matrix-org/complement/must"
+)
+
+type CallbackData struct {
+	Method       string `json:"method"`
+	URL          string `json:"url"`
+	AccessToken  string `json:"access_token"`
+	ResponseCode int    `json:"response_code"`
+}
+
+// NewCallbackServer runs a local HTTP server that can read callbacks from mitmproxy.
+// Returns the URL of the callback server for use with WithMITMOptions, along with a close function
+// which should be called when the test finishes to shut down the HTTP server.
+func NewCallbackServer(t *testing.T, cb func(CallbackData)) (callbackURL string, close func()) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		var data CallbackData
+		if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
+			api.Errorf(t, "error decoding json: %s", err)
+			w.WriteHeader(500)
+			return
+		}
+		t.Logf("CallbackServer: %v %+v", time.Now(), data)
+		cb(data)
+		w.WriteHeader(200)
+	})
+	// listen on a random high numbered port
+	ln, err := net.Listen("tcp", ":0") //nolint
+	must.NotError(t, "failed to listen on a tcp port", err)
+	port := ln.Addr().(*net.TCPAddr).Port
+	srv := http.Server{
+		Addr:    fmt.Sprintf(":%d", port),
+		Handler: mux,
+	}
+	go srv.Serve(ln)
+	return fmt.Sprintf("http://host.docker.internal:%d", port), func() {
+		srv.Close()
+	}
+}

tests/client_connectivity_test.go

@@ -1,7 +1,6 @@
 package tests
 
 import (
-	"encoding/json"
 	"net/http"
 	"os"
 	"sync"
@@ -10,20 +9,11 @@ import (
 	"time"
 
 	"github.com/matrix-org/complement-crypto/internal/api"
+	"github.com/matrix-org/complement-crypto/internal/deploy"
 	templates "github.com/matrix-org/complement-crypto/tests/go_templates"
 	"github.com/matrix-org/complement/helpers"
 )
 
-// TODO: move to internal? or addons?!
-type CallbackData struct {
-	Method       string `json:"method"`
-	URL          string `json:"url"`
-	AccessToken  string `json:"access_token"`
-	ResponseCode int    `json:"response_code"`
-}
-
-// TODO: move internally
-
 // Test that if the client is restarted BEFORE getting the /keys/upload response but
 // AFTER the server has processed the request, the keys are not regenerated (which would
 // cause duplicate key IDs with different keys). Requires persistent storage.
@@ -33,60 +23,48 @@ func TestSigkillBeforeKeysUploadResponse(t *testing.T) {
 			var mu sync.Mutex
 			var terminated atomic.Bool
 			var terminateClient func()
-			// TODO: factor out to helper
-			mux := http.NewServeMux()
-			mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-				var data CallbackData
-				if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
-					t.Logf("error decoding json: %s", err)
-					w.WriteHeader(500)
-					return
-				}
-				t.Logf("%v %+v", time.Now(), data)
+			callbackURL, close := deploy.NewCallbackServer(t, func(cd deploy.CallbackData) {
 				if terminated.Load() {
 					// make sure the 2nd upload 200 OKs
-					if data.ResponseCode != 200 {
+					if cd.ResponseCode != 200 {
 						// TODO: Errorf
-						t.Logf("2nd /keys/upload did not 200 OK => got %v", data.ResponseCode)
+						t.Logf("2nd /keys/upload did not 200 OK => got %v", cd.ResponseCode)
 					}
-					w.WriteHeader(200)
-					return // 2nd /keys/upload should go through
+					return
 				}
 				// destroy the client
 				mu.Lock()
 				terminateClient()
 				mu.Unlock()
-				w.WriteHeader(200)
 			})
-			srv := http.Server{
-				Addr:    ":6879",
-				Handler: mux,
-			}
-			defer srv.Close()
-			go srv.ListenAndServe()
+			defer close()
 
 			tc := CreateTestContext(t, clientType, clientType)
 			tc.Deployment.WithMITMOptions(t, map[string]interface{}{
 				"callback": map[string]interface{}{
-					"callback_url": "http://host.docker.internal:6879",
+					"callback_url": callbackURL,
 					"filter":       "~u .*\\/keys\\/upload.*",
 				},
 			}, func() {
 				cfg := api.FromComplementClient(tc.Alice, "complement-crypto-password")
+				cfg.BaseURL = tc.Deployment.ReverseProxyURLForHS(clientType.HS)
+				cfg.PersistentStorage = true
 				// run some code in a separate process so we can kill it later
 				cmd, close := templates.PrepareGoScript(t, "login_rust_client/login_rust_client.go",
 					struct {
-						UserID   string
-						DeviceID string
-						Password string
-						BaseURL  string
-						SSURL    string
+						UserID            string
+						DeviceID          string
+						Password          string
+						BaseURL           string
+						SSURL             string
+						PersistentStorage bool
 					}{
-						UserID:   cfg.UserID,
-						Password: cfg.Password,
-						DeviceID: cfg.DeviceID,
-						BaseURL:  tc.Deployment.ReverseProxyURLForHS(clientType.HS),
-						SSURL:    tc.Deployment.SlidingSyncURL(t),
+						UserID:            cfg.UserID,
+						Password:          cfg.Password,
+						DeviceID:          cfg.DeviceID,
+						BaseURL:           cfg.BaseURL,
+						PersistentStorage: cfg.PersistentStorage,
+						SSURL:             tc.Deployment.SlidingSyncURL(t),
 					})
 				cmd.WaitDelay = 3 * time.Second
 				defer close()
@@ -107,8 +85,6 @@ func TestSigkillBeforeKeysUploadResponse(t *testing.T) {
 				waiter.Waitf(t, 5*time.Second, "failed to terminate process")
 				t.Logf("terminated process, making new client")
 				// now make the same client
-				cfg.BaseURL = tc.Deployment.ReverseProxyURLForHS(clientType.HS)
-				cfg.PersistentStorage = true
 				alice := MustCreateClient(t, clientType, cfg, tc.Deployment.SlidingSyncURL(t))
 				alice.Login(t, cfg) // login should work
 				alice.Close(t)

tests/go_templates/login_rust_client/login_rust_client.go

@@ -16,7 +16,7 @@ func main() {
 		UserID:            "{{.UserID}}",
 		DeviceID:          "{{.DeviceID}}",
 		Password:          "{{.Password}}",
-		PersistentStorage: true,
+		PersistentStorage: {{.PersistentStorage}},
 	}
 	client, err := rust.NewRustClient(t, cfg, "{{.SSURL}}")
 	if err != nil {