Merge pull request #429 from Ryanmello07/checkpoint/provider-probing-health

Provider probing checkpoint: egress geolocation, verdicts, bandwidth, health-gated offers and counts, blank-name repair

Author: Ryanmello07

api/api.go

@@ -60,6 +60,35 @@ func Routes() []*router.Route {
 		router.NewRoute("POST", "/network/auth-client", handlers.AuthNetworkClient),
 		router.NewRoute("POST", "/network/remove-client", handlers.RemoveNetworkClient),
 		router.NewRoute("POST", "/network/remove-clients", handlers.RemoveNetworkClients),
+		router.NewRoute("POST", "/network/provider-egress-location", handlers.ProviderEgressLocationSubmit),
+		router.NewRoute("GET", "/network/provider-egress-due", handlers.ProviderEgressLocationDue),
+		router.NewRoute("POST", "/network/provider-egress-attempt", handlers.ProviderEgressLocationAttempt),
+		// operator-to-server, same operator secret: the certificate pins this
+		// server observed DIRECTLY for the geolocation source hosts. The
+		// prober fetches them here instead of carrying a compile-time
+		// constant, and refuses to probe at all if it cannot get a complete
+		// set -- probing unpinned would let the provider under test forge its
+		// own location, which is the thing the probe exists to catch.
+		router.NewRoute("GET", "/network/geolocation-source-pins", handlers.GeolocationSourcePins),
+		// operator-to-server, gated by the same operator secret as the egress
+		// location ingest above: the active bandwidth probe's download target,
+		// its result submission, and the byte-budget reservation the prober
+		// takes before spending any probe bytes
+		router.NewRoute("GET", "/network/provider-bandwidth-test", handlers.ProviderBandwidthTest),
+		router.NewRoute("POST", "/network/provider-bandwidth-result", handlers.ProviderBandwidthResult),
+		router.NewRoute("POST", "/network/provider-bandwidth-reserve", handlers.ProviderBandwidthReserve),
+		// operator-to-server, same operator secret again: the egress-health
+		// run the prober takes over the tunnel the geolocation probe already
+		// opened. Until this existed the result was a log line and nothing
+		// else.
+		router.NewRoute("POST", "/network/provider-egress-health", handlers.ProviderEgressHealthResult),
+		// client-to-server, and the only route in this group that is NOT
+		// operator-secret authed: a real client network reporting that a
+		// provider carried nothing. The reporting network is taken from the
+		// session jwt, never from the body, because the quorum counts distinct
+		// networks. A met quorum only brings the provider's next probe
+		// forward -- see model.ProviderClientVerdictQuorumMet.
+		router.NewRoute("POST", "/network/provider-verdict", handlers.ProviderClientVerdictSubmit),
 		router.NewRoute("GET", "/network/clients", handlers.NetworkClients),
 		router.NewRoute("GET", "/network/peers", handlers.NetworkPeers),
 		router.NewRoute("GET", "/network/provider-locations", handlers.NetworkGetProviderLocations),

api/handlers/geolocation_source_pin_handlers.go

@@ -0,0 +1,95 @@
+package handlers
+
+import (
+	"crypto/hmac"
+	"encoding/json"
+	"net/http"
+
+	"github.com/urnetwork/glog"
+
+	"github.com/urnetwork/server/model"
+)
+
+// GeolocationSourcePin is one host's observed certificate pin as served to the
+// prober: the SPKI hash of the leaf certificate and of its issuing
+// intermediate, both base64 sha-256, exactly as the observation job recorded
+// them from a DIRECT, WebPKI-validated connection on this server's own network.
+//
+// Both are served, not just the leaf, because the prober's check
+// (providertunnel.checkPin) accepts a match anywhere in the verified chain: the
+// intermediate is what absorbs routine leaf renewal between two observations,
+// and the leaf is the tighter of the two while it lasts.
+type GeolocationSourcePin struct {
+	Leaf         string `json:"leaf"`
+	Intermediate string `json:"intermediate"`
+}
+
+// GeolocationSourcePinsResult is the response body: a BARE map from host to its
+// pin, `{"ipinfo.io": {"leaf": "...", "intermediate": "..."}}`.
+//
+// The other operator endpoints wrap their payload in a named field
+// (`{"client_ids": [...]}`); this one deliberately does not, because the plan
+// specifies this shape and because the map IS the whole answer -- there is no
+// second field this response could ever grow that would not be better as its
+// own endpoint. A host absent from the map has never been successfully
+// observed, and the prober's correct response to that is to refuse to probe,
+// so the absence has to survive the wire rather than being padded out to a
+// placeholder entry here.
+type GeolocationSourcePinsResult map[string]GeolocationSourcePin
+
+// GeolocationSourcePins serves the certificate pins this server has observed
+// for the geolocation source hosts, so the prober does not have to carry them
+// as a compile-time constant.
+//
+// # Why serving pins is safe, and where the line is
+//
+// The geolocation lookup the prober makes is issued THROUGH the provider under
+// test. The pin is what stops that provider substituting a certificate and
+// forging its own apparent location, which is the entire point of the probe.
+// Handing the prober a pin the SERVER chose is therefore only sound because the
+// server observed it directly, on its own network, with no provider anywhere in
+// the path and full chain validation (see work.RefreshGeolocationSourcePins).
+// A provider cannot influence what this server saw, so it cannot influence what
+// this endpoint says. Nothing in this file may ever accept a pin from a
+// request: this endpoint is read-only, and the table it reads has exactly one
+// writer, the observation job.
+//
+// # It serves what was observed, and nothing else
+//
+// It does not synthesize a row for a source host that has not been observed,
+// and it does not fall back to any built-in default. An empty or partial answer
+// is a truthful one, and the prober treats it as a hard stop rather than
+// probing unpinned -- which is the whole reason the shortfall must be visible
+// rather than papered over. That is also why an empty table returns `{}` with
+// 200 rather than 404: 404 would be indistinguishable from "this server does
+// not implement the endpoint", and the prober does distinguish those in its
+// message even though both are fatal.
+//
+// Same auth as the operator endpoints beside it: the shared secret header
+// rather than a network jwt, fail-closed when the vault resource is missing.
+// The pins are not secret -- anyone can open a TLS connection to ipinfo.io and
+// compute them -- but the endpoint is operator-to-server like the rest of the
+// probe control plane, and there is no reason to give it a wider door than the
+// due list it is fetched alongside.
+func GeolocationSourcePins(w http.ResponseWriter, r *http.Request) {
+	secret := operatorIngestSecret()
+	provided := r.Header.Get(operatorSecretHeader)
+	if secret == "" || provided == "" || !hmac.Equal([]byte(secret), []byte(provided)) {
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	pins := model.GetGeolocationSourcePins(r.Context())
+	result := GeolocationSourcePinsResult{}
+	for host, pin := range pins {
+		result[host] = GeolocationSourcePin{
+			Leaf:         pin.LeafSpki,
+			Intermediate: pin.IntermediateSpki,
+		}
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(result); err != nil {
+		glog.Infof("[gsp]could not write response. err = %s\n", err)
+	}
+}

api/handlers/geolocation_source_pin_handlers_test.go

@@ -0,0 +1,170 @@
+package handlers
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/urnetwork/server"
+	"github.com/urnetwork/server/model"
+)
+
+// The pin endpoint is what stands between the prober and probing unpinned, so
+// its auth has to fail closed in exactly the two ways an operator gets wrong:
+// no header at all (a deployment that never configured the secret) and a wrong
+// one (a rotated secret on one side only). Both are 401, and neither reaches
+// the database.
+func TestGeolocationSourcePinsRejectsMissingSecret(t *testing.T) {
+	req := httptest.NewRequest(http.MethodGet, "/network/geolocation-source-pins", nil)
+	w := httptest.NewRecorder()
+
+	GeolocationSourcePins(w, req)
+
+	if w.Code != http.StatusUnauthorized {
+		t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code)
+	}
+}
+
+func TestGeolocationSourcePinsRejectsWrongSecret(t *testing.T) {
+	req := httptest.NewRequest(http.MethodGet, "/network/geolocation-source-pins", nil)
+	req.Header.Set(operatorSecretHeader, "definitely-not-the-secret")
+	w := httptest.NewRecorder()
+
+	GeolocationSourcePins(w, req)
+
+	if w.Code != http.StatusUnauthorized {
+		t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code)
+	}
+}
+
+// TestGeolocationSourcePinsRejectsAlteredSecret pins the comparison itself.
+// The two tests above both run with the vault unconfigured and take the
+// secret=="" short-circuit, so they would still pass if the comparison were
+// `strings.HasPrefix` or dropped entirely. This one configures a real secret
+// and offers a near miss.
+func TestGeolocationSourcePinsRejectsAlteredSecret(t *testing.T) {
+	const secret = "correct-operator-secret-0123456789"
+	defer withStubOperatorIngestSecret(secret)()
+
+	for _, wrong := range []string{
+		secret + "x",
+		secret[:len(secret)-1],
+		"",
+	} {
+		req := httptest.NewRequest(http.MethodGet, "/network/geolocation-source-pins", nil)
+		if wrong != "" {
+			req.Header.Set(operatorSecretHeader, wrong)
+		}
+		w := httptest.NewRecorder()
+
+		GeolocationSourcePins(w, req)
+
+		if w.Code != http.StatusUnauthorized {
+			t.Errorf("status = %d for secret %q, want 401", w.Code, wrong)
+		}
+	}
+}
+
+// TestGeolocationSourcePinsServesTheObservedSet is the other half: the auth
+// gate must be able to ACCEPT, and what comes back must be what the
+// observation job stored, keyed by host, on the exact wire shape the prober
+// decodes (`{host: {leaf, intermediate}}`).
+//
+// It asserts on the decoded JSON rather than on the handler's Go types,
+// because the prober is a separate repository that only ever sees the bytes:
+// a renamed json tag would be invisible to a Go-level assertion and would take
+// the fleet's probing offline.
+func TestGeolocationSourcePinsServesTheObservedSet(t *testing.T) {
+	t.Setenv("WARP_ENV", "local")
+	server.DefaultTestEnv().Run(t, func(t testing.TB) {
+		const secret = "correct-operator-secret-0123456789"
+		defer withStubOperatorIngestSecret(secret)()
+
+		observedAt := server.NowUtc()
+		model.SetGeolocationSourcePin(t.Context(), &model.GeolocationSourcePin{
+			Host:             "ipinfo.io",
+			LeafSpki:         "leaf-ipinfo",
+			IntermediateSpki: "int-ipinfo",
+			ObservedAt:       observedAt,
+		})
+		model.SetGeolocationSourcePin(t.Context(), &model.GeolocationSourcePin{
+			Host:             "api.i.pn",
+			LeafSpki:         "leaf-ipn",
+			IntermediateSpki: "int-ipn",
+			ObservedAt:       observedAt,
+		})
+
+		req := httptest.NewRequest(http.MethodGet, "/network/geolocation-source-pins", nil)
+		req.Header.Set(operatorSecretHeader, secret)
+		w := httptest.NewRecorder()
+
+		GeolocationSourcePins(w, req)
+
+		if w.Code != http.StatusOK {
+			t.Fatalf("status = %d, want 200 with the correct operator secret. body = %s", w.Code, w.Body.String())
+		}
+
+		var got map[string]map[string]string
+		if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
+			t.Fatalf("decode response %q: %s", w.Body.String(), err)
+		}
+		for host, want := range map[string][2]string{
+			"ipinfo.io": {"leaf-ipinfo", "int-ipinfo"},
+			"api.i.pn":  {"leaf-ipn", "int-ipn"},
+		} {
+			pin, ok := got[host]
+			if !ok {
+				t.Fatalf("host %q missing from the served set %v; the prober treats a missing source host as a hard stop", host, got)
+			}
+			if pin["leaf"] != want[0] {
+				t.Errorf("%s leaf = %q, want %q", host, pin["leaf"], want[0])
+			}
+			if pin["intermediate"] != want[1] {
+				t.Errorf("%s intermediate = %q, want %q", host, pin["intermediate"], want[1])
+			}
+		}
+	})
+}
+
+// A host that has never been observed must stay ABSENT from the answer. The
+// endpoint must not invent a placeholder row to make the map look complete:
+// the prober decides what to do about a missing host (refuse to probe), and it
+// can only decide that if the absence reaches it.
+func TestGeolocationSourcePinsOmitsUnobservedHosts(t *testing.T) {
+	t.Setenv("WARP_ENV", "local")
+	server.DefaultTestEnv().Run(t, func(t testing.TB) {
+		const secret = "correct-operator-secret-0123456789"
+		defer withStubOperatorIngestSecret(secret)()
+
+		// exactly one of the source hosts observed
+		model.SetGeolocationSourcePin(t.Context(), &model.GeolocationSourcePin{
+			Host:             model.GeolocationSourceHosts[0],
+			LeafSpki:         "leaf-only",
+			IntermediateSpki: "int-only",
+			ObservedAt:       server.NowUtc(),
+		})
+
+		req := httptest.NewRequest(http.MethodGet, "/network/geolocation-source-pins", nil)
+		req.Header.Set(operatorSecretHeader, secret)
+		w := httptest.NewRecorder()
+
+		GeolocationSourcePins(w, req)
+
+		if w.Code != http.StatusOK {
+			t.Fatalf("status = %d, want 200. body = %s", w.Code, w.Body.String())
+		}
+		var got map[string]map[string]string
+		if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
+			t.Fatalf("decode response %q: %s", w.Body.String(), err)
+		}
+		if len(got) != 1 {
+			t.Fatalf("served %d host(s) %v, want only the one that was observed", len(got), got)
+		}
+		for _, host := range model.GeolocationSourceHosts[1:] {
+			if _, ok := got[host]; ok {
+				t.Errorf("unobserved host %q appears in the served set; a placeholder pin here would take the fail-closed decision away from the prober", host)
+			}
+		}
+	})
+}