Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions internal/embed/embed_crd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -741,6 +741,56 @@ func bindingHasSubject(doc map[string]any, name, namespace string) bool {
return false
}

func TestX402VerifierRBAC_CanReadAgentAPISecrets(t *testing.T) {
data, err := ReadInfrastructureFile("base/templates/x402.yaml")
if err != nil {
t.Fatalf("ReadInfrastructureFile: %v", err)
}
docs := multiDoc(data)

role := findDocByName(docs, "ClusterRole", "x402-verifier")
if role == nil {
t.Fatal("no ClusterRole 'x402-verifier' found")
}
rules, ok := role["rules"].([]any)
if !ok {
t.Fatal("x402-verifier ClusterRole has no rules")
}

for _, r := range rules {
rm := r.(map[string]any)
if !stringSet(rm["apiGroups"])[""] || !stringSet(rm["resources"])["secrets"] {
continue
}
verbs := stringSet(rm["verbs"])
if !verbs["get"] || !verbs["list"] || !verbs["watch"] {
continue
}
names := stringSet(rm["resourceNames"])
if !names["litellm-secrets"] {
t.Fatal("x402-verifier secret rule lost litellm-secrets")
}
if !names["hermes-api-server"] {
t.Fatal("x402-verifier secret rule must include hermes-api-server for agent upstream auth")
}
return
}

t.Fatal("x402-verifier ClusterRole missing scoped secret get/list/watch rule")
}

func TestX402VerifierImage_CarriesAgentAuthFix(t *testing.T) {
data, err := ReadInfrastructureFile("base/templates/x402.yaml")
if err != nil {
t.Fatalf("ReadInfrastructureFile: %v", err)
}

const ref = "ghcr.io/obolnetwork/x402-verifier:46e63fd@sha256:a8cd7946884c9a702b5cfcfad28d1f5eac1037899303eb4e0157e3ffab7a572c"
if !strings.Contains(string(data), "image: "+ref) {
t.Fatalf("x402-verifier image must carry agent upstream auth fix: %s", ref)
}
}

func TestAgentRBAC_NoOverlyBroadPermissions(t *testing.T) {
data, err := ReadInfrastructureFile("base/templates/obol-agent-monetize-rbac.yaml")
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions internal/embed/infrastructure/base/templates/x402.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ rules:
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["litellm-secrets"]
resourceNames: ["litellm-secrets", "hermes-api-server"]
verbs: ["get", "list", "watch"]

---
Expand Down Expand Up @@ -234,7 +234,7 @@ spec:
type: RuntimeDefault
containers:
- name: verifier
image: ghcr.io/obolnetwork/x402-verifier:b13254e@sha256:a8a7aa0ca4c35b0ddf6983fa6e3e5f8a3f64e44d8e506ebfd55e39de2bc0342d
image: ghcr.io/obolnetwork/x402-verifier:46e63fd@sha256:a8cd7946884c9a702b5cfcfad28d1f5eac1037899303eb4e0157e3ffab7a572c
imagePullPolicy: IfNotPresent
# PSS Restricted: per-container hardening. Verifier is a Go binary
# reading two RO ConfigMaps; no writeable rootfs paths required.
Expand Down
64 changes: 47 additions & 17 deletions internal/x402/serviceoffer_source.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"k8s.io/client-go/tools/cache"
)

// WatchServiceOffers runs the ServiceOffer + litellm-secrets informers and
// WatchServiceOffers runs the ServiceOffer + upstream-auth Secret informers and
// pushes rendered RouteRules to apply on every change. The optional
// onFirstApply callback is invoked exactly once after the post-cache-sync
// refresh succeeds; it is the signal that the route source has produced its
Expand All @@ -33,14 +33,20 @@ func WatchServiceOffers(ctx context.Context, cfg *rest.Config, apply func([]Rout
}

offerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, metav1.NamespaceAll, nil)
secretFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, metav1.NamespaceAll, func(options *metav1.ListOptions) {
litellmSecretFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, metav1.NamespaceAll, func(options *metav1.ListOptions) {
options.FieldSelector = fields.OneTermEqualSelector("metadata.name", "litellm-secrets").String()
})
hermesSecretFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, metav1.NamespaceAll, func(options *metav1.ListOptions) {
options.FieldSelector = fields.OneTermEqualSelector("metadata.name", "hermes-api-server").String()
})
offers := offerFactory.ForResource(monetizeapi.ServiceOfferGVR).Informer()
secrets := secretFactory.ForResource(monetizeapi.SecretGVR).Informer()
litellmSecrets := litellmSecretFactory.ForResource(monetizeapi.SecretGVR).Informer()
hermesSecrets := hermesSecretFactory.ForResource(monetizeapi.SecretGVR).Informer()

refresh := func() (ok bool) {
routes, err := routesFromStore(offers.GetStore().List(), secrets.GetStore().List())
secretItems := append([]any{}, litellmSecrets.GetStore().List()...)
secretItems = append(secretItems, hermesSecrets.GetStore().List()...)
routes, err := routesFromStore(offers.GetStore().List(), secretItems)
if err != nil {
log.Printf("x402-serviceoffer-source: render routes: %v", err)
return false
Expand All @@ -59,11 +65,13 @@ func WatchServiceOffers(ctx context.Context, cfg *rest.Config, apply func([]Rout
DeleteFunc: func(any) { refresh() },
}
offers.AddEventHandler(handler)
secrets.AddEventHandler(handler)
litellmSecrets.AddEventHandler(handler)
hermesSecrets.AddEventHandler(handler)

go offers.Run(ctx.Done())
go secrets.Run(ctx.Done())
if !cache.WaitForCacheSync(ctx.Done(), offers.HasSynced, secrets.HasSynced) {
go litellmSecrets.Run(ctx.Done())
go hermesSecrets.Run(ctx.Done())
if !cache.WaitForCacheSync(ctx.Done(), offers.HasSynced, litellmSecrets.HasSynced, hermesSecrets.HasSynced) {
return fmt.Errorf("wait for serviceoffer informer sync")
}

Expand All @@ -75,7 +83,7 @@ func WatchServiceOffers(ctx context.Context, cfg *rest.Config, apply func([]Rout
}

func routesFromStore(offerItems, secretItems []any) ([]RouteRule, error) {
upstreamAuthByNamespace, err := upstreamAuthByNamespace(secretItems)
litellmAuthByNamespace, hermesAuthByNamespace, err := upstreamAuthByNamespace(secretItems)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -103,7 +111,11 @@ func routesFromStore(offerItems, secretItems []any) ([]RouteRule, error) {
continue
}

rule, err := routeRuleFromOffer(&offer, upstreamAuthByNamespace[offer.EffectiveNamespace()])
upstreamAuth := litellmAuthByNamespace[offer.EffectiveNamespace()]
if offer.IsAgent() {
upstreamAuth = hermesAuthByNamespace[offer.Spec.Agent.Ref.Namespace]
}
rule, err := routeRuleFromOffer(&offer, upstreamAuth)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -194,36 +206,54 @@ func effectivePrice(offer *monetizeapi.ServiceOffer) (price, priceModel, perMTok
}
}

func upstreamAuthByNamespace(items []any) (map[string]string, error) {
result := make(map[string]string)
func upstreamAuthByNamespace(items []any) (map[string]string, map[string]string, error) {
litellmAuth := make(map[string]string)
hermesAuth := make(map[string]string)
for _, item := range items {
obj, ok := item.(*unstructured.Unstructured)
if !ok || obj.GetName() != "litellm-secrets" {
if !ok {
continue
}
dataKey := ""
switch obj.GetName() {
case "litellm-secrets":
dataKey = "LITELLM_MASTER_KEY"
case "hermes-api-server":
dataKey = "API_SERVER_KEY"
default:
continue
}

value, found, err := unstructured.NestedString(obj.Object, "data", "LITELLM_MASTER_KEY")
value, found, err := unstructured.NestedString(obj.Object, "data", dataKey)
if err != nil {
return nil, err
return nil, nil, err
}
if !found || value == "" {
continue
}

decoded, err := base64.StdEncoding.DecodeString(value)
if err != nil {
return nil, err
return nil, nil, err
}
token := strings.TrimSpace(string(decoded))
if token == "" {
continue
}
result[obj.GetNamespace()] = "Bearer " + token
switch obj.GetName() {
case "litellm-secrets":
litellmAuth[obj.GetNamespace()] = "Bearer " + token
case "hermes-api-server":
hermesAuth[obj.GetNamespace()] = "Bearer " + token
}
}
return result, nil
return litellmAuth, hermesAuth, nil
}

func effectiveUpstreamAuth(offer *monetizeapi.ServiceOffer, upstreamAuth string) string {
if offer.IsAgent() {
return upstreamAuth
}
if !strings.EqualFold(offer.Spec.Upstream.Service, "litellm") {
return ""
}
Expand Down
90 changes: 90 additions & 0 deletions internal/x402/serviceoffer_source_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,96 @@ func TestRouteRuleFromOffer_AgentResolutionAdvertisesRuntimeModelSkills(t *testi
}
}

func TestRoutesFromStore_AgentOfferInjectsHermesAPIKey(t *testing.T) {
items := []any{
mustOfferObject(t, monetizeapi.ServiceOffer{
ObjectMeta: metav1.ObjectMeta{Name: "demo-quant", Namespace: "seller"},
Spec: monetizeapi.ServiceOfferSpec{
Type: "agent",
Agent: monetizeapi.ServiceOfferAgent{
Ref: monetizeapi.ServiceOfferAgentRef{Name: "demo-quant", Namespace: "agent-demo-quant"},
},
Payment: monetizeapi.ServiceOfferPayment{
PayTo: "0x1111111111111111111111111111111111111111",
Price: monetizeapi.ServiceOfferPriceTable{PerRequest: "10"},
},
},
Status: monetizeapi.ServiceOfferStatus{
Conditions: []monetizeapi.Condition{{Type: "RoutePublished", Status: "True"}},
AgentResolution: &monetizeapi.ServiceOfferAgentResolution{
Model: "qwen3.5:9b",
Runtime: "hermes",
Endpoint: "http://hermes.agent-demo-quant.svc.cluster.local:8642",
},
},
}),
}
secrets := []any{
mustSecretObject(t, "agent-demo-quant", "hermes-api-server", map[string]string{
"API_SERVER_KEY": base64.StdEncoding.EncodeToString([]byte("agent-api-key")),
}),
mustSecretObject(t, "seller", "litellm-secrets", map[string]string{
"LITELLM_MASTER_KEY": base64.StdEncoding.EncodeToString([]byte("wrong-secret")),
}),
}

routes, err := routesFromStore(items, secrets)
if err != nil {
t.Fatalf("routesFromStore: %v", err)
}
if len(routes) != 1 {
t.Fatalf("len(routes) = %d, want 1", len(routes))
}
if routes[0].UpstreamAuth != "Bearer agent-api-key" {
t.Fatalf("agent UpstreamAuth = %q, want Bearer agent-api-key", routes[0].UpstreamAuth)
}
}

func TestRoutesFromStore_AgentAuthUsesReferencedAgentNamespace(t *testing.T) {
items := []any{
mustOfferObject(t, monetizeapi.ServiceOffer{
ObjectMeta: metav1.ObjectMeta{Name: "cross-ns-agent", Namespace: "seller-ns"},
Spec: monetizeapi.ServiceOfferSpec{
Type: "agent",
Agent: monetizeapi.ServiceOfferAgent{
Ref: monetizeapi.ServiceOfferAgentRef{Name: "quant", Namespace: "agent-quant"},
},
Payment: monetizeapi.ServiceOfferPayment{
PayTo: "0x1111111111111111111111111111111111111111",
Price: monetizeapi.ServiceOfferPriceTable{PerRequest: "1"},
},
},
Status: monetizeapi.ServiceOfferStatus{
Conditions: []monetizeapi.Condition{{Type: "RoutePublished", Status: "True"}},
AgentResolution: &monetizeapi.ServiceOfferAgentResolution{
Model: "qwen3.5:9b",
Runtime: "hermes",
Endpoint: "http://hermes.agent-quant.svc.cluster.local:8642",
},
},
}),
}
secrets := []any{
mustSecretObject(t, "seller-ns", "hermes-api-server", map[string]string{
"API_SERVER_KEY": base64.StdEncoding.EncodeToString([]byte("seller-ns-key")),
}),
mustSecretObject(t, "agent-quant", "hermes-api-server", map[string]string{
"API_SERVER_KEY": base64.StdEncoding.EncodeToString([]byte("agent-ns-key")),
}),
}

routes, err := routesFromStore(items, secrets)
if err != nil {
t.Fatalf("routesFromStore: %v", err)
}
if len(routes) != 1 {
t.Fatalf("len(routes) = %d, want 1", len(routes))
}
if routes[0].UpstreamAuth != "Bearer agent-ns-key" {
t.Fatalf("agent UpstreamAuth = %q, want referenced agent namespace key", routes[0].UpstreamAuth)
}
}

func mustOfferObject(t *testing.T, offer monetizeapi.ServiceOffer) *unstructured.Unstructured {
t.Helper()
offer.TypeMeta = metav1.TypeMeta{
Expand Down
Loading