Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,18 @@ jobs:
} >> "$GITHUB_OUTPUT"
fi

- name: Upload test artifacts
if: always()
id: upload_artifacts
uses: actions/upload-artifact@v4
with:
name: e2e-test-results
path: |
e2e-tests/mcpchecker/mcpchecker-stackrox-mcp-e2e-out.json
e2e-tests/mcpchecker/*-error.txt
wiremock/wiremock.log
if-no-files-found: ignore

- name: Find existing comment
if: always()
uses: peter-evans/find-comment@v3
Expand All @@ -102,6 +114,7 @@ jobs:

**Commit:** ${{ github.event.pull_request.head.sha }}
**Workflow Run:** [View Details](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
**Artifacts:** [Download test results & logs](${{ steps.upload_artifacts.outputs.artifact-url }})

```
${{ steps.parse_results.outputs.summary }}
Expand All @@ -120,3 +133,4 @@ jobs:

**Commit:** ${{ github.event.pull_request.head.sha }}
**Workflow Run:** [View Details](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
**Artifacts:** [Download test results & logs](${{ steps.upload_artifacts.outputs.artifact-url }})
2 changes: 1 addition & 1 deletion e2e-tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ OPENAI_API_KEY=<OpenAI API Key>
CLOUD_ML_REGION=us-east5

# Optional: Judge configuration (defaults to OpenAI)
JUDGE_MODEL_NAME=gpt-5-nano
JUDGE_MODEL_NAME=openai:gpt-5-nano
```

Note: No StackRox API token required - tests use WireMock mock service.
Expand Down
2 changes: 1 addition & 1 deletion e2e-tests/mcpchecker/eval.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ metadata:
config:
agent:
type: "builtin.openai-agent"
model: "gpt-5-nano"
model: "openai:gpt-5-nano"
llmJudge:
env:
typeKey: JUDGE_TYPE
Expand Down
4 changes: 2 additions & 2 deletions e2e-tests/scripts/run-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,12 @@ fi
# Set agent environment variables (use OpenAI)
export MODEL_BASE_URL="${MODEL_BASE_URL:-https://api.openai.com/v1}"
export MODEL_KEY="${MODEL_KEY:-${OPENAI_API_KEY}}"
export MODEL_NAME="${MODEL_NAME:-gpt-5-nano}"
export MODEL_NAME="${MODEL_NAME:-openai:gpt-5-nano}"

# Set judge environment variables (use OpenAI)
export JUDGE_BASE_URL="${JUDGE_BASE_URL:-https://api.openai.com/v1}"
export JUDGE_API_KEY="${JUDGE_API_KEY:-${OPENAI_API_KEY}}"
export JUDGE_MODEL_NAME="${JUDGE_MODEL_NAME:-gpt-5-nano}"
export JUDGE_MODEL_NAME="${JUDGE_MODEL_NAME:-openai:gpt-5-nano}"

echo "Configuration:"
echo " Central URL: ${STACKROX_MCP__CENTRAL__URL} (WireMock)"
Expand Down
85 changes: 62 additions & 23 deletions e2e-tests/tools/go.mod
Original file line number Diff line number Diff line change
@@ -1,35 +1,59 @@
module github.com/stackrox/stackrox-mcp/e2e-tests

go 1.25.5
go 1.26.1

require (
github.com/fullstorydev/grpcurl v1.9.3
github.com/mcpchecker/mcpchecker v0.0.10
github.com/mcpchecker/mcpchecker v0.0.11
github.com/rhysd/actionlint v1.7.11
)

require (
cel.dev/expr v0.24.0 // indirect
cel.dev/expr v0.25.1 // indirect
charm.land/fantasy v0.12.3 // indirect
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/auth v0.18.2 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2 v1.41.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.11 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
github.com/aws/smithy-go v1.24.2 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect
github.com/bufbuild/protocompile v0.14.1 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
github.com/charmbracelet/anthropic-sdk-go v0.0.0-20260223140439-63879b0b8dab // indirect
github.com/charmbracelet/x/exp/slice v0.0.0-20250904123553-b4e2667e5ad5 // indirect
github.com/charmbracelet/x/json v0.2.0 // indirect
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
github.com/coder/acp-go-sdk v0.6.3 // indirect
github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect
github.com/digitorus/pkcs7 v0.0.0-20250730155240-ffadbf3f398c // indirect
github.com/digitorus/timestamp v0.0.0-20250524132541-c45532741eea // indirect
github.com/dlclark/regexp2 v1.11.0 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/genmcp/gen-mcp v0.2.3 // indirect
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/analysis v0.24.2 // indirect
Expand All @@ -54,25 +78,35 @@ require (
github.com/go-openapi/swag/yamlutils v0.25.4 // indirect
github.com/go-openapi/validate v0.25.1 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/goccy/go-yaml v1.19.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/certificate-transparency-go v1.3.2 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/go-containerregistry v0.20.7 // indirect
github.com/google/jsonschema-go v0.4.2 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.12 // indirect
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/in-toto/attestation v1.1.2 // indirect
github.com/in-toto/in-toto-golang v0.9.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/invopop/jsonschema v0.13.0 // indirect
github.com/jhump/protoreflect v1.17.0 // indirect
github.com/kaptinlin/go-i18n v0.2.12 // indirect
github.com/kaptinlin/jsonpointer v0.4.17 // indirect
github.com/kaptinlin/jsonschema v0.7.5 // indirect
github.com/kaptinlin/messageformat-go v0.4.18 // indirect
github.com/mailru/easyjson v0.9.1 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.17 // indirect
github.com/mattn/go-shellwords v1.0.12 // indirect
github.com/modelcontextprotocol/go-sdk v1.3.1 // indirect
github.com/modelcontextprotocol/go-sdk v1.4.1 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/openai/openai-go/v2 v2.7.1 // indirect
github.com/openai/openai-go/v3 v3.26.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkoukk/tiktoken-go v0.1.8 // indirect
Expand All @@ -81,7 +115,7 @@ require (
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/secure-systems-lab/go-securesystemslib v0.10.0 // indirect
github.com/segmentio/asm v1.2.1 // indirect
github.com/segmentio/encoding v0.5.3 // indirect
github.com/segmentio/encoding v0.5.4 // indirect
github.com/shibumi/go-pathspec v1.3.0 // indirect
github.com/sigstore/protobuf-specs v0.5.0 // indirect
github.com/sigstore/rekor v1.4.3 // indirect
Expand All @@ -103,26 +137,31 @@ require (
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
go.mongodb.org/mongo-driver v1.17.6 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel v1.39.0 // indirect
go.opentelemetry.io/otel/metric v1.39.0 // indirect
go.opentelemetry.io/otel/trace v1.39.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
go.opentelemetry.io/otel v1.40.0 // indirect
go.opentelemetry.io/otel/metric v1.40.0 // indirect
go.opentelemetry.io/otel/trace v1.40.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.3 // indirect
golang.org/x/crypto v0.47.0 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/exp/event v0.0.0-20260112195511-716be5621a96 // indirect
golang.org/x/exp/jsonrpc2 v0.0.0-20260112195511-716be5621a96 // indirect
golang.org/x/mod v0.32.0 // indirect
golang.org/x/net v0.49.0 // indirect
golang.org/x/oauth2 v0.34.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/term v0.39.0 // indirect
golang.org/x/text v0.33.0 // indirect
golang.org/x/net v0.51.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.41.0 // indirect
golang.org/x/term v0.40.0 // indirect
golang.org/x/text v0.34.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260120174246-409b4a993575 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260120174246-409b4a993575 // indirect
google.golang.org/grpc v1.78.0 // indirect
google.golang.org/api v0.269.0 // indirect
google.golang.org/genai v1.49.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
google.golang.org/grpc v1.79.1 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
Expand Down
Loading
Loading