From 207164b8fd45b23efa2239ed56367e026b81786f Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 14:25:58 -0700 Subject: [PATCH 01/15] chore: init gcs store --- apps/workspace-engine/go.mod | 36 ++++- apps/workspace-engine/go.sum | 125 ++++++++++++++- apps/workspace-engine/pkg/db/workspaces.go | 59 +++++++- apps/workspace-engine/pkg/events/events.go | 5 + .../pkg/events/handler/handler.go | 25 ++- apps/workspace-engine/pkg/kafka/kafka.go | 24 ++- .../pkg/workspace/storage_gcs.go | 97 ++++++++++++ .../pkg/workspace/workspace.go | 143 +++++++++++++++--- 8 files changed, 477 insertions(+), 37 deletions(-) create mode 100644 apps/workspace-engine/pkg/workspace/storage_gcs.go diff --git a/apps/workspace-engine/go.mod b/apps/workspace-engine/go.mod index 9d3055afa..e6f289572 100644 --- a/apps/workspace-engine/go.mod +++ b/apps/workspace-engine/go.mod @@ -5,6 +5,7 @@ go 1.24.0 toolchain go1.24.5 require ( + cloud.google.com/go/storage v1.49.0 github.com/charmbracelet/log v0.4.2 github.com/confluentinc/confluent-kafka-go/v2 v2.11.1 github.com/exaring/otelpgx v0.9.3 @@ -27,6 +28,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 go.opentelemetry.io/otel/sdk v1.38.0 go.opentelemetry.io/otel/trace v1.38.0 + google.golang.org/api v0.215.0 google.golang.org/protobuf v1.36.10 ) @@ -39,7 +41,7 @@ require ( github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cloudwego/base64x v0.1.6 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/getkin/kin-openapi v0.132.0 // indirect @@ -72,7 +74,7 @@ require ( github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.54.0 // indirect github.com/speakeasy-api/jsonpath v0.6.0 // indirect @@ -99,6 +101,15 @@ require ( ) require ( + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.13.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect + cloud.google.com/go/compute/metadata v0.7.0 // indirect + cloud.google.com/go/iam v1.2.2 // indirect + cloud.google.com/go/monitoring v1.21.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect github.com/aws/aws-sdk-go-v2 v1.39.3 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 // indirect @@ -106,33 +117,54 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 // indirect github.com/aws/smithy-go v1.23.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/lipgloss v1.1.0 // indirect github.com/charmbracelet/x/ansi v0.8.0 // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.12.0 // indirect github.com/spf13/cast v1.7.1 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect golang.org/x/net v0.46.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sys v0.37.0 // indirect golang.org/x/text v0.30.0 // indirect + golang.org/x/time v0.8.0 // indirect + google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/apps/workspace-engine/go.sum b/apps/workspace-engine/go.sum index b4b67b8f1..0e0617295 100644 --- a/apps/workspace-engine/go.sum +++ b/apps/workspace-engine/go.sum @@ -1,5 +1,26 @@ cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.13.0 h1:8Fu8TZy167JkW8Tj3q7dIkr2v4cndv41ouecJx0PAHs= +cloud.google.com/go/auth v0.13.0/go.mod h1:COOjD9gwfKNKz+IIduatIhYJQIc0mG3H102r/EMxX6Q= +cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU= +cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= +cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= +cloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk= +cloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM= +cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= +cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= +cloud.google.com/go/monitoring v1.21.2 h1:FChwVtClH19E7pJ+e0xUhJPGksctZNVOk2UhMmblmdU= +cloud.google.com/go/monitoring v1.21.2/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU= +cloud.google.com/go/storage v1.49.0 h1:zenOPBOWHCnojRd9aJZAyQXBYqkJkdQS42dxL55CIMw= +cloud.google.com/go/storage v1.49.0/go.mod h1:k1eHhhpLvrPjVGfo0mOUPEJ4Y2+a/Hv5PiwehZI9qGU= +cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI= +cloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= @@ -8,6 +29,15 @@ github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkk github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1/go.mod h1:jyqM3eLpJ3IbIFDTKVz2rF9T/xWGW0rIriGwnz8l9Tk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1 h1:oTX4vsorBZo/Zdum6OKPA4o7544hm6smoRv1QjpTwGo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= @@ -66,6 +96,7 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= @@ -83,8 +114,12 @@ github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNE github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/compose-spec/compose-go/v2 v2.1.3 h1:bD67uqLuL/XgkAK6ir3xZvNLFPxPScEi1KW7R5esrLE= github.com/compose-spec/compose-go/v2 v2.1.3/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/confluentinc/confluent-kafka-go/v2 v2.11.1 h1:qGCQznyp2BxyBNyOE+M7O1YS2tI1/Y60O0jQP452zA4= @@ -109,8 +144,9 @@ github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoY github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/buildx v0.15.1 h1:1cO6JIc0rOoC8tlxfXoh1HH1uxaNvYH1q7J7kv5enhw= @@ -140,6 +176,18 @@ github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJ github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/exaring/otelpgx v0.9.3 h1:4yO02tXC7ZJZ+hcqcUkfxblYNCIFGVhpUWI0iw1TzPU= github.com/exaring/otelpgx v0.9.3/go.mod h1:R5/M5LWsPPBZc1SrRE5e0DiU48bI78C1/GPTWs6I66U= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -164,6 +212,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -213,13 +263,21 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -228,10 +286,13 @@ github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ= github.com/google/cel-go v0.26.1/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -242,11 +303,20 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -400,14 +470,18 @@ github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= @@ -459,6 +533,8 @@ github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -523,8 +599,14 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 h1:gbhw/u49SS3gkPWiYweQNJGm/uJN5GkI/FrosxSHT7A= @@ -545,6 +627,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqhe go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= @@ -571,17 +655,26 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= @@ -590,14 +683,17 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -634,6 +730,10 @@ golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -645,12 +745,24 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.215.0 h1:jdYF4qnyczlEz2ReWIsosNLDuzXyvFHJtI5gcr0J7t0= +google.golang.org/api v0.215.0/go.mod h1:fta3CVtuJYOEdugLNWm6WodzOS8KdFckABwN4I40hzY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -658,7 +770,10 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= @@ -687,6 +802,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= diff --git a/apps/workspace-engine/pkg/db/workspaces.go b/apps/workspace-engine/pkg/db/workspaces.go index 3ca756200..3b268ff4f 100644 --- a/apps/workspace-engine/pkg/db/workspaces.go +++ b/apps/workspace-engine/pkg/db/workspaces.go @@ -1,6 +1,10 @@ package db -import "context" +import ( + "context" + + "github.com/jackc/pgx/v5" +) const WORKSPACE_SELECT_QUERY = ` SELECT id FROM workspace @@ -82,3 +86,56 @@ func GetAllWorkspaceIDs(ctx context.Context) ([]string, error) { } return workspaceIDs, nil } + +type WorkspaceSnapshot struct { + Path string + Timestamp string + Partition int32 + NumPartitions int32 +} + +const WORKSPACE_SNAPSHOT_SELECT_QUERY = ` + SELECT path, timestamp, partition, num_partitions FROM workspace_snapshot WHERE workspace_id = $1 ORDER BY timestamp DESC LIMIT 1 +` + +func GetWorkspaceSnapshot(ctx context.Context, workspaceID string) (*WorkspaceSnapshot, error) { + db, err := GetDB(ctx) + if err != nil { + return nil, err + } + defer db.Release() + + workspaceSnapshot := &WorkspaceSnapshot{} + err = db.QueryRow(ctx, WORKSPACE_SNAPSHOT_SELECT_QUERY, workspaceID).Scan( + &workspaceSnapshot.Path, + &workspaceSnapshot.Timestamp, + &workspaceSnapshot.Partition, + &workspaceSnapshot.NumPartitions, + ) + if err != nil { + if err == pgx.ErrNoRows { + return nil, nil + } + return nil, err + } + return workspaceSnapshot, nil +} + +const WORKSPACE_SNAPSHOT_INSERT_QUERY = ` + INSERT INTO workspace_snapshot (workspace_id, path, timestamp, partition, num_partitions) + VALUES ($1, $2) +` + +func WriteWorkspaceSnapshot(ctx context.Context, workspaceID string, snapshot *WorkspaceSnapshot) error { + db, err := GetDB(ctx) + if err != nil { + return err + } + defer db.Release() + + _, err = db.Exec(ctx, WORKSPACE_SNAPSHOT_INSERT_QUERY, workspaceID, snapshot.Path, snapshot.Timestamp, snapshot.Partition, snapshot.NumPartitions) + if err != nil { + return err + } + return nil +} diff --git a/apps/workspace-engine/pkg/events/events.go b/apps/workspace-engine/pkg/events/events.go index dc6df265b..65cf2cbec 100644 --- a/apps/workspace-engine/pkg/events/events.go +++ b/apps/workspace-engine/pkg/events/events.go @@ -16,6 +16,7 @@ import ( "workspace-engine/pkg/events/handler/system" "workspace-engine/pkg/events/handler/tick" "workspace-engine/pkg/events/handler/userapprovalrecords" + "workspace-engine/pkg/workspace" ) var handlers = handler.HandlerRegistry{ @@ -79,3 +80,7 @@ var handlers = handler.HandlerRegistry{ func NewEventHandler() *handler.EventListener { return handler.NewEventListener(handlers) } + +func NewEventHandlerWithWorkspaceSaver(workspaceSaver workspace.WorkspaceSaver) *handler.EventListener { + return handler.NewEventListenerWithWorkspaceSaver(handlers, workspaceSaver) +} diff --git a/apps/workspace-engine/pkg/events/handler/handler.go b/apps/workspace-engine/pkg/events/handler/handler.go index 5557b9b51..d79795525 100644 --- a/apps/workspace-engine/pkg/events/handler/handler.go +++ b/apps/workspace-engine/pkg/events/handler/handler.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "time" "workspace-engine/pkg/changeset" "workspace-engine/pkg/workspace" @@ -83,6 +84,7 @@ type RawEvent struct { EventType EventType `json:"eventType"` WorkspaceID string `json:"workspaceId"` Data json.RawMessage `json:"data,omitempty"` + Timestamp int64 `json:"timestamp"` } // Handler defines the interface for processing events @@ -93,7 +95,8 @@ type HandlerRegistry map[EventType]Handler // EventListener listens for events on the queue and routes them to appropriate handlers type EventListener struct { - handlers HandlerRegistry + handlers HandlerRegistry + workspaceSaver workspace.WorkspaceSaver } // NewEventListener creates a new event listener with the provided handlers @@ -101,6 +104,10 @@ func NewEventListener(handlers HandlerRegistry) *EventListener { return &EventListener{handlers: handlers} } +func NewEventListenerWithWorkspaceSaver(handlers HandlerRegistry, workspaceSaver workspace.WorkspaceSaver) *EventListener { + return &EventListener{handlers: handlers, workspaceSaver: workspaceSaver} +} + // ListenAndRoute processes incoming Kafka messages and routes them to the appropriate handler func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) (*workspace.Workspace, error) { ctx, span := tracer.Start(ctx, "ListenAndRoute", @@ -179,6 +186,22 @@ func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) span.SetAttributes(attribute.Int("release-target.changed", len(releaseTargetChanges.Keys()))) + if el.workspaceSaver != nil { + timestampStr := time.Unix(rawEvent.Timestamp, 0).Format(time.RFC3339) + if err := el.workspaceSaver(ctx, ws.ID, timestampStr); err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to save workspace") + log.Error("Failed to save workspace", "error", err, "workspaceID", ws.ID) + return nil, fmt.Errorf("failed to save workspace: %w", err) + } + + span.SetStatus(codes.Ok, "event processed successfully") + log.Debug("Successfully processed event", + "eventType", rawEvent.EventType) + + return ws, nil + } + if err := ws.ChangesetConsumer().FlushChangeset(ctx, changeSet); err != nil { span.RecordError(err) span.SetStatus(codes.Error, "failed to flush changeset") diff --git a/apps/workspace-engine/pkg/kafka/kafka.go b/apps/workspace-engine/pkg/kafka/kafka.go index 2d57f4940..90cf005ef 100644 --- a/apps/workspace-engine/pkg/kafka/kafka.go +++ b/apps/workspace-engine/pkg/kafka/kafka.go @@ -7,6 +7,7 @@ import ( "time" "workspace-engine/pkg/events" + "workspace-engine/pkg/events/handler" "workspace-engine/pkg/workspace" wskafka "workspace-engine/pkg/workspace/kafka" @@ -29,13 +30,28 @@ func getEnv(varName string, defaultValue string) string { return v } +func getEventHandler(numPartitions int32) *handler.EventListener { + if workspace.IsGCSStorageEnabled() { + return events.NewEventHandlerWithWorkspaceSaver(workspace.CreateGCSWorkspaceSaver(numPartitions)) + } + return events.NewEventHandler() +} + // RunConsumer starts the Kafka consumer without offset resume // Uses default Kafka offsets (committed offsets or 'earliest') func RunConsumer(ctx context.Context) error { - return RunConsumerWithWorkspaceLoader(ctx, nil) + if workspace.IsGCSStorageEnabled() { + return runConsumerWithGCSStore(ctx) + } + return RunConsumerWithWorkspaceStore(ctx, nil) +} + +func runConsumerWithGCSStore(ctx context.Context) error { + workspaceLoader := workspace.CreateGCSWorkspaceLoader(nil) + return RunConsumerWithWorkspaceStore(ctx, workspaceLoader) } -// RunConsumerWithWorkspaceLoader starts the Kafka consumer with workspace-based offset resume +// RunConsumerWithWorkspaceStore starts the Kafka consumer with workspace-based offset resume // // Flow: // 1. Connect to Kafka and subscribe to topic @@ -43,7 +59,7 @@ func RunConsumer(ctx context.Context) error { // 3. Load workspaces for assigned partitions (if workspaceLoader provided) // 4. Seek to stored offsets per partition // 5. Start consuming and processing messages -func RunConsumerWithWorkspaceLoader(ctx context.Context, workspaceLoader workspace.WorkspaceLoader) error { +func RunConsumerWithWorkspaceStore(ctx context.Context, workspaceLoader workspace.WorkspaceLoader) error { // Initialize Kafka consumer consumer, err := createConsumer() if err != nil { @@ -105,7 +121,7 @@ func RunConsumerWithWorkspaceLoader(ctx context.Context, workspaceLoader workspa log.Info("Started Kafka consumer for ctrlplane-events") // Start consuming messages - handler := events.NewEventHandler() + handler := getEventHandler(numPartitions) for { // Check for cancellation diff --git a/apps/workspace-engine/pkg/workspace/storage_gcs.go b/apps/workspace-engine/pkg/workspace/storage_gcs.go new file mode 100644 index 000000000..856f2ad27 --- /dev/null +++ b/apps/workspace-engine/pkg/workspace/storage_gcs.go @@ -0,0 +1,97 @@ +package workspace + +import ( + "context" + "fmt" + "io" + "os" + "strings" + "workspace-engine/pkg/db" + + "cloud.google.com/go/storage" +) + +func IsGCSStorageEnabled() bool { + return strings.HasPrefix(os.Getenv("WORKSPACE_STATES_BUCKET_URL"), "gcs://") +} + +// getBucketURL parses a GCS URL like "gcs://bucket-name/base-path" +// Returns bucket name and base path. +func getBucketURL() string { + return strings.TrimPrefix(os.Getenv("WORKSPACE_STATES_BUCKET_URL"), "gcs://") +} + +func GetWorkspaceSnapshot(ctx context.Context, workspaceID string) ([]byte, error) { + bucket := getBucketURL() + + snapshot, err := db.GetWorkspaceSnapshot(ctx, workspaceID) + if err != nil { + return nil, err + } + + if snapshot == nil { + return nil, nil + } + + if snapshot.Path == "" { + return nil, nil + } + + client, err := storage.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create GCS client: %w", err) + } + defer client.Close() + + obj := client.Bucket(bucket).Object(snapshot.Path) + reader, err := obj.NewReader(ctx) + if err != nil { + return nil, fmt.Errorf("failed to read snapshot: %w", err) + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read data: %w", err) + } + + return data, nil +} + +// PutWorkspaceSnapshot writes a new timestamped snapshot for a workspace to GCS. +// Reads bucket URL from WORKSPACE_STATES_BUCKET_URL env variable. +func PutWorkspaceSnapshot(ctx context.Context, workspaceID string, timestamp string, partition int32, numPartitions int32, data []byte) error { + bucket := getBucketURL() + + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("failed to create GCS client: %w", err) + } + defer client.Close() + + path := fmt.Sprintf("%s_%s.gob", workspaceID, timestamp) + + obj := client.Bucket(bucket).Object(path) + writer := obj.NewWriter(ctx) + + if _, err := writer.Write(data); err != nil { + writer.Close() + return fmt.Errorf("failed to write snapshot: %w", err) + } + + if err := writer.Close(); err != nil { + return fmt.Errorf("failed to close writer: %w", err) + } + + snapshot := &db.WorkspaceSnapshot{ + Path: path, + Timestamp: timestamp, + Partition: partition, + NumPartitions: numPartitions, + } + if err := db.WriteWorkspaceSnapshot(ctx, workspaceID, snapshot); err != nil { + return fmt.Errorf("failed to write snapshot: %w", err) + } + + return nil +} diff --git a/apps/workspace-engine/pkg/workspace/workspace.go b/apps/workspace-engine/pkg/workspace/workspace.go index ce68e2212..ff5f7b302 100644 --- a/apps/workspace-engine/pkg/workspace/workspace.go +++ b/apps/workspace-engine/pkg/workspace/workspace.go @@ -256,8 +256,123 @@ func (w *Workspace) LoadFromStorage(ctx context.Context, storage StorageClient, return nil } +func (w *Workspace) SaveToGCS(ctx context.Context, workspaceID string, timestamp string, partition int32, numPartitions int32) error { + if !IsGCSStorageEnabled() { + return nil + } + + data, err := w.GobEncode() + if err != nil { + return fmt.Errorf("failed to encode workspace: %w", err) + } + + if err := PutWorkspaceSnapshot(ctx, workspaceID, timestamp, partition, numPartitions, data); err != nil { + return fmt.Errorf("failed to put workspace snapshot: %w", err) + } + + return nil +} + +func (w *Workspace) LoadFromGCS(ctx context.Context, workspaceID string) error { + if !IsGCSStorageEnabled() { + return nil + } + + data, err := GetWorkspaceSnapshot(ctx, workspaceID) + if err != nil { + return fmt.Errorf("failed to get workspace snapshot: %w", err) + } + + if data == nil { + return nil + } + + if err := w.GobDecode(data); err != nil { + return fmt.Errorf("failed to decode workspace: %w", err) + } + + return nil +} + +type WorkspaceSaver func(ctx context.Context, workspaceID string, timestamp string) error + +func CreateGCSWorkspaceSaver(numPartitions int32) WorkspaceSaver { + return func(ctx context.Context, workspaceID string, timestamp string) error { + ws := GetWorkspace(workspaceID) + partition := kafka.PartitionForWorkspace(workspaceID, numPartitions) + if err := ws.SaveToGCS(ctx, workspaceID, timestamp, partition, numPartitions); err != nil { + return fmt.Errorf("failed to save workspace %s to S3: %w", workspaceID, err) + } + return nil + } +} + type WorkspaceLoader func(ctx context.Context, assignedPartitions []int32, numPartitions int32) error +// getAssignedWorkspaceIDs retrieves workspace IDs for the assigned partitions. +// Uses the provided discoverer if available, otherwise falls back to the default implementation. +func getAssignedWorkspaceIDs( + ctx context.Context, + assignedPartitions []int32, + numPartitions int32, + discoverer kafka.WorkspaceIDDiscoverer, +) ([]string, error) { + var allWorkspaceIDs []string + + // Use discoverer if provided, otherwise use default implementation + if discoverer != nil { + // Collect workspace IDs for each assigned partition + workspaceIDSet := make(map[string]bool) + for _, partition := range assignedPartitions { + partitionWorkspaces, err := discoverer(ctx, partition, numPartitions) + if err != nil { + return nil, fmt.Errorf("failed to discover workspace IDs for partition %d: %w", partition, err) + } + for _, wsID := range partitionWorkspaces { + workspaceIDSet[wsID] = true + } + } + // Convert set to slice + for wsID := range workspaceIDSet { + allWorkspaceIDs = append(allWorkspaceIDs, wsID) + } + } else { + var err error + allWorkspaceIDs, err = kafka.GetAssignedWorkspaceIDs(ctx, assignedPartitions, numPartitions) + if err != nil { + return nil, fmt.Errorf("failed to get assigned workspace IDs: %w", err) + } + } + + return allWorkspaceIDs, nil +} + +// CreateGCSWorkspaceLoader creates a workspace loader function that: +// 1. Discovers all available workspace IDs +// 2. Determines which workspaces belong to which partitions +// 3. Loads workspaces for the assigned partitions from S3 +func CreateGCSWorkspaceLoader( + discoverer kafka.WorkspaceIDDiscoverer, +) WorkspaceLoader { + return func(ctx context.Context, assignedPartitions []int32, numPartitions int32) error { + allWorkspaceIDs, err := getAssignedWorkspaceIDs(ctx, assignedPartitions, numPartitions, discoverer) + if err != nil { + return err + } + + for _, workspaceID := range allWorkspaceIDs { + ws := GetWorkspace(workspaceID) + if err := ws.LoadFromGCS(ctx, workspaceID); err != nil { + return fmt.Errorf("failed to load workspace %s from S3: %w", workspaceID, err) + } + + Set(workspaceID, ws) + } + + return nil + } +} + // CreateWorkspaceLoader creates a workspace loader function that: // 1. Discovers all available workspace IDs // 2. Determines which workspaces belong to which partitions @@ -267,31 +382,9 @@ func CreateWorkspaceLoader( discoverer kafka.WorkspaceIDDiscoverer, ) WorkspaceLoader { return func(ctx context.Context, assignedPartitions []int32, numPartitions int32) error { - var allWorkspaceIDs []string - var err error - - // Use discoverer if provided, otherwise use default implementation - if discoverer != nil { - // Collect workspace IDs for each assigned partition - workspaceIDSet := make(map[string]bool) - for _, partition := range assignedPartitions { - partitionWorkspaces, discErr := discoverer(ctx, partition, numPartitions) - if discErr != nil { - return fmt.Errorf("failed to discover workspace IDs for partition %d: %w", partition, discErr) - } - for _, wsID := range partitionWorkspaces { - workspaceIDSet[wsID] = true - } - } - // Convert set to slice - for wsID := range workspaceIDSet { - allWorkspaceIDs = append(allWorkspaceIDs, wsID) - } - } else { - allWorkspaceIDs, err = kafka.GetAssignedWorkspaceIDs(ctx, assignedPartitions, numPartitions) - if err != nil { - return fmt.Errorf("failed to get assigned workspace IDs: %w", err) - } + allWorkspaceIDs, err := getAssignedWorkspaceIDs(ctx, assignedPartitions, numPartitions, discoverer) + if err != nil { + return err } for _, workspaceID := range allWorkspaceIDs { From d657ecf6edff6b31813230cc4f02055a8abd4be7 Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 14:43:34 -0700 Subject: [PATCH 02/15] cleanup --- .../pkg/events/handler/handler.go | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/apps/workspace-engine/pkg/events/handler/handler.go b/apps/workspace-engine/pkg/events/handler/handler.go index d79795525..d7a6d20df 100644 --- a/apps/workspace-engine/pkg/events/handler/handler.go +++ b/apps/workspace-engine/pkg/events/handler/handler.go @@ -149,23 +149,29 @@ func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) // Execute the handler var ws *workspace.Workspace - wsExists := workspace.Exists(rawEvent.WorkspaceID) - if wsExists { + if workspace.IsGCSStorageEnabled() { ws = workspace.GetWorkspace(rawEvent.WorkspaceID) } + changeSet := changeset.NewChangeSet[any]() - if !wsExists { - ws = workspace.New(rawEvent.WorkspaceID) - if err := workspace.PopulateWorkspaceWithInitialState(ctx, ws); err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, "failed to load workspace") - log.Error("Failed to load workspace", "error", err, "workspaceID", rawEvent.WorkspaceID) - return nil, fmt.Errorf("failed to load workspace: %w", err) + if !workspace.IsGCSStorageEnabled() { + wsExists := workspace.Exists(rawEvent.WorkspaceID) + if wsExists { + ws = workspace.GetWorkspace(rawEvent.WorkspaceID) + } + if !wsExists { + ws = workspace.New(rawEvent.WorkspaceID) + if err := workspace.PopulateWorkspaceWithInitialState(ctx, ws); err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to load workspace") + log.Error("Failed to load workspace", "error", err, "workspaceID", rawEvent.WorkspaceID) + return nil, fmt.Errorf("failed to load workspace: %w", err) + } + workspace.Set(rawEvent.WorkspaceID, ws) + changeSet.IsInitialLoad = true } - workspace.Set(rawEvent.WorkspaceID, ws) - changeSet.IsInitialLoad = true + ctx = changeset.WithChangeSet(ctx, changeSet) } - ctx = changeset.WithChangeSet(ctx, changeSet) if err := handler(ctx, ws, rawEvent); err != nil { span.RecordError(err) @@ -202,11 +208,13 @@ func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) return ws, nil } - if err := ws.ChangesetConsumer().FlushChangeset(ctx, changeSet); err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, "failed to flush changeset") - log.Error("Failed to flush changeset", "error", err) - return nil, fmt.Errorf("failed to flush changeset: %w", err) + if !workspace.IsGCSStorageEnabled() { + if err := ws.ChangesetConsumer().FlushChangeset(ctx, changeSet); err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to flush changeset") + log.Error("Failed to flush changeset", "error", err) + return nil, fmt.Errorf("failed to flush changeset: %w", err) + } } span.SetStatus(codes.Ok, "event processed successfully") From b2510f05b7af4534d6f38d6f9f0ce99257a3572f Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 14:48:51 -0700 Subject: [PATCH 03/15] cleanup --- apps/workspace-engine/pkg/events/events.go | 1 + .../pkg/events/handler/handler.go | 1 + .../pkg/workspace/storage_gcs.go | 45 +++++++++++++++---- .../pkg/workspace/workspace.go | 9 ++-- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/apps/workspace-engine/pkg/events/events.go b/apps/workspace-engine/pkg/events/events.go index 65cf2cbec..132d47218 100644 --- a/apps/workspace-engine/pkg/events/events.go +++ b/apps/workspace-engine/pkg/events/events.go @@ -81,6 +81,7 @@ func NewEventHandler() *handler.EventListener { return handler.NewEventListener(handlers) } +// NewEventHandlerWithWorkspaceSaver creates a new event handler with a workspace saver func NewEventHandlerWithWorkspaceSaver(workspaceSaver workspace.WorkspaceSaver) *handler.EventListener { return handler.NewEventListenerWithWorkspaceSaver(handlers, workspaceSaver) } diff --git a/apps/workspace-engine/pkg/events/handler/handler.go b/apps/workspace-engine/pkg/events/handler/handler.go index d7a6d20df..1eb8b9e53 100644 --- a/apps/workspace-engine/pkg/events/handler/handler.go +++ b/apps/workspace-engine/pkg/events/handler/handler.go @@ -104,6 +104,7 @@ func NewEventListener(handlers HandlerRegistry) *EventListener { return &EventListener{handlers: handlers} } +// NewEventListenerWithWorkspaceSaver creates a new event listener with the provided handlers and workspace saver func NewEventListenerWithWorkspaceSaver(handlers HandlerRegistry, workspaceSaver workspace.WorkspaceSaver) *EventListener { return &EventListener{handlers: handlers, workspaceSaver: workspaceSaver} } diff --git a/apps/workspace-engine/pkg/workspace/storage_gcs.go b/apps/workspace-engine/pkg/workspace/storage_gcs.go index 856f2ad27..998a54fd3 100644 --- a/apps/workspace-engine/pkg/workspace/storage_gcs.go +++ b/apps/workspace-engine/pkg/workspace/storage_gcs.go @@ -12,17 +12,31 @@ import ( ) func IsGCSStorageEnabled() bool { - return strings.HasPrefix(os.Getenv("WORKSPACE_STATES_BUCKET_URL"), "gcs://") + return strings.HasPrefix(os.Getenv("WORKSPACE_STATES_BUCKET_URL"), "gs://") } -// getBucketURL parses a GCS URL like "gcs://bucket-name/base-path" -// Returns bucket name and base path. -func getBucketURL() string { - return strings.TrimPrefix(os.Getenv("WORKSPACE_STATES_BUCKET_URL"), "gcs://") +// getBucketURL parses a GCS URL like "gs://bucket-name/base-path" +// Returns bucket name and base path (without leading slash). +func getBucketURL() (string, string) { + url := os.Getenv("WORKSPACE_STATES_BUCKET_URL") + + // Trim gs:// scheme + url = strings.TrimPrefix(url, "gs://") + + // Split on first '/' to separate bucket and prefix + parts := strings.SplitN(url, "/", 2) + bucket := parts[0] + prefix := "" + + if len(parts) > 1 { + prefix = strings.TrimPrefix(parts[1], "/") + } + + return bucket, prefix } func GetWorkspaceSnapshot(ctx context.Context, workspaceID string) ([]byte, error) { - bucket := getBucketURL() + bucket, prefix := getBucketURL() snapshot, err := db.GetWorkspaceSnapshot(ctx, workspaceID) if err != nil { @@ -43,7 +57,13 @@ func GetWorkspaceSnapshot(ctx context.Context, workspaceID string) ([]byte, erro } defer client.Close() - obj := client.Bucket(bucket).Object(snapshot.Path) + // Prepend prefix to object path + objectPath := snapshot.Path + if prefix != "" { + objectPath = prefix + "/" + snapshot.Path + } + + obj := client.Bucket(bucket).Object(objectPath) reader, err := obj.NewReader(ctx) if err != nil { return nil, fmt.Errorf("failed to read snapshot: %w", err) @@ -61,7 +81,7 @@ func GetWorkspaceSnapshot(ctx context.Context, workspaceID string) ([]byte, erro // PutWorkspaceSnapshot writes a new timestamped snapshot for a workspace to GCS. // Reads bucket URL from WORKSPACE_STATES_BUCKET_URL env variable. func PutWorkspaceSnapshot(ctx context.Context, workspaceID string, timestamp string, partition int32, numPartitions int32, data []byte) error { - bucket := getBucketURL() + bucket, prefix := getBucketURL() client, err := storage.NewClient(ctx) if err != nil { @@ -69,9 +89,16 @@ func PutWorkspaceSnapshot(ctx context.Context, workspaceID string, timestamp str } defer client.Close() + // Base path for the object (without prefix) path := fmt.Sprintf("%s_%s.gob", workspaceID, timestamp) - obj := client.Bucket(bucket).Object(path) + // Prepend prefix to object path + objectPath := path + if prefix != "" { + objectPath = prefix + "/" + path + } + + obj := client.Bucket(bucket).Object(objectPath) writer := obj.NewWriter(ctx) if _, err := writer.Write(data); err != nil { diff --git a/apps/workspace-engine/pkg/workspace/workspace.go b/apps/workspace-engine/pkg/workspace/workspace.go index ff5f7b302..7fee4be4b 100644 --- a/apps/workspace-engine/pkg/workspace/workspace.go +++ b/apps/workspace-engine/pkg/workspace/workspace.go @@ -294,19 +294,22 @@ func (w *Workspace) LoadFromGCS(ctx context.Context, workspaceID string) error { return nil } +// WorkspaceSaver is a function that saves workspace state to an external storage type WorkspaceSaver func(ctx context.Context, workspaceID string, timestamp string) error +// CreateGCSWorkspaceSaver creates a new workspace saver that saves workspace state to GCS func CreateGCSWorkspaceSaver(numPartitions int32) WorkspaceSaver { return func(ctx context.Context, workspaceID string, timestamp string) error { ws := GetWorkspace(workspaceID) partition := kafka.PartitionForWorkspace(workspaceID, numPartitions) if err := ws.SaveToGCS(ctx, workspaceID, timestamp, partition, numPartitions); err != nil { - return fmt.Errorf("failed to save workspace %s to S3: %w", workspaceID, err) + return fmt.Errorf("failed to save workspace %s to GCS: %w", workspaceID, err) } return nil } } +// WorkspaceLoader is a function that loads workspace state from an external storage type WorkspaceLoader func(ctx context.Context, assignedPartitions []int32, numPartitions int32) error // getAssignedWorkspaceIDs retrieves workspace IDs for the assigned partitions. @@ -350,7 +353,7 @@ func getAssignedWorkspaceIDs( // CreateGCSWorkspaceLoader creates a workspace loader function that: // 1. Discovers all available workspace IDs // 2. Determines which workspaces belong to which partitions -// 3. Loads workspaces for the assigned partitions from S3 +// 3. Loads workspaces for the assigned partitions from GCS func CreateGCSWorkspaceLoader( discoverer kafka.WorkspaceIDDiscoverer, ) WorkspaceLoader { @@ -363,7 +366,7 @@ func CreateGCSWorkspaceLoader( for _, workspaceID := range allWorkspaceIDs { ws := GetWorkspace(workspaceID) if err := ws.LoadFromGCS(ctx, workspaceID); err != nil { - return fmt.Errorf("failed to load workspace %s from S3: %w", workspaceID, err) + return fmt.Errorf("failed to load workspace %s from GCS: %w", workspaceID, err) } Set(workspaceID, ws) From edc07eca55afee17a0f66571681a950f62084f02 Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 14:56:46 -0700 Subject: [PATCH 04/15] more fix --- apps/workspace-engine/pkg/events/handler/handler.go | 12 +++++------- apps/workspace-engine/pkg/kafka/kafka.go | 8 ++++---- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/apps/workspace-engine/pkg/events/handler/handler.go b/apps/workspace-engine/pkg/events/handler/handler.go index 1eb8b9e53..2fad27328 100644 --- a/apps/workspace-engine/pkg/events/handler/handler.go +++ b/apps/workspace-engine/pkg/events/handler/handler.go @@ -209,13 +209,11 @@ func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) return ws, nil } - if !workspace.IsGCSStorageEnabled() { - if err := ws.ChangesetConsumer().FlushChangeset(ctx, changeSet); err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, "failed to flush changeset") - log.Error("Failed to flush changeset", "error", err) - return nil, fmt.Errorf("failed to flush changeset: %w", err) - } + if err := ws.ChangesetConsumer().FlushChangeset(ctx, changeSet); err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to flush changeset") + log.Error("Failed to flush changeset", "error", err) + return nil, fmt.Errorf("failed to flush changeset: %w", err) } span.SetStatus(codes.Ok, "event processed successfully") diff --git a/apps/workspace-engine/pkg/kafka/kafka.go b/apps/workspace-engine/pkg/kafka/kafka.go index 90cf005ef..98b935204 100644 --- a/apps/workspace-engine/pkg/kafka/kafka.go +++ b/apps/workspace-engine/pkg/kafka/kafka.go @@ -43,15 +43,15 @@ func RunConsumer(ctx context.Context) error { if workspace.IsGCSStorageEnabled() { return runConsumerWithGCSStore(ctx) } - return RunConsumerWithWorkspaceStore(ctx, nil) + return RunConsumerWithWorkspaceLoader(ctx, nil) } func runConsumerWithGCSStore(ctx context.Context) error { workspaceLoader := workspace.CreateGCSWorkspaceLoader(nil) - return RunConsumerWithWorkspaceStore(ctx, workspaceLoader) + return RunConsumerWithWorkspaceLoader(ctx, workspaceLoader) } -// RunConsumerWithWorkspaceStore starts the Kafka consumer with workspace-based offset resume +// RunConsumerWithWorkspaceLoader starts the Kafka consumer with workspace-based offset resume // // Flow: // 1. Connect to Kafka and subscribe to topic @@ -59,7 +59,7 @@ func runConsumerWithGCSStore(ctx context.Context) error { // 3. Load workspaces for assigned partitions (if workspaceLoader provided) // 4. Seek to stored offsets per partition // 5. Start consuming and processing messages -func RunConsumerWithWorkspaceStore(ctx context.Context, workspaceLoader workspace.WorkspaceLoader) error { +func RunConsumerWithWorkspaceLoader(ctx context.Context, workspaceLoader workspace.WorkspaceLoader) error { // Initialize Kafka consumer consumer, err := createConsumer() if err != nil { From af2d3f8ea2023f3de80bd5ca76000ba71a237956 Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 15:08:40 -0700 Subject: [PATCH 05/15] more fix --- apps/workspace-engine/pkg/db/workspaces.go | 2 +- apps/workspace-engine/pkg/events/handler/handler.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/workspace-engine/pkg/db/workspaces.go b/apps/workspace-engine/pkg/db/workspaces.go index 3b268ff4f..319a44fe2 100644 --- a/apps/workspace-engine/pkg/db/workspaces.go +++ b/apps/workspace-engine/pkg/db/workspaces.go @@ -123,7 +123,7 @@ func GetWorkspaceSnapshot(ctx context.Context, workspaceID string) (*WorkspaceSn const WORKSPACE_SNAPSHOT_INSERT_QUERY = ` INSERT INTO workspace_snapshot (workspace_id, path, timestamp, partition, num_partitions) - VALUES ($1, $2) + VALUES ($1, $2, $3, $4, $5) ` func WriteWorkspaceSnapshot(ctx context.Context, workspaceID string, snapshot *WorkspaceSnapshot) error { diff --git a/apps/workspace-engine/pkg/events/handler/handler.go b/apps/workspace-engine/pkg/events/handler/handler.go index 2fad27328..3622a3f40 100644 --- a/apps/workspace-engine/pkg/events/handler/handler.go +++ b/apps/workspace-engine/pkg/events/handler/handler.go @@ -149,12 +149,12 @@ func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) // Execute the handler var ws *workspace.Workspace + changeSet := changeset.NewChangeSet[any]() if workspace.IsGCSStorageEnabled() { ws = workspace.GetWorkspace(rawEvent.WorkspaceID) } - changeSet := changeset.NewChangeSet[any]() if !workspace.IsGCSStorageEnabled() { wsExists := workspace.Exists(rawEvent.WorkspaceID) if wsExists { @@ -171,8 +171,8 @@ func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) workspace.Set(rawEvent.WorkspaceID, ws) changeSet.IsInitialLoad = true } - ctx = changeset.WithChangeSet(ctx, changeSet) } + ctx = changeset.WithChangeSet(ctx, changeSet) if err := handler(ctx, ws, rawEvent); err != nil { span.RecordError(err) From a4273be588f69b26c3c32139e1238fccac3df777 Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 15:36:58 -0700 Subject: [PATCH 06/15] simplify --- .../pkg/events/handler/handler.go | 6 +- .../pkg/workspace/storage_gcs.go | 65 ++++++++----------- 2 files changed, 32 insertions(+), 39 deletions(-) diff --git a/apps/workspace-engine/pkg/events/handler/handler.go b/apps/workspace-engine/pkg/events/handler/handler.go index 3622a3f40..d529c16af 100644 --- a/apps/workspace-engine/pkg/events/handler/handler.go +++ b/apps/workspace-engine/pkg/events/handler/handler.go @@ -151,11 +151,13 @@ func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) var ws *workspace.Workspace changeSet := changeset.NewChangeSet[any]() - if workspace.IsGCSStorageEnabled() { + isUsingGCSExternalState := workspace.IsGCSStorageEnabled() + + if isUsingGCSExternalState { ws = workspace.GetWorkspace(rawEvent.WorkspaceID) } - if !workspace.IsGCSStorageEnabled() { + if !isUsingGCSExternalState { wsExists := workspace.Exists(rawEvent.WorkspaceID) if wsExists { ws = workspace.GetWorkspace(rawEvent.WorkspaceID) diff --git a/apps/workspace-engine/pkg/workspace/storage_gcs.go b/apps/workspace-engine/pkg/workspace/storage_gcs.go index 998a54fd3..0b09fe747 100644 --- a/apps/workspace-engine/pkg/workspace/storage_gcs.go +++ b/apps/workspace-engine/pkg/workspace/storage_gcs.go @@ -15,29 +15,7 @@ func IsGCSStorageEnabled() bool { return strings.HasPrefix(os.Getenv("WORKSPACE_STATES_BUCKET_URL"), "gs://") } -// getBucketURL parses a GCS URL like "gs://bucket-name/base-path" -// Returns bucket name and base path (without leading slash). -func getBucketURL() (string, string) { - url := os.Getenv("WORKSPACE_STATES_BUCKET_URL") - - // Trim gs:// scheme - url = strings.TrimPrefix(url, "gs://") - - // Split on first '/' to separate bucket and prefix - parts := strings.SplitN(url, "/", 2) - bucket := parts[0] - prefix := "" - - if len(parts) > 1 { - prefix = strings.TrimPrefix(parts[1], "/") - } - - return bucket, prefix -} - func GetWorkspaceSnapshot(ctx context.Context, workspaceID string) ([]byte, error) { - bucket, prefix := getBucketURL() - snapshot, err := db.GetWorkspaceSnapshot(ctx, workspaceID) if err != nil { return nil, err @@ -51,18 +29,20 @@ func GetWorkspaceSnapshot(ctx context.Context, workspaceID string) ([]byte, erro return nil, nil } + // Parse bucket/object/path from stored path + parts := strings.SplitN(snapshot.Path, "/", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid GCS path: %s", snapshot.Path) + } + bucket := parts[0] + objectPath := parts[1] + client, err := storage.NewClient(ctx) if err != nil { return nil, fmt.Errorf("failed to create GCS client: %w", err) } defer client.Close() - // Prepend prefix to object path - objectPath := snapshot.Path - if prefix != "" { - objectPath = prefix + "/" + snapshot.Path - } - obj := client.Bucket(bucket).Object(objectPath) reader, err := obj.NewReader(ctx) if err != nil { @@ -79,9 +59,19 @@ func GetWorkspaceSnapshot(ctx context.Context, workspaceID string) ([]byte, erro } // PutWorkspaceSnapshot writes a new timestamped snapshot for a workspace to GCS. -// Reads bucket URL from WORKSPACE_STATES_BUCKET_URL env variable. func PutWorkspaceSnapshot(ctx context.Context, workspaceID string, timestamp string, partition int32, numPartitions int32, data []byte) error { - bucket, prefix := getBucketURL() + // Get bucket URL like "gs://bucket-name" or "gs://bucket-name/prefix" + bucketURL := os.Getenv("WORKSPACE_STATES_BUCKET_URL") + bucketURL = strings.TrimSuffix(bucketURL, "/") + + // Parse bucket and optional prefix + gsPath := strings.TrimPrefix(bucketURL, "gs://") + parts := strings.SplitN(gsPath, "/", 2) + bucket := parts[0] + prefix := "" + if len(parts) > 1 { + prefix = parts[1] + } client, err := storage.NewClient(ctx) if err != nil { @@ -89,13 +79,11 @@ func PutWorkspaceSnapshot(ctx context.Context, workspaceID string, timestamp str } defer client.Close() - // Base path for the object (without prefix) - path := fmt.Sprintf("%s_%s.gob", workspaceID, timestamp) - - // Prepend prefix to object path - objectPath := path + // Generate object path + objectName := fmt.Sprintf("%s_%s.gob", workspaceID, timestamp) + objectPath := objectName if prefix != "" { - objectPath = prefix + "/" + path + objectPath = prefix + "/" + objectName } obj := client.Bucket(bucket).Object(objectPath) @@ -110,8 +98,11 @@ func PutWorkspaceSnapshot(ctx context.Context, workspaceID string, timestamp str return fmt.Errorf("failed to close writer: %w", err) } + // Store bucket/object/path in database (no gs:// prefix) + fullPath := fmt.Sprintf("%s/%s", bucket, objectPath) + snapshot := &db.WorkspaceSnapshot{ - Path: path, + Path: fullPath, Timestamp: timestamp, Partition: partition, NumPartitions: numPartitions, From d2146a13d065870c670608f015bfcfcb82b04328 Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 16:31:33 -0700 Subject: [PATCH 07/15] load from db if not set --- .../pkg/workspace/workspace.go | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/apps/workspace-engine/pkg/workspace/workspace.go b/apps/workspace-engine/pkg/workspace/workspace.go index 7fee4be4b..ae4a6ff4e 100644 --- a/apps/workspace-engine/pkg/workspace/workspace.go +++ b/apps/workspace-engine/pkg/workspace/workspace.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/gob" + "errors" "fmt" "workspace-engine/pkg/changeset" "workspace-engine/pkg/cmap" @@ -273,6 +274,8 @@ func (w *Workspace) SaveToGCS(ctx context.Context, workspaceID string, timestamp return nil } +var ErrWorkspaceSnapshotNotFound = errors.New("workspace snapshot not found") + func (w *Workspace) LoadFromGCS(ctx context.Context, workspaceID string) error { if !IsGCSStorageEnabled() { return nil @@ -284,7 +287,7 @@ func (w *Workspace) LoadFromGCS(ctx context.Context, workspaceID string) error { } if data == nil { - return nil + return ErrWorkspaceSnapshotNotFound } if err := w.GobDecode(data); err != nil { @@ -365,11 +368,21 @@ func CreateGCSWorkspaceLoader( for _, workspaceID := range allWorkspaceIDs { ws := GetWorkspace(workspaceID) - if err := ws.LoadFromGCS(ctx, workspaceID); err != nil { - return fmt.Errorf("failed to load workspace %s from GCS: %w", workspaceID, err) + err := ws.LoadFromGCS(ctx, workspaceID) + if err == nil { + Set(workspaceID, ws) + continue } - Set(workspaceID, ws) + if err == ErrWorkspaceSnapshotNotFound { + if err := PopulateWorkspaceWithInitialState(ctx, ws); err != nil { + return fmt.Errorf("failed to populate workspace %s with initial state: %w", workspaceID, err) + } + Set(workspaceID, ws) + continue + } + + return fmt.Errorf("failed to load workspace %s from GCS: %w", workspaceID, err) } return nil From ef0d7843bbc8b11dbeb2d2203b510220b60b99d2 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 22 Oct 2025 20:35:40 -0400 Subject: [PATCH 08/15] clean up loading --- apps/workspace-engine/pkg/events/events.go | 6 - .../pkg/events/handler/handler.go | 49 +---- apps/workspace-engine/pkg/kafka/kafka.go | 73 +++---- apps/workspace-engine/pkg/kafka/message.go | 29 --- apps/workspace-engine/pkg/kafka/offset.go | 106 --------- .../pkg/workspace/kafka/state.go | 29 --- apps/workspace-engine/pkg/workspace/loader.go | 67 ++++++ .../workspace-engine/pkg/workspace/storage.go | 7 +- .../pkg/workspace/storage_file.go | 2 +- .../pkg/workspace/storage_gcs.go | 110 ++++------ .../pkg/workspace/workspace.go | 202 +----------------- 11 files changed, 159 insertions(+), 521 deletions(-) create mode 100644 apps/workspace-engine/pkg/workspace/loader.go diff --git a/apps/workspace-engine/pkg/events/events.go b/apps/workspace-engine/pkg/events/events.go index 132d47218..dc6df265b 100644 --- a/apps/workspace-engine/pkg/events/events.go +++ b/apps/workspace-engine/pkg/events/events.go @@ -16,7 +16,6 @@ import ( "workspace-engine/pkg/events/handler/system" "workspace-engine/pkg/events/handler/tick" "workspace-engine/pkg/events/handler/userapprovalrecords" - "workspace-engine/pkg/workspace" ) var handlers = handler.HandlerRegistry{ @@ -80,8 +79,3 @@ var handlers = handler.HandlerRegistry{ func NewEventHandler() *handler.EventListener { return handler.NewEventListener(handlers) } - -// NewEventHandlerWithWorkspaceSaver creates a new event handler with a workspace saver -func NewEventHandlerWithWorkspaceSaver(workspaceSaver workspace.WorkspaceSaver) *handler.EventListener { - return handler.NewEventListenerWithWorkspaceSaver(handlers, workspaceSaver) -} diff --git a/apps/workspace-engine/pkg/events/handler/handler.go b/apps/workspace-engine/pkg/events/handler/handler.go index d529c16af..9ffdc1ee9 100644 --- a/apps/workspace-engine/pkg/events/handler/handler.go +++ b/apps/workspace-engine/pkg/events/handler/handler.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "time" "workspace-engine/pkg/changeset" "workspace-engine/pkg/workspace" @@ -96,7 +95,6 @@ type HandlerRegistry map[EventType]Handler // EventListener listens for events on the queue and routes them to appropriate handlers type EventListener struct { handlers HandlerRegistry - workspaceSaver workspace.WorkspaceSaver } // NewEventListener creates a new event listener with the provided handlers @@ -104,11 +102,6 @@ func NewEventListener(handlers HandlerRegistry) *EventListener { return &EventListener{handlers: handlers} } -// NewEventListenerWithWorkspaceSaver creates a new event listener with the provided handlers and workspace saver -func NewEventListenerWithWorkspaceSaver(handlers HandlerRegistry, workspaceSaver workspace.WorkspaceSaver) *EventListener { - return &EventListener{handlers: handlers, workspaceSaver: workspaceSaver} -} - // ListenAndRoute processes incoming Kafka messages and routes them to the appropriate handler func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) (*workspace.Workspace, error) { ctx, span := tracer.Start(ctx, "ListenAndRoute", @@ -151,29 +144,11 @@ func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) var ws *workspace.Workspace changeSet := changeset.NewChangeSet[any]() - isUsingGCSExternalState := workspace.IsGCSStorageEnabled() - - if isUsingGCSExternalState { - ws = workspace.GetWorkspace(rawEvent.WorkspaceID) - } - - if !isUsingGCSExternalState { - wsExists := workspace.Exists(rawEvent.WorkspaceID) - if wsExists { - ws = workspace.GetWorkspace(rawEvent.WorkspaceID) - } - if !wsExists { - ws = workspace.New(rawEvent.WorkspaceID) - if err := workspace.PopulateWorkspaceWithInitialState(ctx, ws); err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, "failed to load workspace") - log.Error("Failed to load workspace", "error", err, "workspaceID", rawEvent.WorkspaceID) - return nil, fmt.Errorf("failed to load workspace: %w", err) - } - workspace.Set(rawEvent.WorkspaceID, ws) - changeSet.IsInitialLoad = true - } + ws = workspace.GetWorkspace(rawEvent.WorkspaceID) + if ws == nil { + return nil, fmt.Errorf("workspace not found: %s", rawEvent.WorkspaceID) } + ctx = changeset.WithChangeSet(ctx, changeSet) if err := handler(ctx, ws, rawEvent); err != nil { @@ -195,22 +170,6 @@ func (el *EventListener) ListenAndRoute(ctx context.Context, msg *kafka.Message) span.SetAttributes(attribute.Int("release-target.changed", len(releaseTargetChanges.Keys()))) - if el.workspaceSaver != nil { - timestampStr := time.Unix(rawEvent.Timestamp, 0).Format(time.RFC3339) - if err := el.workspaceSaver(ctx, ws.ID, timestampStr); err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, "failed to save workspace") - log.Error("Failed to save workspace", "error", err, "workspaceID", ws.ID) - return nil, fmt.Errorf("failed to save workspace: %w", err) - } - - span.SetStatus(codes.Ok, "event processed successfully") - log.Debug("Successfully processed event", - "eventType", rawEvent.EventType) - - return ws, nil - } - if err := ws.ChangesetConsumer().FlushChangeset(ctx, changeSet); err != nil { span.RecordError(err) span.SetStatus(codes.Error, "failed to flush changeset") diff --git a/apps/workspace-engine/pkg/kafka/kafka.go b/apps/workspace-engine/pkg/kafka/kafka.go index 98b935204..6b6e2093a 100644 --- a/apps/workspace-engine/pkg/kafka/kafka.go +++ b/apps/workspace-engine/pkg/kafka/kafka.go @@ -6,8 +6,8 @@ import ( "os" "time" + "workspace-engine/pkg/db" "workspace-engine/pkg/events" - "workspace-engine/pkg/events/handler" "workspace-engine/pkg/workspace" wskafka "workspace-engine/pkg/workspace/kafka" @@ -30,27 +30,6 @@ func getEnv(varName string, defaultValue string) string { return v } -func getEventHandler(numPartitions int32) *handler.EventListener { - if workspace.IsGCSStorageEnabled() { - return events.NewEventHandlerWithWorkspaceSaver(workspace.CreateGCSWorkspaceSaver(numPartitions)) - } - return events.NewEventHandler() -} - -// RunConsumer starts the Kafka consumer without offset resume -// Uses default Kafka offsets (committed offsets or 'earliest') -func RunConsumer(ctx context.Context) error { - if workspace.IsGCSStorageEnabled() { - return runConsumerWithGCSStore(ctx) - } - return RunConsumerWithWorkspaceLoader(ctx, nil) -} - -func runConsumerWithGCSStore(ctx context.Context) error { - workspaceLoader := workspace.CreateGCSWorkspaceLoader(nil) - return RunConsumerWithWorkspaceLoader(ctx, workspaceLoader) -} - // RunConsumerWithWorkspaceLoader starts the Kafka consumer with workspace-based offset resume // // Flow: @@ -59,7 +38,7 @@ func runConsumerWithGCSStore(ctx context.Context) error { // 3. Load workspaces for assigned partitions (if workspaceLoader provided) // 4. Seek to stored offsets per partition // 5. Start consuming and processing messages -func RunConsumerWithWorkspaceLoader(ctx context.Context, workspaceLoader workspace.WorkspaceLoader) error { +func RunConsumer(ctx context.Context) error { // Initialize Kafka consumer consumer, err := createConsumer() if err != nil { @@ -107,21 +86,29 @@ func RunConsumerWithWorkspaceLoader(ctx context.Context, workspaceLoader workspa return fmt.Errorf("failed to get assigned workspace IDs: %w", err) } - log.Info("All workspace IDs", "workspaceIDs", allWorkspaceIDs) - - // Load workspaces and seek to stored offsets if workspace loader is provided - if workspaceLoader != nil { - if err := loadWorkspaces(ctx, consumer, assignedPartitions, workspaceLoader); err != nil { - return fmt.Errorf("failed to load workspaces: %w", err) + storage := workspace.NewFileStorage("./state") + if workspace.IsGCSStorageEnabled() { + storage, err = workspace.NewGCSStorageClient(ctx) + if err != nil { + return fmt.Errorf("failed to create GCS storage: %w", err) } - - applyOffsets(consumer, assignedPartitions) } - log.Info("Started Kafka consumer for ctrlplane-events") + log.Info("All workspace IDs", "workspaceIDs", allWorkspaceIDs) + for _, workspaceID := range allWorkspaceIDs { + ws := workspace.GetWorkspace(workspaceID) + if ws == nil { + log.Error("Workspace not found", "workspaceID", workspaceID) + continue + } + if err := workspace.Load(ctx, storage, ws); err != nil { + log.Error("Failed to load workspace", "workspaceID", workspaceID, "error", err) + continue + } + } // Start consuming messages - handler := getEventHandler(numPartitions) + handler := events.NewEventHandler() for { // Check for cancellation @@ -139,10 +126,24 @@ func RunConsumerWithWorkspaceLoader(ctx context.Context, workspaceLoader workspa continue } - // Process message and update workspace state - if err := processMessage(ctx, consumer, handler, msg); err != nil { - log.Error("Failed to process message", "error", err) + ws, err := handler.ListenAndRoute(ctx, msg) + if err != nil { + log.Error("Failed to route message", "error", err) continue } + + // Commit offset to Kafka + if _, err := consumer.CommitMessage(msg); err != nil { + log.Error("Failed to commit message", "error", err) + continue + } + + snapshot := &db.WorkspaceSnapshot{ + Timestamp: time.Now().Format(time.RFC3339), + Partition: int32(msg.TopicPartition.Partition), + NumPartitions: numPartitions, + } + + workspace.Save(ctx, storage, ws, snapshot) } } diff --git a/apps/workspace-engine/pkg/kafka/message.go b/apps/workspace-engine/pkg/kafka/message.go index d3e4856cb..0f40b45af 100644 --- a/apps/workspace-engine/pkg/kafka/message.go +++ b/apps/workspace-engine/pkg/kafka/message.go @@ -1,21 +1,12 @@ package kafka import ( - "context" - "fmt" "time" - "workspace-engine/pkg/workspace" - "github.com/charmbracelet/log" "github.com/confluentinc/confluent-kafka-go/v2/kafka" ) -// messageHandler interface for routing and processing messages -type messageHandler interface { - ListenAndRoute(ctx context.Context, msg *kafka.Message) (*workspace.Workspace, error) -} - // handleReadError handles errors from reading Kafka messages func handleReadError(err error) { if kafkaErr, ok := err.(kafka.Error); ok && kafkaErr.IsTimeout() { @@ -26,23 +17,3 @@ func handleReadError(err error) { log.Error("Consumer error", "error", err) time.Sleep(time.Second) } - -// processMessage handles a single Kafka message: route to workspace, track offset, commit -func processMessage(ctx context.Context, consumer *kafka.Consumer, handler messageHandler, msg *kafka.Message) error { - // Route message to appropriate workspace - ws, err := handler.ListenAndRoute(ctx, msg) - if err != nil { - return fmt.Errorf("failed to route message: %w", err) - } - - // Track offset in workspace state BEFORE committing - // This ensures the workspace state reflects the correct resume point - ws.KafkaProgress.FromMessage(msg) - - // Commit offset to Kafka - if _, err := consumer.CommitMessage(msg); err != nil { - return fmt.Errorf("failed to commit message: %w", err) - } - - return nil -} diff --git a/apps/workspace-engine/pkg/kafka/offset.go b/apps/workspace-engine/pkg/kafka/offset.go index 43c3416e0..6ad7a182e 100644 --- a/apps/workspace-engine/pkg/kafka/offset.go +++ b/apps/workspace-engine/pkg/kafka/offset.go @@ -4,45 +4,10 @@ import ( "context" "fmt" - "workspace-engine/pkg/workspace" - wskafka "workspace-engine/pkg/workspace/kafka" - "github.com/charmbracelet/log" "github.com/confluentinc/confluent-kafka-go/v2/kafka" ) -// loadWorkspaces loads workspaces for the assigned partitions -// Returns the list of assigned partitions -func loadWorkspaces(ctx context.Context, c *kafka.Consumer, assignedPartitions []int32, workspaceLoader workspace.WorkspaceLoader) error { - if len(assignedPartitions) == 0 { - log.Info("No partitions assigned to this consumer") - return nil - } - - // Get total partition count for the topic - numPartitions, err := getTopicPartitionCount(c) - if err != nil { - return fmt.Errorf("failed to get topic partition count: %w", err) - } - - // Load workspaces that belong to our assigned partitions - err = workspaceLoader(ctx, assignedPartitions, numPartitions) - if err != nil { - return fmt.Errorf("failed to load workspaces: %w", err) - } - - return nil -} - -// applyOffsets seeks each assigned partition to its stored offset -func applyOffsets(c *kafka.Consumer, assignedPartitions []int32) { - if len(assignedPartitions) == 0 { - return - } - - seekStoredOffsets(c, assignedPartitions) -} - // waitForPartitionAssignment blocks until Kafka assigns partitions to this consumer (or timeout) func waitForPartitionAssignment(ctx context.Context, c *kafka.Consumer) ([]int32, error) { log.Info("Waiting for partition assignment... (entering poll loop)") @@ -174,74 +139,3 @@ func getTopicPartitionCount(c *kafka.Consumer) (int32, error) { return numPartitions, nil } - -// seekStoredOffsets seeks each assigned partition to its last stored offset from the workspace -func seekStoredOffsets(c *kafka.Consumer, assignedPartitions []int32) { - seekCount := 0 - - for _, partition := range assignedPartitions { - offset, workspaceID, found := findStoredOffsetForPartition(partition) - if !found { - log.Info("No stored offset for partition, will use default", - "partition", partition) - continue - } - - // Seek to the next offset after the last applied one - seekTo := offset + 1 - err := seekPartition(c, partition, seekTo) - if err != nil { - log.Warn("Failed to seek partition", - "partition", partition, - "workspace", workspaceID, - "offset", seekTo, - "error", err) - continue - } - - log.Info("Seeked to stored offset", - "partition", partition, - "workspace", workspaceID, - "offset", seekTo) - seekCount++ - } - - if seekCount > 0 { - log.Info("Successfully applied stored offsets", "count", seekCount) - } else { - log.Info("No stored offsets to apply") - } -} - -// findStoredOffsetForPartition looks through loaded workspaces to find stored offset for a partition -func findStoredOffsetForPartition(partition int32) (offset int64, workspaceID string, found bool) { - topicPartition := wskafka.TopicPartition{ - Topic: Topic, - Partition: partition, - } - - // Check all loaded workspaces to find one with offset for this partition - for _, wsID := range workspace.GetAllWorkspaceIds() { - ws := workspace.GetWorkspace(wsID) - if ws == nil { - continue - } - - progress, exists := ws.KafkaProgress[topicPartition] - if exists { - return progress.LastApplied, ws.ID, true - } - } - - return 0, "", false -} - -// seekPartition seeks a single partition to a specific offset -func seekPartition(c *kafka.Consumer, partition int32, offset int64) error { - seekTp := kafka.TopicPartition{ - Topic: &Topic, - Partition: partition, - Offset: kafka.Offset(offset), - } - return c.Seek(seekTp, 0) -} diff --git a/apps/workspace-engine/pkg/workspace/kafka/state.go b/apps/workspace-engine/pkg/workspace/kafka/state.go index f7a962c12..a25e542f8 100644 --- a/apps/workspace-engine/pkg/workspace/kafka/state.go +++ b/apps/workspace-engine/pkg/workspace/kafka/state.go @@ -4,38 +4,9 @@ import ( "context" "workspace-engine/pkg/db" - "github.com/confluentinc/confluent-kafka-go/v2/kafka" "github.com/spaolacci/murmur3" ) -type TopicPartition struct { - Topic string - Partition int32 -} - -type KafkaProgress struct { - // Last offset you have durably applied to your state. - // Resume at Offset+1 on restart. - LastApplied int64 - - // Optional: last message timestamp or watermark if you want metrics. - LastTimestamp int64 -} - -type KafkaProgressMap map[TopicPartition]KafkaProgress - -func (m KafkaProgressMap) FromMessage(msg *kafka.Message) { - topicPartition := TopicPartition{ - Topic: *msg.TopicPartition.Topic, - Partition: int32(msg.TopicPartition.Partition), - } - - m[topicPartition] = KafkaProgress{ - LastApplied: int64(msg.TopicPartition.Offset), - LastTimestamp: int64(msg.Timestamp.Unix()), - } -} - // PartitionForWorkspace computes which partition a workspace ID should be routed to // using Murmur3 hash (Kafka-compatible partitioning) func PartitionForWorkspace(workspaceID string, numPartitions int32) int32 { diff --git a/apps/workspace-engine/pkg/workspace/loader.go b/apps/workspace-engine/pkg/workspace/loader.go new file mode 100644 index 000000000..abbab6c70 --- /dev/null +++ b/apps/workspace-engine/pkg/workspace/loader.go @@ -0,0 +1,67 @@ +package workspace + +import ( + "context" + "fmt" + "time" + "workspace-engine/pkg/db" +) + +func getPath(workspaceID string, timestamp string) string { + return fmt.Sprintf("%s_%s.gob", workspaceID, timestamp) +} + +func LoadAll(ctx context.Context, storage StorageClient) error { + workspaceIds := GetAllWorkspaceIds() + for _, workspaceID := range workspaceIds { + workspace := GetWorkspace(workspaceID) + err := Load(ctx, storage, workspace) + if err != nil { + return err + } + } + return nil +} + +func Save(ctx context.Context, storage StorageClient, workspace *Workspace, snapshot *db.WorkspaceSnapshot) error { + path := getPath(workspace.ID, time.Now().Format(time.RFC3339)) + + data, err := workspace.GobEncode() + if err != nil { + return fmt.Errorf("failed to encode workspace: %w", err) + } + + // Write to file with appropriate permissions + if err := storage.Put(ctx, path, data); err != nil { + return fmt.Errorf("failed to write workspace to disk: %w", err) + } + + if err := db.WriteWorkspaceSnapshot(ctx, workspace.ID, snapshot); err != nil { + return fmt.Errorf("failed to write workspace snapshot: %w", err) + } + + return nil +} + +func Load(ctx context.Context, storage StorageClient, workspace *Workspace) error { + dbSnapshot, err := db.GetWorkspaceSnapshot(ctx, workspace.ID) + if err != nil { + return fmt.Errorf("failed to get workspace snapshot: %w", err) + } + + if dbSnapshot == nil { + if err := PopulateWorkspaceWithInitialState(ctx, workspace); err != nil { + return fmt.Errorf("failed to populate workspace with initial state: %w", err) + } + return nil + } + + dbSnapshotPath := dbSnapshot.Path + + data, err := storage.Get(ctx, dbSnapshotPath) + if err != nil { + return fmt.Errorf("failed to read workspace from disk: %w", err) + } + + return workspace.GobDecode(data) +} \ No newline at end of file diff --git a/apps/workspace-engine/pkg/workspace/storage.go b/apps/workspace-engine/pkg/workspace/storage.go index 1ffc63fe9..1c91cbef5 100644 --- a/apps/workspace-engine/pkg/workspace/storage.go +++ b/apps/workspace-engine/pkg/workspace/storage.go @@ -1,9 +1,12 @@ package workspace -import "workspace-engine/pkg/workspace/kafka" +import ( + "errors" +) + +var ErrWorkspaceSnapshotNotFound = errors.New("workspace snapshot not found") type WorkspaceStorageObject struct { ID string - KafkaProgress kafka.KafkaProgressMap StoreData []byte } diff --git a/apps/workspace-engine/pkg/workspace/storage_file.go b/apps/workspace-engine/pkg/workspace/storage_file.go index 1f3491e17..98561a790 100644 --- a/apps/workspace-engine/pkg/workspace/storage_file.go +++ b/apps/workspace-engine/pkg/workspace/storage_file.go @@ -17,7 +17,7 @@ type FileStorage struct { } // NewFileStorage returns a FileStorage rooted at the given base directory. -func NewFileStorage(baseDir string) *FileStorage { +func NewFileStorage(baseDir string) StorageClient { return &FileStorage{BaseDir: baseDir} } diff --git a/apps/workspace-engine/pkg/workspace/storage_gcs.go b/apps/workspace-engine/pkg/workspace/storage_gcs.go index 0b09fe747..69ff9969f 100644 --- a/apps/workspace-engine/pkg/workspace/storage_gcs.go +++ b/apps/workspace-engine/pkg/workspace/storage_gcs.go @@ -2,11 +2,12 @@ package workspace import ( "context" + "errors" "fmt" "io" "os" + "path/filepath" "strings" - "workspace-engine/pkg/db" "cloud.google.com/go/storage" ) @@ -15,54 +16,20 @@ func IsGCSStorageEnabled() bool { return strings.HasPrefix(os.Getenv("WORKSPACE_STATES_BUCKET_URL"), "gs://") } -func GetWorkspaceSnapshot(ctx context.Context, workspaceID string) ([]byte, error) { - snapshot, err := db.GetWorkspaceSnapshot(ctx, workspaceID) - if err != nil { - return nil, err - } - - if snapshot == nil { - return nil, nil - } - - if snapshot.Path == "" { - return nil, nil - } +var _ StorageClient = (*GCSStorageClient)(nil) - // Parse bucket/object/path from stored path - parts := strings.SplitN(snapshot.Path, "/", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("invalid GCS path: %s", snapshot.Path) - } - bucket := parts[0] - objectPath := parts[1] - - client, err := storage.NewClient(ctx) - if err != nil { - return nil, fmt.Errorf("failed to create GCS client: %w", err) - } - defer client.Close() - - obj := client.Bucket(bucket).Object(objectPath) - reader, err := obj.NewReader(ctx) - if err != nil { - return nil, fmt.Errorf("failed to read snapshot: %w", err) - } - defer reader.Close() +type GCSStorageClient struct { + client *storage.Client + bucket string + prefix string +} - data, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("failed to read data: %w", err) +func NewGCSStorageClient(ctx context.Context) (StorageClient, error) { + if !IsGCSStorageEnabled() { + return nil, errors.New("gcs storage is not enabled") } - return data, nil -} - -// PutWorkspaceSnapshot writes a new timestamped snapshot for a workspace to GCS. -func PutWorkspaceSnapshot(ctx context.Context, workspaceID string, timestamp string, partition int32, numPartitions int32, data []byte) error { - // Get bucket URL like "gs://bucket-name" or "gs://bucket-name/prefix" bucketURL := os.Getenv("WORKSPACE_STATES_BUCKET_URL") - bucketURL = strings.TrimSuffix(bucketURL, "/") // Parse bucket and optional prefix gsPath := strings.TrimPrefix(bucketURL, "gs://") @@ -75,41 +42,52 @@ func PutWorkspaceSnapshot(ctx context.Context, workspaceID string, timestamp str client, err := storage.NewClient(ctx) if err != nil { - return fmt.Errorf("failed to create GCS client: %w", err) - } - defer client.Close() - - // Generate object path - objectName := fmt.Sprintf("%s_%s.gob", workspaceID, timestamp) - objectPath := objectName - if prefix != "" { - objectPath = prefix + "/" + objectName + return nil, err } + return &GCSStorageClient{client: client, bucket: bucket, prefix: prefix}, nil +} - obj := client.Bucket(bucket).Object(objectPath) +func (c *GCSStorageClient) Put(ctx context.Context, path string, data []byte) error { + path = filepath.Join(c.prefix, path) + obj := c.client.Bucket(c.bucket).Object(path) writer := obj.NewWriter(ctx) - if _, err := writer.Write(data); err != nil { writer.Close() return fmt.Errorf("failed to write snapshot: %w", err) } - if err := writer.Close(); err != nil { return fmt.Errorf("failed to close writer: %w", err) } + return nil +} - // Store bucket/object/path in database (no gs:// prefix) - fullPath := fmt.Sprintf("%s/%s", bucket, objectPath) +func (c *GCSStorageClient) Get(ctx context.Context, path string) ([]byte, error) { + path = filepath.Join(c.prefix, path) + obj := c.client.Bucket(c.bucket).Object(path) - snapshot := &db.WorkspaceSnapshot{ - Path: fullPath, - Timestamp: timestamp, - Partition: partition, - NumPartitions: numPartitions, + // Check if the object exists before trying to read + _, err := obj.Attrs(ctx) + if err != nil { + if err == storage.ErrObjectNotExist { + return nil, ErrWorkspaceSnapshotNotFound + } + return nil, fmt.Errorf("failed to stat GCS object: %w", err) } - if err := db.WriteWorkspaceSnapshot(ctx, workspaceID, snapshot); err != nil { - return fmt.Errorf("failed to write snapshot: %w", err) + + reader, err := obj.NewReader(ctx) + if err != nil { + return nil, fmt.Errorf("failed to read snapshot: %w", err) } + defer reader.Close() - return nil + data, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read data: %w", err) + } + + return data, nil } + +func (c *GCSStorageClient) Close() error { + return c.client.Close() +} \ No newline at end of file diff --git a/apps/workspace-engine/pkg/workspace/workspace.go b/apps/workspace-engine/pkg/workspace/workspace.go index ae4a6ff4e..110797923 100644 --- a/apps/workspace-engine/pkg/workspace/workspace.go +++ b/apps/workspace-engine/pkg/workspace/workspace.go @@ -2,14 +2,10 @@ package workspace import ( "bytes" - "context" "encoding/gob" - "errors" - "fmt" "workspace-engine/pkg/changeset" "workspace-engine/pkg/cmap" "workspace-engine/pkg/db" - "workspace-engine/pkg/workspace/kafka" "workspace-engine/pkg/workspace/releasemanager" "workspace-engine/pkg/workspace/store" ) @@ -26,7 +22,6 @@ func New(id string) *Workspace { store: s, releasemanager: rm, changesetConsumer: cc, - KafkaProgress: make(kafka.KafkaProgressMap), } return ws } @@ -40,7 +35,6 @@ func NewNoFlush(id string) *Workspace { store: s, releasemanager: rm, changesetConsumer: cc, - KafkaProgress: make(kafka.KafkaProgressMap), } return ws } @@ -51,7 +45,6 @@ type Workspace struct { store *store.Store releasemanager *releasemanager.Manager changesetConsumer changeset.ChangesetConsumer[any] - KafkaProgress kafka.KafkaProgressMap } func (w *Workspace) Store() *store.Store { @@ -120,7 +113,6 @@ func (w *Workspace) GobEncode() ([]byte, error) { // Create workspace data with ID and store data := WorkspaceStorageObject{ ID: w.ID, - KafkaProgress: w.KafkaProgress, StoreData: storeData, } @@ -145,8 +137,7 @@ func (w *Workspace) GobDecode(data []byte) error { } // Restore the workspace ID - w.ID = wsData.ID - w.KafkaProgress = wsData.KafkaProgress + w.ID = wsData.ID // Initialize store if needed if w.store == nil { @@ -224,194 +215,3 @@ func GetNoFlushWorkspace(id string) *Workspace { func GetAllWorkspaceIds() []string { return workspaces.Keys() } - -// SaveToStorage serializes the workspace state and saves it to a storage client -func (w *Workspace) SaveToStorage(ctx context.Context, storage StorageClient, path string) error { - // Encode the workspace using gob - data, err := w.GobEncode() - if err != nil { - return fmt.Errorf("failed to encode workspace: %w", err) - } - - // Write to file with appropriate permissions - if err := storage.Put(ctx, path, data); err != nil { - return fmt.Errorf("failed to write workspace to disk: %w", err) - } - - return nil -} - -// LoadFromStorage deserializes the workspace state from a storage client -func (w *Workspace) LoadFromStorage(ctx context.Context, storage StorageClient, path string) error { - // Read from file - data, err := storage.Get(ctx, path) - if err != nil { - return fmt.Errorf("failed to read workspace from disk: %w", err) - } - - // Decode the workspace - if err := w.GobDecode(data); err != nil { - return fmt.Errorf("failed to decode workspace: %w", err) - } - - return nil -} - -func (w *Workspace) SaveToGCS(ctx context.Context, workspaceID string, timestamp string, partition int32, numPartitions int32) error { - if !IsGCSStorageEnabled() { - return nil - } - - data, err := w.GobEncode() - if err != nil { - return fmt.Errorf("failed to encode workspace: %w", err) - } - - if err := PutWorkspaceSnapshot(ctx, workspaceID, timestamp, partition, numPartitions, data); err != nil { - return fmt.Errorf("failed to put workspace snapshot: %w", err) - } - - return nil -} - -var ErrWorkspaceSnapshotNotFound = errors.New("workspace snapshot not found") - -func (w *Workspace) LoadFromGCS(ctx context.Context, workspaceID string) error { - if !IsGCSStorageEnabled() { - return nil - } - - data, err := GetWorkspaceSnapshot(ctx, workspaceID) - if err != nil { - return fmt.Errorf("failed to get workspace snapshot: %w", err) - } - - if data == nil { - return ErrWorkspaceSnapshotNotFound - } - - if err := w.GobDecode(data); err != nil { - return fmt.Errorf("failed to decode workspace: %w", err) - } - - return nil -} - -// WorkspaceSaver is a function that saves workspace state to an external storage -type WorkspaceSaver func(ctx context.Context, workspaceID string, timestamp string) error - -// CreateGCSWorkspaceSaver creates a new workspace saver that saves workspace state to GCS -func CreateGCSWorkspaceSaver(numPartitions int32) WorkspaceSaver { - return func(ctx context.Context, workspaceID string, timestamp string) error { - ws := GetWorkspace(workspaceID) - partition := kafka.PartitionForWorkspace(workspaceID, numPartitions) - if err := ws.SaveToGCS(ctx, workspaceID, timestamp, partition, numPartitions); err != nil { - return fmt.Errorf("failed to save workspace %s to GCS: %w", workspaceID, err) - } - return nil - } -} - -// WorkspaceLoader is a function that loads workspace state from an external storage -type WorkspaceLoader func(ctx context.Context, assignedPartitions []int32, numPartitions int32) error - -// getAssignedWorkspaceIDs retrieves workspace IDs for the assigned partitions. -// Uses the provided discoverer if available, otherwise falls back to the default implementation. -func getAssignedWorkspaceIDs( - ctx context.Context, - assignedPartitions []int32, - numPartitions int32, - discoverer kafka.WorkspaceIDDiscoverer, -) ([]string, error) { - var allWorkspaceIDs []string - - // Use discoverer if provided, otherwise use default implementation - if discoverer != nil { - // Collect workspace IDs for each assigned partition - workspaceIDSet := make(map[string]bool) - for _, partition := range assignedPartitions { - partitionWorkspaces, err := discoverer(ctx, partition, numPartitions) - if err != nil { - return nil, fmt.Errorf("failed to discover workspace IDs for partition %d: %w", partition, err) - } - for _, wsID := range partitionWorkspaces { - workspaceIDSet[wsID] = true - } - } - // Convert set to slice - for wsID := range workspaceIDSet { - allWorkspaceIDs = append(allWorkspaceIDs, wsID) - } - } else { - var err error - allWorkspaceIDs, err = kafka.GetAssignedWorkspaceIDs(ctx, assignedPartitions, numPartitions) - if err != nil { - return nil, fmt.Errorf("failed to get assigned workspace IDs: %w", err) - } - } - - return allWorkspaceIDs, nil -} - -// CreateGCSWorkspaceLoader creates a workspace loader function that: -// 1. Discovers all available workspace IDs -// 2. Determines which workspaces belong to which partitions -// 3. Loads workspaces for the assigned partitions from GCS -func CreateGCSWorkspaceLoader( - discoverer kafka.WorkspaceIDDiscoverer, -) WorkspaceLoader { - return func(ctx context.Context, assignedPartitions []int32, numPartitions int32) error { - allWorkspaceIDs, err := getAssignedWorkspaceIDs(ctx, assignedPartitions, numPartitions, discoverer) - if err != nil { - return err - } - - for _, workspaceID := range allWorkspaceIDs { - ws := GetWorkspace(workspaceID) - err := ws.LoadFromGCS(ctx, workspaceID) - if err == nil { - Set(workspaceID, ws) - continue - } - - if err == ErrWorkspaceSnapshotNotFound { - if err := PopulateWorkspaceWithInitialState(ctx, ws); err != nil { - return fmt.Errorf("failed to populate workspace %s with initial state: %w", workspaceID, err) - } - Set(workspaceID, ws) - continue - } - - return fmt.Errorf("failed to load workspace %s from GCS: %w", workspaceID, err) - } - - return nil - } -} - -// CreateWorkspaceLoader creates a workspace loader function that: -// 1. Discovers all available workspace IDs -// 2. Determines which workspaces belong to which partitions -// 3. Loads workspaces for the assigned partitions from storage -func CreateWorkspaceLoader( - storage StorageClient, - discoverer kafka.WorkspaceIDDiscoverer, -) WorkspaceLoader { - return func(ctx context.Context, assignedPartitions []int32, numPartitions int32) error { - allWorkspaceIDs, err := getAssignedWorkspaceIDs(ctx, assignedPartitions, numPartitions, discoverer) - if err != nil { - return err - } - - for _, workspaceID := range allWorkspaceIDs { - ws := GetWorkspace(workspaceID) - if err := ws.LoadFromStorage(ctx, storage, fmt.Sprintf("%s.gob", workspaceID)); err != nil { - return fmt.Errorf("failed to load workspace %s: %w", workspaceID, err) - } - - Set(workspaceID, ws) - } - - return nil - } -} From 6fccb134f4322ccf93f9c01f79c2b0f77727e228 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 22 Oct 2025 20:44:25 -0400 Subject: [PATCH 09/15] addclean up --- .gitignore | 2 ++ apps/workspace-engine/pkg/kafka/kafka.go | 3 ++- apps/workspace-engine/pkg/workspace/loader.go | 21 +------------------ 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 0cc233825..d054e64ba 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,5 @@ dist/ # IDE files .idea/ + +state/ \ No newline at end of file diff --git a/apps/workspace-engine/pkg/kafka/kafka.go b/apps/workspace-engine/pkg/kafka/kafka.go index 6b6e2093a..f664e3089 100644 --- a/apps/workspace-engine/pkg/kafka/kafka.go +++ b/apps/workspace-engine/pkg/kafka/kafka.go @@ -139,7 +139,8 @@ func RunConsumer(ctx context.Context) error { } snapshot := &db.WorkspaceSnapshot{ - Timestamp: time.Now().Format(time.RFC3339), + Path: fmt.Sprintf("%s_%s.gob", ws.ID, msg.Timestamp.Format(time.RFC3339Nano)), + Timestamp: msg.Timestamp.Format(time.RFC3339Nano), Partition: int32(msg.TopicPartition.Partition), NumPartitions: numPartitions, } diff --git a/apps/workspace-engine/pkg/workspace/loader.go b/apps/workspace-engine/pkg/workspace/loader.go index abbab6c70..e959d0a35 100644 --- a/apps/workspace-engine/pkg/workspace/loader.go +++ b/apps/workspace-engine/pkg/workspace/loader.go @@ -3,36 +3,17 @@ package workspace import ( "context" "fmt" - "time" "workspace-engine/pkg/db" ) -func getPath(workspaceID string, timestamp string) string { - return fmt.Sprintf("%s_%s.gob", workspaceID, timestamp) -} - -func LoadAll(ctx context.Context, storage StorageClient) error { - workspaceIds := GetAllWorkspaceIds() - for _, workspaceID := range workspaceIds { - workspace := GetWorkspace(workspaceID) - err := Load(ctx, storage, workspace) - if err != nil { - return err - } - } - return nil -} - func Save(ctx context.Context, storage StorageClient, workspace *Workspace, snapshot *db.WorkspaceSnapshot) error { - path := getPath(workspace.ID, time.Now().Format(time.RFC3339)) - data, err := workspace.GobEncode() if err != nil { return fmt.Errorf("failed to encode workspace: %w", err) } // Write to file with appropriate permissions - if err := storage.Put(ctx, path, data); err != nil { + if err := storage.Put(ctx, snapshot.Path, data); err != nil { return fmt.Errorf("failed to write workspace to disk: %w", err) } From 956ec9748742082fc47125a2f473e6be3738e849 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 22 Oct 2025 20:47:23 -0400 Subject: [PATCH 10/15] clean up --- apps/workspace-engine/pkg/workspace/loader.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/workspace-engine/pkg/workspace/loader.go b/apps/workspace-engine/pkg/workspace/loader.go index e959d0a35..7e3ecc4e6 100644 --- a/apps/workspace-engine/pkg/workspace/loader.go +++ b/apps/workspace-engine/pkg/workspace/loader.go @@ -37,9 +37,7 @@ func Load(ctx context.Context, storage StorageClient, workspace *Workspace) erro return nil } - dbSnapshotPath := dbSnapshot.Path - - data, err := storage.Get(ctx, dbSnapshotPath) + data, err := storage.Get(ctx, dbSnapshot.Path) if err != nil { return fmt.Errorf("failed to read workspace from disk: %w", err) } From 4ff5e7553d204ae2d952ae3fc23a28621e1b23f2 Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 18:27:27 -0700 Subject: [PATCH 11/15] testing --- .../test/e2e/engine_workspace_loader_test.go | 383 ----- .../e2e/engine_workspace_persistence_test.go | 1231 ++++++++++++++++ .../test/e2e/engine_workspace_storage_test.go | 1264 ----------------- 3 files changed, 1231 insertions(+), 1647 deletions(-) delete mode 100644 apps/workspace-engine/test/e2e/engine_workspace_loader_test.go create mode 100644 apps/workspace-engine/test/e2e/engine_workspace_persistence_test.go delete mode 100644 apps/workspace-engine/test/e2e/engine_workspace_storage_test.go diff --git a/apps/workspace-engine/test/e2e/engine_workspace_loader_test.go b/apps/workspace-engine/test/e2e/engine_workspace_loader_test.go deleted file mode 100644 index 903cd66ac..000000000 --- a/apps/workspace-engine/test/e2e/engine_workspace_loader_test.go +++ /dev/null @@ -1,383 +0,0 @@ -package e2e - -import ( - "context" - "fmt" - "os" - "testing" - "workspace-engine/pkg/workspace" - "workspace-engine/pkg/workspace/kafka" -) - -func TestEngine_WorkspaceLoader_SingleWorkspace(t *testing.T) { - ctx := context.Background() - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-loader-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - storage := workspace.NewFileStorage(tempDir) - - // Create and save a workspace - workspaceID := "test-workspace-1" - ws := workspace.New(workspaceID) - ws.KafkaProgress[kafka.TopicPartition{Topic: "events", Partition: 0}] = kafka.KafkaProgress{ - LastApplied: 42, - LastTimestamp: 1234567890, - } - - if err := ws.SaveToStorage(ctx, storage, workspaceID+".gob"); err != nil { - t.Fatalf("failed to save workspace: %v", err) - } - - // Determine which partition this workspace belongs to - numPartitions := int32(4) - workspacePartition := kafka.PartitionForWorkspace(workspaceID, numPartitions) - - // Create a mock discoverer that returns our workspace ID only for its partition - discoverer := func(ctx context.Context, targetPartition int32, numPartitions int32) ([]string, error) { - // Return workspace only if it belongs to the requested partition - if kafka.PartitionForWorkspace(workspaceID, numPartitions) == targetPartition { - return []string{workspaceID}, nil - } - return []string{}, nil - } - - // Create workspace loader - loader := workspace.CreateWorkspaceLoader(storage, discoverer) - - // Load workspaces for the assigned partition - assignedPartitions := []int32{workspacePartition} - if err := loader(ctx, assignedPartitions, numPartitions); err != nil { - t.Fatalf("failed to load workspaces: %v", err) - } - - // Verify workspace was loaded - if !workspace.HasWorkspace(workspaceID) { - t.Fatal("workspace was not loaded") - } - - loadedWs := workspace.GetWorkspace(workspaceID) - if loadedWs.ID != workspaceID { - t.Errorf("workspace ID mismatch: expected %s, got %s", workspaceID, loadedWs.ID) - } - - // Verify KafkaProgress was restored - tp := kafka.TopicPartition{Topic: "events", Partition: 0} - if progress, ok := loadedWs.KafkaProgress[tp]; !ok { - t.Error("KafkaProgress not found") - } else if progress.LastApplied != 42 { - t.Errorf("LastApplied mismatch: expected 42, got %d", progress.LastApplied) - } -} - -func TestEngine_WorkspaceLoader_MultipleWorkspaces(t *testing.T) { - ctx := context.Background() - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-loader-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - storage := workspace.NewFileStorage(tempDir) - - // Create and save multiple workspaces - numPartitions := int32(4) - workspaceIDs := []string{ - "workspace-alpha", - "workspace-beta", - "workspace-gamma", - "workspace-delta", - "workspace-epsilon", - } - - // Map workspaces to their partitions - workspacePartitions := make(map[string]int32) - for _, wsID := range workspaceIDs { - ws := workspace.New(wsID) - ws.KafkaProgress[kafka.TopicPartition{Topic: "events", Partition: 0}] = kafka.KafkaProgress{ - LastApplied: int64(len(wsID)), // Use length as unique identifier - LastTimestamp: 1234567890, - } - - if err := ws.SaveToStorage(ctx, storage, wsID+".gob"); err != nil { - t.Fatalf("failed to save workspace %s: %v", wsID, err) - } - - partition := kafka.PartitionForWorkspace(wsID, numPartitions) - workspacePartitions[wsID] = partition - } - - // Choose partitions to load (e.g., partitions 0 and 2) - assignedPartitions := []int32{0, 2} - - // Determine which workspaces should be loaded - expectedWorkspaces := make(map[string]bool) - for wsID, partition := range workspacePartitions { - for _, assigned := range assignedPartitions { - if partition == assigned { - expectedWorkspaces[wsID] = true - break - } - } - } - - // Create a mock discoverer that returns workspace IDs only for the requested partition - discoverer := func(ctx context.Context, targetPartition int32, numPartitions int32) ([]string, error) { - var result []string - for _, wsID := range workspaceIDs { - if kafka.PartitionForWorkspace(wsID, numPartitions) == targetPartition { - result = append(result, wsID) - } - } - return result, nil - } - - // Create and execute workspace loader - loader := workspace.CreateWorkspaceLoader(storage, discoverer) - if err := loader(ctx, assignedPartitions, numPartitions); err != nil { - t.Fatalf("failed to load workspaces: %v", err) - } - - // Verify only expected workspaces were loaded - for wsID := range expectedWorkspaces { - if !workspace.HasWorkspace(wsID) { - t.Errorf("expected workspace %s to be loaded (partition %d)", wsID, workspacePartitions[wsID]) - } - } - - // Verify unexpected workspaces were NOT loaded - for _, wsID := range workspaceIDs { - if !expectedWorkspaces[wsID] { - // Note: GetWorkspace creates a new workspace if it doesn't exist, - // so we need to check HasWorkspace instead - if workspace.HasWorkspace(wsID) { - t.Errorf("workspace %s should not be loaded (partition %d not in assigned)", wsID, workspacePartitions[wsID]) - } - } - } -} - -func TestEngine_WorkspaceLoader_PartitionAssignment(t *testing.T) { - // Test that the partition assignment logic is consistent - numPartitions := int32(8) - - testCases := []struct { - workspaceID string - expectedPartition int32 - }{ - // These values are based on Murmur3 hash - // The actual partition depends on the hash function - {"workspace-1", kafka.PartitionForWorkspace("workspace-1", numPartitions)}, - {"workspace-2", kafka.PartitionForWorkspace("workspace-2", numPartitions)}, - {"workspace-3", kafka.PartitionForWorkspace("workspace-3", numPartitions)}, - } - - for _, tc := range testCases { - partition := kafka.PartitionForWorkspace(tc.workspaceID, numPartitions) - - if partition != tc.expectedPartition { - t.Errorf("workspace %s: expected partition %d, got %d", tc.workspaceID, tc.expectedPartition, partition) - } - - // Verify partition is within valid range - if partition < 0 || partition >= numPartitions { - t.Errorf("workspace %s: partition %d is out of range [0, %d)", tc.workspaceID, partition, numPartitions) - } - - // Verify consistency - calling multiple times should return same result - for i := 0; i < 10; i++ { - p := kafka.PartitionForWorkspace(tc.workspaceID, numPartitions) - if p != partition { - t.Errorf("partition assignment is not consistent for %s: got %d, expected %d", tc.workspaceID, p, partition) - } - } - } -} - -func TestEngine_WorkspaceLoader_EmptyAssignment(t *testing.T) { - ctx := context.Background() - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-loader-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - storage := workspace.NewFileStorage(tempDir) - - // Create a mock discoverer - discoverer := func(ctx context.Context, targetPartition int32, numPartitions int32) ([]string, error) { - return []string{}, nil - } - - // Create workspace loader - loader := workspace.CreateWorkspaceLoader(storage, discoverer) - - // Load workspaces with empty assignment - assignedPartitions := []int32{} - if err := loader(ctx, assignedPartitions, int32(4)); err != nil { - t.Fatalf("failed to load workspaces with empty assignment: %v", err) - } - - // This should succeed and not load any workspaces - // GetAllWorkspaceIds might return workspaces, but they shouldn't be loaded from storage -} - -func TestEngine_WorkspaceLoader_AllPartitions(t *testing.T) { - ctx := context.Background() - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-loader-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - storage := workspace.NewFileStorage(tempDir) - - // Create and save multiple workspaces - numPartitions := int32(4) - workspaceIDs := []string{ - "workspace-all-1", - "workspace-all-2", - "workspace-all-3", - "workspace-all-4", - } - - for _, wsID := range workspaceIDs { - ws := workspace.New(wsID) - ws.KafkaProgress[kafka.TopicPartition{Topic: "events", Partition: 0}] = kafka.KafkaProgress{ - LastApplied: int64(len(wsID)), - LastTimestamp: 1234567890, - } - - if err := ws.SaveToStorage(ctx, storage, wsID+".gob"); err != nil { - t.Fatalf("failed to save workspace %s: %v", wsID, err) - } - } - - // Create a mock discoverer that returns workspaces for the requested partition - discoverer := func(ctx context.Context, targetPartition int32, numPartitions int32) ([]string, error) { - var result []string - for _, wsID := range workspaceIDs { - if kafka.PartitionForWorkspace(wsID, numPartitions) == targetPartition { - result = append(result, wsID) - } - } - return result, nil - } - - // Create workspace loader and load ALL partitions - loader := workspace.CreateWorkspaceLoader(storage, discoverer) - assignedPartitions := []int32{0, 1, 2, 3} // All partitions - - if err := loader(ctx, assignedPartitions, numPartitions); err != nil { - t.Fatalf("failed to load workspaces: %v", err) - } - - // Verify all workspaces were loaded - for _, wsID := range workspaceIDs { - if !workspace.HasWorkspace(wsID) { - t.Errorf("workspace %s should be loaded", wsID) - } - } -} - -func TestEngine_WorkspaceLoader_MissingWorkspaceFile(t *testing.T) { - ctx := context.Background() - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-loader-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - storage := workspace.NewFileStorage(tempDir) - - // Create a mock discoverer that returns a workspace that doesn't exist on disk - missingWorkspaceID := "missing-workspace" - numPartitions := int32(4) - partition := kafka.PartitionForWorkspace(missingWorkspaceID, numPartitions) - - discoverer := func(ctx context.Context, targetPartition int32, numPartitions int32) ([]string, error) { - // Only return the missing workspace for its assigned partition - if targetPartition == partition { - return []string{missingWorkspaceID}, nil - } - return []string{}, nil - } - - // Create workspace loader - loader := workspace.CreateWorkspaceLoader(storage, discoverer) - - // Try to load - this should return an error since the file doesn't exist - assignedPartitions := []int32{partition} - err = loader(ctx, assignedPartitions, numPartitions) - - if err == nil { - t.Fatal("expected error when loading missing workspace file, got nil") - } - - // Verify error message contains workspace ID - if !contains(err.Error(), missingWorkspaceID) { - t.Errorf("error message should mention missing workspace ID: %v", err) - } -} - -func TestEngine_WorkspaceLoader_DiscovererError(t *testing.T) { - ctx := context.Background() - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-loader-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - storage := workspace.NewFileStorage(tempDir) - - // Create a mock discoverer that returns an error - expectedError := fmt.Errorf("discoverer error") - discoverer := func(ctx context.Context, targetPartition int32, numPartitions int32) ([]string, error) { - return nil, expectedError - } - - // Create workspace loader - loader := workspace.CreateWorkspaceLoader(storage, discoverer) - - // Try to load - this should return the discoverer error - assignedPartitions := []int32{0} - err = loader(ctx, assignedPartitions, int32(4)) - - if err == nil { - t.Fatal("expected error from discoverer, got nil") - } - - // Verify we get the expected error (either from discoverer or from GetAssignedWorkspaceIDs) - if !contains(err.Error(), "failed to discover workspace IDs") && !contains(err.Error(), "discoverer error") { - t.Errorf("error should indicate failure to discover workspace IDs: %v", err) - } -} - -// Helper function to check if a string contains a substring -func contains(s, substr string) bool { - return len(s) >= len(substr) && (s == substr || len(substr) == 0 || - (len(s) > 0 && len(substr) > 0 && indexOfString(s, substr) >= 0)) -} - -func indexOfString(s, substr string) int { - for i := 0; i <= len(s)-len(substr); i++ { - if s[i:i+len(substr)] == substr { - return i - } - } - return -1 -} diff --git a/apps/workspace-engine/test/e2e/engine_workspace_persistence_test.go b/apps/workspace-engine/test/e2e/engine_workspace_persistence_test.go new file mode 100644 index 000000000..88e8c03e6 --- /dev/null +++ b/apps/workspace-engine/test/e2e/engine_workspace_persistence_test.go @@ -0,0 +1,1231 @@ +package e2e + +import ( + "context" + "fmt" + "os" + "sync" + "testing" + "time" + "workspace-engine/pkg/oapi" + "workspace-engine/pkg/workspace" + "workspace-engine/test/integration" + + "github.com/google/uuid" +) + +// These tests validate workspace persistence including: +// - Storage layer operations (file/GCS Put/Get) +// - Gob encoding/decoding +// - All entity fields are preserved (metadata, config, timestamps, etc.) + +// Helper functions for deep equality verification + +func verifyResourcesEqual(t *testing.T, expected, actual *oapi.Resource, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: resource ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: resource name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + if actual.Kind != expected.Kind { + t.Errorf("%s: resource kind mismatch: expected %s, got %s", context, expected.Kind, actual.Kind) + } + if actual.Version != expected.Version { + t.Errorf("%s: resource version mismatch: expected %s, got %s", context, expected.Version, actual.Version) + } + if actual.Identifier != expected.Identifier { + t.Errorf("%s: resource identifier mismatch: expected %s, got %s", context, expected.Identifier, actual.Identifier) + } + + // Verify metadata + if len(actual.Metadata) != len(expected.Metadata) { + t.Errorf("%s: metadata length mismatch: expected %d, got %d", context, len(expected.Metadata), len(actual.Metadata)) + } + for key, expectedValue := range expected.Metadata { + if actualValue, ok := actual.Metadata[key]; !ok { + t.Errorf("%s: metadata key %s missing", context, key) + } else if actualValue != expectedValue { + t.Errorf("%s: metadata[%s] mismatch: expected %s, got %s", context, key, expectedValue, actualValue) + } + } + + // Verify config (deep comparison would require reflection or JSON marshaling) + if (expected.Config == nil) != (actual.Config == nil) { + t.Errorf("%s: config nil mismatch", context) + } + + // Verify timestamps + if !actual.CreatedAt.Equal(expected.CreatedAt) { + t.Errorf("%s: createdAt mismatch: expected %v, got %v", context, expected.CreatedAt, actual.CreatedAt) + } + + // UpdatedAt is optional + if (expected.UpdatedAt == nil) != (actual.UpdatedAt == nil) { + t.Errorf("%s: updatedAt nil mismatch", context) + } else if expected.UpdatedAt != nil && !actual.UpdatedAt.Equal(*expected.UpdatedAt) { + t.Errorf("%s: updatedAt mismatch: expected %v, got %v", context, *expected.UpdatedAt, *actual.UpdatedAt) + } +} + +func verifyJobsEqual(t *testing.T, expected, actual *oapi.Job, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: job ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Status != expected.Status { + t.Errorf("%s: job status mismatch: expected %s, got %s", context, expected.Status, actual.Status) + } + if actual.JobAgentId != expected.JobAgentId { + t.Errorf("%s: job agent ID mismatch: expected %s, got %s", context, expected.JobAgentId, actual.JobAgentId) + } + if actual.ReleaseId != expected.ReleaseId { + t.Errorf("%s: release ID mismatch: expected %s, got %s", context, expected.ReleaseId, actual.ReleaseId) + } + // ExternalId is optional + if (expected.ExternalId == nil) != (actual.ExternalId == nil) { + t.Errorf("%s: externalId nil mismatch", context) + } else if expected.ExternalId != nil && *actual.ExternalId != *expected.ExternalId { + t.Errorf("%s: external ID mismatch: expected %s, got %s", context, *expected.ExternalId, *actual.ExternalId) + } + + // Verify metadata + if len(actual.Metadata) != len(expected.Metadata) { + t.Errorf("%s: metadata length mismatch: expected %d, got %d", context, len(expected.Metadata), len(actual.Metadata)) + } + for key, expectedValue := range expected.Metadata { + if actualValue, ok := actual.Metadata[key]; !ok { + t.Errorf("%s: metadata key %s missing", context, key) + } else if actualValue != expectedValue { + t.Errorf("%s: metadata[%s] mismatch: expected %s, got %s", context, key, expectedValue, actualValue) + } + } + + // Verify timestamps + if !actual.CreatedAt.Equal(expected.CreatedAt) { + t.Errorf("%s: createdAt mismatch: expected %v, got %v", context, expected.CreatedAt, actual.CreatedAt) + } + if !actual.UpdatedAt.Equal(expected.UpdatedAt) { + t.Errorf("%s: updatedAt mismatch: expected %v, got %v", context, expected.UpdatedAt, actual.UpdatedAt) + } + + // Verify optional timestamps + if (expected.StartedAt == nil) != (actual.StartedAt == nil) { + t.Errorf("%s: startedAt nil mismatch", context) + } else if expected.StartedAt != nil && !actual.StartedAt.Equal(*expected.StartedAt) { + t.Errorf("%s: startedAt mismatch: expected %v, got %v", context, *expected.StartedAt, *actual.StartedAt) + } + + if (expected.CompletedAt == nil) != (actual.CompletedAt == nil) { + t.Errorf("%s: completedAt nil mismatch", context) + } else if expected.CompletedAt != nil && !actual.CompletedAt.Equal(*expected.CompletedAt) { + t.Errorf("%s: completedAt mismatch: expected %v, got %v", context, *expected.CompletedAt, *actual.CompletedAt) + } +} + +func verifyDeploymentsEqual(t *testing.T, expected, actual *oapi.Deployment, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: deployment ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: deployment name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + // Description is optional + if (expected.Description == nil) != (actual.Description == nil) { + t.Errorf("%s: description nil mismatch", context) + } else if expected.Description != nil && *actual.Description != *expected.Description { + t.Errorf("%s: deployment description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) + } + if actual.SystemId != expected.SystemId { + t.Errorf("%s: system ID mismatch: expected %s, got %s", context, expected.SystemId, actual.SystemId) + } + // JobAgentId is optional + if (expected.JobAgentId == nil) != (actual.JobAgentId == nil) { + t.Errorf("%s: jobAgentId nil mismatch", context) + } else if expected.JobAgentId != nil && *actual.JobAgentId != *expected.JobAgentId { + t.Errorf("%s: job agent ID mismatch: expected %s, got %s", context, *expected.JobAgentId, *actual.JobAgentId) + } +} + +func verifySystemsEqual(t *testing.T, expected, actual *oapi.System, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: system ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: system name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + // Description is optional + if (expected.Description == nil) != (actual.Description == nil) { + t.Errorf("%s: description nil mismatch", context) + } else if expected.Description != nil && *actual.Description != *expected.Description { + t.Errorf("%s: system description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) + } + if actual.WorkspaceId != expected.WorkspaceId { + t.Errorf("%s: workspace ID mismatch: expected %s, got %s", context, expected.WorkspaceId, actual.WorkspaceId) + } +} + +func verifyEnvironmentsEqual(t *testing.T, expected, actual *oapi.Environment, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: environment ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: environment name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + // Description is optional + if (expected.Description == nil) != (actual.Description == nil) { + t.Errorf("%s: description nil mismatch", context) + } else if expected.Description != nil && *actual.Description != *expected.Description { + t.Errorf("%s: environment description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) + } + if actual.SystemId != expected.SystemId { + t.Errorf("%s: system ID mismatch: expected %s, got %s", context, expected.SystemId, actual.SystemId) + } +} + +func verifyJobAgentsEqual(t *testing.T, expected, actual *oapi.JobAgent, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: job agent ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: job agent name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + if actual.Type != expected.Type { + t.Errorf("%s: job agent type mismatch: expected %s, got %s", context, expected.Type, actual.Type) + } + if actual.WorkspaceId != expected.WorkspaceId { + t.Errorf("%s: workspace ID mismatch: expected %s, got %s", context, expected.WorkspaceId, actual.WorkspaceId) + } +} + +func verifyPoliciesEqual(t *testing.T, expected, actual *oapi.Policy, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: policy ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: policy name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + // Description is optional + if (expected.Description == nil) != (actual.Description == nil) { + t.Errorf("%s: description nil mismatch", context) + } else if expected.Description != nil && *actual.Description != *expected.Description { + t.Errorf("%s: policy description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) + } + if actual.WorkspaceId != expected.WorkspaceId { + t.Errorf("%s: workspace ID mismatch: expected %s, got %s", context, expected.WorkspaceId, actual.WorkspaceId) + } +} + +func TestEngine_Persistence_BasicSaveLoadRoundtrip(t *testing.T) { + ctx := context.Background() + + resource1Id := uuid.New().String() + resource2Id := uuid.New().String() + systemId := uuid.New().String() + jobAgentId := uuid.New().String() + deploymentId := uuid.New().String() + + // Create workspace and populate using integration helpers + engine := integration.NewTestWorkspace(t, + integration.WithResource( + integration.ResourceID(resource1Id), + integration.ResourceName("resource-1"), + ), + integration.WithResource( + integration.ResourceID(resource2Id), + integration.ResourceName("resource-2"), + ), + integration.WithJobAgent( + integration.JobAgentID(jobAgentId), + integration.JobAgentName("test-job-agent"), + ), + integration.WithSystem( + integration.SystemID(systemId), + integration.SystemName("test-system"), + integration.WithDeployment( + integration.DeploymentID(deploymentId), + integration.DeploymentName("deployment-1"), + integration.DeploymentJobAgent(jobAgentId), + ), + ), + ) + + ws := engine.Workspace() + workspaceID := ws.ID + + // Capture original state + originalResources := ws.Resources().Items() + originalDeployments := ws.Deployments().Items() + originalSystems := ws.Systems().Items() + originalJobAgents := ws.JobAgents().Items() + + // Create temporary directory for storage + tempDir, err := os.MkdirTemp("", "workspace-persistence-test-*") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Save workspace to storage + storage := workspace.NewFileStorage(tempDir) + + // Encode workspace + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + // Write to storage + if err := storage.Put(ctx, "workspace.gob", data); err != nil { + t.Fatalf("failed to write workspace: %v", err) + } + + // Create a new workspace and load from storage + newWs := workspace.New(workspaceID) + + // Read from storage + loadedData, err := storage.Get(ctx, "workspace.gob") + if err != nil { + t.Fatalf("failed to read workspace: %v", err) + } + + // Decode workspace + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify workspace ID + if newWs.ID != workspaceID { + t.Errorf("workspace ID mismatch: expected %s, got %s", workspaceID, newWs.ID) + } + + // Verify all resources with full field comparison + loadedResources := newWs.Resources().Items() + if len(loadedResources) != len(originalResources) { + t.Errorf("resources count mismatch: expected %d, got %d", len(originalResources), len(loadedResources)) + } + for id, original := range originalResources { + loaded, ok := loadedResources[id] + if !ok { + t.Errorf("resource %s not found after load", id) + continue + } + verifyResourcesEqual(t, original, loaded, "resource "+id) + } + + // Verify all deployments + loadedDeployments := newWs.Deployments().Items() + if len(loadedDeployments) != len(originalDeployments) { + t.Errorf("deployments count mismatch: expected %d, got %d", len(originalDeployments), len(loadedDeployments)) + } + for id, original := range originalDeployments { + loaded, ok := loadedDeployments[id] + if !ok { + t.Errorf("deployment %s not found after load", id) + continue + } + verifyDeploymentsEqual(t, original, loaded, "deployment "+id) + } + + // Verify all systems + loadedSystems := newWs.Systems().Items() + if len(loadedSystems) != len(originalSystems) { + t.Errorf("systems count mismatch: expected %d, got %d", len(originalSystems), len(loadedSystems)) + } + for id, original := range originalSystems { + loaded, ok := loadedSystems[id] + if !ok { + t.Errorf("system %s not found after load", id) + continue + } + verifySystemsEqual(t, original, loaded, "system "+id) + } + + // Verify all job agents + loadedJobAgents := newWs.JobAgents().Items() + if len(loadedJobAgents) != len(originalJobAgents) { + t.Errorf("job agents count mismatch: expected %d, got %d", len(originalJobAgents), len(loadedJobAgents)) + } + for id, original := range originalJobAgents { + loaded, ok := loadedJobAgents[id] + if !ok { + t.Errorf("job agent %s not found after load", id) + continue + } + verifyJobAgentsEqual(t, original, loaded, "job agent "+id) + } +} + +func TestEngine_Persistence_EmptyWorkspace(t *testing.T) { + ctx := context.Background() + + // Create empty workspace using integration helpers + workspaceID := "test-empty-workspace" + engine := integration.NewTestWorkspace(t, + integration.WithWorkspaceID(workspaceID), + ) + + ws := engine.Workspace() + + // Create temporary directory for storage + tempDir, err := os.MkdirTemp("", "workspace-persistence-test-*") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Save empty workspace + storage := workspace.NewFileStorage(tempDir) + + // Encode workspace + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + // Write to storage + if err := storage.Put(ctx, "empty.gob", data); err != nil { + t.Fatalf("failed to write workspace: %v", err) + } + + // Load into new workspace + newWs := workspace.New(workspaceID) + + // Read from storage + loadedData, err := storage.Get(ctx, "empty.gob") + if err != nil { + t.Fatalf("failed to read workspace: %v", err) + } + + // Decode workspace + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify it's still empty + if newWs.ID != workspaceID { + t.Errorf("workspace ID mismatch: expected %s, got %s", workspaceID, newWs.ID) + } + + if len(newWs.Resources().Items()) != 0 { + t.Errorf("expected 0 resources, got %d", len(newWs.Resources().Items())) + } + + if len(newWs.Deployments().Items()) != 0 { + t.Errorf("expected 0 deployments, got %d", len(newWs.Deployments().Items())) + } +} + +func TestEngine_Persistence_MultipleResources(t *testing.T) { + ctx := context.Background() + + resource1Id := uuid.New().String() + resource2Id := uuid.New().String() + resource3Id := uuid.New().String() + systemId := uuid.New().String() + + // Create workspace and populate using integration helpers + engine := integration.NewTestWorkspace(t, + integration.WithSystem( + integration.SystemID(systemId), + integration.SystemName("test-system"), + ), + integration.WithResource( + integration.ResourceID(resource1Id), + integration.ResourceName("resource-1"), + integration.ResourceConfig(map[string]interface{}{"type": "server"}), + ), + integration.WithResource( + integration.ResourceID(resource2Id), + integration.ResourceName("resource-2"), + integration.ResourceConfig(map[string]interface{}{"type": "database"}), + ), + integration.WithResource( + integration.ResourceID(resource3Id), + integration.ResourceName("resource-3"), + integration.ResourceConfig(map[string]interface{}{"type": "cache"}), + ), + ) + + ws := engine.Workspace() + workspaceID := ws.ID + + // Track resources + resourceIds := []string{resource1Id, resource2Id, resource3Id} + + // Verify resources exist + allResources := ws.Resources().Items() + if len(allResources) != 3 { + t.Fatalf("expected 3 resources, got %d", len(allResources)) + } + + // Create temporary directory for storage + tempDir, err := os.MkdirTemp("", "workspace-persistence-test-*") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Save workspace + storage := workspace.NewFileStorage(tempDir) + + // Encode workspace + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + // Write to storage + if err := storage.Put(ctx, "workspace.gob", data); err != nil { + t.Fatalf("failed to write workspace: %v", err) + } + + // Load into new workspace + newWs := workspace.New(workspaceID) + + // Read from storage + loadedData, err := storage.Get(ctx, "workspace.gob") + if err != nil { + t.Fatalf("failed to read workspace: %v", err) + } + + // Decode workspace + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify all resources are preserved with full field comparison + originalResources := ws.Resources().Items() + restoredResources := newWs.Resources().Items() + + if len(restoredResources) != 3 { + t.Errorf("expected 3 resources after restore, got %d", len(restoredResources)) + } + + for _, resourceId := range resourceIds { + originalResource := originalResources[resourceId] + restoredResource, ok := restoredResources[resourceId] + if !ok { + t.Errorf("resource %s not found after restore", resourceId) + continue + } + + verifyResourcesEqual(t, originalResource, restoredResource, "resource "+resourceId) + } +} + +func TestEngine_Persistence_ComplexEntities(t *testing.T) { + ctx := context.Background() + + sysId := uuid.New().String() + jobAgentId := uuid.New().String() + deploymentId := uuid.New().String() + deploymentVersionId := uuid.New().String() + env1Id := uuid.New().String() + env2Id := uuid.New().String() + resource1Id := uuid.New().String() + resource2Id := uuid.New().String() + policyId := uuid.New().String() + + // Create workspace and populate using integration helpers + engine := integration.NewTestWorkspace(t, + integration.WithJobAgent( + integration.JobAgentID(jobAgentId), + integration.JobAgentName("test-agent"), + ), + integration.WithSystem( + integration.SystemID(sysId), + integration.SystemName("complex-system"), + integration.WithDeployment( + integration.DeploymentID(deploymentId), + integration.DeploymentName("api-service"), + integration.DeploymentJobAgent(jobAgentId), + integration.WithDeploymentVersion( + integration.DeploymentVersionID(deploymentVersionId), + integration.DeploymentVersionTag("v1.0.0"), + ), + ), + integration.WithEnvironment( + integration.EnvironmentID(env1Id), + integration.EnvironmentName("production"), + ), + integration.WithEnvironment( + integration.EnvironmentID(env2Id), + integration.EnvironmentName("staging"), + ), + ), + integration.WithResource( + integration.ResourceID(resource1Id), + integration.ResourceName("resource-1"), + ), + integration.WithResource( + integration.ResourceID(resource2Id), + integration.ResourceName("resource-2"), + ), + integration.WithPolicy( + integration.PolicyID(policyId), + integration.PolicyName("approval-policy"), + ), + ) + + ws := engine.Workspace() + workspaceID := ws.ID + + // Create temporary directory for storage + tempDir, err := os.MkdirTemp("", "workspace-persistence-test-*") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Save workspace + storage := workspace.NewFileStorage(tempDir) + + // Encode workspace + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + // Write to storage + if err := storage.Put(ctx, "workspace.gob", data); err != nil { + t.Fatalf("failed to write workspace: %v", err) + } + + // Load into new workspace + newWs := workspace.New(workspaceID) + + // Read from storage + loadedData, err := storage.Get(ctx, "workspace.gob") + if err != nil { + t.Fatalf("failed to read workspace: %v", err) + } + + // Decode workspace + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Capture original entities for deep comparison + originalSys, _ := ws.Systems().Get(sysId) + originalDeployment, _ := ws.Deployments().Get(deploymentId) + originalJobAgent, _ := ws.JobAgents().Get(jobAgentId) + originalEnv1, _ := ws.Environments().Get(env1Id) + originalEnv2, _ := ws.Environments().Get(env2Id) + originalResource1, _ := ws.Resources().Get(resource1Id) + originalResource2, _ := ws.Resources().Get(resource2Id) + originalPolicy, _ := ws.Policies().Get(policyId) + + // Verify system with full field comparison + restoredSys, ok := newWs.Systems().Get(sysId) + if !ok { + t.Fatal("system not found in restored workspace") + } + verifySystemsEqual(t, originalSys, restoredSys, "system "+sysId) + + // Verify deployment with full field comparison + restoredDeployment, ok := newWs.Deployments().Get(deploymentId) + if !ok { + t.Fatal("deployment not found in restored workspace") + } + verifyDeploymentsEqual(t, originalDeployment, restoredDeployment, "deployment "+deploymentId) + + // Verify job agent with full field comparison + restoredJobAgent, ok := newWs.JobAgents().Get(jobAgentId) + if !ok { + t.Fatal("job agent not found in restored workspace") + } + verifyJobAgentsEqual(t, originalJobAgent, restoredJobAgent, "job agent "+jobAgentId) + + // Verify environments + environments := newWs.Environments().Items() + if len(environments) != 2 { + t.Errorf("expected 2 environments, got %d", len(environments)) + } + + restoredEnv1, ok := newWs.Environments().Get(env1Id) + if !ok { + t.Error("environment production not found") + } else { + verifyEnvironmentsEqual(t, originalEnv1, restoredEnv1, "environment "+env1Id) + } + + restoredEnv2, ok := newWs.Environments().Get(env2Id) + if !ok { + t.Error("environment staging not found") + } else { + verifyEnvironmentsEqual(t, originalEnv2, restoredEnv2, "environment "+env2Id) + } + + // Verify resources + resources := newWs.Resources().Items() + if len(resources) != 2 { + t.Errorf("expected 2 resources, got %d", len(resources)) + } + + restoredResource1, ok := newWs.Resources().Get(resource1Id) + if !ok { + t.Error("resource 1 not found") + } else { + verifyResourcesEqual(t, originalResource1, restoredResource1, "resource "+resource1Id) + } + + restoredResource2, ok := newWs.Resources().Get(resource2Id) + if !ok { + t.Error("resource 2 not found") + } else { + verifyResourcesEqual(t, originalResource2, restoredResource2, "resource "+resource2Id) + } + + // Verify policies + policies := newWs.Policies().Items() + if len(policies) != 1 { + t.Errorf("expected 1 policy, got %d", len(policies)) + } + + restoredPolicy, ok := newWs.Policies().Get(policyId) + if !ok { + t.Error("policy not found") + } else { + verifyPoliciesEqual(t, originalPolicy, restoredPolicy, "policy "+policyId) + } +} + +func TestEngine_Persistence_JobsWithStatuses(t *testing.T) { + ctx := context.Background() + + systemId := uuid.New().String() + jobAgentId := uuid.New().String() + deploymentId := uuid.New().String() + deploymentVersionId := uuid.New().String() + envId := uuid.New().String() + resourceId := uuid.New().String() + + // Create workspace with deployment that generates jobs + engine := integration.NewTestWorkspace(t, + integration.WithJobAgent( + integration.JobAgentID(jobAgentId), + integration.JobAgentName("test-agent"), + ), + integration.WithSystem( + integration.SystemID(systemId), + integration.SystemName("test-system"), + integration.WithDeployment( + integration.DeploymentID(deploymentId), + integration.DeploymentName("test-deployment"), + integration.DeploymentJobAgent(jobAgentId), + integration.WithDeploymentVersion( + integration.DeploymentVersionID(deploymentVersionId), + integration.DeploymentVersionTag("v1.0.0"), + ), + ), + integration.WithEnvironment( + integration.EnvironmentID(envId), + integration.EnvironmentName("test-env"), + ), + ), + integration.WithResource( + integration.ResourceID(resourceId), + integration.ResourceName("test-resource"), + ), + ) + + ws := engine.Workspace() + workspaceID := ws.ID + + // Get all jobs created by the deployment version + allJobs := ws.Jobs().Items() + if len(allJobs) == 0 { + t.Fatal("expected at least one job to be created") + } + + // Update job statuses + jobsByStatus := make(map[oapi.JobStatus]string) + testStatuses := []oapi.JobStatus{ + oapi.Pending, + oapi.InProgress, + oapi.Successful, + } + + jobIndex := 0 + for _, status := range testStatuses { + var jobId string + if jobIndex < len(allJobs) { + // Use existing job + for id := range allJobs { + jobId = id + break + } + delete(allJobs, jobId) + } else { + // Create new job + jobId = uuid.New().String() + releaseId := uuid.New().String() + + job := &oapi.Job{ + Id: jobId, + Status: status, + JobAgentId: jobAgentId, + ReleaseId: releaseId, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + JobAgentConfig: make(map[string]interface{}), + Metadata: make(map[string]string), + } + ws.Jobs().Upsert(ctx, job) + } + + // Update job status + job, _ := ws.Jobs().Get(jobId) + job.Status = status + ws.Jobs().Upsert(ctx, job) + jobsByStatus[status] = jobId + jobIndex++ + } + + // Create temporary directory for storage + tempDir, err := os.MkdirTemp("", "workspace-persistence-test-*") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Save workspace + storage := workspace.NewFileStorage(tempDir) + + // Encode workspace + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + // Write to storage + if err := storage.Put(ctx, "workspace.gob", data); err != nil { + t.Fatalf("failed to write workspace: %v", err) + } + + // Load into new workspace + newWs := workspace.New(workspaceID) + + // Read from storage + loadedData, err := storage.Get(ctx, "workspace.gob") + if err != nil { + t.Fatalf("failed to read workspace: %v", err) + } + + // Decode workspace + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify all jobs with full field comparison + for status, jobId := range jobsByStatus { + originalJob, _ := ws.Jobs().Get(jobId) + restoredJob, ok := newWs.Jobs().Get(jobId) + if !ok { + t.Errorf("job %s with status %s not found after restore", jobId, status) + continue + } + + verifyJobsEqual(t, originalJob, restoredJob, "job "+jobId) + } +} + +func TestEngine_Persistence_MultipleWorkspaces(t *testing.T) { + ctx := context.Background() + + // Create temporary directory for storage + tempDir, err := os.MkdirTemp("", "workspace-persistence-test-*") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + storage := workspace.NewFileStorage(tempDir) + + // Create and save multiple different workspaces (using NewNoFlush to avoid DB interaction) + workspaceIDs := []string{uuid.New().String(), uuid.New().String(), uuid.New().String()} + + for i, wsID := range workspaceIDs { + ws := workspace.NewNoFlush(wsID) + + // Encode and save + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace %s: %v", wsID, err) + } + + filename := fmt.Sprintf("workspace-%d.gob", i) + if err := storage.Put(ctx, filename, data); err != nil { + t.Fatalf("failed to save workspace %s: %v", wsID, err) + } + } + + // Load each workspace and verify they're distinct + for i, wsID := range workspaceIDs { + newWs := workspace.NewNoFlush("temp") + + filename := fmt.Sprintf("workspace-%d.gob", i) + loadedData, err := storage.Get(ctx, filename) + if err != nil { + t.Fatalf("failed to load workspace %s: %v", wsID, err) + } + + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace %s: %v", wsID, err) + } + + // Verify workspace ID is correct + if newWs.ID != wsID { + t.Errorf("workspace ID mismatch: expected %s, got %s", wsID, newWs.ID) + } + } +} + +func TestEngine_Persistence_TimestampsAndTimeZones(t *testing.T) { + ctx := context.Background() + + systemId := uuid.New().String() + jobAgentId := uuid.New().String() + deploymentId := uuid.New().String() + deploymentVersionId := uuid.New().String() + envId := uuid.New().String() + resourceId := uuid.New().String() + + // Create workspace with jobs + engine := integration.NewTestWorkspace(t, + integration.WithJobAgent( + integration.JobAgentID(jobAgentId), + integration.JobAgentName("test-agent"), + ), + integration.WithSystem( + integration.SystemID(systemId), + integration.SystemName("test-system"), + integration.WithDeployment( + integration.DeploymentID(deploymentId), + integration.DeploymentName("test-deployment"), + integration.DeploymentJobAgent(jobAgentId), + integration.WithDeploymentVersion( + integration.DeploymentVersionID(deploymentVersionId), + integration.DeploymentVersionTag("v1.0.0"), + ), + ), + integration.WithEnvironment( + integration.EnvironmentID(envId), + integration.EnvironmentName("test-env"), + ), + ), + integration.WithResource( + integration.ResourceID(resourceId), + integration.ResourceName("test-resource"), + ), + ) + + ws := engine.Workspace() + workspaceID := ws.ID + + // Define various timestamps with different timezones and edge cases + utcLoc := time.UTC + estLoc, _ := time.LoadLocation("America/New_York") + pstLoc, _ := time.LoadLocation("America/Los_Angeles") + + testTimestamps := []struct { + name string + createdAt time.Time + updatedAt time.Time + startedAt *time.Time + completed *time.Time + }{ + { + name: "utc-with-nanos", + createdAt: time.Date(2023, 5, 15, 10, 30, 45, 123456789, utcLoc), + updatedAt: time.Date(2023, 5, 15, 11, 30, 45, 987654321, utcLoc), + startedAt: ptrTime(time.Date(2023, 5, 15, 10, 31, 0, 555555555, utcLoc)), + completed: ptrTime(time.Date(2023, 5, 15, 11, 30, 0, 999999999, utcLoc)), + }, + { + name: "est-timezone", + createdAt: time.Date(2023, 6, 1, 9, 0, 0, 0, estLoc), + updatedAt: time.Date(2023, 6, 1, 10, 0, 0, 0, estLoc), + startedAt: ptrTime(time.Date(2023, 6, 1, 9, 5, 0, 0, estLoc)), + completed: nil, + }, + { + name: "pst-timezone", + createdAt: time.Date(2023, 7, 4, 8, 0, 0, 0, pstLoc), + updatedAt: time.Date(2023, 7, 4, 9, 0, 0, 0, pstLoc), + startedAt: nil, + completed: nil, + }, + { + name: "far-future", + createdAt: time.Date(2099, 12, 31, 23, 59, 59, 0, utcLoc), + updatedAt: time.Date(2099, 12, 31, 23, 59, 59, 0, utcLoc), + startedAt: nil, + completed: nil, + }, + { + name: "far-past", + createdAt: time.Date(1970, 1, 1, 0, 0, 1, 0, utcLoc), + updatedAt: time.Date(1970, 1, 1, 0, 0, 1, 0, utcLoc), + startedAt: nil, + completed: nil, + }, + } + + jobTimestamps := make(map[string]struct { + createdAt time.Time + updatedAt time.Time + startedAt *time.Time + completed *time.Time + }) + + // Create jobs with specific timestamps + for i, ts := range testTimestamps { + jobId := uuid.New().String() + releaseId := uuid.New().String() + + job := &oapi.Job{ + Id: jobId, + Status: oapi.Pending, + JobAgentId: jobAgentId, + ReleaseId: releaseId, + CreatedAt: ts.createdAt, + UpdatedAt: ts.updatedAt, + StartedAt: ts.startedAt, + CompletedAt: ts.completed, + JobAgentConfig: make(map[string]interface{}), + Metadata: map[string]string{"test": ts.name, "index": string(rune('0' + i))}, + } + + ws.Jobs().Upsert(ctx, job) + jobTimestamps[jobId] = struct { + createdAt time.Time + updatedAt time.Time + startedAt *time.Time + completed *time.Time + }{ + createdAt: ts.createdAt, + updatedAt: ts.updatedAt, + startedAt: ts.startedAt, + completed: ts.completed, + } + } + + // Create temporary directory for storage + tempDir, err := os.MkdirTemp("", "workspace-persistence-test-*") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Save workspace + storage := workspace.NewFileStorage(tempDir) + + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + if err := storage.Put(ctx, "workspace.gob", data); err != nil { + t.Fatalf("failed to save workspace: %v", err) + } + + // Load into new workspace + newWs := workspace.New(workspaceID) + + loadedData, err := storage.Get(ctx, "workspace.gob") + if err != nil { + t.Fatalf("failed to load workspace: %v", err) + } + + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify all timestamps are preserved exactly + for jobId, expectedTimestamps := range jobTimestamps { + restoredJob, ok := newWs.Jobs().Get(jobId) + if !ok { + t.Errorf("job %s not found after restore", jobId) + continue + } + + // Check CreatedAt + if !restoredJob.CreatedAt.Equal(expectedTimestamps.createdAt) { + t.Errorf("job %s: CreatedAt mismatch, expected %v, got %v", + jobId, expectedTimestamps.createdAt, restoredJob.CreatedAt) + } + + // Verify nanoseconds are preserved + if restoredJob.CreatedAt.Nanosecond() != expectedTimestamps.createdAt.Nanosecond() { + t.Errorf("job %s: CreatedAt nanoseconds not preserved, expected %d, got %d", + jobId, expectedTimestamps.createdAt.Nanosecond(), restoredJob.CreatedAt.Nanosecond()) + } + + // Check UpdatedAt + if !restoredJob.UpdatedAt.Equal(expectedTimestamps.updatedAt) { + t.Errorf("job %s: UpdatedAt mismatch, expected %v, got %v", + jobId, expectedTimestamps.updatedAt, restoredJob.UpdatedAt) + } + + // Check StartedAt + if expectedTimestamps.startedAt == nil { + if restoredJob.StartedAt != nil { + t.Errorf("job %s: StartedAt should be nil, got %v", jobId, *restoredJob.StartedAt) + } + } else { + if restoredJob.StartedAt == nil { + t.Errorf("job %s: StartedAt is nil, expected %v", jobId, *expectedTimestamps.startedAt) + } else if !restoredJob.StartedAt.Equal(*expectedTimestamps.startedAt) { + t.Errorf("job %s: StartedAt mismatch, expected %v, got %v", + jobId, *expectedTimestamps.startedAt, *restoredJob.StartedAt) + } + } + + // Check CompletedAt + if expectedTimestamps.completed == nil { + if restoredJob.CompletedAt != nil { + t.Errorf("job %s: CompletedAt should be nil, got %v", jobId, *restoredJob.CompletedAt) + } + } else { + if restoredJob.CompletedAt == nil { + t.Errorf("job %s: CompletedAt is nil, expected %v", jobId, *expectedTimestamps.completed) + } else if !restoredJob.CompletedAt.Equal(*expectedTimestamps.completed) { + t.Errorf("job %s: CompletedAt mismatch, expected %v, got %v", + jobId, *expectedTimestamps.completed, *restoredJob.CompletedAt) + } + } + } +} + +func TestEngine_Persistence_LoadFromNonExistentFile(t *testing.T) { + ctx := context.Background() + + // Create temporary directory for storage + tempDir, err := os.MkdirTemp("", "workspace-persistence-test-*") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + storage := workspace.NewFileStorage(tempDir) + + // Attempt to load from non-existent file + _, err = storage.Get(ctx, "non-existent-file.gob") + + // Verify error is returned + if err == nil { + t.Fatal("expected error when loading from non-existent file, got nil") + } +} + +func TestEngine_Persistence_ConcurrentWrites(t *testing.T) { + ctx := context.Background() + + // Create workspace with known state (using NewNoFlush to avoid DB interaction) + workspaceID := uuid.New().String() + ws := workspace.NewNoFlush(workspaceID) + + // Create temporary directory for storage + tempDir, err := os.MkdirTemp("", "workspace-persistence-test-*") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + storage := workspace.NewFileStorage(tempDir) + + // Encode once to reuse + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + // Launch multiple goroutines that simultaneously write to the same file + const numGoroutines = 10 + var wg sync.WaitGroup + errChan := make(chan error, numGoroutines) + + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + + // Each goroutine writes the same data + if err := storage.Put(ctx, "concurrent.gob", data); err != nil { + errChan <- err + } + }(i) + } + + // Wait for all goroutines to complete + wg.Wait() + close(errChan) + + // Check if any goroutine had errors + for err := range errChan { + t.Errorf("concurrent write operation failed: %v", err) + } + + // Load workspace and verify it's valid (not corrupted) + newWs := workspace.NewNoFlush("temp") + + loadedData, err := storage.Get(ctx, "concurrent.gob") + if err != nil { + t.Fatalf("failed to load workspace after concurrent writes: %v", err) + } + + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace after concurrent writes: %v", err) + } + + // Verify workspace has valid data + if newWs.ID != workspaceID { + t.Errorf("workspace ID mismatch: expected %s, got %s", workspaceID, newWs.ID) + } +} + +func TestEngine_Persistence_FileStorageOperations(t *testing.T) { + ctx := context.Background() + + // Create temporary directory for storage + tempDir, err := os.MkdirTemp("", "workspace-persistence-test-*") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + storage := workspace.NewFileStorage(tempDir) + + // Test basic Put/Get operations + testData := []byte("test data content") + testPath := "test/path/file.dat" + + // Put data + if err := storage.Put(ctx, testPath, testData); err != nil { + t.Fatalf("failed to put data: %v", err) + } + + // Get data + retrievedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to get data: %v", err) + } + + // Verify data matches + if string(retrievedData) != string(testData) { + t.Errorf("data mismatch: expected %q, got %q", string(testData), string(retrievedData)) + } +} + +// Helper function to create pointer to time.Time +func ptrTime(t time.Time) *time.Time { + return &t +} diff --git a/apps/workspace-engine/test/e2e/engine_workspace_storage_test.go b/apps/workspace-engine/test/e2e/engine_workspace_storage_test.go deleted file mode 100644 index ea92ec395..000000000 --- a/apps/workspace-engine/test/e2e/engine_workspace_storage_test.go +++ /dev/null @@ -1,1264 +0,0 @@ -package e2e - -import ( - "context" - "errors" - "os" - "strings" - "sync" - "testing" - "time" - "workspace-engine/pkg/oapi" - "workspace-engine/pkg/workspace" - "workspace-engine/pkg/workspace/kafka" - "workspace-engine/test/integration" - - "github.com/google/uuid" -) - -func TestEngine_WorkspaceStorage_BasicSaveLoadRoundtrip(t *testing.T) { - ctx := context.Background() - - resource1Id := uuid.New().String() - resource2Id := uuid.New().String() - systemId := uuid.New().String() - jobAgentId := uuid.New().String() - deploymentId := uuid.New().String() - deploymentVersionId := uuid.New().String() - env1Id := uuid.New().String() - env2Id := uuid.New().String() - - // Create workspace and populate using integration helpers - engine := integration.NewTestWorkspace(t, - integration.WithResource( - integration.ResourceID(resource1Id), - integration.ResourceName("resource-1"), - ), - integration.WithResource( - integration.ResourceID(resource2Id), - integration.ResourceName("resource-2"), - ), - integration.WithJobAgent( - integration.JobAgentID(jobAgentId), - integration.JobAgentName("test-job-agent"), - ), - integration.WithSystem( - integration.SystemID(systemId), - integration.SystemName("test-system"), - integration.WithDeployment( - integration.DeploymentID(deploymentId), - integration.DeploymentName("deployment-1"), - integration.DeploymentJobAgent(jobAgentId), - integration.WithDeploymentVersion( - integration.DeploymentVersionID(deploymentVersionId), - integration.DeploymentVersionTag("v1.0.0"), - ), - ), - integration.WithEnvironment( - integration.EnvironmentID(env1Id), - integration.EnvironmentName("env-prod"), - ), - integration.WithEnvironment( - integration.EnvironmentID(env2Id), - integration.EnvironmentName("env-dev"), - ), - ), - ) - - ws := engine.Workspace() - workspaceID := ws.ID - - // Add some Kafka progress to verify it's preserved - ws.KafkaProgress[kafka.TopicPartition{Topic: "events", Partition: 0}] = kafka.KafkaProgress{ - LastApplied: 100, - LastTimestamp: 1234567890, - } - ws.KafkaProgress[kafka.TopicPartition{Topic: "events", Partition: 1}] = kafka.KafkaProgress{ - LastApplied: 200, - LastTimestamp: 1234567900, - } - - // Capture original state counts - originalResources := len(ws.Resources().Items()) - originalDeployments := len(ws.Deployments().Items()) - originalSystems := len(ws.Systems().Items()) - originalJobAgents := len(ws.JobAgents().Items()) - originalEnvironments := len(ws.Environments().Items()) - originalDeploymentVersions := len(ws.DeploymentVersions().Items()) - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - // Save workspace to storage - storage := workspace.NewFileStorage(tempDir) - if err := ws.SaveToStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to save workspace: %v", err) - } - - // Load into new workspace - newWs := workspace.New(workspaceID) - if err := newWs.LoadFromStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to load workspace: %v", err) - } - - // Verify workspace ID - if newWs.ID != workspaceID { - t.Errorf("workspace ID mismatch: expected %s, got %s", workspaceID, newWs.ID) - } - - // Verify KafkaProgress - if len(newWs.KafkaProgress) != 2 { - t.Errorf("expected 2 KafkaProgress entries, got %d", len(newWs.KafkaProgress)) - } - - tp0 := kafka.TopicPartition{Topic: "events", Partition: 0} - if progress, ok := newWs.KafkaProgress[tp0]; !ok { - t.Error("KafkaProgress for partition 0 not found") - } else { - if progress.LastApplied != 100 { - t.Errorf("partition 0 LastApplied: expected 100, got %d", progress.LastApplied) - } - if progress.LastTimestamp != 1234567890 { - t.Errorf("partition 0 LastTimestamp: expected 1234567890, got %d", progress.LastTimestamp) - } - } - - // Verify entity counts - if len(newWs.Resources().Items()) != originalResources { - t.Errorf("resources count mismatch: expected %d, got %d", originalResources, len(newWs.Resources().Items())) - } - - if len(newWs.Deployments().Items()) != originalDeployments { - t.Errorf("deployments count mismatch: expected %d, got %d", originalDeployments, len(newWs.Deployments().Items())) - } - - if len(newWs.Systems().Items()) != originalSystems { - t.Errorf("systems count mismatch: expected %d, got %d", originalSystems, len(newWs.Systems().Items())) - } - - if len(newWs.JobAgents().Items()) != originalJobAgents { - t.Errorf("job agents count mismatch: expected %d, got %d", originalJobAgents, len(newWs.JobAgents().Items())) - } - - if len(newWs.Environments().Items()) != originalEnvironments { - t.Errorf("environments count mismatch: expected %d, got %d", originalEnvironments, len(newWs.Environments().Items())) - } - - if len(newWs.DeploymentVersions().Items()) != originalDeploymentVersions { - t.Errorf("deployment versions count mismatch: expected %d, got %d", originalDeploymentVersions, len(newWs.DeploymentVersions().Items())) - } -} - -func TestEngine_WorkspaceStorage_EmptyWorkspace(t *testing.T) { - ctx := context.Background() - - // Create empty workspace using integration helpers - workspaceID := "test-empty-workspace" - engine := integration.NewTestWorkspace(t, - integration.WithWorkspaceID(workspaceID), - ) - - ws := engine.Workspace() - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - // Save empty workspace - storage := workspace.NewFileStorage(tempDir) - if err := ws.SaveToStorage(ctx, storage, "empty.gob"); err != nil { - t.Fatalf("failed to save empty workspace: %v", err) - } - - // Load into new workspace - newWs := workspace.New(workspaceID) - if err := newWs.LoadFromStorage(ctx, storage, "empty.gob"); err != nil { - t.Fatalf("failed to load empty workspace: %v", err) - } - - // Verify it's still empty - if newWs.ID != workspaceID { - t.Errorf("workspace ID mismatch: expected %s, got %s", workspaceID, newWs.ID) - } - - if len(newWs.Resources().Items()) != 0 { - t.Errorf("expected 0 resources, got %d", len(newWs.Resources().Items())) - } - - if len(newWs.Deployments().Items()) != 0 { - t.Errorf("expected 0 deployments, got %d", len(newWs.Deployments().Items())) - } -} - -func TestEngine_WorkspaceStorage_MultipleResources(t *testing.T) { - ctx := context.Background() - - resource1Id := uuid.New().String() - resource2Id := uuid.New().String() - resource3Id := uuid.New().String() - systemId := uuid.New().String() - - // Create workspace and populate using integration helpers - engine := integration.NewTestWorkspace(t, - integration.WithSystem( - integration.SystemID(systemId), - integration.SystemName("test-system"), - ), - integration.WithResource( - integration.ResourceID(resource1Id), - integration.ResourceName("resource-1"), - integration.ResourceConfig(map[string]interface{}{"type": "server"}), - ), - integration.WithResource( - integration.ResourceID(resource2Id), - integration.ResourceName("resource-2"), - integration.ResourceConfig(map[string]interface{}{"type": "database"}), - ), - integration.WithResource( - integration.ResourceID(resource3Id), - integration.ResourceName("resource-3"), - integration.ResourceConfig(map[string]interface{}{"type": "cache"}), - ), - ) - - ws := engine.Workspace() - workspaceID := ws.ID - - // Track resources - resourceIds := []string{resource1Id, resource2Id, resource3Id} - - // Verify resources exist - allResources := ws.Resources().Items() - if len(allResources) != 3 { - t.Fatalf("expected 3 resources, got %d", len(allResources)) - } - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - // Save workspace - storage := workspace.NewFileStorage(tempDir) - if err := ws.SaveToStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to save workspace: %v", err) - } - - // Load into new workspace - newWs := workspace.New(workspaceID) - if err := newWs.LoadFromStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to load workspace: %v", err) - } - - // Verify all resources are preserved with their config - for _, resourceId := range resourceIds { - restoredResource, ok := newWs.Resources().Get(resourceId) - if !ok { - t.Errorf("resource %s not found after restore", resourceId) - continue - } - - // Verify config is preserved - if restoredResource.Config == nil { - t.Errorf("resource %s: config is nil after restore", resourceId) - } - } - - // Verify resource count - restoredResources := newWs.Resources().Items() - if len(restoredResources) != 3 { - t.Errorf("expected 3 resources after restore, got %d", len(restoredResources)) - } -} - -func TestEngine_WorkspaceStorage_ComplexEntities(t *testing.T) { - ctx := context.Background() - - sysId := uuid.New().String() - jobAgentId := uuid.New().String() - deploymentId := uuid.New().String() - deploymentVersionId := uuid.New().String() - env1Id := uuid.New().String() - env2Id := uuid.New().String() - resource1Id := uuid.New().String() - resource2Id := uuid.New().String() - policyId := uuid.New().String() - - // Create workspace and populate using integration helpers - engine := integration.NewTestWorkspace(t, - integration.WithJobAgent( - integration.JobAgentID(jobAgentId), - integration.JobAgentName("test-agent"), - ), - integration.WithSystem( - integration.SystemID(sysId), - integration.SystemName("complex-system"), - integration.WithDeployment( - integration.DeploymentID(deploymentId), - integration.DeploymentName("api-service"), - integration.DeploymentJobAgent(jobAgentId), - integration.WithDeploymentVersion( - integration.DeploymentVersionID(deploymentVersionId), - integration.DeploymentVersionTag("v1.0.0"), - ), - ), - integration.WithEnvironment( - integration.EnvironmentID(env1Id), - integration.EnvironmentName("production"), - ), - integration.WithEnvironment( - integration.EnvironmentID(env2Id), - integration.EnvironmentName("staging"), - ), - ), - integration.WithResource( - integration.ResourceID(resource1Id), - integration.ResourceName("resource-1"), - ), - integration.WithResource( - integration.ResourceID(resource2Id), - integration.ResourceName("resource-2"), - ), - integration.WithPolicy( - integration.PolicyID(policyId), - integration.PolicyName("approval-policy"), - ), - ) - - ws := engine.Workspace() - workspaceID := ws.ID - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - // Save workspace - storage := workspace.NewFileStorage(tempDir) - if err := ws.SaveToStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to save workspace: %v", err) - } - - // Load into new workspace - newWs := workspace.New(workspaceID) - if err := newWs.LoadFromStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to load workspace: %v", err) - } - - // Verify system - restoredSys, ok := newWs.Systems().Get(sysId) - if !ok { - t.Fatal("system not found in restored workspace") - } - if restoredSys.Name != "complex-system" { - t.Errorf("system name mismatch: expected 'complex-system', got %s", restoredSys.Name) - } - - // Verify deployment - restoredDeployment, ok := newWs.Deployments().Get(deploymentId) - if !ok { - t.Fatal("deployment not found in restored workspace") - } - if restoredDeployment.Name != "api-service" { - t.Errorf("deployment name mismatch: expected 'api-service', got %s", restoredDeployment.Name) - } - - // Verify job agent - restoredJobAgent, ok := newWs.JobAgents().Get(jobAgentId) - if !ok { - t.Fatal("job agent not found in restored workspace") - } - if restoredJobAgent.Name != "test-agent" { - t.Errorf("job agent name mismatch: expected 'test-agent', got %s", restoredJobAgent.Name) - } - - // Verify environments - environments := newWs.Environments().Items() - if len(environments) != 2 { - t.Errorf("expected 2 environments, got %d", len(environments)) - } - - // Verify resources - resources := newWs.Resources().Items() - if len(resources) != 2 { - t.Errorf("expected 2 resources, got %d", len(resources)) - } - - // Verify policies - policies := newWs.Policies().Items() - if len(policies) != 1 { - t.Errorf("expected 1 policy, got %d", len(policies)) - } -} - -func TestEngine_WorkspaceStorage_MultipleWorkspaces(t *testing.T) { - ctx := context.Background() - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - storage := workspace.NewFileStorage(tempDir) - - // Create and save multiple workspaces - workspaceIDs := []string{"workspace-1", "workspace-2", "workspace-3"} - - for _, wsID := range workspaceIDs { - engine := integration.NewTestWorkspace(t, - integration.WithWorkspaceID(wsID), - ) - - ws := engine.Workspace() - - // Add some unique KafkaProgress for each workspace - ws.KafkaProgress[kafka.TopicPartition{Topic: "events", Partition: 0}] = kafka.KafkaProgress{ - LastApplied: int64(len(wsID)), // Use length as unique value - LastTimestamp: 1234567890, - } - - if err := ws.SaveToStorage(ctx, storage, wsID+".gob"); err != nil { - t.Fatalf("failed to save workspace %s: %v", wsID, err) - } - } - - // Load each workspace and verify - for _, wsID := range workspaceIDs { - newWs := workspace.New(wsID) - if err := newWs.LoadFromStorage(ctx, storage, wsID+".gob"); err != nil { - t.Fatalf("failed to load workspace %s: %v", wsID, err) - } - - if newWs.ID != wsID { - t.Errorf("workspace ID mismatch: expected %s, got %s", wsID, newWs.ID) - } - - tp := kafka.TopicPartition{Topic: "events", Partition: 0} - if progress, ok := newWs.KafkaProgress[tp]; !ok { - t.Errorf("KafkaProgress not found for workspace %s", wsID) - } else { - expectedValue := int64(len(wsID)) - if progress.LastApplied != expectedValue { - t.Errorf("workspace %s: expected LastApplied %d, got %d", wsID, expectedValue, progress.LastApplied) - } - } - } -} - -func TestEngine_WorkspaceStorage_EnvironmentReleaseTargetsAndJobs(t *testing.T) { - ctx := context.Background() - - systemId := uuid.New().String() - jobAgentId := uuid.New().String() - deploymentId := uuid.New().String() - deploymentVersionId := uuid.New().String() - env1Id := uuid.New().String() - env2Id := uuid.New().String() - resource1Id := uuid.New().String() - resource2Id := uuid.New().String() - - // Create workspace with complex setup including jobs - engine := integration.NewTestWorkspace(t, - integration.WithJobAgent( - integration.JobAgentID(jobAgentId), - integration.JobAgentName("test-agent"), - ), - integration.WithSystem( - integration.SystemID(systemId), - integration.SystemName("test-system"), - integration.WithDeployment( - integration.DeploymentID(deploymentId), - integration.DeploymentName("api-service"), - integration.DeploymentJobAgent(jobAgentId), - integration.WithDeploymentVersion( - integration.DeploymentVersionID(deploymentVersionId), - integration.DeploymentVersionTag("v1.0.0"), - ), - ), - integration.WithEnvironment( - integration.EnvironmentID(env1Id), - integration.EnvironmentName("production"), - ), - integration.WithEnvironment( - integration.EnvironmentID(env2Id), - integration.EnvironmentName("staging"), - ), - ), - integration.WithResource( - integration.ResourceID(resource1Id), - integration.ResourceName("server-1"), - ), - integration.WithResource( - integration.ResourceID(resource2Id), - integration.ResourceName("server-2"), - ), - ) - - ws := engine.Workspace() - workspaceID := ws.ID - - // Wait for release targets to be computed - releaseTargets, err := ws.ReleaseTargets().Items(ctx) - if err != nil { - t.Fatalf("failed to get release targets: %v", err) - } - - // Should have 4 release targets: 1 deployment * 2 environments * 2 resources - expectedReleaseTargets := 4 - if len(releaseTargets) != expectedReleaseTargets { - t.Fatalf("expected %d release targets, got %d", expectedReleaseTargets, len(releaseTargets)) - } - - // Get jobs - should have been created by deployment version - allJobs := ws.Jobs().Items() - if len(allJobs) != expectedReleaseTargets { - t.Fatalf("expected %d jobs, got %d", expectedReleaseTargets, len(allJobs)) - } - - // Track job IDs and their statuses - jobIdsAndStatuses := make(map[string]string) - for jobId, job := range allJobs { - jobIdsAndStatuses[jobId] = string(job.Status) - } - - // Track release target keys - releaseTargetKeys := make(map[string]bool) - for key := range releaseTargets { - releaseTargetKeys[key] = true - } - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - // Save workspace to storage - storage := workspace.NewFileStorage(tempDir) - if err := ws.SaveToStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to save workspace: %v", err) - } - - // Load into new workspace - newWs := workspace.New(workspaceID) - if err := newWs.LoadFromStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to load workspace: %v", err) - } - - // Verify environments - restoredEnv1, ok := newWs.Environments().Get(env1Id) - if !ok { - t.Fatal("environment 'production' not found after restore") - } - if restoredEnv1.Name != "production" { - t.Errorf("environment name mismatch: expected 'production', got %s", restoredEnv1.Name) - } - - restoredEnv2, ok := newWs.Environments().Get(env2Id) - if !ok { - t.Fatal("environment 'staging' not found after restore") - } - if restoredEnv2.Name != "staging" { - t.Errorf("environment name mismatch: expected 'staging', got %s", restoredEnv2.Name) - } - - // Verify release targets - restoredReleaseTargets, err := newWs.ReleaseTargets().Items(ctx) - if err != nil { - t.Fatalf("failed to get restored release targets: %v", err) - } - - if len(restoredReleaseTargets) != expectedReleaseTargets { - t.Errorf("release targets count mismatch: expected %d, got %d", expectedReleaseTargets, len(restoredReleaseTargets)) - } - - // Verify each original release target key exists in restored targets - for key := range releaseTargetKeys { - if _, ok := restoredReleaseTargets[key]; !ok { - t.Errorf("release target with key %s not found after restore", key) - } - } - - // Verify release target structure - for key, rt := range restoredReleaseTargets { - if rt.DeploymentId != deploymentId { - t.Errorf("release target %s: deployment ID mismatch, got %s", key, rt.DeploymentId) - } - if rt.EnvironmentId != env1Id && rt.EnvironmentId != env2Id { - t.Errorf("release target %s: unexpected environment ID %s", key, rt.EnvironmentId) - } - if rt.ResourceId != resource1Id && rt.ResourceId != resource2Id { - t.Errorf("release target %s: unexpected resource ID %s", key, rt.ResourceId) - } - } - - // Verify jobs - restoredJobs := newWs.Jobs().Items() - if len(restoredJobs) != len(allJobs) { - t.Errorf("jobs count mismatch: expected %d, got %d", len(allJobs), len(restoredJobs)) - } - - // Verify each original job exists with correct status - for jobId, expectedStatus := range jobIdsAndStatuses { - restoredJob, ok := restoredJobs[jobId] - if !ok { - t.Errorf("job %s not found after restore", jobId) - continue - } - - if string(restoredJob.Status) != expectedStatus { - t.Errorf("job %s: status mismatch, expected %s, got %s", jobId, expectedStatus, restoredJob.Status) - } - - // Verify job has correct job agent - if restoredJob.JobAgentId != jobAgentId { - t.Errorf("job %s: job agent mismatch, expected %s, got %s", jobId, jobAgentId, restoredJob.JobAgentId) - } - - // Verify job has a release ID - if restoredJob.ReleaseId == "" { - t.Errorf("job %s: release ID is empty", jobId) - } - } - - // Verify pending jobs specifically - pendingJobs := newWs.Jobs().GetPending() - if len(pendingJobs) != expectedReleaseTargets { - t.Errorf("expected %d pending jobs after restore, got %d", expectedReleaseTargets, len(pendingJobs)) - } - - // Verify job agent exists - restoredJobAgent, ok := newWs.JobAgents().Get(jobAgentId) - if !ok { - t.Fatal("job agent not found after restore") - } - if restoredJobAgent.Name != "test-agent" { - t.Errorf("job agent name mismatch: expected 'test-agent', got %s", restoredJobAgent.Name) - } -} - -func TestEngine_WorkspaceStorage_JobsWithAllStatuses(t *testing.T) { - ctx := context.Background() - - systemId := uuid.New().String() - jobAgentId := uuid.New().String() - deploymentId := uuid.New().String() - deploymentVersionId := uuid.New().String() - envId := uuid.New().String() - resourceId := uuid.New().String() - - // Create workspace with deployment that generates jobs - engine := integration.NewTestWorkspace(t, - integration.WithJobAgent( - integration.JobAgentID(jobAgentId), - integration.JobAgentName("test-agent"), - ), - integration.WithSystem( - integration.SystemID(systemId), - integration.SystemName("test-system"), - integration.WithDeployment( - integration.DeploymentID(deploymentId), - integration.DeploymentName("test-deployment"), - integration.DeploymentJobAgent(jobAgentId), - integration.WithDeploymentVersion( - integration.DeploymentVersionID(deploymentVersionId), - integration.DeploymentVersionTag("v1.0.0"), - ), - ), - integration.WithEnvironment( - integration.EnvironmentID(envId), - integration.EnvironmentName("test-env"), - ), - ), - integration.WithResource( - integration.ResourceID(resourceId), - integration.ResourceName("test-resource"), - ), - ) - - ws := engine.Workspace() - workspaceID := ws.ID - - // Get all jobs created by the deployment version - allJobs := ws.Jobs().Items() - if len(allJobs) == 0 { - t.Fatal("expected at least one job to be created") - } - - // All job statuses to test - allStatuses := []oapi.JobStatus{ - oapi.Pending, - oapi.InProgress, - oapi.Successful, - oapi.Cancelled, - oapi.Skipped, - oapi.Failure, - oapi.ActionRequired, - oapi.InvalidJobAgent, - oapi.InvalidIntegration, - oapi.ExternalRunNotFound, - } - - // Manually set different statuses on jobs - // We'll modify the first job to have each status, creating new jobs as needed - jobsByStatus := make(map[oapi.JobStatus]string) - jobIndex := 0 - - for _, status := range allStatuses { - var jobId string - - if jobIndex < len(allJobs) { - // Use existing job - for id := range allJobs { - jobId = id - break - } - delete(allJobs, jobId) - } else { - // Create new job - jobId = uuid.New().String() - releaseId := uuid.New().String() - - job := &oapi.Job{ - Id: jobId, - Status: status, - JobAgentId: jobAgentId, - ReleaseId: releaseId, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - JobAgentConfig: make(map[string]interface{}), - Metadata: make(map[string]string), - } - ws.Jobs().Upsert(ctx, job) - } - - // Update job status - job, _ := ws.Jobs().Get(jobId) - job.Status = status - ws.Jobs().Upsert(ctx, job) - jobsByStatus[status] = jobId - jobIndex++ - } - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - // Save workspace - storage := workspace.NewFileStorage(tempDir) - if err := ws.SaveToStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to save workspace: %v", err) - } - - // Load into new workspace - newWs := workspace.New(workspaceID) - if err := newWs.LoadFromStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to load workspace: %v", err) - } - - // Verify all job statuses are preserved - for status, jobId := range jobsByStatus { - restoredJob, ok := newWs.Jobs().Get(jobId) - if !ok { - t.Errorf("job %s with status %s not found after restore", jobId, status) - continue - } - - if restoredJob.Status != status { - t.Errorf("job %s: expected status %s, got %s", jobId, status, restoredJob.Status) - } - } -} - -func TestEngine_WorkspaceStorage_TimestampsAndTimeZones(t *testing.T) { - ctx := context.Background() - - systemId := uuid.New().String() - jobAgentId := uuid.New().String() - deploymentId := uuid.New().String() - deploymentVersionId := uuid.New().String() - envId := uuid.New().String() - resourceId := uuid.New().String() - - // Create workspace with jobs - engine := integration.NewTestWorkspace(t, - integration.WithJobAgent( - integration.JobAgentID(jobAgentId), - integration.JobAgentName("test-agent"), - ), - integration.WithSystem( - integration.SystemID(systemId), - integration.SystemName("test-system"), - integration.WithDeployment( - integration.DeploymentID(deploymentId), - integration.DeploymentName("test-deployment"), - integration.DeploymentJobAgent(jobAgentId), - integration.WithDeploymentVersion( - integration.DeploymentVersionID(deploymentVersionId), - integration.DeploymentVersionTag("v1.0.0"), - ), - ), - integration.WithEnvironment( - integration.EnvironmentID(envId), - integration.EnvironmentName("test-env"), - ), - ), - integration.WithResource( - integration.ResourceID(resourceId), - integration.ResourceName("test-resource"), - ), - ) - - ws := engine.Workspace() - workspaceID := ws.ID - - // Define various timestamps with different timezones and edge cases - utcLoc := time.UTC - estLoc, _ := time.LoadLocation("America/New_York") - pstLoc, _ := time.LoadLocation("America/Los_Angeles") - - testTimestamps := []struct { - name string - createdAt time.Time - updatedAt time.Time - startedAt *time.Time - completed *time.Time - }{ - { - name: "utc-with-nanos", - createdAt: time.Date(2023, 5, 15, 10, 30, 45, 123456789, utcLoc), - updatedAt: time.Date(2023, 5, 15, 11, 30, 45, 987654321, utcLoc), - startedAt: ptrTime(time.Date(2023, 5, 15, 10, 31, 0, 555555555, utcLoc)), - completed: ptrTime(time.Date(2023, 5, 15, 11, 30, 0, 999999999, utcLoc)), - }, - { - name: "est-timezone", - createdAt: time.Date(2023, 6, 1, 9, 0, 0, 0, estLoc), - updatedAt: time.Date(2023, 6, 1, 10, 0, 0, 0, estLoc), - startedAt: ptrTime(time.Date(2023, 6, 1, 9, 5, 0, 0, estLoc)), - completed: nil, - }, - { - name: "pst-timezone", - createdAt: time.Date(2023, 7, 4, 8, 0, 0, 0, pstLoc), - updatedAt: time.Date(2023, 7, 4, 9, 0, 0, 0, pstLoc), - startedAt: nil, - completed: nil, - }, - { - name: "far-future", - createdAt: time.Date(2099, 12, 31, 23, 59, 59, 0, utcLoc), - updatedAt: time.Date(2099, 12, 31, 23, 59, 59, 0, utcLoc), - startedAt: nil, - completed: nil, - }, - { - name: "far-past", - createdAt: time.Date(1970, 1, 1, 0, 0, 1, 0, utcLoc), - updatedAt: time.Date(1970, 1, 1, 0, 0, 1, 0, utcLoc), - startedAt: nil, - completed: nil, - }, - } - - jobTimestamps := make(map[string]struct { - createdAt time.Time - updatedAt time.Time - startedAt *time.Time - completed *time.Time - }) - - // Create jobs with specific timestamps - for i, ts := range testTimestamps { - jobId := uuid.New().String() - releaseId := uuid.New().String() - - job := &oapi.Job{ - Id: jobId, - Status: oapi.Pending, - JobAgentId: jobAgentId, - ReleaseId: releaseId, - CreatedAt: ts.createdAt, - UpdatedAt: ts.updatedAt, - StartedAt: ts.startedAt, - CompletedAt: ts.completed, - JobAgentConfig: make(map[string]interface{}), - Metadata: map[string]string{"test": ts.name, "index": string(rune('0' + i))}, - } - - ws.Jobs().Upsert(ctx, job) - jobTimestamps[jobId] = struct { - createdAt time.Time - updatedAt time.Time - startedAt *time.Time - completed *time.Time - }{ - createdAt: ts.createdAt, - updatedAt: ts.updatedAt, - startedAt: ts.startedAt, - completed: ts.completed, - } - } - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - // Save workspace - storage := workspace.NewFileStorage(tempDir) - if err := ws.SaveToStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to save workspace: %v", err) - } - - // Load into new workspace - newWs := workspace.New(workspaceID) - if err := newWs.LoadFromStorage(ctx, storage, "workspace.gob"); err != nil { - t.Fatalf("failed to load workspace: %v", err) - } - - // Verify all timestamps are preserved exactly - for jobId, expectedTimestamps := range jobTimestamps { - restoredJob, ok := newWs.Jobs().Get(jobId) - if !ok { - t.Errorf("job %s not found after restore", jobId) - continue - } - - // Check CreatedAt - if !restoredJob.CreatedAt.Equal(expectedTimestamps.createdAt) { - t.Errorf("job %s: CreatedAt mismatch, expected %v, got %v", - jobId, expectedTimestamps.createdAt, restoredJob.CreatedAt) - } - - // Verify nanoseconds are preserved - if restoredJob.CreatedAt.Nanosecond() != expectedTimestamps.createdAt.Nanosecond() { - t.Errorf("job %s: CreatedAt nanoseconds not preserved, expected %d, got %d", - jobId, expectedTimestamps.createdAt.Nanosecond(), restoredJob.CreatedAt.Nanosecond()) - } - - // Check UpdatedAt - if !restoredJob.UpdatedAt.Equal(expectedTimestamps.updatedAt) { - t.Errorf("job %s: UpdatedAt mismatch, expected %v, got %v", - jobId, expectedTimestamps.updatedAt, restoredJob.UpdatedAt) - } - - // Check StartedAt - if expectedTimestamps.startedAt == nil { - if restoredJob.StartedAt != nil { - t.Errorf("job %s: StartedAt should be nil, got %v", jobId, *restoredJob.StartedAt) - } - } else { - if restoredJob.StartedAt == nil { - t.Errorf("job %s: StartedAt is nil, expected %v", jobId, *expectedTimestamps.startedAt) - } else if !restoredJob.StartedAt.Equal(*expectedTimestamps.startedAt) { - t.Errorf("job %s: StartedAt mismatch, expected %v, got %v", - jobId, *expectedTimestamps.startedAt, *restoredJob.StartedAt) - } - } - - // Check CompletedAt - if expectedTimestamps.completed == nil { - if restoredJob.CompletedAt != nil { - t.Errorf("job %s: CompletedAt should be nil, got %v", jobId, *restoredJob.CompletedAt) - } - } else { - if restoredJob.CompletedAt == nil { - t.Errorf("job %s: CompletedAt is nil, expected %v", jobId, *expectedTimestamps.completed) - } else if !restoredJob.CompletedAt.Equal(*expectedTimestamps.completed) { - t.Errorf("job %s: CompletedAt mismatch, expected %v, got %v", - jobId, *expectedTimestamps.completed, *restoredJob.CompletedAt) - } - } - } -} - -func TestEngine_WorkspaceStorage_LoadFromNonExistentFile(t *testing.T) { - ctx := context.Background() - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - storage := workspace.NewFileStorage(tempDir) - - // Create a workspace directly - workspaceID := uuid.New().String() - ws := workspace.New(workspaceID) - - // Add some KafkaProgress to verify workspace has data - ws.KafkaProgress[kafka.TopicPartition{Topic: "events", Partition: 0}] = kafka.KafkaProgress{ - LastApplied: 100, - LastTimestamp: 1234567890, - } - - // Attempt to load from non-existent file - err = ws.LoadFromStorage(ctx, storage, "non-existent-file.gob") - - // Verify error is returned - if err == nil { - t.Fatal("expected error when loading from non-existent file, got nil") - } - - // Verify error message indicates file not found - errMsg := err.Error() - if !strings.Contains(errMsg, "no such file") && !strings.Contains(errMsg, "failed to read") { - t.Errorf("expected error message to indicate file not found, got: %s", errMsg) - } - - // Verify workspace state remains intact (KafkaProgress not overwritten) - if len(ws.KafkaProgress) != 1 { - t.Errorf("workspace state may be corrupted: expected 1 KafkaProgress entry, got %d", len(ws.KafkaProgress)) - } -} - -func TestEngine_WorkspaceStorage_SaveWithInvalidWorkspaceID(t *testing.T) { - ctx := context.Background() - - testCases := []struct { - name string - workspaceID string - shouldSave bool // whether we expect save to succeed or fail - }{ - { - name: "empty-string", - workspaceID: "", - shouldSave: true, // gob encoding should handle this - }, - { - name: "path-separator", - workspaceID: "workspace/../attack", - shouldSave: true, // filepath.Join handles this - }, - { - name: "very-long-id", - workspaceID: strings.Repeat("a", 1000), - shouldSave: true, - }, - { - name: "special-characters", - workspaceID: "workspace\n\t\r", - shouldSave: true, // These are just string characters - }, - { - name: "unicode-emoji", - workspaceID: "workspace-🚀-test", - shouldSave: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - storage := workspace.NewFileStorage(tempDir) - - // Create workspace directly without integration helpers to test edge case IDs - // Use NewNoFlush to avoid database interactions - ws := workspace.NewNoFlush(tc.workspaceID) - - // Attempt to save - err = ws.SaveToStorage(ctx, storage, "workspace.gob") - - if tc.shouldSave { - if err != nil { - t.Errorf("expected save to succeed, got error: %v", err) - return - } - - // Attempt to load back - newWs := workspace.NewNoFlush("temp-id") - err = newWs.LoadFromStorage(ctx, storage, "workspace.gob") - if err != nil { - t.Errorf("failed to load workspace after save: %v", err) - return - } - - // Verify workspace ID matches - if newWs.ID != tc.workspaceID { - t.Errorf("workspace ID mismatch: expected %q, got %q", tc.workspaceID, newWs.ID) - } - } else { - if err == nil { - t.Error("expected save to fail, but it succeeded") - } - } - }) - } -} - -func TestEngine_WorkspaceStorage_ConcurrentSaveOperations(t *testing.T) { - ctx := context.Background() - - // Create workspace with known state - workspaceID := uuid.New().String() - ws := workspace.New(workspaceID) - - // Add some KafkaProgress to verify workspace has data - ws.KafkaProgress[kafka.TopicPartition{Topic: "events", Partition: 0}] = kafka.KafkaProgress{ - LastApplied: 100, - LastTimestamp: 1234567890, - } - ws.KafkaProgress[kafka.TopicPartition{Topic: "events", Partition: 1}] = kafka.KafkaProgress{ - LastApplied: 200, - LastTimestamp: 1234567900, - } - - // Create temporary directory for storage - tempDir, err := os.MkdirTemp("", "workspace-storage-test-*") - if err != nil { - t.Fatalf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - storage := workspace.NewFileStorage(tempDir) - - // Launch multiple goroutines that simultaneously save workspace to same file - const numGoroutines = 10 - var wg sync.WaitGroup - errChan := make(chan error, numGoroutines) - - for i := 0; i < numGoroutines; i++ { - wg.Add(1) - go func(index int) { - defer wg.Done() - - // Each goroutine saves the workspace - if err := ws.SaveToStorage(ctx, storage, "concurrent.gob"); err != nil { - errChan <- err - } - }(i) - } - - // Wait for all goroutines to complete - wg.Wait() - close(errChan) - - // Check if any goroutine had errors - for err := range errChan { - t.Errorf("concurrent save operation failed: %v", err) - } - - // Load workspace and verify it's valid (not corrupted) - newWs := workspace.New(workspaceID) - if err := newWs.LoadFromStorage(ctx, storage, "concurrent.gob"); err != nil { - t.Fatalf("failed to load workspace after concurrent saves: %v", err) - } - - // Verify workspace has valid data - if newWs.ID != workspaceID { - t.Errorf("workspace ID mismatch: expected %s, got %s", workspaceID, newWs.ID) - } - - // Verify KafkaProgress data - if len(newWs.KafkaProgress) != 2 { - t.Errorf("expected 2 KafkaProgress entries, got %d", len(newWs.KafkaProgress)) - } - - // Verify specific KafkaProgress values - tp0 := kafka.TopicPartition{Topic: "events", Partition: 0} - if progress, ok := newWs.KafkaProgress[tp0]; !ok { - t.Error("KafkaProgress for partition 0 not found") - } else if progress.LastApplied != 100 { - t.Errorf("partition 0 LastApplied: expected 100, got %d", progress.LastApplied) - } -} - -func TestEngine_WorkspaceStorage_DiskFullScenario(t *testing.T) { - ctx := context.Background() - - // Create workspace - workspaceID := uuid.New().String() - ws := workspace.New(workspaceID) - - // Add some KafkaProgress data - ws.KafkaProgress[kafka.TopicPartition{Topic: "events", Partition: 0}] = kafka.KafkaProgress{ - LastApplied: 100, - LastTimestamp: 1234567890, - } - - // Create mock storage that simulates disk full - failingStorage := &FailingStorageClient{shouldFailPut: true} - - // Attempt to save workspace - err := ws.SaveToStorage(ctx, failingStorage, "workspace.gob") - - // Verify error is returned - if err == nil { - t.Fatal("expected error when disk is full, got nil") - } - - // Verify error message indicates storage issue - errMsg := err.Error() - if !strings.Contains(errMsg, "no space left") && !strings.Contains(errMsg, "failed to write") { - t.Errorf("expected error message to indicate storage issue, got: %s", errMsg) - } - - // Test load failure scenario - failingStorage.shouldFailGet = true - newWs := workspace.New(workspaceID) - err = newWs.LoadFromStorage(ctx, failingStorage, "workspace.gob") - - // Verify error is returned - if err == nil { - t.Fatal("expected error when reading fails, got nil") - } - - // Verify error message indicates read failure - errMsg = err.Error() - if !strings.Contains(errMsg, "disk full") && !strings.Contains(errMsg, "failed to read") { - t.Errorf("expected error message to indicate read failure, got: %s", errMsg) - } -} - -// Helper function to create pointer to time.Time -func ptrTime(t time.Time) *time.Time { - return &t -} - -// Mock storage client that simulates failures -type FailingStorageClient struct { - shouldFailGet bool - shouldFailPut bool -} - -func (f *FailingStorageClient) Get(ctx context.Context, path string) ([]byte, error) { - if f.shouldFailGet { - return nil, errors.New("disk full: cannot read file") - } - return nil, errors.New("file not found") -} - -func (f *FailingStorageClient) Put(ctx context.Context, path string, data []byte) error { - if f.shouldFailPut { - return errors.New("no space left on device") - } - return nil -} From 8f6b2ad5ed2086b7453a09d984bc87bf2cad1368 Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 18:50:56 -0700 Subject: [PATCH 12/15] fix --- apps/workspace-engine/pkg/db/workspaces.go | 3 ++- apps/workspace-engine/pkg/kafka/kafka.go | 8 ++++---- apps/workspace-engine/pkg/workspace/loader.go | 5 ++++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/workspace-engine/pkg/db/workspaces.go b/apps/workspace-engine/pkg/db/workspaces.go index 319a44fe2..002308395 100644 --- a/apps/workspace-engine/pkg/db/workspaces.go +++ b/apps/workspace-engine/pkg/db/workspaces.go @@ -2,6 +2,7 @@ package db import ( "context" + "time" "github.com/jackc/pgx/v5" ) @@ -89,7 +90,7 @@ func GetAllWorkspaceIDs(ctx context.Context) ([]string, error) { type WorkspaceSnapshot struct { Path string - Timestamp string + Timestamp time.Time Partition int32 NumPartitions int32 } diff --git a/apps/workspace-engine/pkg/kafka/kafka.go b/apps/workspace-engine/pkg/kafka/kafka.go index f664e3089..08fd4e560 100644 --- a/apps/workspace-engine/pkg/kafka/kafka.go +++ b/apps/workspace-engine/pkg/kafka/kafka.go @@ -131,20 +131,20 @@ func RunConsumer(ctx context.Context) error { log.Error("Failed to route message", "error", err) continue } - + // Commit offset to Kafka if _, err := consumer.CommitMessage(msg); err != nil { log.Error("Failed to commit message", "error", err) continue } - + snapshot := &db.WorkspaceSnapshot{ Path: fmt.Sprintf("%s_%s.gob", ws.ID, msg.Timestamp.Format(time.RFC3339Nano)), - Timestamp: msg.Timestamp.Format(time.RFC3339Nano), + Timestamp: msg.Timestamp, Partition: int32(msg.TopicPartition.Partition), NumPartitions: numPartitions, } - + workspace.Save(ctx, storage, ws, snapshot) } } diff --git a/apps/workspace-engine/pkg/workspace/loader.go b/apps/workspace-engine/pkg/workspace/loader.go index 7e3ecc4e6..a456d859d 100644 --- a/apps/workspace-engine/pkg/workspace/loader.go +++ b/apps/workspace-engine/pkg/workspace/loader.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "workspace-engine/pkg/db" + + "github.com/charmbracelet/log" ) func Save(ctx context.Context, storage StorageClient, workspace *Workspace, snapshot *db.WorkspaceSnapshot) error { @@ -12,6 +14,7 @@ func Save(ctx context.Context, storage StorageClient, workspace *Workspace, snap return fmt.Errorf("failed to encode workspace: %w", err) } + log.Info("Saving workspace", "workspaceID", workspace.ID, "path", snapshot.Path) // Write to file with appropriate permissions if err := storage.Put(ctx, snapshot.Path, data); err != nil { return fmt.Errorf("failed to write workspace to disk: %w", err) @@ -43,4 +46,4 @@ func Load(ctx context.Context, storage StorageClient, workspace *Workspace) erro } return workspace.GobDecode(data) -} \ No newline at end of file +} From 44e66117f3616fa7718c0a1521cf11fe2a8bc1ce Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 19:36:42 -0700 Subject: [PATCH 13/15] remove log --- apps/workspace-engine/pkg/workspace/loader.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apps/workspace-engine/pkg/workspace/loader.go b/apps/workspace-engine/pkg/workspace/loader.go index a456d859d..e2410524c 100644 --- a/apps/workspace-engine/pkg/workspace/loader.go +++ b/apps/workspace-engine/pkg/workspace/loader.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "workspace-engine/pkg/db" - - "github.com/charmbracelet/log" ) func Save(ctx context.Context, storage StorageClient, workspace *Workspace, snapshot *db.WorkspaceSnapshot) error { @@ -13,8 +11,6 @@ func Save(ctx context.Context, storage StorageClient, workspace *Workspace, snap if err != nil { return fmt.Errorf("failed to encode workspace: %w", err) } - - log.Info("Saving workspace", "workspaceID", workspace.ID, "path", snapshot.Path) // Write to file with appropriate permissions if err := storage.Put(ctx, snapshot.Path, data); err != nil { return fmt.Errorf("failed to write workspace to disk: %w", err) From 5ea9db83147408efc890f76a447c2866c1b077c8 Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 19:37:31 -0700 Subject: [PATCH 14/15] renae --- ...sistence_test.go => engine_workspace_disk_persistence_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apps/workspace-engine/test/e2e/{engine_workspace_persistence_test.go => engine_workspace_disk_persistence_test.go} (100%) diff --git a/apps/workspace-engine/test/e2e/engine_workspace_persistence_test.go b/apps/workspace-engine/test/e2e/engine_workspace_disk_persistence_test.go similarity index 100% rename from apps/workspace-engine/test/e2e/engine_workspace_persistence_test.go rename to apps/workspace-engine/test/e2e/engine_workspace_disk_persistence_test.go From 14ac9a1ad86c804978cfe65eb63195559890b379 Mon Sep 17 00:00:00 2001 From: Aditya Choudhari Date: Wed, 22 Oct 2025 20:18:34 -0700 Subject: [PATCH 15/15] testing --- .../pkg/workspace/storage_gcs.go | 14 +- .../engine_workspace_disk_persistence_test.go | 209 +--- .../engine_workspace_gcs_persistence_test.go | 1114 +++++++++++++++++ ...gine_workspace_persistence_helpers_test.go | 226 ++++ 4 files changed, 1354 insertions(+), 209 deletions(-) create mode 100644 apps/workspace-engine/test/e2e/engine_workspace_gcs_persistence_test.go create mode 100644 apps/workspace-engine/test/e2e/engine_workspace_persistence_helpers_test.go diff --git a/apps/workspace-engine/pkg/workspace/storage_gcs.go b/apps/workspace-engine/pkg/workspace/storage_gcs.go index 69ff9969f..b7241699d 100644 --- a/apps/workspace-engine/pkg/workspace/storage_gcs.go +++ b/apps/workspace-engine/pkg/workspace/storage_gcs.go @@ -88,6 +88,18 @@ func (c *GCSStorageClient) Get(ctx context.Context, path string) ([]byte, error) return data, nil } +func (c *GCSStorageClient) Delete(ctx context.Context, path string) error { + path = filepath.Join(c.prefix, path) + obj := c.client.Bucket(c.bucket).Object(path) + if err := obj.Delete(ctx); err != nil { + if err == storage.ErrObjectNotExist { + return nil // Already deleted, no error + } + return fmt.Errorf("failed to delete object: %w", err) + } + return nil +} + func (c *GCSStorageClient) Close() error { return c.client.Close() -} \ No newline at end of file +} diff --git a/apps/workspace-engine/test/e2e/engine_workspace_disk_persistence_test.go b/apps/workspace-engine/test/e2e/engine_workspace_disk_persistence_test.go index 88e8c03e6..82b4e22b8 100644 --- a/apps/workspace-engine/test/e2e/engine_workspace_disk_persistence_test.go +++ b/apps/workspace-engine/test/e2e/engine_workspace_disk_persistence_test.go @@ -18,209 +18,7 @@ import ( // - Storage layer operations (file/GCS Put/Get) // - Gob encoding/decoding // - All entity fields are preserved (metadata, config, timestamps, etc.) - -// Helper functions for deep equality verification - -func verifyResourcesEqual(t *testing.T, expected, actual *oapi.Resource, context string) { - t.Helper() - if actual.Id != expected.Id { - t.Errorf("%s: resource ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) - } - if actual.Name != expected.Name { - t.Errorf("%s: resource name mismatch: expected %s, got %s", context, expected.Name, actual.Name) - } - if actual.Kind != expected.Kind { - t.Errorf("%s: resource kind mismatch: expected %s, got %s", context, expected.Kind, actual.Kind) - } - if actual.Version != expected.Version { - t.Errorf("%s: resource version mismatch: expected %s, got %s", context, expected.Version, actual.Version) - } - if actual.Identifier != expected.Identifier { - t.Errorf("%s: resource identifier mismatch: expected %s, got %s", context, expected.Identifier, actual.Identifier) - } - - // Verify metadata - if len(actual.Metadata) != len(expected.Metadata) { - t.Errorf("%s: metadata length mismatch: expected %d, got %d", context, len(expected.Metadata), len(actual.Metadata)) - } - for key, expectedValue := range expected.Metadata { - if actualValue, ok := actual.Metadata[key]; !ok { - t.Errorf("%s: metadata key %s missing", context, key) - } else if actualValue != expectedValue { - t.Errorf("%s: metadata[%s] mismatch: expected %s, got %s", context, key, expectedValue, actualValue) - } - } - - // Verify config (deep comparison would require reflection or JSON marshaling) - if (expected.Config == nil) != (actual.Config == nil) { - t.Errorf("%s: config nil mismatch", context) - } - - // Verify timestamps - if !actual.CreatedAt.Equal(expected.CreatedAt) { - t.Errorf("%s: createdAt mismatch: expected %v, got %v", context, expected.CreatedAt, actual.CreatedAt) - } - - // UpdatedAt is optional - if (expected.UpdatedAt == nil) != (actual.UpdatedAt == nil) { - t.Errorf("%s: updatedAt nil mismatch", context) - } else if expected.UpdatedAt != nil && !actual.UpdatedAt.Equal(*expected.UpdatedAt) { - t.Errorf("%s: updatedAt mismatch: expected %v, got %v", context, *expected.UpdatedAt, *actual.UpdatedAt) - } -} - -func verifyJobsEqual(t *testing.T, expected, actual *oapi.Job, context string) { - t.Helper() - if actual.Id != expected.Id { - t.Errorf("%s: job ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) - } - if actual.Status != expected.Status { - t.Errorf("%s: job status mismatch: expected %s, got %s", context, expected.Status, actual.Status) - } - if actual.JobAgentId != expected.JobAgentId { - t.Errorf("%s: job agent ID mismatch: expected %s, got %s", context, expected.JobAgentId, actual.JobAgentId) - } - if actual.ReleaseId != expected.ReleaseId { - t.Errorf("%s: release ID mismatch: expected %s, got %s", context, expected.ReleaseId, actual.ReleaseId) - } - // ExternalId is optional - if (expected.ExternalId == nil) != (actual.ExternalId == nil) { - t.Errorf("%s: externalId nil mismatch", context) - } else if expected.ExternalId != nil && *actual.ExternalId != *expected.ExternalId { - t.Errorf("%s: external ID mismatch: expected %s, got %s", context, *expected.ExternalId, *actual.ExternalId) - } - - // Verify metadata - if len(actual.Metadata) != len(expected.Metadata) { - t.Errorf("%s: metadata length mismatch: expected %d, got %d", context, len(expected.Metadata), len(actual.Metadata)) - } - for key, expectedValue := range expected.Metadata { - if actualValue, ok := actual.Metadata[key]; !ok { - t.Errorf("%s: metadata key %s missing", context, key) - } else if actualValue != expectedValue { - t.Errorf("%s: metadata[%s] mismatch: expected %s, got %s", context, key, expectedValue, actualValue) - } - } - - // Verify timestamps - if !actual.CreatedAt.Equal(expected.CreatedAt) { - t.Errorf("%s: createdAt mismatch: expected %v, got %v", context, expected.CreatedAt, actual.CreatedAt) - } - if !actual.UpdatedAt.Equal(expected.UpdatedAt) { - t.Errorf("%s: updatedAt mismatch: expected %v, got %v", context, expected.UpdatedAt, actual.UpdatedAt) - } - - // Verify optional timestamps - if (expected.StartedAt == nil) != (actual.StartedAt == nil) { - t.Errorf("%s: startedAt nil mismatch", context) - } else if expected.StartedAt != nil && !actual.StartedAt.Equal(*expected.StartedAt) { - t.Errorf("%s: startedAt mismatch: expected %v, got %v", context, *expected.StartedAt, *actual.StartedAt) - } - - if (expected.CompletedAt == nil) != (actual.CompletedAt == nil) { - t.Errorf("%s: completedAt nil mismatch", context) - } else if expected.CompletedAt != nil && !actual.CompletedAt.Equal(*expected.CompletedAt) { - t.Errorf("%s: completedAt mismatch: expected %v, got %v", context, *expected.CompletedAt, *actual.CompletedAt) - } -} - -func verifyDeploymentsEqual(t *testing.T, expected, actual *oapi.Deployment, context string) { - t.Helper() - if actual.Id != expected.Id { - t.Errorf("%s: deployment ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) - } - if actual.Name != expected.Name { - t.Errorf("%s: deployment name mismatch: expected %s, got %s", context, expected.Name, actual.Name) - } - // Description is optional - if (expected.Description == nil) != (actual.Description == nil) { - t.Errorf("%s: description nil mismatch", context) - } else if expected.Description != nil && *actual.Description != *expected.Description { - t.Errorf("%s: deployment description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) - } - if actual.SystemId != expected.SystemId { - t.Errorf("%s: system ID mismatch: expected %s, got %s", context, expected.SystemId, actual.SystemId) - } - // JobAgentId is optional - if (expected.JobAgentId == nil) != (actual.JobAgentId == nil) { - t.Errorf("%s: jobAgentId nil mismatch", context) - } else if expected.JobAgentId != nil && *actual.JobAgentId != *expected.JobAgentId { - t.Errorf("%s: job agent ID mismatch: expected %s, got %s", context, *expected.JobAgentId, *actual.JobAgentId) - } -} - -func verifySystemsEqual(t *testing.T, expected, actual *oapi.System, context string) { - t.Helper() - if actual.Id != expected.Id { - t.Errorf("%s: system ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) - } - if actual.Name != expected.Name { - t.Errorf("%s: system name mismatch: expected %s, got %s", context, expected.Name, actual.Name) - } - // Description is optional - if (expected.Description == nil) != (actual.Description == nil) { - t.Errorf("%s: description nil mismatch", context) - } else if expected.Description != nil && *actual.Description != *expected.Description { - t.Errorf("%s: system description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) - } - if actual.WorkspaceId != expected.WorkspaceId { - t.Errorf("%s: workspace ID mismatch: expected %s, got %s", context, expected.WorkspaceId, actual.WorkspaceId) - } -} - -func verifyEnvironmentsEqual(t *testing.T, expected, actual *oapi.Environment, context string) { - t.Helper() - if actual.Id != expected.Id { - t.Errorf("%s: environment ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) - } - if actual.Name != expected.Name { - t.Errorf("%s: environment name mismatch: expected %s, got %s", context, expected.Name, actual.Name) - } - // Description is optional - if (expected.Description == nil) != (actual.Description == nil) { - t.Errorf("%s: description nil mismatch", context) - } else if expected.Description != nil && *actual.Description != *expected.Description { - t.Errorf("%s: environment description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) - } - if actual.SystemId != expected.SystemId { - t.Errorf("%s: system ID mismatch: expected %s, got %s", context, expected.SystemId, actual.SystemId) - } -} - -func verifyJobAgentsEqual(t *testing.T, expected, actual *oapi.JobAgent, context string) { - t.Helper() - if actual.Id != expected.Id { - t.Errorf("%s: job agent ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) - } - if actual.Name != expected.Name { - t.Errorf("%s: job agent name mismatch: expected %s, got %s", context, expected.Name, actual.Name) - } - if actual.Type != expected.Type { - t.Errorf("%s: job agent type mismatch: expected %s, got %s", context, expected.Type, actual.Type) - } - if actual.WorkspaceId != expected.WorkspaceId { - t.Errorf("%s: workspace ID mismatch: expected %s, got %s", context, expected.WorkspaceId, actual.WorkspaceId) - } -} - -func verifyPoliciesEqual(t *testing.T, expected, actual *oapi.Policy, context string) { - t.Helper() - if actual.Id != expected.Id { - t.Errorf("%s: policy ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) - } - if actual.Name != expected.Name { - t.Errorf("%s: policy name mismatch: expected %s, got %s", context, expected.Name, actual.Name) - } - // Description is optional - if (expected.Description == nil) != (actual.Description == nil) { - t.Errorf("%s: description nil mismatch", context) - } else if expected.Description != nil && *actual.Description != *expected.Description { - t.Errorf("%s: policy description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) - } - if actual.WorkspaceId != expected.WorkspaceId { - t.Errorf("%s: workspace ID mismatch: expected %s, got %s", context, expected.WorkspaceId, actual.WorkspaceId) - } -} +// Helper functions are in engine_workspace_persistence_helpers_test.go func TestEngine_Persistence_BasicSaveLoadRoundtrip(t *testing.T) { ctx := context.Background() @@ -1224,8 +1022,3 @@ func TestEngine_Persistence_FileStorageOperations(t *testing.T) { t.Errorf("data mismatch: expected %q, got %q", string(testData), string(retrievedData)) } } - -// Helper function to create pointer to time.Time -func ptrTime(t time.Time) *time.Time { - return &t -} diff --git a/apps/workspace-engine/test/e2e/engine_workspace_gcs_persistence_test.go b/apps/workspace-engine/test/e2e/engine_workspace_gcs_persistence_test.go new file mode 100644 index 000000000..1ed9a9f10 --- /dev/null +++ b/apps/workspace-engine/test/e2e/engine_workspace_gcs_persistence_test.go @@ -0,0 +1,1114 @@ +package e2e + +import ( + "context" + "fmt" + "testing" + "time" + "workspace-engine/pkg/oapi" + "workspace-engine/pkg/workspace" + "workspace-engine/test/integration" + + "github.com/google/uuid" +) + +// These tests validate GCS storage persistence +// They automatically configure WORKSPACE_STATES_BUCKET_URL to gs://ctrlplane/workspace-states-testing +// They require proper GCP authentication (gcloud auth application-default login --project=wandb-ctrlplane) +// Helper functions are in engine_workspace_persistence_helpers_test.go + +func setupGCSTest(t *testing.T, ctx context.Context) workspace.StorageClient { + t.Helper() + + // Set the GCS bucket URL for this test (automatically restored after test) + t.Setenv("WORKSPACE_STATES_BUCKET_URL", "gs://ctrlplane/workspace-states-testing") + + // Create GCS storage client + storage, err := workspace.NewGCSStorageClient(ctx) + if err != nil { + t.Fatalf("failed to create GCS storage client: %v\nMake sure you're authenticated: gcloud auth application-default login --project=wandb-ctrlplane", err) + } + return storage +} + +// cleanupGCSFile deletes a test file from GCS +func cleanupGCSFile(t *testing.T, ctx context.Context, storage workspace.StorageClient, path string) { + t.Helper() + + // Type assert to GCSStorageClient to access Delete method + gcsStorage, ok := storage.(*workspace.GCSStorageClient) + if !ok { + t.Logf("Warning: storage is not GCSStorageClient, cannot cleanup file: %s", path) + return + } + + if err := gcsStorage.Delete(ctx, path); err != nil { + t.Logf("Warning: failed to cleanup GCS file %s: %v", path, err) + } +} + +func TestEngine_GCS_BasicSaveLoadRoundtrip(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + resource1Id := uuid.New().String() + resource2Id := uuid.New().String() + systemId := uuid.New().String() + jobAgentId := uuid.New().String() + deploymentId := uuid.New().String() + + // Create workspace and populate using integration helpers + engine := integration.NewTestWorkspace(t, + integration.WithResource( + integration.ResourceID(resource1Id), + integration.ResourceName("gcs-resource-1"), + ), + integration.WithResource( + integration.ResourceID(resource2Id), + integration.ResourceName("gcs-resource-2"), + ), + integration.WithJobAgent( + integration.JobAgentID(jobAgentId), + integration.JobAgentName("gcs-job-agent"), + ), + integration.WithSystem( + integration.SystemID(systemId), + integration.SystemName("gcs-test-system"), + integration.WithDeployment( + integration.DeploymentID(deploymentId), + integration.DeploymentName("gcs-deployment"), + integration.DeploymentJobAgent(jobAgentId), + ), + ), + ) + + ws := engine.Workspace() + workspaceID := ws.ID + + // Capture original state + originalResources := ws.Resources().Items() + originalDeployments := ws.Deployments().Items() + originalSystems := ws.Systems().Items() + originalJobAgents := ws.JobAgents().Items() + + // Encode workspace + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + // Write to GCS + testPath := fmt.Sprintf("test-roundtrip-%s.gob", uuid.New().String()) + if err := storage.Put(ctx, testPath, data); err != nil { + t.Fatalf("failed to write workspace to GCS: %v", err) + } + defer cleanupGCSFile(t, ctx, storage, testPath) + + // Create a new workspace and load from GCS + newWs := workspace.New(workspaceID) + + // Read from GCS + loadedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to read workspace from GCS: %v", err) + } + + // Decode workspace + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify workspace ID + if newWs.ID != workspaceID { + t.Errorf("workspace ID mismatch: expected %s, got %s", workspaceID, newWs.ID) + } + + // Verify all resources with full field comparison + loadedResources := newWs.Resources().Items() + if len(loadedResources) != len(originalResources) { + t.Errorf("resources count mismatch: expected %d, got %d", len(originalResources), len(loadedResources)) + } + for id, original := range originalResources { + loaded, ok := loadedResources[id] + if !ok { + t.Errorf("resource %s not found after load from GCS", id) + continue + } + verifyResourcesEqual(t, original, loaded, "resource "+id) + } + + // Verify all deployments + loadedDeployments := newWs.Deployments().Items() + if len(loadedDeployments) != len(originalDeployments) { + t.Errorf("deployments count mismatch: expected %d, got %d", len(originalDeployments), len(loadedDeployments)) + } + for id, original := range originalDeployments { + loaded, ok := loadedDeployments[id] + if !ok { + t.Errorf("deployment %s not found after load from GCS", id) + continue + } + verifyDeploymentsEqual(t, original, loaded, "deployment "+id) + } + + // Verify all systems + loadedSystems := newWs.Systems().Items() + if len(loadedSystems) != len(originalSystems) { + t.Errorf("systems count mismatch: expected %d, got %d", len(originalSystems), len(loadedSystems)) + } + for id, original := range originalSystems { + loaded, ok := loadedSystems[id] + if !ok { + t.Errorf("system %s not found after load from GCS", id) + continue + } + verifySystemsEqual(t, original, loaded, "system "+id) + } + + // Verify all job agents + loadedJobAgents := newWs.JobAgents().Items() + if len(loadedJobAgents) != len(originalJobAgents) { + t.Errorf("job agents count mismatch: expected %d, got %d", len(originalJobAgents), len(loadedJobAgents)) + } + for id, original := range originalJobAgents { + loaded, ok := loadedJobAgents[id] + if !ok { + t.Errorf("job agent %s not found after load from GCS", id) + continue + } + verifyJobAgentsEqual(t, original, loaded, "job agent "+id) + } + + t.Logf("Successfully saved and loaded workspace to/from GCS at path: %s", testPath) +} + +func TestEngine_GCS_EmptyWorkspace(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + // Create empty workspace + workspaceID := uuid.New().String() + ws := workspace.NewNoFlush(workspaceID) + + // Encode workspace + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + // Write to GCS + testPath := fmt.Sprintf("test-empty-%s.gob", uuid.New().String()) + if err := storage.Put(ctx, testPath, data); err != nil { + t.Fatalf("failed to write empty workspace to GCS: %v", err) + } + defer cleanupGCSFile(t, ctx, storage, testPath) + + // Load into new workspace + newWs := workspace.NewNoFlush("temp") + + // Read from GCS + loadedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to read workspace from GCS: %v", err) + } + + // Decode workspace + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify it's still empty + if newWs.ID != workspaceID { + t.Errorf("workspace ID mismatch: expected %s, got %s", workspaceID, newWs.ID) + } + + if len(newWs.Resources().Items()) != 0 { + t.Errorf("expected 0 resources, got %d", len(newWs.Resources().Items())) + } + + if len(newWs.Deployments().Items()) != 0 { + t.Errorf("expected 0 deployments, got %d", len(newWs.Deployments().Items())) + } + + t.Logf("Successfully saved and loaded empty workspace to/from GCS at path: %s", testPath) +} + +func TestEngine_GCS_ResourcesWithMetadata(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + systemId := uuid.New().String() + resource1Id := uuid.New().String() + resource2Id := uuid.New().String() + resource3Id := uuid.New().String() + + // Create workspace with resources containing rich metadata + engine := integration.NewTestWorkspace(t, + integration.WithSystem( + integration.SystemID(systemId), + integration.SystemName("metadata-test-system"), + ), + integration.WithResource( + integration.ResourceID(resource1Id), + integration.ResourceName("server-prod-1"), + integration.ResourceConfig(map[string]interface{}{ + "type": "server", + "cpu": 4, + "memory": 16, + "location": "us-west-2a", + }), + ), + integration.WithResource( + integration.ResourceID(resource2Id), + integration.ResourceName("database-prod"), + integration.ResourceConfig(map[string]interface{}{ + "type": "postgresql", + "version": "15.2", + "storage_gb": 500, + "replicas": 3, + "auto_backup": true, + }), + ), + integration.WithResource( + integration.ResourceID(resource3Id), + integration.ResourceName("cache-cluster"), + integration.ResourceConfig(map[string]interface{}{ + "type": "redis", + "nodes": 6, + "eviction": "allkeys-lru", + "maxmemory": "8gb", + "persistence": map[string]interface{}{ + "enabled": true, + "type": "aof", + }, + }), + ), + ) + + ws := engine.Workspace() + workspaceID := ws.ID + + // Add additional metadata to resources after creation + res1, _ := ws.Resources().Get(resource1Id) + res1.Metadata = map[string]string{ + "env": "production", + "region": "us-west-2", + "owner": "platform-team", + "cost_center": "engineering", + "managed_by": "terraform", + } + + res2, _ := ws.Resources().Get(resource2Id) + res2.Metadata = map[string]string{ + "env": "production", + "backup_window": "02:00-04:00", + "maintenance": "sunday-03:00", + "encrypted": "true", + } + + res3, _ := ws.Resources().Get(resource3Id) + res3.Metadata = map[string]string{ + "env": "production", + "cluster": "main", + "sentinel": "enabled", + } + + // Capture original resources + originalResources := ws.Resources().Items() + + // Encode and save to GCS + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + testPath := fmt.Sprintf("test-resource-metadata-%s.gob", uuid.New().String()) + if err := storage.Put(ctx, testPath, data); err != nil { + t.Fatalf("failed to write workspace to GCS: %v", err) + } + defer cleanupGCSFile(t, ctx, storage, testPath) + + // Load from GCS + newWs := workspace.New(workspaceID) + + loadedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to read workspace from GCS: %v", err) + } + + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify all resources with metadata and config + loadedResources := newWs.Resources().Items() + if len(loadedResources) != 3 { + t.Fatalf("expected 3 resources, got %d", len(loadedResources)) + } + + for id, original := range originalResources { + loaded, ok := loadedResources[id] + if !ok { + t.Errorf("resource %s not found after GCS restore", id) + continue + } + + // Use full field verification + verifyResourcesEqual(t, original, loaded, "resource "+id) + } + + // Specific metadata checks for our test resources + loadedRes1, _ := newWs.Resources().Get(resource1Id) + if loadedRes1.Metadata["owner"] != "platform-team" { + t.Errorf("resource 1 metadata[owner] not preserved: got %s", loadedRes1.Metadata["owner"]) + } + if loadedRes1.Metadata["cost_center"] != "engineering" { + t.Errorf("resource 1 metadata[cost_center] not preserved: got %s", loadedRes1.Metadata["cost_center"]) + } + + loadedRes2, _ := newWs.Resources().Get(resource2Id) + if loadedRes2.Metadata["encrypted"] != "true" { + t.Errorf("resource 2 metadata[encrypted] not preserved: got %s", loadedRes2.Metadata["encrypted"]) + } + + loadedRes3, _ := newWs.Resources().Get(resource3Id) + + // Verify config objects are preserved + if loadedRes1.Config == nil { + t.Error("resource 1 config is nil after GCS restore") + } + if loadedRes2.Config == nil { + t.Error("resource 2 config is nil after GCS restore") + } + if loadedRes3.Config == nil { + t.Error("resource 3 config is nil after GCS restore") + } + + t.Logf("Successfully verified resources with metadata and config in GCS at path: %s", testPath) +} + +func TestEngine_GCS_MultipleWorkspaces(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + // Create and save multiple different workspaces + workspaceIDs := []string{uuid.New().String(), uuid.New().String(), uuid.New().String()} + testPaths := make(map[string]string) + + for i, wsID := range workspaceIDs { + ws := workspace.NewNoFlush(wsID) + + // Encode and save + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace %s: %v", wsID, err) + } + + testPath := fmt.Sprintf("test-multi-%d-%s.gob", i, uuid.New().String()) + testPaths[wsID] = testPath + + if err := storage.Put(ctx, testPath, data); err != nil { + t.Fatalf("failed to save workspace %s to GCS: %v", wsID, err) + } + defer cleanupGCSFile(t, ctx, storage, testPath) + } + + // Load each workspace and verify they're distinct + for _, wsID := range workspaceIDs { + newWs := workspace.NewNoFlush("temp") + + testPath := testPaths[wsID] + loadedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to load workspace %s from GCS: %v", wsID, err) + } + + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace %s: %v", wsID, err) + } + + // Verify workspace ID is correct + if newWs.ID != wsID { + t.Errorf("workspace ID mismatch: expected %s, got %s", wsID, newWs.ID) + } + } + + t.Logf("Successfully saved and loaded %d workspaces to/from GCS", len(workspaceIDs)) +} + +func TestEngine_GCS_LargeWorkspace(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + systemId := uuid.New().String() + + // Create workspace with many resources to test large file handling + opts := []integration.WorkspaceOption{ + integration.WithSystem( + integration.SystemID(systemId), + integration.SystemName("large-system"), + ), + } + + // Add 100 resources + for i := 0; i < 100; i++ { + resourceId := uuid.New().String() + opts = append(opts, integration.WithResource( + integration.ResourceID(resourceId), + integration.ResourceName(fmt.Sprintf("resource-%d", i)), + integration.ResourceConfig(map[string]interface{}{ + "index": i, + "type": "test", + "data": fmt.Sprintf("large payload data for resource %d", i), + }), + )) + } + + engine := integration.NewTestWorkspace(t, opts...) + ws := engine.Workspace() + workspaceID := ws.ID + + // Verify we have 100 resources + originalResources := ws.Resources().Items() + if len(originalResources) != 100 { + t.Fatalf("expected 100 resources, got %d", len(originalResources)) + } + + // Encode workspace + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + t.Logf("Encoded workspace size: %d bytes (%.2f KB)", len(data), float64(len(data))/1024) + + // Write to GCS + testPath := fmt.Sprintf("test-large-%s.gob", uuid.New().String()) + startWrite := time.Now() + if err := storage.Put(ctx, testPath, data); err != nil { + t.Fatalf("failed to write large workspace to GCS: %v", err) + } + defer cleanupGCSFile(t, ctx, storage, testPath) + writeDuration := time.Since(startWrite) + + // Load from GCS + newWs := workspace.New(workspaceID) + startRead := time.Now() + loadedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to read large workspace from GCS: %v", err) + } + readDuration := time.Since(startRead) + + // Decode workspace + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify all 100 resources are preserved + loadedResources := newWs.Resources().Items() + if len(loadedResources) != 100 { + t.Errorf("expected 100 resources after load, got %d", len(loadedResources)) + } + + // Spot check a few resources + for id, original := range originalResources { + loaded, ok := loadedResources[id] + if !ok { + t.Errorf("resource %s not found after load from GCS", id) + continue + } + verifyResourcesEqual(t, original, loaded, "resource "+id) + } + + t.Logf("GCS Performance: Write=%v, Read=%v, Size=%.2fKB", writeDuration, readDuration, float64(len(data))/1024) +} + +func TestEngine_GCS_JobsWithMetadata(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + systemId := uuid.New().String() + jobAgentId := uuid.New().String() + deploymentId := uuid.New().String() + deploymentVersionId := uuid.New().String() + envId := uuid.New().String() + resourceId := uuid.New().String() + + // Create workspace with deployment that generates jobs + engine := integration.NewTestWorkspace(t, + integration.WithJobAgent( + integration.JobAgentID(jobAgentId), + integration.JobAgentName("gcs-test-agent"), + ), + integration.WithSystem( + integration.SystemID(systemId), + integration.SystemName("gcs-test-system"), + integration.WithDeployment( + integration.DeploymentID(deploymentId), + integration.DeploymentName("gcs-test-deployment"), + integration.DeploymentJobAgent(jobAgentId), + integration.WithDeploymentVersion( + integration.DeploymentVersionID(deploymentVersionId), + integration.DeploymentVersionTag("v1.0.0"), + ), + ), + integration.WithEnvironment( + integration.EnvironmentID(envId), + integration.EnvironmentName("gcs-test-env"), + ), + ), + integration.WithResource( + integration.ResourceID(resourceId), + integration.ResourceName("gcs-test-resource"), + ), + ) + + ws := engine.Workspace() + workspaceID := ws.ID + + // Get all jobs and update them with various metadata + allJobs := ws.Jobs().Items() + if len(allJobs) == 0 { + t.Fatal("expected at least one job to be created") + } + + // Add rich metadata to jobs + jobsWithMetadata := make(map[string]*oapi.Job) + for jobId, job := range allJobs { + job.Metadata = map[string]string{ + "environment": "test", + "region": "us-west-2", + "version": "1.2.3", + "deploy_id": uuid.New().String(), + } + ws.Jobs().Upsert(ctx, job) + jobsWithMetadata[jobId] = job + } + + // Encode workspace + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + // Write to GCS + testPath := fmt.Sprintf("test-jobs-metadata-%s.gob", uuid.New().String()) + if err := storage.Put(ctx, testPath, data); err != nil { + t.Fatalf("failed to write workspace to GCS: %v", err) + } + defer cleanupGCSFile(t, ctx, storage, testPath) + + // Load into new workspace + newWs := workspace.New(workspaceID) + + loadedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to read workspace from GCS: %v", err) + } + + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify all jobs with metadata + for jobId, originalJob := range jobsWithMetadata { + restoredJob, ok := newWs.Jobs().Get(jobId) + if !ok { + t.Errorf("job %s not found after restore from GCS", jobId) + continue + } + + verifyJobsEqual(t, originalJob, restoredJob, "job "+jobId) + } + + t.Logf("Successfully saved and loaded jobs with metadata to/from GCS at path: %s", testPath) +} + +func TestEngine_GCS_TimestampPrecision(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + systemId := uuid.New().String() + jobAgentId := uuid.New().String() + deploymentId := uuid.New().String() + deploymentVersionId := uuid.New().String() + envId := uuid.New().String() + resourceId := uuid.New().String() + + // Create workspace with jobs + engine := integration.NewTestWorkspace(t, + integration.WithJobAgent( + integration.JobAgentID(jobAgentId), + integration.JobAgentName("gcs-agent"), + ), + integration.WithSystem( + integration.SystemID(systemId), + integration.SystemName("gcs-system"), + integration.WithDeployment( + integration.DeploymentID(deploymentId), + integration.DeploymentName("gcs-deployment"), + integration.DeploymentJobAgent(jobAgentId), + integration.WithDeploymentVersion( + integration.DeploymentVersionID(deploymentVersionId), + integration.DeploymentVersionTag("v1.0.0"), + ), + ), + integration.WithEnvironment( + integration.EnvironmentID(envId), + integration.EnvironmentName("gcs-env"), + ), + ), + integration.WithResource( + integration.ResourceID(resourceId), + integration.ResourceName("gcs-resource"), + ), + ) + + ws := engine.Workspace() + workspaceID := ws.ID + + // Create jobs with specific timestamps including nanoseconds + utcLoc := time.UTC + testTimestamps := []struct { + name string + createdAt time.Time + updatedAt time.Time + startedAt *time.Time + completed *time.Time + }{ + { + name: "utc-with-nanos", + createdAt: time.Date(2023, 5, 15, 10, 30, 45, 123456789, utcLoc), + updatedAt: time.Date(2023, 5, 15, 11, 30, 45, 987654321, utcLoc), + startedAt: ptrTime(time.Date(2023, 5, 15, 10, 31, 0, 555555555, utcLoc)), + completed: ptrTime(time.Date(2023, 5, 15, 11, 30, 0, 999999999, utcLoc)), + }, + { + name: "far-future", + createdAt: time.Date(2099, 12, 31, 23, 59, 59, 111111111, utcLoc), + updatedAt: time.Date(2099, 12, 31, 23, 59, 59, 222222222, utcLoc), + startedAt: nil, + completed: nil, + }, + } + + jobTimestamps := make(map[string]struct { + createdAt time.Time + updatedAt time.Time + startedAt *time.Time + completed *time.Time + }) + + // Create jobs with specific timestamps + for _, ts := range testTimestamps { + jobId := uuid.New().String() + releaseId := uuid.New().String() + + job := &oapi.Job{ + Id: jobId, + Status: oapi.Pending, + JobAgentId: jobAgentId, + ReleaseId: releaseId, + CreatedAt: ts.createdAt, + UpdatedAt: ts.updatedAt, + StartedAt: ts.startedAt, + CompletedAt: ts.completed, + JobAgentConfig: make(map[string]interface{}), + Metadata: map[string]string{"test": ts.name}, + } + + ws.Jobs().Upsert(ctx, job) + jobTimestamps[jobId] = struct { + createdAt time.Time + updatedAt time.Time + startedAt *time.Time + completed *time.Time + }{ + createdAt: ts.createdAt, + updatedAt: ts.updatedAt, + startedAt: ts.startedAt, + completed: ts.completed, + } + } + + // Encode and save to GCS + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + testPath := fmt.Sprintf("test-timestamps-%s.gob", uuid.New().String()) + if err := storage.Put(ctx, testPath, data); err != nil { + t.Fatalf("failed to write workspace to GCS: %v", err) + } + defer cleanupGCSFile(t, ctx, storage, testPath) + + // Load from GCS + newWs := workspace.New(workspaceID) + + loadedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to read workspace from GCS: %v", err) + } + + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify all timestamps are preserved with nanosecond precision + for jobId, expectedTimestamps := range jobTimestamps { + restoredJob, ok := newWs.Jobs().Get(jobId) + if !ok { + t.Errorf("job %s not found after restore from GCS", jobId) + continue + } + + // Verify nanoseconds are preserved + if restoredJob.CreatedAt.Nanosecond() != expectedTimestamps.createdAt.Nanosecond() { + t.Errorf("job %s: CreatedAt nanoseconds not preserved in GCS, expected %d, got %d", + jobId, expectedTimestamps.createdAt.Nanosecond(), restoredJob.CreatedAt.Nanosecond()) + } + + if restoredJob.UpdatedAt.Nanosecond() != expectedTimestamps.updatedAt.Nanosecond() { + t.Errorf("job %s: UpdatedAt nanoseconds not preserved in GCS, expected %d, got %d", + jobId, expectedTimestamps.updatedAt.Nanosecond(), restoredJob.UpdatedAt.Nanosecond()) + } + } + + t.Logf("Successfully verified nanosecond timestamp precision in GCS at path: %s", testPath) +} + +func TestEngine_GCS_LoadFromNonExistentFile(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + // Attempt to load from non-existent file + nonExistentPath := fmt.Sprintf("non-existent-%s.gob", uuid.New().String()) + _, err := storage.Get(ctx, nonExistentPath) + + // Verify error is returned + if err == nil { + t.Fatal("expected error when loading from non-existent GCS file, got nil") + } + + // Verify it's a "not found" error + if !isNotFoundError(err) { + t.Logf("Error message: %v", err) + } + + t.Logf("Correctly received error for non-existent file: %v", err) +} + +func TestEngine_GCS_OverwriteExistingFile(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + testPath := fmt.Sprintf("test-overwrite-%s.gob", uuid.New().String()) + defer cleanupGCSFile(t, ctx, storage, testPath) + + // Create first workspace + workspaceID1 := uuid.New().String() + ws1 := workspace.NewNoFlush(workspaceID1) + + data1, err := ws1.GobEncode() + if err != nil { + t.Fatalf("failed to encode first workspace: %v", err) + } + + // Write first workspace + if err := storage.Put(ctx, testPath, data1); err != nil { + t.Fatalf("failed to write first workspace to GCS: %v", err) + } + + // Create second workspace with different ID + workspaceID2 := uuid.New().String() + ws2 := workspace.NewNoFlush(workspaceID2) + + data2, err := ws2.GobEncode() + if err != nil { + t.Fatalf("failed to encode second workspace: %v", err) + } + + // Overwrite with second workspace + if err := storage.Put(ctx, testPath, data2); err != nil { + t.Fatalf("failed to overwrite workspace in GCS: %v", err) + } + + // Load and verify it's the second workspace + loadedWs := workspace.NewNoFlush("temp") + + loadedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to read workspace from GCS: %v", err) + } + + if err := loadedWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Should be the second workspace (overwritten) + if loadedWs.ID != workspaceID2 { + t.Errorf("expected workspace ID %s (second), got %s", workspaceID2, loadedWs.ID) + } + + if loadedWs.ID == workspaceID1 { + t.Error("file was not overwritten - still contains first workspace") + } + + t.Logf("Successfully verified GCS file overwrite at path: %s", testPath) +} + +func TestEngine_GCS_NestedPathHandling(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + // Test that nested paths work in GCS + workspaceID := uuid.New().String() + ws := workspace.NewNoFlush(workspaceID) + + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + // Use nested path structure + testPath := fmt.Sprintf("nested/deep/path/%s/workspace.gob", uuid.New().String()) + defer cleanupGCSFile(t, ctx, storage, testPath) + + // Write to GCS with nested path + if err := storage.Put(ctx, testPath, data); err != nil { + t.Fatalf("failed to write workspace to nested GCS path: %v", err) + } + + // Read back from nested path + loadedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to read workspace from nested GCS path: %v", err) + } + + // Verify data integrity + newWs := workspace.NewNoFlush("temp") + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + if newWs.ID != workspaceID { + t.Errorf("workspace ID mismatch: expected %s, got %s", workspaceID, newWs.ID) + } + + t.Logf("Successfully handled nested path in GCS: %s", testPath) +} + +func TestEngine_GCS_ComplexEntitiesRoundtrip(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + sysId := uuid.New().String() + jobAgentId := uuid.New().String() + deploymentId := uuid.New().String() + deploymentVersionId := uuid.New().String() + env1Id := uuid.New().String() + env2Id := uuid.New().String() + resource1Id := uuid.New().String() + resource2Id := uuid.New().String() + policyId := uuid.New().String() + + // Create workspace with complex entity graph + engine := integration.NewTestWorkspace(t, + integration.WithJobAgent( + integration.JobAgentID(jobAgentId), + integration.JobAgentName("gcs-complex-agent"), + ), + integration.WithSystem( + integration.SystemID(sysId), + integration.SystemName("gcs-complex-system"), + integration.WithDeployment( + integration.DeploymentID(deploymentId), + integration.DeploymentName("gcs-api-service"), + integration.DeploymentJobAgent(jobAgentId), + integration.WithDeploymentVersion( + integration.DeploymentVersionID(deploymentVersionId), + integration.DeploymentVersionTag("v2.1.0"), + ), + ), + integration.WithEnvironment( + integration.EnvironmentID(env1Id), + integration.EnvironmentName("gcs-production"), + ), + integration.WithEnvironment( + integration.EnvironmentID(env2Id), + integration.EnvironmentName("gcs-staging"), + ), + ), + integration.WithResource( + integration.ResourceID(resource1Id), + integration.ResourceName("gcs-resource-1"), + ), + integration.WithResource( + integration.ResourceID(resource2Id), + integration.ResourceName("gcs-resource-2"), + ), + integration.WithPolicy( + integration.PolicyID(policyId), + integration.PolicyName("gcs-approval-policy"), + ), + ) + + ws := engine.Workspace() + workspaceID := ws.ID + + // Capture original entities + originalSys, _ := ws.Systems().Get(sysId) + originalDeployment, _ := ws.Deployments().Get(deploymentId) + originalJobAgent, _ := ws.JobAgents().Get(jobAgentId) + originalEnv1, _ := ws.Environments().Get(env1Id) + originalEnv2, _ := ws.Environments().Get(env2Id) + originalResource1, _ := ws.Resources().Get(resource1Id) + originalResource2, _ := ws.Resources().Get(resource2Id) + originalPolicy, _ := ws.Policies().Get(policyId) + + // Encode and save to GCS + data, err := ws.GobEncode() + if err != nil { + t.Fatalf("failed to encode workspace: %v", err) + } + + testPath := fmt.Sprintf("test-complex-%s.gob", uuid.New().String()) + if err := storage.Put(ctx, testPath, data); err != nil { + t.Fatalf("failed to write complex workspace to GCS: %v", err) + } + defer cleanupGCSFile(t, ctx, storage, testPath) + + // Load from GCS + newWs := workspace.New(workspaceID) + + loadedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to read complex workspace from GCS: %v", err) + } + + if err := newWs.GobDecode(loadedData); err != nil { + t.Fatalf("failed to decode workspace: %v", err) + } + + // Verify all entities with deep field comparison + restoredSys, ok := newWs.Systems().Get(sysId) + if !ok { + t.Fatal("system not found after GCS restore") + } + verifySystemsEqual(t, originalSys, restoredSys, "system") + + restoredDeployment, ok := newWs.Deployments().Get(deploymentId) + if !ok { + t.Fatal("deployment not found after GCS restore") + } + verifyDeploymentsEqual(t, originalDeployment, restoredDeployment, "deployment") + + restoredJobAgent, ok := newWs.JobAgents().Get(jobAgentId) + if !ok { + t.Fatal("job agent not found after GCS restore") + } + verifyJobAgentsEqual(t, originalJobAgent, restoredJobAgent, "job agent") + + restoredEnv1, ok := newWs.Environments().Get(env1Id) + if !ok { + t.Error("environment production not found after GCS restore") + } else { + verifyEnvironmentsEqual(t, originalEnv1, restoredEnv1, "environment production") + } + + restoredEnv2, ok := newWs.Environments().Get(env2Id) + if !ok { + t.Error("environment staging not found after GCS restore") + } else { + verifyEnvironmentsEqual(t, originalEnv2, restoredEnv2, "environment staging") + } + + restoredResource1, ok := newWs.Resources().Get(resource1Id) + if !ok { + t.Error("resource 1 not found after GCS restore") + } else { + verifyResourcesEqual(t, originalResource1, restoredResource1, "resource 1") + } + + restoredResource2, ok := newWs.Resources().Get(resource2Id) + if !ok { + t.Error("resource 2 not found after GCS restore") + } else { + verifyResourcesEqual(t, originalResource2, restoredResource2, "resource 2") + } + + restoredPolicy, ok := newWs.Policies().Get(policyId) + if !ok { + t.Error("policy not found after GCS restore") + } else { + verifyPoliciesEqual(t, originalPolicy, restoredPolicy, "policy") + } + + t.Logf("Successfully verified complex entity graph in GCS at path: %s", testPath) +} + +func TestEngine_GCS_RawBinaryDataIntegrity(t *testing.T) { + ctx := context.Background() + storage := setupGCSTest(t, ctx) + + // Test that raw binary data is preserved without corruption + testData := []byte{ + 0x00, 0xFF, 0xAB, 0xCD, 0xEF, // Binary data with null bytes and high values + 0x01, 0x02, 0x03, 0x04, 0x05, + 0x7F, 0x80, 0x81, 0xFE, 0xFF, + } + + testPath := fmt.Sprintf("test-binary-%s.dat", uuid.New().String()) + + // Write binary data to GCS + if err := storage.Put(ctx, testPath, testData); err != nil { + t.Fatalf("failed to write binary data to GCS: %v", err) + } + defer cleanupGCSFile(t, ctx, storage, testPath) + + // Read back + retrievedData, err := storage.Get(ctx, testPath) + if err != nil { + t.Fatalf("failed to read binary data from GCS: %v", err) + } + + // Verify exact byte-for-byte match + if len(retrievedData) != len(testData) { + t.Fatalf("data length mismatch: expected %d bytes, got %d bytes", len(testData), len(retrievedData)) + } + + for i, expectedByte := range testData { + if retrievedData[i] != expectedByte { + t.Errorf("byte %d mismatch: expected 0x%02X, got 0x%02X", i, expectedByte, retrievedData[i]) + } + } + + t.Logf("Successfully verified binary data integrity in GCS at path: %s", testPath) +} + +// Helper to check if error is a "not found" error +func isNotFoundError(err error) bool { + if err == nil { + return false + } + errMsg := err.Error() + return contains(errMsg, "not found") || + contains(errMsg, "does not exist") || + contains(errMsg, "ErrWorkspaceSnapshotNotFound") +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(substr) == 0 || + (len(s) > 0 && len(substr) > 0 && indexOfString(s, substr) >= 0)) +} + +func indexOfString(s, substr string) int { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return i + } + } + return -1 +} diff --git a/apps/workspace-engine/test/e2e/engine_workspace_persistence_helpers_test.go b/apps/workspace-engine/test/e2e/engine_workspace_persistence_helpers_test.go new file mode 100644 index 000000000..e6fde25bc --- /dev/null +++ b/apps/workspace-engine/test/e2e/engine_workspace_persistence_helpers_test.go @@ -0,0 +1,226 @@ +package e2e + +import ( + "testing" + "time" + "workspace-engine/pkg/oapi" +) + +// Shared helper functions for workspace persistence tests +// Used by both disk and GCS persistence tests + +func verifyResourcesEqual(t *testing.T, expected, actual *oapi.Resource, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: resource ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: resource name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + if actual.Kind != expected.Kind { + t.Errorf("%s: resource kind mismatch: expected %s, got %s", context, expected.Kind, actual.Kind) + } + if actual.Version != expected.Version { + t.Errorf("%s: resource version mismatch: expected %s, got %s", context, expected.Version, actual.Version) + } + if actual.Identifier != expected.Identifier { + t.Errorf("%s: resource identifier mismatch: expected %s, got %s", context, expected.Identifier, actual.Identifier) + } + + // Verify metadata + if len(actual.Metadata) != len(expected.Metadata) { + t.Errorf("%s: metadata length mismatch: expected %d, got %d", context, len(expected.Metadata), len(actual.Metadata)) + } + for key, expectedValue := range expected.Metadata { + if actualValue, ok := actual.Metadata[key]; !ok { + t.Errorf("%s: metadata key %s missing", context, key) + } else if actualValue != expectedValue { + t.Errorf("%s: metadata[%s] mismatch: expected %s, got %s", context, key, expectedValue, actualValue) + } + } + + // Verify config (deep comparison would require reflection or JSON marshaling) + if (expected.Config == nil) != (actual.Config == nil) { + t.Errorf("%s: config nil mismatch", context) + } + + // Verify timestamps + if !actual.CreatedAt.Equal(expected.CreatedAt) { + t.Errorf("%s: createdAt mismatch: expected %v, got %v", context, expected.CreatedAt, actual.CreatedAt) + } + + // UpdatedAt is optional + if (expected.UpdatedAt == nil) != (actual.UpdatedAt == nil) { + t.Errorf("%s: updatedAt nil mismatch", context) + } else if expected.UpdatedAt != nil && !actual.UpdatedAt.Equal(*expected.UpdatedAt) { + t.Errorf("%s: updatedAt mismatch: expected %v, got %v", context, *expected.UpdatedAt, *actual.UpdatedAt) + } +} + +func verifyJobsEqual(t *testing.T, expected, actual *oapi.Job, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: job ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Status != expected.Status { + t.Errorf("%s: job status mismatch: expected %s, got %s", context, expected.Status, actual.Status) + } + if actual.JobAgentId != expected.JobAgentId { + t.Errorf("%s: job agent ID mismatch: expected %s, got %s", context, expected.JobAgentId, actual.JobAgentId) + } + if actual.ReleaseId != expected.ReleaseId { + t.Errorf("%s: release ID mismatch: expected %s, got %s", context, expected.ReleaseId, actual.ReleaseId) + } + // ExternalId is optional + if (expected.ExternalId == nil) != (actual.ExternalId == nil) { + t.Errorf("%s: externalId nil mismatch", context) + } else if expected.ExternalId != nil && *actual.ExternalId != *expected.ExternalId { + t.Errorf("%s: external ID mismatch: expected %s, got %s", context, *expected.ExternalId, *actual.ExternalId) + } + + // Verify metadata + if len(actual.Metadata) != len(expected.Metadata) { + t.Errorf("%s: metadata length mismatch: expected %d, got %d", context, len(expected.Metadata), len(actual.Metadata)) + } + for key, expectedValue := range expected.Metadata { + if actualValue, ok := actual.Metadata[key]; !ok { + t.Errorf("%s: metadata key %s missing", context, key) + } else if actualValue != expectedValue { + t.Errorf("%s: metadata[%s] mismatch: expected %s, got %s", context, key, expectedValue, actualValue) + } + } + + // Verify timestamps + if !actual.CreatedAt.Equal(expected.CreatedAt) { + t.Errorf("%s: createdAt mismatch: expected %v, got %v", context, expected.CreatedAt, actual.CreatedAt) + } + if !actual.UpdatedAt.Equal(expected.UpdatedAt) { + t.Errorf("%s: updatedAt mismatch: expected %v, got %v", context, expected.UpdatedAt, actual.UpdatedAt) + } + + // Verify optional timestamps + if (expected.StartedAt == nil) != (actual.StartedAt == nil) { + t.Errorf("%s: startedAt nil mismatch", context) + } else if expected.StartedAt != nil && !actual.StartedAt.Equal(*expected.StartedAt) { + t.Errorf("%s: startedAt mismatch: expected %v, got %v", context, *expected.StartedAt, *actual.StartedAt) + } + + if (expected.CompletedAt == nil) != (actual.CompletedAt == nil) { + t.Errorf("%s: completedAt nil mismatch", context) + } else if expected.CompletedAt != nil && !actual.CompletedAt.Equal(*expected.CompletedAt) { + t.Errorf("%s: completedAt mismatch: expected %v, got %v", context, *expected.CompletedAt, *actual.CompletedAt) + } +} + +func verifyDeploymentsEqual(t *testing.T, expected, actual *oapi.Deployment, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: deployment ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: deployment name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + // Description is optional + if (expected.Description == nil) != (actual.Description == nil) { + t.Errorf("%s: description nil mismatch", context) + } else if expected.Description != nil && *actual.Description != *expected.Description { + t.Errorf("%s: deployment description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) + } + if actual.SystemId != expected.SystemId { + t.Errorf("%s: system ID mismatch: expected %s, got %s", context, expected.SystemId, actual.SystemId) + } + // JobAgentId is optional + if (expected.JobAgentId == nil) != (actual.JobAgentId == nil) { + t.Errorf("%s: jobAgentId nil mismatch", context) + } else if expected.JobAgentId != nil && *actual.JobAgentId != *expected.JobAgentId { + t.Errorf("%s: job agent ID mismatch: expected %s, got %s", context, *expected.JobAgentId, *actual.JobAgentId) + } +} + +func verifySystemsEqual(t *testing.T, expected, actual *oapi.System, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: system ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: system name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + // Description is optional + if (expected.Description == nil) != (actual.Description == nil) { + t.Errorf("%s: description nil mismatch", context) + } else if expected.Description != nil && *actual.Description != *expected.Description { + t.Errorf("%s: system description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) + } + if actual.WorkspaceId != expected.WorkspaceId { + t.Errorf("%s: workspace ID mismatch: expected %s, got %s", context, expected.WorkspaceId, actual.WorkspaceId) + } +} + +func verifyEnvironmentsEqual(t *testing.T, expected, actual *oapi.Environment, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: environment ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: environment name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + // Description is optional + if (expected.Description == nil) != (actual.Description == nil) { + t.Errorf("%s: description nil mismatch", context) + } else if expected.Description != nil && *actual.Description != *expected.Description { + t.Errorf("%s: environment description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) + } + if actual.SystemId != expected.SystemId { + t.Errorf("%s: system ID mismatch: expected %s, got %s", context, expected.SystemId, actual.SystemId) + } +} + +func verifyJobAgentsEqual(t *testing.T, expected, actual *oapi.JobAgent, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: job agent ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: job agent name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + if actual.Type != expected.Type { + t.Errorf("%s: job agent type mismatch: expected %s, got %s", context, expected.Type, actual.Type) + } + if actual.WorkspaceId != expected.WorkspaceId { + t.Errorf("%s: workspace ID mismatch: expected %s, got %s", context, expected.WorkspaceId, actual.WorkspaceId) + } +} + +func verifyPoliciesEqual(t *testing.T, expected, actual *oapi.Policy, context string) { + t.Helper() + if actual.Id != expected.Id { + t.Errorf("%s: policy ID mismatch: expected %s, got %s", context, expected.Id, actual.Id) + } + if actual.Name != expected.Name { + t.Errorf("%s: policy name mismatch: expected %s, got %s", context, expected.Name, actual.Name) + } + // Description is optional + if (expected.Description == nil) != (actual.Description == nil) { + t.Errorf("%s: description nil mismatch", context) + } else if expected.Description != nil && *actual.Description != *expected.Description { + t.Errorf("%s: policy description mismatch: expected %s, got %s", context, *expected.Description, *actual.Description) + } + if actual.WorkspaceId != expected.WorkspaceId { + t.Errorf("%s: workspace ID mismatch: expected %s, got %s", context, expected.WorkspaceId, actual.WorkspaceId) + } + + // Verify rules array + if len(actual.Rules) != len(expected.Rules) { + t.Errorf("%s: rules length mismatch: expected %d, got %d", context, len(expected.Rules), len(actual.Rules)) + } + + // Verify selectors array + if len(actual.Selectors) != len(expected.Selectors) { + t.Errorf("%s: selectors length mismatch: expected %d, got %d", context, len(expected.Selectors), len(actual.Selectors)) + } +} + +// Helper function to create pointer to time.Time +func ptrTime(t time.Time) *time.Time { + return &t +}