From cc38cb3af3979e23401022e2e220bf1264d56b4a Mon Sep 17 00:00:00 2001 From: Milan Pavlik Date: Tue, 13 Sep 2022 21:02:21 +0000 Subject: [PATCH] [content-service] Remove UsageReport service --- .../content-service-api/go/config/config.go | 7 +- components/content-service/cmd/run.go | 6 - .../pkg/service/usage-report-service.go | 105 ------------------ .../pkg/service/usage-report-service_test.go | 61 ---------- .../testdata/render/aws-setup/output.golden | 5 +- .../testdata/render/azure-setup/output.golden | 5 +- .../render/customization/output.golden | 5 +- .../render/external-registry/output.golden | 5 +- .../testdata/render/gcp-setup/output.golden | 5 +- .../testdata/render/http-proxy/output.golden | 5 +- .../cmd/testdata/render/minimal/output.golden | 5 +- .../statefulset-customization/output.golden | 5 +- .../use-pod-security-policies/output.golden | 5 +- .../workspace-requests-limits/output.golden | 5 +- .../components/content-service/configmap.go | 13 --- .../content-service/configmap_test.go | 63 ----------- .../config/v1/experimental/experimental.go | 1 + 17 files changed, 15 insertions(+), 291 deletions(-) delete mode 100644 components/content-service/pkg/service/usage-report-service.go delete mode 100644 components/content-service/pkg/service/usage-report-service_test.go delete mode 100644 install/installer/pkg/components/content-service/configmap_test.go diff --git a/components/content-service-api/go/config/config.go b/components/content-service-api/go/config/config.go index bedbe4baa65af9..22f426f0f7977c 100644 --- a/components/content-service-api/go/config/config.go +++ b/components/content-service-api/go/config/config.go @@ -112,7 +112,8 @@ type UsageReportConfig struct { } type ServiceConfig struct { - Service baseserver.ServerConfiguration `json:"service"` - Storage StorageConfig `json:"storage"` - UsageReports UsageReportConfig `json:"usageReport"` + Service baseserver.ServerConfiguration `json:"service"` + Storage StorageConfig `json:"storage"` + // Deprecated + _ UsageReportConfig `json:"usageReport"` } diff --git a/components/content-service/cmd/run.go b/components/content-service/cmd/run.go index 7ee580462c6bb0..6519ef12202b4e 100644 --- a/components/content-service/cmd/run.go +++ b/components/content-service/cmd/run.go @@ -57,12 +57,6 @@ var runCmd = &cobra.Command{ } api.RegisterIDEPluginServiceServer(srv.GRPC(), idePluginService) - usageReportService, err := service.NewUsageReportService(cfg.Storage, cfg.UsageReports.BucketName) - if err != nil { - log.WithError(err).Fatalf("Cannot create usage report service") - } - api.RegisterUsageReportServiceServer(srv.GRPC(), usageReportService) - err = srv.ListenAndServe() if err != nil { log.WithError(err).Fatal("Cannot start server") diff --git a/components/content-service/pkg/service/usage-report-service.go b/components/content-service/pkg/service/usage-report-service.go deleted file mode 100644 index c2820d0877f542..00000000000000 --- a/components/content-service/pkg/service/usage-report-service.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2022 Gitpod GmbH. All rights reserved. -// Licensed under the GNU Affero General Public License (AGPL). -// See License-AGPL.txt in the project root for license information. - -package service - -import ( - "context" - "errors" - "github.com/opentracing/opentracing-go" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/gitpod-io/gitpod/common-go/log" - "github.com/gitpod-io/gitpod/common-go/tracing" - "github.com/gitpod-io/gitpod/content-service/api" - "github.com/gitpod-io/gitpod/content-service/api/config" - "github.com/gitpod-io/gitpod/content-service/pkg/storage" -) - -// UsageReportService implements UsageReportServiceServer -type UsageReportService struct { - cfg config.StorageConfig - s storage.PresignedAccess - bucketName string - - api.UnimplementedUsageReportServiceServer -} - -// NewUsageReportService create a new usagereport service -func NewUsageReportService(cfg config.StorageConfig, bucketName string) (res *UsageReportService, err error) { - s, err := storage.NewPresignedAccess(&cfg) - if err != nil { - return nil, err - } - return &UsageReportService{cfg: cfg, s: s, bucketName: bucketName}, nil -} - -// UploadURL provides a URL to which clients can upload the content via HTTP PUT. -func (us *UsageReportService) UploadURL(ctx context.Context, req *api.UsageReportUploadURLRequest) (resp *api.UsageReportUploadURLResponse, err error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "UsageReport.UploadURL") - span.SetTag("name", req.Name) - defer tracing.FinishSpan(span, &err) - - if req.GetName() == "" { - return nil, status.Error(codes.InvalidArgument, "Name is required but got empty.") - } - - logger := log.WithField("name", req.Name). - WithField("bucket", us.bucketName) - - err = us.ensureBucketExists(ctx) - if err != nil { - return nil, err - } - - info, err := us.s.SignUpload(ctx, us.bucketName, req.Name, &storage.SignedURLOptions{ - ContentType: "application/json", - }) - if err != nil { - logger.WithError(err).Error("Error getting UsageReport SignUpload URL") - return nil, status.Error(codes.Internal, err.Error()) - } - return &api.UsageReportUploadURLResponse{Url: info.URL}, nil -} - -func (us *UsageReportService) DownloadURL(ctx context.Context, req *api.UsageReportDownloadURLRequest) (resp *api.UsageReportDownloadURLResponse, err error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "UsageReport.DownloadURL") - span.SetTag("name", req.Name) - defer tracing.FinishSpan(span, &err) - - if req.GetName() == "" { - return nil, status.Error(codes.InvalidArgument, "Name is required but got empty.") - } - - err = us.ensureBucketExists(ctx) - if err != nil { - return nil, err - } - - download, err := us.s.SignDownload(ctx, us.bucketName, req.GetName(), &storage.SignedURLOptions{ - ContentType: "application/json", - }) - if err != nil { - if errors.Is(err, storage.ErrNotFound) { - return nil, status.Errorf(codes.NotFound, "Object %s does not exist.", req.GetName()) - } - - return nil, status.Errorf(codes.Internal, "Failed to generate download URL for usage report: %s", err.Error()) - } - - return &api.UsageReportDownloadURLResponse{ - Url: download.URL, - }, nil -} - -func (us *UsageReportService) ensureBucketExists(ctx context.Context) error { - err := us.s.EnsureExists(ctx, us.bucketName) - if err != nil { - log.WithError(err).Errorf("Bucket %s does not exist", us.bucketName) - return status.Error(codes.Internal, err.Error()) - } - - return nil -} diff --git a/components/content-service/pkg/service/usage-report-service_test.go b/components/content-service/pkg/service/usage-report-service_test.go deleted file mode 100644 index a8e34a1a170f8f..00000000000000 --- a/components/content-service/pkg/service/usage-report-service_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2022 Gitpod GmbH. All rights reserved. -// Licensed under the GNU Affero General Public License (AGPL). -// See License-AGPL.txt in the project root for license information. - -package service - -import ( - "context" - "testing" - - "github.com/gitpod-io/gitpod/content-service/api" - "github.com/gitpod-io/gitpod/content-service/api/config" - "github.com/gitpod-io/gitpod/content-service/pkg/storage" - storagemock "github.com/gitpod-io/gitpod/content-service/pkg/storage/mock" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/require" -) - -// TestUploadURL tests that usageReportService.UploadURL interacts with PresignedAccess -// correctly to produce an upload URL for the correct bucket and filename. -func TestUploadURL(t *testing.T) { - const ( - fileName = "some-report-filename" - bucketName = "gitpod-usage-reports" - ) - - ctrl := gomock.NewController(t) - s := storagemock.NewMockPresignedAccess(ctrl) - - s.EXPECT().EnsureExists(gomock.Any(), bucketName). - Return(nil) - s.EXPECT().SignUpload(gomock.Any(), bucketName, fileName, gomock.Any()). - Return(&storage.UploadInfo{URL: "http://example.com/some-path"}, nil) - - svc := &UsageReportService{cfg: config.StorageConfig{}, s: s, bucketName: bucketName} - resp, err := svc.UploadURL(context.Background(), &api.UsageReportUploadURLRequest{Name: fileName}) - - require.NoError(t, err) - require.Equal(t, "http://example.com/some-path", resp.Url) -} - -func TestDownloadURL(t *testing.T) { - const ( - fileName = "some-report-filename" - bucketName = "gitpod-usage-reports" - ) - - ctrl := gomock.NewController(t) - s := storagemock.NewMockPresignedAccess(ctrl) - - s.EXPECT().EnsureExists(gomock.Any(), bucketName). - Return(nil) - s.EXPECT().SignDownload(gomock.Any(), bucketName, fileName, gomock.Any()). - Return(&storage.DownloadInfo{URL: "http://example.com/some-path"}, nil) - - svc := &UsageReportService{cfg: config.StorageConfig{}, s: s, bucketName: bucketName} - resp, err := svc.DownloadURL(context.Background(), &api.UsageReportDownloadURLRequest{Name: fileName}) - - require.NoError(t, err) - require.Equal(t, "http://example.com/some-path", resp.Url) -} diff --git a/install/installer/cmd/testdata/render/aws-setup/output.golden b/install/installer/cmd/testdata/render/aws-setup/output.golden index 74e8fc5c638b44..ec6228aab390c5 100644 --- a/install/installer/cmd/testdata/render/aws-setup/output.golden +++ b/install/installer/cmd/testdata/render/aws-setup/output.golden @@ -1240,9 +1240,6 @@ data: "bucket": "s3-bucket" }, "blobQuota": 5368709120 - }, - "usageReport": { - "bucketName": "gitpod-usage-reports" } } kind: ConfigMap @@ -7227,7 +7224,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 2e45691c9b7dbd15eefd950854dfc5d89500012dc70f6a624069a404c72f9010 + gitpod.io/checksum_config: 8a160b6e8470e7a514961a4530576b4fa6bfb60eae6a275d05f2b488542a9e9b creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/azure-setup/output.golden b/install/installer/cmd/testdata/render/azure-setup/output.golden index 913d947f88176d..70aaed8cddfcb0 100644 --- a/install/installer/cmd/testdata/render/azure-setup/output.golden +++ b/install/installer/cmd/testdata/render/azure-setup/output.golden @@ -1202,9 +1202,6 @@ data: "parallelUpload": 6 }, "blobQuota": 5368709120 - }, - "usageReport": { - "bucketName": "gitpod-usage-reports" } } kind: ConfigMap @@ -7067,7 +7064,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 9f2fefeff4c2a12ff62db1ffafb3d9af6b91ba0b536b6e8664b55c43a1aec3c0 + gitpod.io/checksum_config: 20782435cc6d6e080cf720b57dca8d4d5cfe40942fc71c4bfd54934fdc34c766 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/customization/output.golden b/install/installer/cmd/testdata/render/customization/output.golden index c95cffa3963e06..7a6f7c59617783 100644 --- a/install/installer/cmd/testdata/render/customization/output.golden +++ b/install/installer/cmd/testdata/render/customization/output.golden @@ -1419,9 +1419,6 @@ data: "parallelUpload": 6 }, "blobQuota": 5368709120 - }, - "usageReport": { - "bucketName": "gitpod-usage-reports" } } kind: ConfigMap @@ -8441,7 +8438,7 @@ spec: metadata: annotations: gitpod.io: hello - gitpod.io/checksum_config: d72d95c5bd425eaabe281cea259139df35cc6343145eadac5f2e95aae30360ab + gitpod.io/checksum_config: efaa4ecb9c874bc91c8f6000604bbbc61f5294f388703b19ec5cdb58c303ea23 hello: world creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/external-registry/output.golden b/install/installer/cmd/testdata/render/external-registry/output.golden index f27198912056b6..3b6484fe965cab 100644 --- a/install/installer/cmd/testdata/render/external-registry/output.golden +++ b/install/installer/cmd/testdata/render/external-registry/output.golden @@ -1233,9 +1233,6 @@ data: "parallelUpload": 6 }, "blobQuota": 5368709120 - }, - "usageReport": { - "bucketName": "gitpod-usage-reports" } } kind: ConfigMap @@ -7508,7 +7505,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 9f2fefeff4c2a12ff62db1ffafb3d9af6b91ba0b536b6e8664b55c43a1aec3c0 + gitpod.io/checksum_config: 20782435cc6d6e080cf720b57dca8d4d5cfe40942fc71c4bfd54934fdc34c766 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/gcp-setup/output.golden b/install/installer/cmd/testdata/render/gcp-setup/output.golden index 10e375a9a805e3..04c12639a59656 100644 --- a/install/installer/cmd/testdata/render/gcp-setup/output.golden +++ b/install/installer/cmd/testdata/render/gcp-setup/output.golden @@ -1178,9 +1178,6 @@ data: "region": "" }, "blobQuota": 5368709120 - }, - "usageReport": { - "bucketName": "gitpod-usage-reports" } } kind: ConfigMap @@ -7116,7 +7113,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 161f3cb1cb0878446fc17e8b20667e54261e2ebe7ac8212e33a63e2674507435 + gitpod.io/checksum_config: facd640dff87a0bd161053705fc8e34204a605c5eeaf445f6f5ef8e5329f486a creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/http-proxy/output.golden b/install/installer/cmd/testdata/render/http-proxy/output.golden index 86c5125da3eed7..5fe849752b1786 100644 --- a/install/installer/cmd/testdata/render/http-proxy/output.golden +++ b/install/installer/cmd/testdata/render/http-proxy/output.golden @@ -1304,9 +1304,6 @@ data: "parallelUpload": 6 }, "blobQuota": 5368709120 - }, - "usageReport": { - "bucketName": "gitpod-usage-reports" } } kind: ConfigMap @@ -8353,7 +8350,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 9f2fefeff4c2a12ff62db1ffafb3d9af6b91ba0b536b6e8664b55c43a1aec3c0 + gitpod.io/checksum_config: 20782435cc6d6e080cf720b57dca8d4d5cfe40942fc71c4bfd54934fdc34c766 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/minimal/output.golden b/install/installer/cmd/testdata/render/minimal/output.golden index d320b0b833da64..c3879a6a9f8267 100644 --- a/install/installer/cmd/testdata/render/minimal/output.golden +++ b/install/installer/cmd/testdata/render/minimal/output.golden @@ -1304,9 +1304,6 @@ data: "parallelUpload": 6 }, "blobQuota": 5368709120 - }, - "usageReport": { - "bucketName": "gitpod-usage-reports" } } kind: ConfigMap @@ -7788,7 +7785,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 9f2fefeff4c2a12ff62db1ffafb3d9af6b91ba0b536b6e8664b55c43a1aec3c0 + gitpod.io/checksum_config: 20782435cc6d6e080cf720b57dca8d4d5cfe40942fc71c4bfd54934fdc34c766 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/statefulset-customization/output.golden b/install/installer/cmd/testdata/render/statefulset-customization/output.golden index 023d7f14754a9d..35158ccde36a34 100644 --- a/install/installer/cmd/testdata/render/statefulset-customization/output.golden +++ b/install/installer/cmd/testdata/render/statefulset-customization/output.golden @@ -1304,9 +1304,6 @@ data: "parallelUpload": 6 }, "blobQuota": 5368709120 - }, - "usageReport": { - "bucketName": "gitpod-usage-reports" } } kind: ConfigMap @@ -7800,7 +7797,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 9f2fefeff4c2a12ff62db1ffafb3d9af6b91ba0b536b6e8664b55c43a1aec3c0 + gitpod.io/checksum_config: 20782435cc6d6e080cf720b57dca8d4d5cfe40942fc71c4bfd54934fdc34c766 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden b/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden index c69e46e760a70a..d8c05447d2516f 100644 --- a/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden +++ b/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden @@ -1526,9 +1526,6 @@ data: "parallelUpload": 6 }, "blobQuota": 5368709120 - }, - "usageReport": { - "bucketName": "gitpod-usage-reports" } } kind: ConfigMap @@ -8232,7 +8229,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 9f2fefeff4c2a12ff62db1ffafb3d9af6b91ba0b536b6e8664b55c43a1aec3c0 + gitpod.io/checksum_config: 20782435cc6d6e080cf720b57dca8d4d5cfe40942fc71c4bfd54934fdc34c766 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden b/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden index 12425c449587ed..0570a67f9b7e32 100644 --- a/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden +++ b/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden @@ -1304,9 +1304,6 @@ data: "parallelUpload": 6 }, "blobQuota": 5368709120 - }, - "usageReport": { - "bucketName": "gitpod-usage-reports" } } kind: ConfigMap @@ -7791,7 +7788,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 9f2fefeff4c2a12ff62db1ffafb3d9af6b91ba0b536b6e8664b55c43a1aec3c0 + gitpod.io/checksum_config: 20782435cc6d6e080cf720b57dca8d4d5cfe40942fc71c4bfd54934fdc34c766 creationTimestamp: null labels: app: gitpod diff --git a/install/installer/pkg/components/content-service/configmap.go b/install/installer/pkg/components/content-service/configmap.go index 9927ac755789a4..b49660abc27ff0 100644 --- a/install/installer/pkg/components/content-service/configmap.go +++ b/install/installer/pkg/components/content-service/configmap.go @@ -11,30 +11,17 @@ import ( "github.com/gitpod-io/gitpod/content-service/api/config" "github.com/gitpod-io/gitpod/installer/pkg/common" - "github.com/gitpod-io/gitpod/installer/pkg/config/v1/experimental" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { - usageReportBucketName := "gitpod-usage-reports" - _ = ctx.WithExperimental(func(cfg *experimental.Config) error { - if cfg.Workspace != nil && cfg.Workspace.ContentService.UsageReportBucketName != "" { - usageReportBucketName = cfg.Workspace.ContentService.UsageReportBucketName - } - return nil - }) - cscfg := config.ServiceConfig{ Service: baseserver.ServerConfiguration{ Address: fmt.Sprintf("0.0.0.0:%d", RPCPort), }, Storage: common.StorageConfig(ctx), - UsageReports: config.UsageReportConfig{ - BucketName: usageReportBucketName, - }, } fc, err := common.ToJSONString(cscfg) diff --git a/install/installer/pkg/components/content-service/configmap_test.go b/install/installer/pkg/components/content-service/configmap_test.go deleted file mode 100644 index e439b510c23b4a..00000000000000 --- a/install/installer/pkg/components/content-service/configmap_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) 2022 Gitpod GmbH. All rights reserved. -// Licensed under the MIT License. See License-MIT.txt in the project root for license information. - -package content_service - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - "k8s.io/utils/pointer" - - csconfig "github.com/gitpod-io/gitpod/content-service/api/config" - "github.com/gitpod-io/gitpod/installer/pkg/common" - config "github.com/gitpod-io/gitpod/installer/pkg/config/v1" - "github.com/gitpod-io/gitpod/installer/pkg/config/v1/experimental" - "github.com/gitpod-io/gitpod/installer/pkg/config/versions" -) - -func TestConfigMap_CanConfigureUsageReportBucketName(t *testing.T) { - ctx := newRenderContext(t) - objs, err := configmap(ctx) - require.NoError(t, err) - require.Len(t, objs, 1, "must only render one configmap") - - expectedUsageReportConfig := csconfig.UsageReportConfig{BucketName: "some-bucket-name"} - expectedJSON, err := common.ToJSONString(expectedUsageReportConfig) - require.NoError(t, err) - - cm, ok := objs[0].(*corev1.ConfigMap) - require.True(t, ok) - - var fullConfig csconfig.ServiceConfig - err = json.Unmarshal([]byte(cm.Data["config.json"]), &fullConfig) - require.NoError(t, err) - - actualUsageReportConfig := fullConfig.UsageReports - actualJSON, err := common.ToJSONString(actualUsageReportConfig) - require.NoError(t, err) - - require.JSONEq(t, string(expectedJSON), string(actualJSON)) -} - -func newRenderContext(t *testing.T) *common.RenderContext { - t.Helper() - - ctx, err := common.NewRenderContext(config.Config{ - Domain: "test.domain.everything.awesome.is", - ObjectStorage: config.ObjectStorage{InCluster: pointer.Bool(true)}, - Experimental: &experimental.Config{ - Workspace: &experimental.WorkspaceConfig{ - ContentService: struct { - UsageReportBucketName string "json:\"usageReportBucketName\"" - }{UsageReportBucketName: "some-bucket-name"}, - }, - }, - }, versions.Manifest{}, "test-namespace") - - require.NoError(t, err) - - return ctx -} diff --git a/install/installer/pkg/config/v1/experimental/experimental.go b/install/installer/pkg/config/v1/experimental/experimental.go index ff6e15816ff0f4..3f03651f525f13 100644 --- a/install/installer/pkg/config/v1/experimental/experimental.go +++ b/install/installer/pkg/config/v1/experimental/experimental.go @@ -115,6 +115,7 @@ type WorkspaceConfig struct { } `json:"wsProxy"` ContentService struct { + // Deprecated UsageReportBucketName string `json:"usageReportBucketName"` } `json:"contentService"` }