Merge remote-tracking branch 'origin/main' into getDlqJobDetailsApi

Makefile

@@ -38,13 +38,13 @@ generate:
 	@make format
 
 generate-mocks:
-	@mockery --srcpkg=buf.build/gen/go/gotocompany/proton/grpc/go/gotocompany/siren/v1beta1/sirenv1beta1grpc --name=SirenServiceClient
-	@mockery --srcpkg=buf.build/gen/go/gotocompany/proton/grpc/go/gotocompany/shield/v1beta1/shieldv1beta1grpc --name=ShieldServiceClient
-	@mockery --srcpkg=buf.build/gen/go/gotocompany/proton/grpc/go/gotocompany/optimus/core/v1beta1/corev1beta1grpc --name=JobSpecificationServiceClient
-	@mockery --srcpkg=buf.build/gen/go/gotocompany/proton/grpc/go/gotocompany/entropy/v1beta1/entropyv1beta1grpc --name=ResourceServiceClient
-	@mockery --srcpkg=./internal/server/gcs --name=BlobStorageClient
-	@mockery --srcpkg=./internal/server/gcs --name=BlobObjectClient
-	@mockery --srcpkg=./internal/server/gcs --name=ObjectIterator
+	@mockery --with-expecter --srcpkg=buf.build/gen/go/gotocompany/proton/grpc/go/gotocompany/siren/v1beta1/sirenv1beta1grpc --name=SirenServiceClient
+	@mockery --with-expecter --srcpkg=buf.build/gen/go/gotocompany/proton/grpc/go/gotocompany/shield/v1beta1/shieldv1beta1grpc --name=ShieldServiceClient
+	@mockery --with-expecter --srcpkg=buf.build/gen/go/gotocompany/proton/grpc/go/gotocompany/optimus/core/v1beta1/corev1beta1grpc --name=JobSpecificationServiceClient
+	@mockery --with-expecter --srcpkg=buf.build/gen/go/gotocompany/proton/grpc/go/gotocompany/entropy/v1beta1/entropyv1beta1grpc --name=ResourceServiceClient
+	@mockery --with-expecter --srcpkg=./internal/server/gcs --name=BlobStorageClient
+	@mockery --with-expecter --srcpkg=./internal/server/gcs --name=BlobObjectClient
+	@mockery --with-expecter --srcpkg=./internal/server/gcs --name=ObjectIterator
 
 clean: tidy
 	@echo "Cleaning up build directories..."

README.md

@@ -1,3 +1,7 @@
 # dex
 
 Data Experience
+
+## Setting up GCloud Credentials
+1. login to cloud - `gcloud auth login`
+2. setup ADC - `gcloud auth application-default login`

cli/server/configs.go

@@ -24,6 +24,11 @@ type serverConfig struct {
 	Optimus     optimusConfig    `mapstructure:"optimus"`
 	StencilAddr string           `mapstructure:"stencil_addr"`
 	Dlq         dlqConfig        `mapstructure:"dlq"`
+	Warden      wardenConfig     `mapstructure:"warden"`
+}
+
+type wardenConfig struct {
+	Addr string `mapstructure:"addr"`
 }
 
 type odinConfig struct {

cli/server/server.go

@@ -20,6 +20,7 @@ import (
 	"github.com/goto/dex/internal/server/v1/optimus"
 	"github.com/goto/dex/pkg/logger"
 	"github.com/goto/dex/pkg/telemetry"
+	"github.com/goto/dex/warden"
 )
 
 func Commands() *cobra.Command {
@@ -102,8 +103,8 @@ func runServer(baseCtx context.Context, nrApp *newrelic.Application, zapLog *zap
 		return err
 	}
 
-	dlqConfig := &dlq.DlqJobConfig{
-		// TODO: map cfg.Dlq\
+	wardenClient := warden.NewClient(cfg.Warden.Addr)
+	dlqConfig := dlq.DlqJobConfig{
 		DlqJobImage:    cfg.Dlq.DlqJobImage,
 		PrometheusHost: cfg.Dlq.PrometheusHost,
 	}
@@ -117,6 +118,7 @@ func runServer(baseCtx context.Context, nrApp *newrelic.Application, zapLog *zap
 		&gcs.Client{StorageClient: gcsClient},
 		cfg.Odin.Addr,
 		cfg.StencilAddr,
+		wardenClient,
 		dlqConfig,
 	)
 }

internal/server/server.go

@@ -20,9 +20,11 @@ import (
 	alertsv1 "github.com/goto/dex/internal/server/v1/alert"
 	dlqv1 "github.com/goto/dex/internal/server/v1/dlq"
 	firehosev1 "github.com/goto/dex/internal/server/v1/firehose"
+	iamv1 "github.com/goto/dex/internal/server/v1/iam"
 	kubernetesv1 "github.com/goto/dex/internal/server/v1/kubernetes"
 	optimusv1 "github.com/goto/dex/internal/server/v1/optimus"
 	projectsv1 "github.com/goto/dex/internal/server/v1/project"
+	"github.com/goto/dex/warden"
 )
 
 // Serve initialises all the HTTP API routes, starts listening for requests at addr, and blocks until
@@ -37,7 +39,8 @@ func Serve(ctx context.Context, addr string,
 	gcsClient gcs.BlobStorageClient,
 	odinAddr string,
 	stencilAddr string,
-	dlqConfig *dlqv1.DlqJobConfig,
+	wardenClient *warden.Client,
+	dlqConfig dlqv1.DlqJobConfig,
 ) error {
 	alertSvc := alertsv1.NewService(sirenClient)
 
@@ -66,6 +69,7 @@ func Serve(ctx context.Context, addr string,
 		r.Route("/dlq", dlqv1.Routes(entropyClient, gcsClient, dlqConfig))
 		r.Route("/firehoses", firehosev1.Routes(entropyClient, shieldClient, alertSvc, compassClient, odinAddr, stencilAddr))
 		r.Route("/kubernetes", kubernetesv1.Routes(entropyClient))
+		r.Route("/iam", iamv1.Routes(shieldClient, wardenClient))
 	})
 
 	logger.Info("starting server", zap.String("addr", addr))

internal/server/v1/dlq/errors.go

@@ -5,5 +5,8 @@ import "errors"
 var (
 	ErrFirehoseNamespaceNotFound = errors.New("could not find firehose namespace from resource output")
 	ErrFirehoseNamespaceInvalid  = errors.New("invalid firehose namespace from resource output")
-	ErrJobNotFound               = errors.New("no job found for this URN")
+	ErrFirehoseNotFound          = errors.New("firehose not found")
+	ErrEmptyConfigImage          = errors.New("empty dlq job image")
+	ErrEmptyConfigPrometheusHost = errors.New("empty prometheus host")
+	ErrJobNotFound               = errors.New("no dlq job is found for this URN")
 )

internal/server/v1/dlq/fixtures/list_dlq_jobs.json

@@ -0,0 +1,38 @@
+{
+    "dlq_jobs": [
+        {
+            "resource_id": "test-resource-id",
+            "resource_type": "test-resource-type",
+            "topic": "test-topic",
+            "date": "2022-10-21",
+            "name": "test1-firehose-test-topic-2022-10-21",
+            "group": "test-group",
+            "kube_cluster": "test-kube-cluster",
+            "project": "test-project-1",
+            "prometheus_host": "prom_host",
+            "urn": "test-urn-1",
+            "status": "STATUS_UNSPECIFIED",
+            "created_at": "2022-12-10T00:00:00.000Z",
+            "created_by": "user@test.com",
+            "updated_at": "2023-12-10T02:00:00.000Z",
+            "updated_by": "user@test.com"
+        },
+        {
+            "resource_id": "test-resource-id",
+            "resource_type": "test-resource-type",
+            "topic": "test-topic",
+            "date": "2022-10-21",
+            "name": "test2-firehose-test-topic-2022-10-21",
+            "group": "test-group",
+            "kube_cluster": "test-kube-cluster",
+            "project": "test-project-1",
+            "prometheus_host": "prom_host2",
+            "urn": "test-urn-2",
+            "status": "STATUS_UNSPECIFIED",
+            "created_at": "2012-10-10T04:00:00.000Z",
+            "created_by": "user@test.com",
+            "updated_at": "2013-02-12T02:04:00.000Z",
+            "updated_by": "user@test.com"
+        }
+    ]
+}