diff --git a/api/api/openapi.bundle.yaml b/api/api/openapi.bundle.yaml index 7a329f9ef..d77a8e746 100644 --- a/api/api/openapi.bundle.yaml +++ b/api/api/openapi.bundle.yaml @@ -1617,6 +1617,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name monitoring_url: monitoring_url environment_name: environment_name properties: @@ -1672,6 +1677,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name properties: artifact_uri: type: string @@ -1680,6 +1690,10 @@ components: service_account_name: type: string x-go-custom-tag: validate:"required" + secrets: + items: + $ref: '#/components/schemas/MountedMLPSecret' + type: array resources: $ref: '#/components/schemas/EnsemblingResources' run_id: @@ -2136,6 +2150,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout routes: - endpoint: endpoint @@ -2186,6 +2205,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout updated_at: 2000-01-23T04:56:07.000+00:00 standard_config: @@ -2220,6 +2244,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout properties: id: @@ -2438,6 +2467,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout properties: id: @@ -2460,6 +2494,10 @@ components: items: $ref: '#/components/schemas/EnvVar' type: array + secrets: + items: + $ref: '#/components/schemas/MountedMLPSecret' + type: array service_account: description: | (Optional) Name of the secret registered in the current MLP project that contains the Google service account JSON key. This secret will be mounted as a file inside the container and the environment variable GOOGLE_APPLICATION_CREDENTIALS will point to the service account file." @@ -2479,6 +2517,7 @@ components: - image - port - resource_request + - secrets - timeout type: object RouterEnsemblerConfig: @@ -2500,6 +2539,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout updated_at: 2000-01-23T04:56:07.000+00:00 standard_config: @@ -2534,6 +2578,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout properties: id: @@ -2609,6 +2658,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout nullable: true properties: @@ -2630,6 +2684,10 @@ components: items: $ref: '#/components/schemas/EnvVar' type: array + secrets: + items: + $ref: '#/components/schemas/MountedMLPSecret' + type: array service_account: description: | (Optional) Name of the secret registered in the current MLP project that contains the Google service account JSON key. This secret will be mounted as a file inside the container and the environment variable GOOGLE_APPLICATION_CREDENTIALS will point to the service account file." @@ -2641,6 +2699,7 @@ components: - image - port - resource_request + - secrets - timeout type: object EnsemblerPyfuncConfig: @@ -2662,6 +2721,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout nullable: true properties: @@ -2680,10 +2744,16 @@ components: items: $ref: '#/components/schemas/EnvVar' type: array + secrets: + items: + $ref: '#/components/schemas/MountedMLPSecret' + type: array required: - ensembler_id + - env - project_id - resource_request + - secrets - timeout type: object TrafficRule: @@ -2794,6 +2864,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout routes: - endpoint: endpoint @@ -2867,6 +2942,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout updated_at: 2000-01-23T04:56:07.000+00:00 standard_config: @@ -2901,6 +2981,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout log_config: bigquery_config: @@ -2957,6 +3042,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout routes: - endpoint: endpoint @@ -3030,6 +3120,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout updated_at: 2000-01-23T04:56:07.000+00:00 standard_config: @@ -3064,6 +3159,11 @@ components: value: value - name: name value: value + secrets: + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + - mlp_secret_name: mlp_secret_name + env_var_name: env_var_name timeout: timeout log_config: bigquery_config: @@ -3476,6 +3576,21 @@ components: format: int32 type: integer type: object + MountedMLPSecret: + example: + mlp_secret_name: mlp_secret_name + env_var_name: env_var_name + properties: + mlp_secret_name: + pattern: ^[-._a-zA-Z0-9]+$ + type: string + env_var_name: + pattern: ^[a-zA-Z0-9_]*$ + type: string + required: + - env_var_name + - mlp_secret_name + type: object EnvVar: example: name: name diff --git a/api/api/specs/common.yaml b/api/api/specs/common.yaml index 9fc3c79e2..045041544 100644 --- a/api/api/specs/common.yaml +++ b/api/api/specs/common.yaml @@ -35,6 +35,19 @@ components: value: type: "string" + MountedMLPSecret: + type: "object" + required: + - mlp_secret_name + - env_var_name + properties: + mlp_secret_name: + type: "string" + pattern: '^[-._a-zA-Z0-9]+$' + env_var_name: + type: "string" + pattern: '^[a-zA-Z0-9_]*$' + pagination.Paging: type: "object" properties: diff --git a/api/api/specs/jobs.yaml b/api/api/specs/jobs.yaml index 6d00223ab..9520e0048 100644 --- a/api/api/specs/jobs.yaml +++ b/api/api/specs/jobs.yaml @@ -417,6 +417,10 @@ components: service_account_name: type: string x-go-custom-tag: validate:"required" + secrets: + type: array + items: + $ref: "common.yaml#/components/schemas/MountedMLPSecret" resources: $ref: "#/components/schemas/EnsemblingResources" run_id: diff --git a/api/api/specs/routers.yaml b/api/api/specs/routers.yaml index b5116e258..e8bce212e 100644 --- a/api/api/specs/routers.yaml +++ b/api/api/specs/routers.yaml @@ -782,6 +782,7 @@ components: - timeout - port - env + - secrets properties: id: $ref: "common.yaml#/components/schemas/Id" @@ -801,6 +802,10 @@ components: type: "array" items: $ref: "common.yaml#/components/schemas/EnvVar" + secrets: + type: "array" + items: + $ref: "common.yaml#/components/schemas/MountedMLPSecret" service_account: type: "string" description: > @@ -888,6 +893,7 @@ components: - timeout - port - env + - secrets properties: image: type: "string" @@ -906,6 +912,10 @@ components: type: "array" items: $ref: "common.yaml#/components/schemas/EnvVar" + secrets: + type: "array" + items: + $ref: "common.yaml#/components/schemas/MountedMLPSecret" service_account: type: "string" description: > @@ -923,6 +933,8 @@ components: - ensembler_id - resource_request - timeout + - env + - secrets properties: project_id: type: "integer" @@ -938,6 +950,10 @@ components: type: "array" items: $ref: "common.yaml#/components/schemas/EnvVar" + secrets: + type: "array" + items: + $ref: "common.yaml#/components/schemas/MountedMLPSecret" ResourceRequest: type: "object" diff --git a/api/db-migrations/000016_add_secrets_columns.down.sql b/api/db-migrations/000016_add_secrets_columns.down.sql new file mode 100644 index 000000000..6506ba8d6 --- /dev/null +++ b/api/db-migrations/000016_add_secrets_columns.down.sql @@ -0,0 +1,7 @@ +-- Remove secrets column for enrichers +ALTER TABLE enrichers DROP COLUMN secrets; + +-- Remove secrets field in docker_config and pyfunc_config columns for ensemblers +UPDATE ensembler_configs set docker_config = docker_config - 'secrets' WHERE docker_config IS NOT NULL; + +UPDATE ensembler_configs set pyfunc_config = pyfunc_config - 'secrets' WHERE pyfunc_config IS NOT NULL; diff --git a/api/db-migrations/000016_add_secrets_columns.up.sql b/api/db-migrations/000016_add_secrets_columns.up.sql new file mode 100644 index 000000000..98c589ce3 --- /dev/null +++ b/api/db-migrations/000016_add_secrets_columns.up.sql @@ -0,0 +1,7 @@ +-- Create secrets column for enrichers +ALTER TABLE enrichers ADD COLUMN secrets jsonb NOT NULL DEFAULT '[]'::jsonb; + +-- Create secrets field in docker_config and pyfunc_config columns for ensemblers +UPDATE ensembler_configs SET docker_config = jsonb_set(docker_config, '{secrets}', '[]'::jsonb) WHERE docker_config IS NOT NULL AND docker_config->'secrets' IS NULL; + +UPDATE ensembler_configs SET pyfunc_config = jsonb_set(pyfunc_config, '{secrets}', '[]'::jsonb) WHERE pyfunc_config IS NOT NULL AND pyfunc_config->'secrets' IS NULL; diff --git a/api/e2e/test/testdata/create_router_nop_logger_proprietary_exp.json.tmpl b/api/e2e/test/testdata/create_router_nop_logger_proprietary_exp.json.tmpl index bb6b7ea37..4ab914f39 100644 --- a/api/e2e/test/testdata/create_router_nop_logger_proprietary_exp.json.tmpl +++ b/api/e2e/test/testdata/create_router_nop_logger_proprietary_exp.json.tmpl @@ -72,7 +72,8 @@ "name": "TEST_ENV", "value": "enricher" } - ] + ], + "secrets": [] }, "ensembler": { "type": "docker", @@ -92,7 +93,8 @@ "name": "TEST_ENV", "value": "ensembler" } - ] + ], + "secrets": [] } } } diff --git a/api/e2e/test/testdata/create_router_with_traffic_rules.json.tmpl b/api/e2e/test/testdata/create_router_with_traffic_rules.json.tmpl index d8a3dcb71..c9dae8468 100644 --- a/api/e2e/test/testdata/create_router_with_traffic_rules.json.tmpl +++ b/api/e2e/test/testdata/create_router_with_traffic_rules.json.tmpl @@ -83,7 +83,8 @@ "name": "TEST_ENV", "value": "ensembler" } - ] + ], + "secrets": [] } } } diff --git a/api/e2e/test/testdata/update_router_high_cpu.json.tmpl b/api/e2e/test/testdata/update_router_high_cpu.json.tmpl index 787ba2fd0..7c1d0d7db 100644 --- a/api/e2e/test/testdata/update_router_high_cpu.json.tmpl +++ b/api/e2e/test/testdata/update_router_high_cpu.json.tmpl @@ -40,7 +40,8 @@ "name": "TEST_ENV", "value": "enricher" } - ] + ], + "secrets": [] }, "ensembler": { "type": "docker", @@ -60,7 +61,8 @@ "name": "TEST_ENV", "value": "ensembler" } - ] + ], + "secrets": [] } } } diff --git a/api/openapi-codegen.yaml b/api/openapi-codegen.yaml index e7f4a10ec..d3435c1f1 100644 --- a/api/openapi-codegen.yaml +++ b/api/openapi-codegen.yaml @@ -25,3 +25,4 @@ globalProperties: - SaveMode - EnsemblerInfraConfig - EnvVar + - MountedMLPSecret diff --git a/api/turing/api/deployment_controller.go b/api/turing/api/deployment_controller.go index b5022e44e..bfa8278b9 100644 --- a/api/turing/api/deployment_controller.go +++ b/api/turing/api/deployment_controller.go @@ -5,11 +5,13 @@ import ( "encoding/json" "errors" "fmt" + "maps" "strings" merlin "github.com/caraml-dev/merlin/client" mlp "github.com/caraml-dev/mlp/api/client" + "github.com/caraml-dev/turing/api/turing/cluster/servicebuilder" "github.com/caraml-dev/turing/api/turing/models" "github.com/caraml-dev/turing/api/turing/service" "github.com/caraml-dev/turing/engines/experiment/manager" @@ -159,41 +161,12 @@ func (c RouterDeploymentController) deployRouterVersion( routerVersion *models.RouterVersion, eventsCh *service.EventChannel, ) (string, error) { - var routerServiceAccountKey, enricherServiceAccountKey, ensemblerServiceAccountKey, - expEngineServiceAccountKey string var experimentConfig json.RawMessage var err error - if routerVersion.LogConfig.ResultLoggerType == models.BigQueryLogger { - routerServiceAccountKey, err = c.MLPService.GetSecret( - models.ID(project.ID), - routerVersion.LogConfig.BigQueryConfig.ServiceAccountSecret, - ) - if err != nil { - return "", c.updateRouterVersionStatusToFailed(err, routerVersion) - } - } - - if routerVersion.Enricher != nil && routerVersion.Enricher.ServiceAccount != "" { - enricherServiceAccountKey, err = c.MLPService.GetSecret( - models.ID(project.ID), - routerVersion.Enricher.ServiceAccount, - ) - if err != nil { - return "", c.updateRouterVersionStatusToFailed(err, routerVersion) - } - } - - if routerVersion.Ensembler != nil && routerVersion.Ensembler.Type == models.EnsemblerDockerType { - if routerVersion.Ensembler.DockerConfig.ServiceAccount != "" { - ensemblerServiceAccountKey, err = c.MLPService.GetSecret( - models.ID(project.ID), - routerVersion.Ensembler.DockerConfig.ServiceAccount, - ) - if err != nil { - return "", c.updateRouterVersionStatusToFailed(err, routerVersion) - } - } + secretMap, err := c.getMLPSecrets(routerVersion, project) + if err != nil { + return "", c.updateRouterVersionStatusToFailed(err, routerVersion) } if routerVersion.ExperimentEngine.Type != models.ExperimentEngineTypeNop { @@ -208,7 +181,7 @@ func (c RouterDeploymentController) deployRouterVersion( if err != nil { return "", c.updateRouterVersionStatusToFailed(err, routerVersion) } - expEngineServiceAccountKey = *serviceAccountKey + secretMap[servicebuilder.SecretKeyNameExpEngine] = *serviceAccountKey } } @@ -258,10 +231,7 @@ func (c RouterDeploymentController) deployRouterVersion( environment, currRouterVersion, routerVersion, - routerServiceAccountKey, - enricherServiceAccountKey, - ensemblerServiceAccountKey, - expEngineServiceAccountKey, + secretMap, pyfuncEnsembler, experimentConfig, eventsCh, @@ -494,3 +464,98 @@ func (c RouterDeploymentController) getExperimentConfig(routerVersion *models.Ro return experimentConfig, nil } + +func (c RouterDeploymentController) getMLPSecrets( + routerVersion *models.RouterVersion, + project *mlp.Project, +) (map[string]string, error) { + secretMap := make(map[string]string) + + if routerVersion.LogConfig.ResultLoggerType == models.BigQueryLogger { + routerSecrets, err := c.getSecretsForComponent( + routerVersion.LogConfig.BigQueryConfig.ServiceAccountSecret, + servicebuilder.SecretKeyNameRouter, + []models.Secret{}, // there are no user-configured secrets for the router + project, + ) + if err != nil { + return nil, err + } + maps.Copy(secretMap, routerSecrets) + } + + if routerVersion.Enricher != nil { + enricherSecrets, err := c.getSecretsForComponent( + routerVersion.Enricher.ServiceAccount, + servicebuilder.SecretKeyNameEnricher, + routerVersion.Enricher.Secrets, + project, + ) + if err != nil { + return nil, err + } + maps.Copy(secretMap, enricherSecrets) + } + + if routerVersion.Ensembler != nil && routerVersion.Ensembler.Type == models.EnsemblerDockerType { + ensemblerSecrets, err := c.getSecretsForComponent( + routerVersion.Ensembler.DockerConfig.ServiceAccount, + servicebuilder.SecretKeyNameEnsembler, + routerVersion.Ensembler.DockerConfig.Secrets, + project, + ) + if err != nil { + return nil, err + } + maps.Copy(secretMap, ensemblerSecrets) + } + + if routerVersion.Ensembler != nil && routerVersion.Ensembler.Type == models.EnsemblerPyFuncType { + ensemblerSecrets, err := c.getSecretsForComponent( + "", // there are no service accounts for pyfunc ensemblers + "", + routerVersion.Ensembler.PyfuncConfig.Secrets, + project, + ) + if err != nil { + return nil, err + } + maps.Copy(secretMap, ensemblerSecrets) + } + + return secretMap, nil +} + +func (c RouterDeploymentController) getSecretsForComponent( + serviceAccountName string, + serviceAccountSecretKey string, + secrets []models.Secret, + project *mlp.Project, +) (map[string]string, error) { + secretMap := make(map[string]string) + // Retrieve Google Service Account secret from MLP + if serviceAccountName != "" { + serviceAccountKey, err := c.MLPService.GetSecret( + models.ID(project.ID), + serviceAccountName, + ) + if err != nil { + return nil, fmt.Errorf("service account %s is not found within %s project: %w", + serviceAccountName, project.Name, err) + } + secretMap[serviceAccountSecretKey] = serviceAccountKey + } + // Retrieve user-configured secrets from MLP + for _, secret := range secrets { + secretString, err := c.MLPService.GetSecret( + models.ID(project.ID), + secret.MLPSecretName, + ) + if err != nil { + return nil, fmt.Errorf("user-configured secret %s is not found within %s project: %w", + secret.MLPSecretName, project.Name, err) + } + secretMap[secret.MLPSecretName] = secretString + } + return secretMap, nil +} diff --git a/api/turing/api/deployment_controller_test.go b/api/turing/api/deployment_controller_test.go index 07277274d..7e1b082ce 100644 --- a/api/turing/api/deployment_controller_test.go +++ b/api/turing/api/deployment_controller_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/caraml-dev/turing/api/turing/cluster/servicebuilder" "github.com/caraml-dev/turing/api/turing/config" "github.com/caraml-dev/turing/api/turing/models" "github.com/caraml-dev/turing/api/turing/service" @@ -173,9 +174,9 @@ func TestDeployVersionSuccess(t *testing.T) { ds := &mocks.DeploymentService{} - ds.On("DeployRouterVersion", project, environment, (*models.RouterVersion)(router.CurrRouterVersion), - data.pendingVersion, "service-acct", "", "", "", mock.Anything, data.expRunnerCfg, eventsCh, - ).Return("test-url", nil) + ds.On("DeployRouterVersion", project, environment, router.CurrRouterVersion, data.pendingVersion, + map[string]string{servicebuilder.SecretKeyNameRouter: "service-acct"}, mock.Anything, data.expRunnerCfg, + eventsCh).Return("test-url", nil) // Create test controller ctrl := RouterDeploymentController{ @@ -286,8 +287,9 @@ func TestRollbackVersionSuccess(t *testing.T) { rvs.On("Save", newVerFailed).Return(newVerFailed, nil) ds := &mocks.DeploymentService{} - ds.On("DeployRouterVersion", project, environment, router.CurrRouterVersion, newVer, testSvcAcct, - "", "", "", mock.Anything, json.RawMessage(nil), mock.Anything).Return("", errors.New("error")) + ds.On("DeployRouterVersion", project, environment, router.CurrRouterVersion, newVer, + map[string]string{servicebuilder.SecretKeyNameRouter: testSvcAcct}, mock.Anything, json.RawMessage(nil), + mock.Anything).Return("", errors.New("error")) ds.On("UndeployRouterVersion", project, environment, newVer, mock.Anything, true). Return(nil) diff --git a/api/turing/api/request/request.go b/api/turing/api/request/request.go index 9456a73d2..c43cf3f1b 100644 --- a/api/turing/api/request/request.go +++ b/api/turing/api/request/request.go @@ -82,6 +82,8 @@ type EnricherEnsemblerConfig struct { Port int `json:"port" validate:"required"` // Environment variables to inject into the pod. Env models.EnvVars `json:"env" validate:"required"` + // MLP secrets to inject into the pod. + Secrets models.Secrets `json:"secrets" validate:"required"` // ServiceAccount specifies the name of the secret registered in the MLP project containing the service account. // The service account will be mounted into the user-container and the environment variable // GOOGLE_APPLICATION_CREDENTIALS will reference the service account file. @@ -98,6 +100,7 @@ func (cfg EnricherEnsemblerConfig) BuildEnricher() *models.Enricher { Timeout: cfg.Timeout, Port: cfg.Port, Env: cfg.Env, + Secrets: cfg.Secrets, ServiceAccount: cfg.ServiceAccount, } } diff --git a/api/turing/batch/ensembling/controller.go b/api/turing/batch/ensembling/controller.go index 5b430510e..ef6a2d117 100644 --- a/api/turing/batch/ensembling/controller.go +++ b/api/turing/batch/ensembling/controller.go @@ -131,20 +131,13 @@ func (c *ensemblingController) Create(request *CreateEnsemblingJobRequest) error ) } - secretString, err := c.mlpService.GetSecret( - request.EnsemblingJob.ProjectID, - request.EnsemblingJob.InfraConfig.GetServiceAccountName(), - ) + // Get MLP secrets for Google service account and user-specified MLP secrets + secretMap, err := c.getMLPSecrets(request.EnsemblingJob, request.Namespace) if err != nil { - return fmt.Errorf( - "service account %s is not found within %s project: %s", - request.EnsemblingJob.InfraConfig.GetServiceAccountName(), - request.Namespace, - err, - ) + return fmt.Errorf("error retrieving secrets: %w", err) } - err = c.createSecret(request, secretString) + err = c.createSecret(request, secretMap) if err != nil { return fmt.Errorf( "failed creating secret for job %s in namespace %s: %v", @@ -199,6 +192,7 @@ func (c *ensemblingController) createSparkApplication( ExecutorReplica: *infraConfig.GetResources().ExecutorReplica, ServiceAccountName: serviceAccount.Name, SparkInfraConfig: c.sparkInfraConfig, + Secrets: jobRequest.EnsemblingJob.InfraConfig.Secrets, EnvVars: jobRequest.EnsemblingJob.InfraConfig.Env, } return c.clusterController.CreateSparkApplication(context.Background(), jobRequest.Namespace, request) @@ -233,14 +227,12 @@ func (c *ensemblingController) createJobConfigMap( return nil } -func (c *ensemblingController) createSecret(request *CreateEnsemblingJobRequest, secretName string) error { +func (c *ensemblingController) createSecret(request *CreateEnsemblingJobRequest, secretMap map[string]string) error { secret := &cluster.Secret{ Name: request.EnsemblingJob.Name, Namespace: request.Namespace, - Data: map[string]string{ - cluster.ServiceAccountFileName: secretName, - }, - Labels: request.Labels, + Data: secretMap, + Labels: request.Labels, } // I'm not sure why we need to pass in a context here but not other kubernetes cluster functions. // Leaving a context.Background() until we figure out what to do with this. @@ -305,6 +297,38 @@ func (c *ensemblingController) createSparkDriverAuthorization( return sa, err } +func (c *ensemblingController) getMLPSecrets( + ensemblingJob *models.EnsemblingJob, + namespace string, +) (map[string]string, error) { + secretMap := make(map[string]string) + // Retrieve Google Service Account secret from MLP + secretString, err := c.mlpService.GetSecret( + ensemblingJob.ProjectID, + ensemblingJob.InfraConfig.GetServiceAccountName(), + ) + if err != nil { + return nil, fmt.Errorf("service account %s is not found within %s project: %w", + ensemblingJob.InfraConfig.GetServiceAccountName(), namespace, err) + } + secretMap[cluster.ServiceAccountFileName] = secretString + // Retrieve user-configured secrets from MLP + if ensemblingJob.InfraConfig.Secrets != nil { + for _, secret := range *ensemblingJob.InfraConfig.Secrets { + secretString, err = c.mlpService.GetSecret( + ensemblingJob.ProjectID, + secret.GetMlpSecretName(), + ) + if err != nil { + return nil, fmt.Errorf("user-configured secret %s is not found within %s project: %w", + secret.GetMlpSecretName(), namespace, err) + } + secretMap[secret.GetMlpSecretName()] = secretString + } + } + return secretMap, nil +} + func createAuthorizationResourceNames(namespace string) (string, string, string) { serviceAccountName := fmt.Sprintf("%s-driver-sa", namespace) driverRoleName := fmt.Sprintf("%s-driver-role", namespace) diff --git a/api/turing/batch/ensembling/controller_test.go b/api/turing/batch/ensembling/controller_test.go index 2c09ea7bb..0620078ab 100644 --- a/api/turing/batch/ensembling/controller_test.go +++ b/api/turing/batch/ensembling/controller_test.go @@ -159,6 +159,21 @@ func generateEnsemblingJobFixture() *models.EnsemblingJob { } } +func generateEnsemblingJobFixtureWithUserDefinedSecrets() *models.EnsemblingJob { + baseEnsemblingJob := generateEnsemblingJobFixture() + baseEnsemblingJob.InfraConfig.Secrets = &[]openapi.MountedMLPSecret{ + { + MlpSecretName: "MLP_SECRET_1", + EnvVarName: "SECRET_ENV_VAR_1", + }, + { + MlpSecretName: "MLP_SECRET_2", + EnvVarName: "SECRET_ENV_VAR_2", + }, + } + return baseEnsemblingJob +} + func TestCreate(t *testing.T) { tests := map[string]struct { expected error @@ -203,12 +218,22 @@ func TestCreate(t *testing.T) { svc.On( "GetSecret", mock.Anything, + "test-service-account", + ).Return("Alright then. Keep your secrets.", nil) + svc.On( + "GetSecret", + mock.Anything, + "MLP_SECRET_1", + ).Return("Alright then. Keep your secrets.", nil) + svc.On( + "GetSecret", mock.Anything, + "MLP_SECRET_2", ).Return("Alright then. Keep your secrets.", nil) return svc }, request: &CreateEnsemblingJobRequest{ - EnsemblingJob: generateEnsemblingJobFixture(), + EnsemblingJob: generateEnsemblingJobFixtureWithUserDefinedSecrets(), Labels: standardLabels, ImageRef: imageRef, Namespace: namespace, @@ -361,8 +386,8 @@ func TestCreate(t *testing.T) { Namespace: namespace, }, }, - "failure | fail to get secret": { - expected: fmt.Errorf("service account %s is not found within %s project: %s", + "failure | fail to get service account secret": { + expected: fmt.Errorf("error retrieving secrets: service account %s is not found within %s project: %w", "test-service-account", namespace, fmt.Errorf("hi"), @@ -412,6 +437,62 @@ func TestCreate(t *testing.T) { Namespace: namespace, }, }, + "failure | fail to get user-configured secret": { + expected: fmt.Errorf("error retrieving secrets: user-configured secret %s is not found within %s project: %w", + "MLP_SECRET_1", + namespace, + fmt.Errorf("hi"), + ), + clusterController: func() cluster.Controller { + ctrler := &clustermock.Controller{} + ctrler.On("CreateNamespace", mock.Anything, mock.Anything).Return(nil) + ctrler.On("CreateServiceAccount", mock.Anything, mock.Anything, mock.Anything).Return( + &apicorev1.ServiceAccount{ + ObjectMeta: apimetav1.ObjectMeta{ + Name: fmt.Sprintf("%s-driver-sa", namespace), + }, + }, + nil, + ) + ctrler.On("CreateRole", mock.Anything, mock.Anything, mock.Anything).Return( + &apirbacv1.Role{ + ObjectMeta: apimetav1.ObjectMeta{ + Name: fmt.Sprintf("%s-driver-role", namespace), + }, + }, + nil, + ) + ctrler.On( + "CreateRoleBinding", + mock.Anything, + mock.Anything, + mock.Anything, + ).Return(nil, nil) + ctrler.On("DeleteSecret", mock.Anything, mock.Anything, mock.Anything, false).Return(nil) + ctrler.On("DeleteConfigMap", mock.Anything, mock.Anything, mock.Anything, false).Return(nil) + return ctrler + }, + mlpService: func() service.MLPService { + svc := &servicemock.MLPService{} + svc.On( + "GetSecret", + mock.Anything, + "test-service-account", + ).Return("Alright then. Keep your secrets.", nil) + svc.On( + "GetSecret", + mock.Anything, + "MLP_SECRET_1", + ).Return("", fmt.Errorf("hi")) + return svc + }, + request: &CreateEnsemblingJobRequest{ + EnsemblingJob: generateEnsemblingJobFixtureWithUserDefinedSecrets(), + Labels: standardLabels, + ImageRef: imageRef, + Namespace: namespace, + }, + }, "failure | fail to create job config map": { expected: fmt.Errorf("failed creating job specification configmap for job %s in namespace %s: %v", "test-ensembler-1", diff --git a/api/turing/cluster/servicebuilder/fluentd.go b/api/turing/cluster/servicebuilder/fluentd.go index 4cf56183e..6a7faee4c 100644 --- a/api/turing/cluster/servicebuilder/fluentd.go +++ b/api/turing/cluster/servicebuilder/fluentd.go @@ -45,7 +45,7 @@ func (sb *clusterSvcBuilder) NewFluentdService( {Name: "FLUENTD_WORKER_COUNT", Value: strconv.Itoa(fluentdConfig.WorkerCount)}, {Name: "FLUENTD_LOG_LEVEL", Value: "info"}, {Name: "FLUENTD_LOG_PATH", Value: "/cache/log/bq_load_logs.*.buffer"}, - {Name: "FLUENTD_GCP_JSON_KEY_PATH", Value: secretMountPath + secretKeyNameRouter}, + {Name: "FLUENTD_GCP_JSON_KEY_PATH", Value: secretMountPath + SecretKeyNameRouter}, {Name: "FLUENTD_BUFFER_LIMIT", Value: "10g"}, {Name: "FLUENTD_FLUSH_INTERVAL_SECONDS", Value: strconv.Itoa(fluentdConfig.FlushIntervalSeconds * FluentdReplicaCount)}, @@ -128,8 +128,8 @@ func buildFluentdVolumes( SecretName: svcAccountSecretName, Items: []corev1.KeyToPath{ { - Key: secretKeyNameRouter, - Path: secretKeyNameRouter, + Key: SecretKeyNameRouter, + Path: SecretKeyNameRouter, }, }, }, diff --git a/api/turing/cluster/servicebuilder/fluentd_test.go b/api/turing/cluster/servicebuilder/fluentd_test.go index 707ad0e9d..e6c2314a2 100644 --- a/api/turing/cluster/servicebuilder/fluentd_test.go +++ b/api/turing/cluster/servicebuilder/fluentd_test.go @@ -102,8 +102,8 @@ func TestNewFluentdService(t *testing.T) { SecretName: "service-account", Items: []corev1.KeyToPath{ { - Key: secretKeyNameRouter, - Path: secretKeyNameRouter, + Key: SecretKeyNameRouter, + Path: SecretKeyNameRouter, }, }, }, diff --git a/api/turing/cluster/servicebuilder/router.go b/api/turing/cluster/servicebuilder/router.go index 87c3ce35a..d7d69d118 100644 --- a/api/turing/cluster/servicebuilder/router.go +++ b/api/turing/cluster/servicebuilder/router.go @@ -210,7 +210,7 @@ func (sb *clusterSvcBuilder) buildRouterEnvs( sentryDSN string, ver *models.RouterVersion, ) ([]corev1.EnvVar, error) { - envs := sb.getEnvVars(ver.ResourceRequest, nil) + envs := sb.getEnvVars(ver.ResourceRequest, nil, nil, "") // Add app name, router timeout, jaeger collector envs = mergeEnvVars(envs, @@ -250,7 +250,7 @@ func (sb *clusterSvcBuilder) buildRouterEnvs( // Add exp engine secret path as env var if service account key file path is specified for exp engine if ver.ExperimentEngine != nil && ver.ExperimentEngine.ServiceAccountKeyFilePath != nil { envs = mergeEnvVars(envs, []corev1.EnvVar{ - {Name: envExpGoogleApplicationCredentials, Value: secretMountPathExpEngine + secretKeyNameExpEngine}, + {Name: envExpGoogleApplicationCredentials, Value: secretMountPathExpEngine + SecretKeyNameExpEngine}, }) } @@ -280,7 +280,7 @@ func (sb *clusterSvcBuilder) buildRouterEnvs( {Name: envBQDataset, Value: bqFQN[1]}, {Name: envBQTable, Value: bqFQN[2]}, {Name: envBQBatchLoad, Value: strconv.FormatBool(logConfig.BigQueryConfig.BatchLoad)}, - {Name: envGoogleApplicationCredentials, Value: secretMountPathRouter + secretKeyNameRouter}, + {Name: envGoogleApplicationCredentials, Value: secretMountPathRouter + SecretKeyNameRouter}, }) if logConfig.BigQueryConfig.BatchLoad { envs = mergeEnvVars(envs, []corev1.EnvVar{ @@ -349,8 +349,8 @@ func buildRouterVolumes( SecretName: secretName, Items: []corev1.KeyToPath{ { - Key: secretKeyNameExpEngine, - Path: secretKeyNameExpEngine, + Key: SecretKeyNameExpEngine, + Path: SecretKeyNameExpEngine, }, }, }, @@ -372,8 +372,8 @@ func buildRouterVolumes( SecretName: secretName, Items: []corev1.KeyToPath{ { - Key: secretKeyNameRouter, - Path: secretKeyNameRouter, + Key: SecretKeyNameRouter, + Path: SecretKeyNameRouter, }, }, }, diff --git a/api/turing/cluster/servicebuilder/router_test.go b/api/turing/cluster/servicebuilder/router_test.go index a4fd0d491..773f5ee97 100644 --- a/api/turing/cluster/servicebuilder/router_test.go +++ b/api/turing/cluster/servicebuilder/router_test.go @@ -164,8 +164,8 @@ func TestNewRouterService(t *testing.T) { SecretName: "service-account", Items: []corev1.KeyToPath{ { - Key: secretKeyNameRouter, - Path: secretKeyNameRouter, + Key: SecretKeyNameRouter, + Path: SecretKeyNameRouter, }, }, }, @@ -267,8 +267,8 @@ func TestNewRouterService(t *testing.T) { SecretName: "service-account", Items: []corev1.KeyToPath{ { - Key: secretKeyNameRouter, - Path: secretKeyNameRouter, + Key: SecretKeyNameRouter, + Path: SecretKeyNameRouter, }, }, }, @@ -377,8 +377,8 @@ func TestNewRouterService(t *testing.T) { SecretName: "service-account", Items: []corev1.KeyToPath{ { - Key: secretKeyNameRouter, - Path: secretKeyNameRouter, + Key: SecretKeyNameRouter, + Path: SecretKeyNameRouter, }, }, }, @@ -478,8 +478,8 @@ func TestNewRouterService(t *testing.T) { SecretName: "service-account", Items: []corev1.KeyToPath{ { - Key: secretKeyNameRouter, - Path: secretKeyNameRouter, + Key: SecretKeyNameRouter, + Path: SecretKeyNameRouter, }, }, }, @@ -579,8 +579,8 @@ func TestNewRouterService(t *testing.T) { SecretName: "service-account", Items: []corev1.KeyToPath{ { - Key: secretKeyNameRouter, - Path: secretKeyNameRouter, + Key: SecretKeyNameRouter, + Path: SecretKeyNameRouter, }, }, }, @@ -680,8 +680,8 @@ func TestNewRouterService(t *testing.T) { SecretName: "service-account", Items: []corev1.KeyToPath{ { - Key: secretKeyNameRouter, - Path: secretKeyNameRouter, + Key: SecretKeyNameRouter, + Path: SecretKeyNameRouter, }, }, }, @@ -781,8 +781,8 @@ func TestNewRouterService(t *testing.T) { SecretName: "service-account", Items: []corev1.KeyToPath{ { - Key: secretKeyNameRouter, - Path: secretKeyNameRouter, + Key: SecretKeyNameRouter, + Path: SecretKeyNameRouter, }, }, }, @@ -885,8 +885,8 @@ func TestNewRouterService(t *testing.T) { SecretName: "service-account", Items: []corev1.KeyToPath{ { - Key: secretKeyNameExpEngine, - Path: secretKeyNameExpEngine, + Key: SecretKeyNameExpEngine, + Path: SecretKeyNameExpEngine, }, }, }, diff --git a/api/turing/cluster/servicebuilder/service_builder.go b/api/turing/cluster/servicebuilder/service_builder.go index 4dd50fdf9..f97488112 100644 --- a/api/turing/cluster/servicebuilder/service_builder.go +++ b/api/turing/cluster/servicebuilder/service_builder.go @@ -28,10 +28,10 @@ const ( // Kubernetes secret key name for usage in: router, ensembler, enricher. // They will share the same Kubernetes secret for every RouterVersion deployment. // Hence, the key name should be used to retrieve different credentials. - secretKeyNameRouter = "router-service-account.json" - secretKeyNameEnsembler = "ensembler-service-account.json" - secretKeyNameEnricher = "enricher-service-account.json" - secretKeyNameExpEngine = "exp-engine-service-account.json" + SecretKeyNameRouter = "router-service-account.json" + SecretKeyNameEnsembler = "ensembler-service-account.json" + SecretKeyNameEnricher = "enricher-service-account.json" + SecretKeyNameExpEngine = "exp-engine-service-account.json" ) var ComponentTypes = struct { @@ -99,10 +99,7 @@ type ClusterServiceBuilder interface { NewSecret( routerVersion *models.RouterVersion, project *mlp.Project, - routerServiceAccountKey string, - enricherServiceAccountKey string, - ensemblerServiceAccountKey string, - expEngineServiceAccountKey string, + secretMap map[string]string, ) *cluster.Secret NewPodDisruptionBudget( routerVersion *models.RouterVersion, @@ -170,7 +167,7 @@ func (sb *clusterSvcBuilder) NewEnricherService( Name: secretVolume, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ SecretName: secretName, - Items: []corev1.KeyToPath{{Key: secretKeyNameEnricher, Path: secretKeyNameEnricher}}, + Items: []corev1.KeyToPath{{Key: SecretKeyNameEnricher, Path: SecretKeyNameEnricher}}, }}, } volumes = append(volumes, v) @@ -182,14 +179,14 @@ func (sb *clusterSvcBuilder) NewEnricherService( existingReplaced := false for _, env := range enricher.Env { if env.Name == envGoogleApplicationCredentials { - env.Value = filepath.Join(secretMountPath, secretKeyNameEnricher) + env.Value = filepath.Join(secretMountPath, SecretKeyNameEnricher) existingReplaced = true } } if !existingReplaced { env := &models.EnvVar{ Name: envGoogleApplicationCredentials, - Value: filepath.Join(secretMountPath, secretKeyNameEnricher), + Value: filepath.Join(secretMountPath, SecretKeyNameEnricher), } enricher.Env = append(enricher.Env, env) } @@ -209,7 +206,7 @@ func (sb *clusterSvcBuilder) NewEnricherService( CPULimit: sb.getCPULimit(enricher.ResourceRequest), MemoryRequests: enricher.ResourceRequest.MemoryRequest, MemoryLimit: sb.getMemoryLimit(enricher.ResourceRequest), - Envs: sb.getEnvVars(enricher.ResourceRequest, &enricher.Env), + Envs: sb.getEnvVars(enricher.ResourceRequest, &enricher.Env, &enricher.Secrets, secretName), Labels: buildLabels(project, routerVersion.Router), Volumes: volumes, VolumeMounts: volumeMounts, @@ -256,7 +253,7 @@ func (sb *clusterSvcBuilder) NewEnsemblerService( Name: secretVolume, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ SecretName: secretName, - Items: []corev1.KeyToPath{{Key: secretKeyNameEnsembler, Path: secretKeyNameEnsembler}}, + Items: []corev1.KeyToPath{{Key: SecretKeyNameEnsembler, Path: SecretKeyNameEnsembler}}, }}, } volumes = append(volumes, v) @@ -268,14 +265,14 @@ func (sb *clusterSvcBuilder) NewEnsemblerService( existingReplaced := false for _, env := range docker.Env { if env.Name == envGoogleApplicationCredentials { - env.Value = filepath.Join(secretMountPath, secretKeyNameEnsembler) + env.Value = filepath.Join(secretMountPath, SecretKeyNameEnsembler) existingReplaced = true } } if !existingReplaced { env := &models.EnvVar{ Name: envGoogleApplicationCredentials, - Value: filepath.Join(secretMountPath, secretKeyNameEnsembler), + Value: filepath.Join(secretMountPath, SecretKeyNameEnsembler), } docker.Env = append(docker.Env, env) } @@ -295,7 +292,7 @@ func (sb *clusterSvcBuilder) NewEnsemblerService( CPULimit: sb.getCPULimit(docker.ResourceRequest), MemoryRequests: docker.ResourceRequest.MemoryRequest, MemoryLimit: sb.getMemoryLimit(docker.ResourceRequest), - Envs: sb.getEnvVars(docker.ResourceRequest, &docker.Env), + Envs: sb.getEnvVars(docker.ResourceRequest, &docker.Env, &docker.Secrets, secretName), Labels: buildLabels(project, routerVersion.Router), Volumes: volumes, VolumeMounts: volumeMounts, @@ -318,17 +315,8 @@ func (sb *clusterSvcBuilder) NewEnsemblerService( func (sb *clusterSvcBuilder) NewSecret( routerVersion *models.RouterVersion, project *mlp.Project, - routerServiceAccountKey string, - enricherServiceAccountKey string, - ensemblerServiceAccountKey string, - expEngineServiceAccountKey string, + secretMap map[string]string, ) *cluster.Secret { - data := map[string]string{ - secretKeyNameRouter: routerServiceAccountKey, - secretKeyNameEnricher: enricherServiceAccountKey, - secretKeyNameEnsembler: ensemblerServiceAccountKey, - secretKeyNameExpEngine: expEngineServiceAccountKey, - } return &cluster.Secret{ Name: fmt.Sprintf( "%s-turing-%s-%d", @@ -337,7 +325,7 @@ func (sb *clusterSvcBuilder) NewSecret( routerVersion.Version, ), Namespace: project.Name, - Data: data, + Data: secretMap, Labels: buildLabels(project, routerVersion.Router), } } @@ -432,7 +420,7 @@ func (sb *clusterSvcBuilder) getMemoryLimit(resourceRequest *models.ResourceRequ } func (sb *clusterSvcBuilder) getEnvVars(resourceRequest *models.ResourceRequest, - userEnvVars *models.EnvVars) (newEnvVars []corev1.EnvVar) { + userEnvVars *models.EnvVars, secrets *models.Secrets, secretName string) (newEnvVars []corev1.EnvVar) { if resourceRequest != nil && (resourceRequest.CPULimit == nil || resourceRequest.CPULimit.IsZero()) && sb.knativeServiceConfig.UserContainerCPULimitRequestFactor == 0 { newEnvVars = mergeEnvVars(newEnvVars, sb.knativeServiceConfig.DefaultEnvVarsWithoutCPULimits) @@ -440,6 +428,10 @@ func (sb *clusterSvcBuilder) getEnvVars(resourceRequest *models.ResourceRequest, if userEnvVars != nil { newEnvVars = mergeEnvVars(newEnvVars, userEnvVars.ToKubernetesEnvVars()) } + + if secrets != nil { + newEnvVars = mergeEnvVars(newEnvVars, secrets.ToKubernetesEnvVars(secretName)) + } return } diff --git a/api/turing/cluster/servicebuilder/service_builder_test.go b/api/turing/cluster/servicebuilder/service_builder_test.go index 7f40cf3ce..e9d8ce21d 100644 --- a/api/turing/cluster/servicebuilder/service_builder_test.go +++ b/api/turing/cluster/servicebuilder/service_builder_test.go @@ -97,7 +97,7 @@ func TestNewEnricherService(t *testing.T) { Name: secretVolume, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ SecretName: "secret", - Items: []corev1.KeyToPath{{Key: secretKeyNameEnricher, Path: secretKeyNameEnricher}}, + Items: []corev1.KeyToPath{{Key: SecretKeyNameEnricher, Path: SecretKeyNameEnricher}}, }}, }, }, @@ -197,7 +197,7 @@ func TestNewEnsemblerService(t *testing.T) { Name: secretVolume, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ SecretName: "secret", - Items: []corev1.KeyToPath{{Key: secretKeyNameEnsembler, Path: secretKeyNameEnsembler}}, + Items: []corev1.KeyToPath{{Key: SecretKeyNameEnsembler, Path: SecretKeyNameEnsembler}}, }}, }, }, @@ -241,7 +241,7 @@ func TestNewEnsemblerService(t *testing.T) { Name: secretVolume, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ SecretName: "secret", - Items: []corev1.KeyToPath{{Key: secretKeyNameEnsembler, Path: secretKeyNameEnsembler}}, + Items: []corev1.KeyToPath{{Key: SecretKeyNameEnsembler, Path: SecretKeyNameEnsembler}}, }}, }, }, @@ -286,15 +286,19 @@ func TestNewEnsemblerService(t *testing.T) { } func TestNewSecret(t *testing.T) { + secretMap := map[string]string{ + SecretKeyNameRouter: "router-key", + SecretKeyNameEnricher: "enricher-key", + SecretKeyNameEnsembler: "ensembler-key", + SecretKeyNameExpEngine: "exp-engine-key", + } + tests := map[string]struct { - version *models.RouterVersion - project *mlp.Project - envType string - routerSvcKey string - enricherSvcKey string - ensemblerSvcKey string - expEngineSvcKey string - expected *cluster.Secret + version *models.RouterVersion + project *mlp.Project + envType string + secretMap map[string]string + expected *cluster.Secret }{ "success": { version: &models.RouterVersion{ @@ -304,21 +308,13 @@ func TestNewSecret(t *testing.T) { Type: "exp-engine", }, }, - project: &mlp.Project{Name: "test-project"}, - envType: "test", - routerSvcKey: "router-key", - enricherSvcKey: "enricher-key", - ensemblerSvcKey: "ensembler-key", - expEngineSvcKey: "exp-engine-key", + project: &mlp.Project{Name: "test-project"}, + envType: "test", + secretMap: secretMap, expected: &cluster.Secret{ Name: "test-router-turing-secret-2", Namespace: "test-project", - Data: map[string]string{ - "router-service-account.json": "router-key", - "enricher-service-account.json": "enricher-key", - "ensembler-service-account.json": "ensembler-key", - "exp-engine-service-account.json": "exp-engine-key", - }, + Data: secretMap, Labels: map[string]string{ "app": "test-router", "environment": "", @@ -332,8 +328,7 @@ func TestNewSecret(t *testing.T) { sb := &clusterSvcBuilder{} for name, tt := range tests { t.Run(name, func(t *testing.T) { - secret := sb.NewSecret(tt.version, tt.project, tt.routerSvcKey, tt.enricherSvcKey, tt.ensemblerSvcKey, - tt.expEngineSvcKey) + secret := sb.NewSecret(tt.version, tt.project, tt.secretMap) assert.Equal(t, tt.expected, secret) }) } diff --git a/api/turing/cluster/spark.go b/api/turing/cluster/spark.go index 83d5fcafd..cb9d9a562 100644 --- a/api/turing/cluster/spark.go +++ b/api/turing/cluster/spark.go @@ -101,6 +101,7 @@ type CreateSparkRequest struct { ExecutorReplica int32 ServiceAccountName string SparkInfraConfig *config.SparkAppConfig + Secrets *[]openapi.MountedMLPSecret EnvVars *[]openapi.EnvVar } @@ -157,6 +158,24 @@ func getEnvVarFromRequest(request *CreateSparkRequest) []apicorev1.EnvVar { }) } + if request.Secrets == nil { + return envVars + } + + for _, secret := range *request.Secrets { + envVars = append(envVars, apicorev1.EnvVar{ + Name: secret.GetEnvVarName(), + ValueFrom: &apicorev1.EnvVarSource{ + SecretKeyRef: &apicorev1.SecretKeySelector{ + LocalObjectReference: apicorev1.LocalObjectReference{ + Name: request.JobName, + }, + Key: secret.GetMlpSecretName(), + }, + }, + }) + } + return envVars } diff --git a/api/turing/cluster/spark_test.go b/api/turing/cluster/spark_test.go index 859f42166..147bcc1c5 100644 --- a/api/turing/cluster/spark_test.go +++ b/api/turing/cluster/spark_test.go @@ -126,6 +126,9 @@ func TestGetCPURequestAndLimit(t *testing.T) { } func TestGetEnvVars(t *testing.T) { + // Create a copy of sparkInfraConfig to avoid modifying the original when running tests in parallel + sparkInfraConfigCopy := *sparkInfraConfig + request := &CreateSparkRequest{ JobName: jobName, JobLabels: jobLabels, @@ -139,12 +142,13 @@ func TestGetEnvVars(t *testing.T) { ExecutorMemoryRequest: memoryValue, ExecutorReplica: executorReplica, ServiceAccountName: serviceAccountName, - SparkInfraConfig: sparkInfraConfig, + SparkInfraConfig: &sparkInfraConfigCopy, EnvVars: &envVars, } tests := map[string]struct { sparkInfraConfigAPIServerEnvVars []string apiServerEnvVars []apicorev1.EnvVar + userConfiguredSecrets *[]openapi.MountedMLPSecret expectedEnvVars []apicorev1.EnvVar }{ "api server env vars specified": { @@ -155,6 +159,7 @@ func TestGetEnvVars(t *testing.T) { Value: "TEST_VALUE_1", }, }, + nil, []apicorev1.EnvVar{ { Name: envServiceAccountPathKey, @@ -178,6 +183,7 @@ func TestGetEnvVars(t *testing.T) { Value: "TEST_VALUE_1", }, }, + nil, []apicorev1.EnvVar{ { Name: envServiceAccountPathKey, @@ -189,6 +195,57 @@ func TestGetEnvVars(t *testing.T) { }, }, }, + "user-configured secrets exist": { + []string{}, + []apicorev1.EnvVar{ + { + Name: "TEST_ENV_VAR_1", + Value: "TEST_VALUE_1", + }, + }, + &[]openapi.MountedMLPSecret{ + { + MlpSecretName: "MLP_SECRET_1", + EnvVarName: "SECRET_ENV_VAR_1", + }, + { + MlpSecretName: "MLP_SECRET_2", + EnvVarName: "SECRET_ENV_VAR_2", + }, + }, + []apicorev1.EnvVar{ + { + Name: envServiceAccountPathKey, + Value: envServiceAccountPath, + }, + { + Name: "foo", + Value: barString, + }, + { + Name: "SECRET_ENV_VAR_1", + ValueFrom: &apicorev1.EnvVarSource{ + SecretKeyRef: &apicorev1.SecretKeySelector{ + LocalObjectReference: apicorev1.LocalObjectReference{ + Name: request.JobName, + }, + Key: "MLP_SECRET_1", + }, + }, + }, + { + Name: "SECRET_ENV_VAR_2", + ValueFrom: &apicorev1.EnvVarSource{ + SecretKeyRef: &apicorev1.SecretKeySelector{ + LocalObjectReference: apicorev1.LocalObjectReference{ + Name: request.JobName, + }, + Key: "MLP_SECRET_2", + }, + }, + }, + }, + }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { @@ -198,6 +255,7 @@ func TestGetEnvVars(t *testing.T) { } request.SparkInfraConfig.APIServerEnvVars = tt.sparkInfraConfigAPIServerEnvVars + request.Secrets = tt.userConfiguredSecrets envVars := getEnvVars(request) assert.Equal(t, tt.expectedEnvVars, envVars) diff --git a/api/turing/generated/model_ensembler_infra_config.go b/api/turing/generated/model_ensembler_infra_config.go index 0f9997d2c..33cefba2d 100644 --- a/api/turing/generated/model_ensembler_infra_config.go +++ b/api/turing/generated/model_ensembler_infra_config.go @@ -16,12 +16,13 @@ import ( // EnsemblerInfraConfig struct for EnsemblerInfraConfig type EnsemblerInfraConfig struct { - ArtifactUri *string `json:"artifact_uri,omitempty"` - EnsemblerName *string `json:"ensembler_name,omitempty"` - ServiceAccountName *string `json:"service_account_name,omitempty" validate:"required"` - Resources NullableEnsemblingResources `json:"resources,omitempty"` - RunId *string `json:"run_id,omitempty"` - Env *[]EnvVar `json:"env,omitempty"` + ArtifactUri *string `json:"artifact_uri,omitempty"` + EnsemblerName *string `json:"ensembler_name,omitempty"` + ServiceAccountName *string `json:"service_account_name,omitempty" validate:"required"` + Secrets *[]MountedMLPSecret `json:"secrets,omitempty"` + Resources NullableEnsemblingResources `json:"resources,omitempty"` + RunId *string `json:"run_id,omitempty"` + Env *[]EnvVar `json:"env,omitempty"` } // NewEnsemblerInfraConfig instantiates a new EnsemblerInfraConfig object @@ -137,6 +138,38 @@ func (o *EnsemblerInfraConfig) SetServiceAccountName(v string) { o.ServiceAccountName = &v } +// GetSecrets returns the Secrets field value if set, zero value otherwise. +func (o *EnsemblerInfraConfig) GetSecrets() []MountedMLPSecret { + if o == nil || o.Secrets == nil { + var ret []MountedMLPSecret + return ret + } + return *o.Secrets +} + +// GetSecretsOk returns a tuple with the Secrets field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *EnsemblerInfraConfig) GetSecretsOk() (*[]MountedMLPSecret, bool) { + if o == nil || o.Secrets == nil { + return nil, false + } + return o.Secrets, true +} + +// HasSecrets returns a boolean if a field has been set. +func (o *EnsemblerInfraConfig) HasSecrets() bool { + if o != nil && o.Secrets != nil { + return true + } + + return false +} + +// SetSecrets gets a reference to the given []MountedMLPSecret and assigns it to the Secrets field. +func (o *EnsemblerInfraConfig) SetSecrets(v []MountedMLPSecret) { + o.Secrets = &v +} + // GetResources returns the Resources field value if set, zero value otherwise (both if not set or set to explicit null). func (o *EnsemblerInfraConfig) GetResources() EnsemblingResources { if o == nil || o.Resources.Get() == nil { @@ -150,7 +183,7 @@ func (o *EnsemblerInfraConfig) GetResources() EnsemblingResources { // and a boolean to check if the value has been set. // NOTE: If the value is an explicit nil, `nil, true` will be returned func (o *EnsemblerInfraConfig) GetResourcesOk() (*EnsemblingResources, bool) { - if o == nil { + if o == nil { return nil, false } return o.Resources.Get(), o.Resources.IsSet() @@ -169,6 +202,7 @@ func (o *EnsemblerInfraConfig) HasResources() bool { func (o *EnsemblerInfraConfig) SetResources(v EnsemblingResources) { o.Resources.Set(&v) } + // SetResourcesNil sets the value for Resources to be an explicit nil func (o *EnsemblerInfraConfig) SetResourcesNil() { o.Resources.Set(nil) @@ -254,6 +288,9 @@ func (o EnsemblerInfraConfig) MarshalJSON() ([]byte, error) { if o.ServiceAccountName != nil { toSerialize["service_account_name"] = o.ServiceAccountName } + if o.Secrets != nil { + toSerialize["secrets"] = o.Secrets + } if o.Resources.IsSet() { toSerialize["resources"] = o.Resources.Get() } @@ -301,5 +338,3 @@ func (v *NullableEnsemblerInfraConfig) UnmarshalJSON(src []byte) error { v.isSet = true return json.Unmarshal(src, &v.value) } - - diff --git a/api/turing/generated/model_mounted_mlp_secret.go b/api/turing/generated/model_mounted_mlp_secret.go new file mode 100644 index 000000000..b72f5738c --- /dev/null +++ b/api/turing/generated/model_mounted_mlp_secret.go @@ -0,0 +1,135 @@ +/* + * Turing Minimal Openapi Spec for SDK + * + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * API version: 0.0.1 + */ + +// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT. + +package openapi + +import ( + "encoding/json" +) + +// MountedMLPSecret struct for MountedMLPSecret +type MountedMLPSecret struct { + MlpSecretName string `json:"mlp_secret_name"` + EnvVarName string `json:"env_var_name"` +} + +// NewMountedMLPSecret instantiates a new MountedMLPSecret object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewMountedMLPSecret(mlpSecretName string, envVarName string) *MountedMLPSecret { + this := MountedMLPSecret{} + this.MlpSecretName = mlpSecretName + this.EnvVarName = envVarName + return &this +} + +// NewMountedMLPSecretWithDefaults instantiates a new MountedMLPSecret object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewMountedMLPSecretWithDefaults() *MountedMLPSecret { + this := MountedMLPSecret{} + return &this +} + +// GetMlpSecretName returns the MlpSecretName field value +func (o *MountedMLPSecret) GetMlpSecretName() string { + if o == nil { + var ret string + return ret + } + + return o.MlpSecretName +} + +// GetMlpSecretNameOk returns a tuple with the MlpSecretName field value +// and a boolean to check if the value has been set. +func (o *MountedMLPSecret) GetMlpSecretNameOk() (*string, bool) { + if o == nil { + return nil, false + } + return &o.MlpSecretName, true +} + +// SetMlpSecretName sets field value +func (o *MountedMLPSecret) SetMlpSecretName(v string) { + o.MlpSecretName = v +} + +// GetEnvVarName returns the EnvVarName field value +func (o *MountedMLPSecret) GetEnvVarName() string { + if o == nil { + var ret string + return ret + } + + return o.EnvVarName +} + +// GetEnvVarNameOk returns a tuple with the EnvVarName field value +// and a boolean to check if the value has been set. +func (o *MountedMLPSecret) GetEnvVarNameOk() (*string, bool) { + if o == nil { + return nil, false + } + return &o.EnvVarName, true +} + +// SetEnvVarName sets field value +func (o *MountedMLPSecret) SetEnvVarName(v string) { + o.EnvVarName = v +} + +func (o MountedMLPSecret) MarshalJSON() ([]byte, error) { + toSerialize := map[string]interface{}{} + if true { + toSerialize["mlp_secret_name"] = o.MlpSecretName + } + if true { + toSerialize["env_var_name"] = o.EnvVarName + } + return json.Marshal(toSerialize) +} + +type NullableMountedMLPSecret struct { + value *MountedMLPSecret + isSet bool +} + +func (v NullableMountedMLPSecret) Get() *MountedMLPSecret { + return v.value +} + +func (v *NullableMountedMLPSecret) Set(val *MountedMLPSecret) { + v.value = val + v.isSet = true +} + +func (v NullableMountedMLPSecret) IsSet() bool { + return v.isSet +} + +func (v *NullableMountedMLPSecret) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableMountedMLPSecret(val *MountedMLPSecret) *NullableMountedMLPSecret { + return &NullableMountedMLPSecret{value: val, isSet: true} +} + +func (v NullableMountedMLPSecret) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableMountedMLPSecret) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} diff --git a/api/turing/models/enricher.go b/api/turing/models/enricher.go index 6af7aac87..83bdc0f85 100644 --- a/api/turing/models/enricher.go +++ b/api/turing/models/enricher.go @@ -17,6 +17,8 @@ type Enricher struct { Timeout string `json:"timeout"` // Port to query. Port int `json:"port"` + // MLP secrets to inject into the pod. + Secrets Secrets `json:"secrets"` // Environment variables to inject into the pod. Env EnvVars `json:"env"` // (optional) ServiceAccount specifies the name of the secret registered in the MLP project containing the service diff --git a/api/turing/models/ensembler.go b/api/turing/models/ensembler.go index f0cf9f7f8..49954493a 100644 --- a/api/turing/models/ensembler.go +++ b/api/turing/models/ensembler.go @@ -54,6 +54,8 @@ type EnsemblerDockerConfig struct { Timeout string `json:"timeout" validate:"required"` // Port number the container listens to for requests Port int `json:"port" validate:"required"` + // MLP secrets to inject into the container + Secrets Secrets `json:"secrets" validate:"required"` // Environment variables to set in the container Env EnvVars `json:"env" validate:"required"` // secret name in MLP containing service account key @@ -69,6 +71,8 @@ type EnsemblerPyfuncConfig struct { AutoscalingPolicy *AutoscalingPolicy `json:"autoscaling_policy" validate:"omitempty,dive"` // Request timeout in duration format e.g. 60s Timeout string `json:"timeout" validate:"required"` + // MLP secrets to inject into the container + Secrets Secrets `json:"secrets" validate:"required"` // Environment variables to set in the container Env EnvVars `json:"env" validate:"required"` } diff --git a/api/turing/models/secret.go b/api/turing/models/secret.go new file mode 100644 index 000000000..fc1ff00da --- /dev/null +++ b/api/turing/models/secret.go @@ -0,0 +1,56 @@ +package models + +import ( + "database/sql/driver" + "encoding/json" + "errors" + + k8scorev1 "k8s.io/api/core/v1" +) + +// Secret represents an MLP secret present in a container that is mounted as an environment variable +type Secret struct { + // Name of the secret as stored in MLP + MLPSecretName string `json:"mlp_secret_name"` + + // Name of the environment variable when the secret is mounted + EnvVarName string `json:"env_var_name"` +} + +// Secret is a list of MLP secrets to set in the container. +type Secrets []Secret + +func (sec Secrets) Value() (driver.Value, error) { + return json.Marshal(sec) +} + +func (sec *Secrets) Scan(value interface{}) error { + b, ok := value.([]byte) + if !ok { + return errors.New("type assertion to []byte failed") + } + + return json.Unmarshal(b, &sec) +} + +// ToKubernetesEnvVars returns the representation of Kubernetes' +// v1.EnvVars. +func (sec Secrets) ToKubernetesEnvVars(secretKeyRefName string) []k8scorev1.EnvVar { + kubeEnvVars := make([]k8scorev1.EnvVar, len(sec)) + + for k, secret := range sec { + kubeEnvVars[k] = k8scorev1.EnvVar{ + Name: secret.EnvVarName, + ValueFrom: &k8scorev1.EnvVarSource{ + SecretKeyRef: &k8scorev1.SecretKeySelector{ + LocalObjectReference: k8scorev1.LocalObjectReference{ + Name: secretKeyRefName, + }, + Key: secret.MLPSecretName, + }, + }, + } + } + + return kubeEnvVars +} diff --git a/api/turing/service/mocks/router_deployment_service.go b/api/turing/service/mocks/router_deployment_service.go index ee33f5341..295540a26 100644 --- a/api/turing/service/mocks/router_deployment_service.go +++ b/api/turing/service/mocks/router_deployment_service.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.28.2. DO NOT EDIT. +// Code generated by mockery v2.52.2. DO NOT EDIT. package mocks @@ -25,6 +25,10 @@ type DeploymentService struct { func (_m *DeploymentService) DeleteRouterEndpoint(project *client.Project, environment *merlinclient.Environment, routerVersion *models.RouterVersion) error { ret := _m.Called(project, environment, routerVersion) + if len(ret) == 0 { + panic("no return value specified for DeleteRouterEndpoint") + } + var r0 error if rf, ok := ret.Get(0).(func(*client.Project, *merlinclient.Environment, *models.RouterVersion) error); ok { r0 = rf(project, environment, routerVersion) @@ -35,23 +39,27 @@ func (_m *DeploymentService) DeleteRouterEndpoint(project *client.Project, envir return r0 } -// DeployRouterVersion provides a mock function with given fields: project, environment, currentRouterVersion, routerVersion, routerServiceAccountKey, enricherServiceAccountKey, ensemblerServiceAccountKey, expEngineServiceAccountKey, pyfuncEnsembler, experimentConfig, eventsCh -func (_m *DeploymentService) DeployRouterVersion(project *client.Project, environment *merlinclient.Environment, currentRouterVersion *models.RouterVersion, routerVersion *models.RouterVersion, routerServiceAccountKey string, enricherServiceAccountKey string, ensemblerServiceAccountKey string, expEngineServiceAccountKey string, pyfuncEnsembler *models.PyFuncEnsembler, experimentConfig json.RawMessage, eventsCh *service.EventChannel) (string, error) { - ret := _m.Called(project, environment, currentRouterVersion, routerVersion, routerServiceAccountKey, enricherServiceAccountKey, ensemblerServiceAccountKey, expEngineServiceAccountKey, pyfuncEnsembler, experimentConfig, eventsCh) +// DeployRouterVersion provides a mock function with given fields: project, environment, currentRouterVersion, routerVersion, secretMap, pyfuncEnsembler, experimentConfig, eventsCh +func (_m *DeploymentService) DeployRouterVersion(project *client.Project, environment *merlinclient.Environment, currentRouterVersion *models.RouterVersion, routerVersion *models.RouterVersion, secretMap map[string]string, pyfuncEnsembler *models.PyFuncEnsembler, experimentConfig json.RawMessage, eventsCh *service.EventChannel) (string, error) { + ret := _m.Called(project, environment, currentRouterVersion, routerVersion, secretMap, pyfuncEnsembler, experimentConfig, eventsCh) + + if len(ret) == 0 { + panic("no return value specified for DeployRouterVersion") + } var r0 string var r1 error - if rf, ok := ret.Get(0).(func(*client.Project, *merlinclient.Environment, *models.RouterVersion, *models.RouterVersion, string, string, string, string, *models.PyFuncEnsembler, json.RawMessage, *service.EventChannel) (string, error)); ok { - return rf(project, environment, currentRouterVersion, routerVersion, routerServiceAccountKey, enricherServiceAccountKey, ensemblerServiceAccountKey, expEngineServiceAccountKey, pyfuncEnsembler, experimentConfig, eventsCh) + if rf, ok := ret.Get(0).(func(*client.Project, *merlinclient.Environment, *models.RouterVersion, *models.RouterVersion, map[string]string, *models.PyFuncEnsembler, json.RawMessage, *service.EventChannel) (string, error)); ok { + return rf(project, environment, currentRouterVersion, routerVersion, secretMap, pyfuncEnsembler, experimentConfig, eventsCh) } - if rf, ok := ret.Get(0).(func(*client.Project, *merlinclient.Environment, *models.RouterVersion, *models.RouterVersion, string, string, string, string, *models.PyFuncEnsembler, json.RawMessage, *service.EventChannel) string); ok { - r0 = rf(project, environment, currentRouterVersion, routerVersion, routerServiceAccountKey, enricherServiceAccountKey, ensemblerServiceAccountKey, expEngineServiceAccountKey, pyfuncEnsembler, experimentConfig, eventsCh) + if rf, ok := ret.Get(0).(func(*client.Project, *merlinclient.Environment, *models.RouterVersion, *models.RouterVersion, map[string]string, *models.PyFuncEnsembler, json.RawMessage, *service.EventChannel) string); ok { + r0 = rf(project, environment, currentRouterVersion, routerVersion, secretMap, pyfuncEnsembler, experimentConfig, eventsCh) } else { r0 = ret.Get(0).(string) } - if rf, ok := ret.Get(1).(func(*client.Project, *merlinclient.Environment, *models.RouterVersion, *models.RouterVersion, string, string, string, string, *models.PyFuncEnsembler, json.RawMessage, *service.EventChannel) error); ok { - r1 = rf(project, environment, currentRouterVersion, routerVersion, routerServiceAccountKey, enricherServiceAccountKey, ensemblerServiceAccountKey, expEngineServiceAccountKey, pyfuncEnsembler, experimentConfig, eventsCh) + if rf, ok := ret.Get(1).(func(*client.Project, *merlinclient.Environment, *models.RouterVersion, *models.RouterVersion, map[string]string, *models.PyFuncEnsembler, json.RawMessage, *service.EventChannel) error); ok { + r1 = rf(project, environment, currentRouterVersion, routerVersion, secretMap, pyfuncEnsembler, experimentConfig, eventsCh) } else { r1 = ret.Error(1) } @@ -63,6 +71,10 @@ func (_m *DeploymentService) DeployRouterVersion(project *client.Project, enviro func (_m *DeploymentService) GetLocalSecret(serviceAccountKeyFilePath string) (*string, error) { ret := _m.Called(serviceAccountKeyFilePath) + if len(ret) == 0 { + panic("no return value specified for GetLocalSecret") + } + var r0 *string var r1 error if rf, ok := ret.Get(0).(func(string) (*string, error)); ok { @@ -89,6 +101,10 @@ func (_m *DeploymentService) GetLocalSecret(serviceAccountKeyFilePath string) (* func (_m *DeploymentService) UndeployRouterVersion(project *client.Project, environment *merlinclient.Environment, routerVersion *models.RouterVersion, eventsCh *service.EventChannel, isCleanUp bool) error { ret := _m.Called(project, environment, routerVersion, eventsCh, isCleanUp) + if len(ret) == 0 { + panic("no return value specified for UndeployRouterVersion") + } + var r0 error if rf, ok := ret.Get(0).(func(*client.Project, *merlinclient.Environment, *models.RouterVersion, *service.EventChannel, bool) error); ok { r0 = rf(project, environment, routerVersion, eventsCh, isCleanUp) @@ -99,13 +115,12 @@ func (_m *DeploymentService) UndeployRouterVersion(project *client.Project, envi return r0 } -type mockConstructorTestingTNewDeploymentService interface { +// NewDeploymentService creates a new instance of DeploymentService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewDeploymentService(t interface { mock.TestingT Cleanup(func()) -} - -// NewDeploymentService creates a new instance of DeploymentService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewDeploymentService(t mockConstructorTestingTNewDeploymentService) *DeploymentService { +}) *DeploymentService { mock := &DeploymentService{} mock.Mock.Test(t) diff --git a/api/turing/service/router_deployment_service.go b/api/turing/service/router_deployment_service.go index 734e2866f..635e390b2 100644 --- a/api/turing/service/router_deployment_service.go +++ b/api/turing/service/router_deployment_service.go @@ -32,10 +32,7 @@ type DeploymentService interface { environment *merlin.Environment, currentRouterVersion *models.RouterVersion, routerVersion *models.RouterVersion, - routerServiceAccountKey string, - enricherServiceAccountKey string, - ensemblerServiceAccountKey string, - expEngineServiceAccountKey string, + secretMap map[string]string, pyfuncEnsembler *models.PyFuncEnsembler, experimentConfig json.RawMessage, eventsCh *EventChannel, @@ -114,10 +111,7 @@ func (ds *deploymentService) DeployRouterVersion( environment *merlin.Environment, currRouterVersion *models.RouterVersion, routerVersion *models.RouterVersion, - routerServiceAccountKey string, - enricherServiceAccountKey string, - ensemblerServiceAccountKey string, - expEngineServiceAccountKey string, + secretMap map[string]string, pyfuncEnsembler *models.PyFuncEnsembler, experimentConfig json.RawMessage, eventsCh *EventChannel, @@ -154,10 +148,7 @@ func (ds *deploymentService) DeployRouterVersion( secret := ds.svcBuilder.NewSecret( routerVersion, project, - routerServiceAccountKey, - enricherServiceAccountKey, - ensemblerServiceAccountKey, - expEngineServiceAccountKey, + secretMap, ) err = createSecret(ctx, controller, secret) if err != nil { @@ -253,7 +244,7 @@ func (ds *deploymentService) UndeployRouterVersion( // Delete secret eventsCh.Write(models.NewInfoEvent(models.EventStageDeletingDependencies, "deleting secrets")) - secret := ds.svcBuilder.NewSecret(routerVersion, project, "", "", "", "") + secret := ds.svcBuilder.NewSecret(routerVersion, project, nil) err = deleteSecret(controller, secret, isCleanUp) if err != nil { return err @@ -480,6 +471,7 @@ func (ds *deploymentService) buildEnsemblerServiceImage( Timeout: routerVersion.Ensembler.PyfuncConfig.Timeout, Endpoint: PyFuncEnsemblerServiceEndpoint, Port: PyFuncEnsemblerServicePort, + Secrets: routerVersion.Ensembler.PyfuncConfig.Secrets, Env: routerVersion.Ensembler.PyfuncConfig.Env, } diff --git a/api/turing/service/router_deployment_service_test.go b/api/turing/service/router_deployment_service_test.go index b77961217..ca3975b81 100644 --- a/api/turing/service/router_deployment_service_test.go +++ b/api/turing/service/router_deployment_service_test.go @@ -55,20 +55,12 @@ func (msb *mockClusterServiceBuilder) NewRouterEndpoint( func (msb *mockClusterServiceBuilder) NewSecret( routerVersion *models.RouterVersion, project *mlp.Project, - routerServiceAccountKey string, - enricherServiceAccountKey string, - ensemblerServiceAccountKey string, - expEngineServiceAccountKey string, + secretMap map[string]string, ) *cluster.Secret { return &cluster.Secret{ Name: fmt.Sprintf("%s-svc-acct-secret-%d", routerVersion.Router.Name, routerVersion.Version), Namespace: project.Name, - Data: map[string]string{ - "SecretKeyNameRouter": routerServiceAccountKey, - "SecretKeyNameEnricher": enricherServiceAccountKey, - "SecretKeyNameEnsembler": ensemblerServiceAccountKey, - "SecretKeyNameExpEngine": expEngineServiceAccountKey, - }, + Data: secretMap, } } @@ -187,6 +179,15 @@ func (msb *mockClusterServiceBuilder) GetRouterServiceName(_ *models.RouterVersi return "test-router-svc" } +var ( + secretMap = map[string]string{ + servicebuilder.SecretKeyNameRouter: "router-service-account-key", + servicebuilder.SecretKeyNameEnricher: "enricher-service-account-key", + servicebuilder.SecretKeyNameEnsembler: "ensembler-service-account-key", + servicebuilder.SecretKeyNameExpEngine: "exp-engine-service-account-key", + } +) + func TestDeployEndpoint(t *testing.T) { testEnv := "test-env" testNamespace := "test-namespace" @@ -258,10 +259,7 @@ func TestDeployEndpoint(t *testing.T) { &merlin.Environment{Name: testEnv}, nil, routerVersion, - "router-service-account-key", - "enricher-service-account-key", - "ensembler-service-account-key", - "exp-engine-service-account-key", + secretMap, nil, nil, eventsCh, @@ -281,10 +279,10 @@ func TestDeployEndpoint(t *testing.T) { Name: fmt.Sprintf("%s-svc-acct-secret-%d", routerVersion.Router.Name, routerVersion.Version), Namespace: testNamespace, Data: map[string]string{ - "SecretKeyNameRouter": "router-service-account-key", - "SecretKeyNameEnricher": "enricher-service-account-key", - "SecretKeyNameEnsembler": "ensembler-service-account-key", - "SecretKeyNameExpEngine": "exp-engine-service-account-key", + servicebuilder.SecretKeyNameRouter: "router-service-account-key", + servicebuilder.SecretKeyNameEnricher: "enricher-service-account-key", + servicebuilder.SecretKeyNameEnsembler: "ensembler-service-account-key", + servicebuilder.SecretKeyNameExpEngine: "exp-engine-service-account-key", }, }) controller.AssertCalled(t, "DeployKnativeService", mock.Anything, &cluster.KnativeService{ @@ -327,10 +325,10 @@ func TestDeployEndpoint(t *testing.T) { Name: fmt.Sprintf("%s-svc-acct-secret-%d", routerVersion.Router.Name, routerVersion.Version), Namespace: testNamespace, Data: map[string]string{ - "SecretKeyNameRouter": "router-service-account-key", - "SecretKeyNameEnricher": "enricher-service-account-key", - "SecretKeyNameEnsembler": "ensembler-service-account-key", - "SecretKeyNameExpEngine": "exp-engine-service-account-key", + servicebuilder.SecretKeyNameRouter: "router-service-account-key", + servicebuilder.SecretKeyNameEnricher: "enricher-service-account-key", + servicebuilder.SecretKeyNameEnsembler: "ensembler-service-account-key", + servicebuilder.SecretKeyNameExpEngine: "exp-engine-service-account-key", }, }) controller.AssertNumberOfCalls(t, "DeployKnativeService", 3) @@ -372,10 +370,7 @@ func TestDeployEndpoint(t *testing.T) { &merlin.Environment{Name: testEnv}, nil, routerVersion, - "router-service-account-key", - "enricher-service-account-key", - "ensembler-service-account-key", - "exp-engine-service-account-key", + secretMap, nil, nil, eventsCh, diff --git a/api/turing/validation/validator_test.go b/api/turing/validation/validator_test.go index d81044902..5986c04fa 100644 --- a/api/turing/validation/validator_test.go +++ b/api/turing/validation/validator_test.go @@ -1008,6 +1008,7 @@ func TestValidateAutoscaling(t *testing.T) { Value: "value", }, }, + Secrets: []models.Secret{}, } makeEnricher := func( enricher request.EnricherEnsemblerConfig, @@ -1043,6 +1044,7 @@ func TestValidateAutoscaling(t *testing.T) { ResourceRequest: &models.ResourceRequest{}, Timeout: "1s", Env: models.EnvVars{}, + Secrets: []models.Secret{}, }, }, }, @@ -1072,6 +1074,7 @@ func TestValidateAutoscaling(t *testing.T) { }, Timeout: "5s", Env: models.EnvVars{}, + Secrets: models.Secrets{}, }, }, }, @@ -1089,6 +1092,7 @@ func TestValidateAutoscaling(t *testing.T) { }, Timeout: "1s", Env: models.EnvVars{}, + Secrets: models.Secrets{}, }, }, }, diff --git a/sdk/e2e/01_create_router_test.py b/sdk/e2e/01_create_router_test.py index 5949a585f..1f939d177 100644 --- a/sdk/e2e/01_create_router_test.py +++ b/sdk/e2e/01_create_router_test.py @@ -71,6 +71,7 @@ def test_create_router(): timeout="3s", port=80, env=[EnvVar(name="TEST_ENV", value="enricher")], + secrets=[], ) # set up ensembler for the router @@ -83,6 +84,7 @@ def test_create_router(): timeout="3s", port=80, env=[EnvVar(name="TEST_ENV", value="ensembler")], + secrets=[], ) # create the RouterConfig instance diff --git a/sdk/e2e/02_update_router_invalid_test.py b/sdk/e2e/02_update_router_invalid_test.py index cd6205244..483b26fff 100644 --- a/sdk/e2e/02_update_router_invalid_test.py +++ b/sdk/e2e/02_update_router_invalid_test.py @@ -35,6 +35,7 @@ def test_update_router_invalid_config(): timeout="2s", port=80, env=[EnvVar(name="TEST_ENV", value="enricher")], + secrets=[], ) # update router diff --git a/sdk/e2e/03_create_router_version_test.py b/sdk/e2e/03_create_router_version_test.py index 05f52fc47..a13588a1b 100644 --- a/sdk/e2e/03_create_router_version_test.py +++ b/sdk/e2e/03_create_router_version_test.py @@ -43,6 +43,7 @@ def test_create_router_version(): timeout="2s", port=80, env=[EnvVar(name="TEST_ENV", value="enricher")], + secrets=[], ) # set up the new ensembler for the router @@ -58,6 +59,7 @@ def test_create_router_version(): timeout="3s", port=80, env=[EnvVar(name="TEST_ENV", value="ensembler")], + secrets=[], ) # update router diff --git a/sdk/e2e/08_deploy_router_with_traffic_rules_test.py b/sdk/e2e/08_deploy_router_with_traffic_rules_test.py index 0db262beb..a58851746 100644 --- a/sdk/e2e/08_deploy_router_with_traffic_rules_test.py +++ b/sdk/e2e/08_deploy_router_with_traffic_rules_test.py @@ -89,6 +89,7 @@ def test_deploy_router_with_traffic_rules(): timeout="3s", port=80, env=[EnvVar(name="TEST_ENV", value="ensembler")], + secrets=[], ) # create the RouterConfig instance diff --git a/sdk/tests/batch/config/config_test.py b/sdk/tests/batch/config/config_test.py index 126f9ded9..32a964a0b 100644 --- a/sdk/tests/batch/config/config_test.py +++ b/sdk/tests/batch/config/config_test.py @@ -1,9 +1,11 @@ import pytest +import turing.mounted_mlp_secret import turing.batch.config.source import turing.batch.config.sink import turing.generated.models from turing.generated.model.env_var import EnvVar +from turing.generated.model.mounted_mlp_secret import MountedMLPSecret @pytest.mark.parametrize( @@ -62,7 +64,7 @@ def test_job_spec(source, predictions, result_config, sink, expected_fn): @pytest.mark.parametrize( - "service_account,resource_request,env_vars,expected_fn", + "service_account,resource_request,env_vars,secrets,expected_fn", [ pytest.param( "service-account@gcp-project.iam.gserviceaccount.com", @@ -74,12 +76,24 @@ def test_job_spec(source, predictions, result_config, sink, expected_fn): executor_memory_request="800M", ), {"SOME_VAR": "SOME_VALUE"}, - lambda service_account, resource_request, env_vars: turing.generated.models.EnsemblerInfraConfig( + [ + turing.mounted_mlp_secret.MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], + lambda service_account, resource_request, env_vars, secrets: turing.generated.models.EnsemblerInfraConfig( service_account_name=service_account, resources=resource_request, env=[ EnvVar(name=name, value=value) for name, value in env_vars.items() ], + secrets=[ + MountedMLPSecret( + mlp_secret_name=secret.mlp_secret_name, + env_var_name=secret.env_var_name, + ) + for secret in secrets + ], ), id="Initialize ensembling job infra spec", ), @@ -87,18 +101,30 @@ def test_job_spec(source, predictions, result_config, sink, expected_fn): "service-account@gcp-project.iam.gserviceaccount.com", None, {"SOME_VAR": "SOME_VALUE"}, - lambda service_account, resource_request, env_vars: turing.generated.models.EnsemblerInfraConfig( + [ + turing.mounted_mlp_secret.MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], + lambda service_account, resource_request, env_vars, secrets: turing.generated.models.EnsemblerInfraConfig( service_account_name=service_account, resources=resource_request, env=[ EnvVar(name=name, value=value) for name, value in env_vars.items() ], + secrets=[ + MountedMLPSecret( + mlp_secret_name=secret.mlp_secret_name, + env_var_name=secret.env_var_name, + ) + for secret in secrets + ], ), id="Initialize ensembling job with default resource request", ), ], ) -def test_infra_spec(service_account, resource_request, env_vars, expected_fn): +def test_infra_spec(service_account, resource_request, env_vars, secrets, expected_fn): job_config = turing.batch.config.EnsemblingJobConfig( source=None, predictions={}, @@ -107,6 +133,7 @@ def test_infra_spec(service_account, resource_request, env_vars, expected_fn): service_account=service_account, resource_request=resource_request, env_vars=env_vars, + secrets=secrets, ) - expected = expected_fn(service_account, resource_request, env_vars) + expected = expected_fn(service_account, resource_request, env_vars, secrets) assert job_config.infra_spec() == expected diff --git a/sdk/tests/conftest.py b/sdk/tests/conftest.py index e75cfd9a0..1293ccc08 100644 --- a/sdk/tests/conftest.py +++ b/sdk/tests/conftest.py @@ -14,6 +14,7 @@ from tests.fixtures.gcs import mock_gcs from tests.fixtures.mlflow import mock_mlflow from turing.ensembler import PyFuncEnsembler +from turing.mounted_mlp_secret import MountedMLPSecret from turing.router.config.autoscaling_policy import AutoscalingPolicy from turing.router.config.common.env_var import EnvVar from turing.router.config.enricher import Enricher @@ -147,6 +148,11 @@ def docker_router_ensembler_config(): timeout="500ms", port=5120, env=[], + secrets=[ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], ) @@ -160,6 +166,11 @@ def pyfunc_router_ensembler_config(): ), timeout="500ms", env=[], + secrets=[ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], ) @@ -277,7 +288,11 @@ def generic_router_version_status(): @pytest.fixture def generic_resource_request(): return turing.generated.models.ResourceRequest( - min_replica=1, max_replica=3, cpu_request="100m", memory_request="512Mi", cpu_limit=None, + min_replica=1, + max_replica=3, + cpu_request="100m", + memory_request="512Mi", + cpu_limit=None, ) @@ -445,7 +460,17 @@ def generic_env_var(): @pytest.fixture -def generic_ensembler_docker_config(generic_resource_request, generic_env_var): +def generic_secret(): + return turing.generated.models.MountedMLPSecret( + mlp_secret_name="mlp_secret_name", + env_var_name="env_var_name", + ) + + +@pytest.fixture +def generic_ensembler_docker_config( + generic_resource_request, generic_env_var, generic_secret +): return turing.generated.models.EnsemblerDockerConfig( image="test.io/just-a-test/turing-ensembler:0.0.0-build.0", resource_request=generic_resource_request, @@ -453,6 +478,7 @@ def generic_ensembler_docker_config(generic_resource_request, generic_env_var): timeout="500ms", port=5120, env=[generic_env_var], + secrets=[generic_secret], service_account="secret-name-for-google-service-account", autoscaling_policy=turing.generated.models.AutoscalingPolicy( metric="memory", target="80" @@ -461,13 +487,16 @@ def generic_ensembler_docker_config(generic_resource_request, generic_env_var): @pytest.fixture -def generic_ensembler_pyfunc_config(generic_resource_request, generic_env_var): +def generic_ensembler_pyfunc_config( + generic_resource_request, generic_env_var, generic_secret +): return turing.generated.models.EnsemblerPyfuncConfig( project_id=77, ensembler_id=11, resource_request=generic_resource_request, timeout="500ms", env=[generic_env_var], + secrets=[generic_secret], autoscaling_policy=turing.generated.models.AutoscalingPolicy( metric="concurrency", target="10" ), @@ -537,7 +566,7 @@ def generic_pyfunc_router_ensembler_config(generic_ensembler_pyfunc_config): @pytest.fixture -def generic_enricher(generic_resource_request, generic_env_var): +def generic_enricher(generic_resource_request, generic_env_var, generic_secret): return turing.generated.models.Enricher( id=1, image="test.io/just-a-test/turing-enricher:0.0.0-build.0", @@ -546,6 +575,7 @@ def generic_enricher(generic_resource_request, generic_env_var): timeout="500ms", port=5180, env=[generic_env_var], + secrets=[generic_secret], service_account="service-account", autoscaling_policy=turing.generated.models.AutoscalingPolicy( metric="rps", target="100" @@ -658,6 +688,11 @@ def generic_router_config(docker_router_ensembler_config): timeout="60ms", port=8080, env=[EnvVar(name="test", value="abc")], + secrets=[ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], ), ensembler=docker_router_ensembler_config, ) diff --git a/sdk/tests/router/config/autoscaling_policy_test.py b/sdk/tests/router/config/autoscaling_policy_test.py index 080a58048..0e0cd40b1 100644 --- a/sdk/tests/router/config/autoscaling_policy_test.py +++ b/sdk/tests/router/config/autoscaling_policy_test.py @@ -41,6 +41,35 @@ def test_set_invalid_metric(): ), ], ) -def test_set_route_with_invalid_endpoint(metric, expected): +def test_set_autoscaling_policy_metric(metric, expected): policy = AutoscalingPolicy(metric=metric, target=DEFAULT_AUTOSCALING_POLICY.target) assert policy.metric == expected + + +@pytest.mark.parametrize( + "target,expected,error", + [ + pytest.param( + "100", + "100", + None, + ), + pytest.param( + "0.5", + "0.5", + None, + ), + pytest.param( + "0.5.5", + None, + AssertionError, + ), + ], +) +def test_set_autoscaling_policy_target(target, expected, error): + if error is not None: + with pytest.raises(error): + AutoscalingPolicy(metric=AutoscalingMetric.CONCURRENCY, target=target) + else: + policy = AutoscalingPolicy(metric=AutoscalingMetric.CONCURRENCY, target=target) + assert policy.target == expected diff --git a/sdk/tests/router/config/enricher_test.py b/sdk/tests/router/config/enricher_test.py index 63ea6b904..ef1b6f053 100644 --- a/sdk/tests/router/config/enricher_test.py +++ b/sdk/tests/router/config/enricher_test.py @@ -1,4 +1,5 @@ import pytest +from turing.mounted_mlp_secret import MountedMLPSecret from turing.router.config.autoscaling_policy import DEFAULT_AUTOSCALING_POLICY from turing.router.config.enricher import Enricher from turing.router.config.common.env_var import EnvVar @@ -7,7 +8,7 @@ @pytest.mark.parametrize( - "id,image,resource_request,autoscaling_policy,endpoint,timeout,port,env,service_account,expected", + "id,image,resource_request,autoscaling_policy,endpoint,timeout,port,env,secrets,service_account,expected", [ pytest.param( 1, @@ -20,6 +21,11 @@ "500ms", 5180, [EnvVar(name="env_name", value="env_val")], + [ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], "service-account", "generic_enricher", ) @@ -34,6 +40,7 @@ def test_create_enricher( timeout, port, env, + secrets, service_account, expected, request, @@ -47,6 +54,7 @@ def test_create_enricher( timeout=timeout, port=port, env=env, + secrets=secrets, service_account=service_account, ).to_open_api() assert actual == request.getfixturevalue(expected) @@ -64,6 +72,11 @@ def test_default_enricher_autoscaling_policy(): timeout="1s", port=8080, env=EnvVar(name="env_name", value="env_val"), + secrets=[ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], service_account="service_account", ).autoscaling_policy == DEFAULT_AUTOSCALING_POLICY diff --git a/sdk/tests/router/config/router_config_test.py b/sdk/tests/router/config/router_config_test.py index fd75d53af..1ab03b6e6 100644 --- a/sdk/tests/router/config/router_config_test.py +++ b/sdk/tests/router/config/router_config_test.py @@ -131,15 +131,17 @@ def test_set_router_config_base_ensembler( ensembler = RouterEnsemblerConfig( type=type, nop_config=None if nop_config is None else request.getfixturevalue(nop_config), - standard_config=None - if standard_config is None - else request.getfixturevalue(standard_config), - docker_config=None - if docker_config is None - else request.getfixturevalue(docker_config), - pyfunc_config=None - if pyfunc_config is None - else request.getfixturevalue(pyfunc_config), + standard_config=( + None + if standard_config is None + else request.getfixturevalue(standard_config) + ), + docker_config=( + None if docker_config is None else request.getfixturevalue(docker_config) + ), + pyfunc_config=( + None if pyfunc_config is None else request.getfixturevalue(pyfunc_config) + ), ) actual.ensembler = ensembler assert isinstance(actual.ensembler, expected_class) @@ -224,6 +226,7 @@ def test_default_router_autoscaling_policy(request): timeout="500ms", port=5120, env=[], + secrets=[], ), InvalidEnsemblerTypeException, ), @@ -240,6 +243,7 @@ def test_default_router_autoscaling_policy(request): ), timeout="60ms", env=[], + secrets=[], ), InvalidEnsemblerTypeException, ), diff --git a/sdk/tests/router/config/router_ensembler_config_test.py b/sdk/tests/router/config/router_ensembler_config_test.py index 878d4cfd1..2d37ab691 100644 --- a/sdk/tests/router/config/router_ensembler_config_test.py +++ b/sdk/tests/router/config/router_ensembler_config_test.py @@ -1,6 +1,7 @@ import pytest import turing from turing.generated.exceptions import ApiValueError +from turing.mounted_mlp_secret import MountedMLPSecret from turing.router.config.common.env_var import EnvVar from turing.router.config.autoscaling_policy import ( AutoscalingPolicy, @@ -19,6 +20,7 @@ InvalidExperimentMappingException, ) + @pytest.mark.parametrize( "id,type,standard_config,docker_config,expected", [ @@ -64,7 +66,7 @@ def test_create_router_ensembler_config( @pytest.mark.parametrize( - "project_id,ensembler_id,resource_request,autoscaling_policy,timeout,env,expected", + "project_id,ensembler_id,resource_request,autoscaling_policy,timeout,env,secrets,expected", [ pytest.param( 77, @@ -75,6 +77,11 @@ def test_create_router_ensembler_config( AutoscalingPolicy(metric="concurrency", target="10"), "500ms", [EnvVar(name="env_name", value="env_val")], + [ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], "generic_pyfunc_router_ensembler_config", ) ], @@ -86,6 +93,7 @@ def test_create_pyfunc_router_ensembler_config( autoscaling_policy, timeout, env, + secrets, expected, request, ): @@ -96,12 +104,13 @@ def test_create_pyfunc_router_ensembler_config( autoscaling_policy=autoscaling_policy, timeout=timeout, env=env, + secrets=secrets, ).to_open_api() assert actual == request.getfixturevalue(expected) @pytest.mark.parametrize( - "project_id,ensembler_id,resource_request,timeout,env,expected", + "project_id,ensembler_id,resource_request,timeout,env,secrets,expected", [ pytest.param( 77, @@ -111,12 +120,17 @@ def test_create_pyfunc_router_ensembler_config( ), "500ks", [EnvVar(name="env_name", value="env_val")], + [ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], ApiValueError, ) ], ) def test_create_pyfunc_router_ensembler_config_with_invalid_timeout( - project_id, ensembler_id, resource_request, timeout, env, expected + project_id, ensembler_id, resource_request, timeout, env, secrets, expected ): with pytest.raises(expected): PyfuncRouterEnsemblerConfig( @@ -125,11 +139,12 @@ def test_create_pyfunc_router_ensembler_config_with_invalid_timeout( resource_request=resource_request, timeout=timeout, env=env, + secrets=secrets, ).to_open_api() @pytest.mark.parametrize( - "image,resource_request,autoscaling_policy,endpoint,timeout,port,env,service_account,expected", + "image,resource_request,autoscaling_policy,endpoint,timeout,port,env,secrets,service_account,expected", [ pytest.param( "test.io/just-a-test/turing-ensembler:0.0.0-build.0", @@ -141,6 +156,11 @@ def test_create_pyfunc_router_ensembler_config_with_invalid_timeout( "500ms", 5120, [EnvVar(name="env_name", value="env_val")], + [ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], "secret-name-for-google-service-account", "generic_docker_router_ensembler_config", ) @@ -154,6 +174,7 @@ def test_create_docker_router_ensembler_config( timeout, port, env, + secrets, service_account, expected, request, @@ -166,13 +187,14 @@ def test_create_docker_router_ensembler_config( timeout=timeout, port=port, env=env, + secrets=secrets, service_account=service_account, ).to_open_api() assert actual == request.getfixturevalue(expected) @pytest.mark.parametrize( - "image,resource_request,endpoint,timeout,port,env,service_account,expected", + "image,resource_request,endpoint,timeout,port,env,secrets,service_account,expected", [ pytest.param( "#@!#!@#@!", @@ -183,13 +205,26 @@ def test_create_docker_router_ensembler_config( "500ms", 5120, [EnvVar(name="env_name", value="env_val")], + [ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], "secret-name-for-google-service-account", ApiValueError, ) ], ) def test_create_docker_router_ensembler_config_with_invalid_image( - image, resource_request, endpoint, timeout, port, env, service_account, expected + image, + resource_request, + endpoint, + timeout, + port, + env, + secrets, + service_account, + expected, ): with pytest.raises(expected): DockerRouterEnsemblerConfig( @@ -199,12 +234,13 @@ def test_create_docker_router_ensembler_config_with_invalid_image( timeout=timeout, port=port, env=env, + secrets=secrets, service_account=service_account, ).to_open_api() @pytest.mark.parametrize( - "image,resource_request,endpoint,timeout,port,env,service_account,expected", + "image,resource_request,endpoint,timeout,port,env,secrets,service_account,expected", [ pytest.param( "test.io/just-a-test/turing-ensembler:0.0.0-build.0", @@ -215,13 +251,26 @@ def test_create_docker_router_ensembler_config_with_invalid_image( "500ks", 5120, [EnvVar(name="env_name", value="env_val")], + [ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], "secret-name-for-google-service-account", ApiValueError, ) ], ) def test_create_docker_router_ensembler_config_with_invalid_timeout( - image, resource_request, endpoint, timeout, port, env, service_account, expected + image, + resource_request, + endpoint, + timeout, + port, + env, + secrets, + service_account, + expected, ): with pytest.raises(expected): DockerRouterEnsemblerConfig( @@ -231,12 +280,13 @@ def test_create_docker_router_ensembler_config_with_invalid_timeout( timeout=timeout, port=port, env=env, + secrets=secrets, service_account=service_account, ).to_open_api() @pytest.mark.parametrize( - "image,resource_request,endpoint,timeout,port,env,service_account,expected", + "image,resource_request,endpoint,timeout,port,env,secrets,service_account,expected", [ pytest.param( "test.io/just-a-test/turing-ensembler:0.0.0-build.0", @@ -247,6 +297,11 @@ def test_create_docker_router_ensembler_config_with_invalid_timeout( "500ms", 5120, [EnvVar(name="env_!@#!@$!", value="env_val")], + [ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], "secret-name-for-google-service-account", ApiValueError, ) @@ -259,6 +314,7 @@ def test_create_docker_router_ensembler_config_with_invalid_env( timeout, port, env, + secrets, service_account, expected, ): @@ -270,6 +326,7 @@ def test_create_docker_router_ensembler_config_with_invalid_env( timeout=timeout, port=port, env=env, + secrets=secrets, service_account=service_account, ).to_open_api() @@ -545,6 +602,7 @@ def test_create_nop_router_ensembler_config_with_invalid_route( with pytest.raises(expected): router.to_open_api() + @pytest.mark.parametrize( "ensembler_type,config,expected", [ @@ -553,7 +611,7 @@ def test_create_nop_router_ensembler_config_with_invalid_route( "nop_router_ensembler_config", { "nop_config": EnsemblerNopConfig(final_response_route_id="test"), - "type": "nop" + "type": "nop", }, ), pytest.param( @@ -564,16 +622,20 @@ def test_create_nop_router_ensembler_config_with_invalid_route( fallback_response_route_id="route-1", experiment_mappings=[ turing.generated.models.EnsemblerStandardConfigExperimentMappings( - experiment="experiment-1", route="route-1", treatment="treatment-1", + experiment="experiment-1", + route="route-1", + treatment="treatment-1", ), turing.generated.models.EnsemblerStandardConfigExperimentMappings( - experiment="experiment-2", route="route-2", treatment="treatment-2", + experiment="experiment-2", + route="route-2", + treatment="treatment-2", ), ], route_name_path=None, lazy_routing=False, ), - "type": "standard" + "type": "standard", }, ), pytest.param( @@ -586,7 +648,7 @@ def test_create_nop_router_ensembler_config_with_invalid_route( route_name_path="route_name", lazy_routing=False, ), - "type": "standard" + "type": "standard", }, ), pytest.param( @@ -594,19 +656,32 @@ def test_create_nop_router_ensembler_config_with_invalid_route( "generic_ensembler_docker_config", { "docker_config": turing.generated.models.EnsemblerDockerConfig( - autoscaling_policy=turing.generated.models.AutoscalingPolicy(metric="memory", target="80"), + autoscaling_policy=turing.generated.models.AutoscalingPolicy( + metric="memory", target="80" + ), endpoint="http://localhost:5000/ensembler_endpoint", - env=[turing.generated.models.EnvVar(name="env_name", value="env_val")], + env=[ + turing.generated.models.EnvVar(name="env_name", value="env_val") + ], + secrets=[ + turing.generated.models.MountedMLPSecret( + mlp_secret_name="mlp_secret_name", + env_var_name="env_var_name", + ) + ], image="test.io/just-a-test/turing-ensembler:0.0.0-build.0", port=5120, resource_request=turing.generated.models.ResourceRequest( - cpu_request="100m", cpu_limit=None, max_replica=3, - memory_request="512Mi", min_replica=1, + cpu_request="100m", + cpu_limit=None, + max_replica=3, + memory_request="512Mi", + min_replica=1, ), service_account="secret-name-for-google-service-account", - timeout="500ms" + timeout="500ms", ), - "type": "docker" + "type": "docker", }, ), pytest.param( @@ -614,17 +689,30 @@ def test_create_nop_router_ensembler_config_with_invalid_route( "generic_ensembler_pyfunc_config", { "pyfunc_config": turing.generated.models.EnsemblerPyfuncConfig( - autoscaling_policy=turing.generated.models.AutoscalingPolicy(metric="concurrency", target="10"), + autoscaling_policy=turing.generated.models.AutoscalingPolicy( + metric="concurrency", target="10" + ), ensembler_id=11, - env=[turing.generated.models.EnvVar(name="env_name", value="env_val")], + env=[ + turing.generated.models.EnvVar(name="env_name", value="env_val") + ], + secrets=[ + turing.generated.models.MountedMLPSecret( + mlp_secret_name="mlp_secret_name", + env_var_name="env_var_name", + ) + ], project_id=77, resource_request=turing.generated.models.ResourceRequest( - cpu_request="100m", cpu_limit=None, max_replica=3, - memory_request="512Mi", min_replica=1, + cpu_request="100m", + cpu_limit=None, + max_replica=3, + memory_request="512Mi", + min_replica=1, ), - timeout="500ms" + timeout="500ms", ), - "type": "pyfunc" + "type": "pyfunc", }, ), ], @@ -633,15 +721,24 @@ def test_create_base_ensembler(ensembler_type, config, expected, request): config_data = request.getfixturevalue(config) ensembler_config = None if ensembler_type == "nop": - ensembler_config = RouterEnsemblerConfig(type=ensembler_type, nop_config=config_data) + ensembler_config = RouterEnsemblerConfig( + type=ensembler_type, nop_config=config_data + ) elif ensembler_type == "standard": - ensembler_config = RouterEnsemblerConfig(type=ensembler_type, standard_config=config_data) + ensembler_config = RouterEnsemblerConfig( + type=ensembler_type, standard_config=config_data + ) elif ensembler_type == "docker": - ensembler_config = RouterEnsemblerConfig(type=ensembler_type, docker_config=config_data) + ensembler_config = RouterEnsemblerConfig( + type=ensembler_type, docker_config=config_data + ) elif ensembler_type == "pyfunc": - ensembler_config = RouterEnsemblerConfig(type=ensembler_type, pyfunc_config=config_data) + ensembler_config = RouterEnsemblerConfig( + type=ensembler_type, pyfunc_config=config_data + ) assert ensembler_config.to_dict() == expected + @pytest.mark.parametrize( "cls,config,expected", [ @@ -696,6 +793,11 @@ def test_create_base_ensembler(ensembler_type, config, expected, request): "timeout": "500ms", "port": 5120, "env": [EnvVar(name="env_name", value="env_val")], + "secrets": [ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], "service_account": "secret-name-for-google-service-account", }, ), @@ -717,6 +819,11 @@ def test_create_base_ensembler(ensembler_type, config, expected, request): ), "timeout": "500ms", "env": [EnvVar(name="env_name", value="env_val")], + "secrets": [ + MountedMLPSecret( + mlp_secret_name="mlp_secret_name", env_var_name="env_var_name" + ) + ], }, ), ], diff --git a/sdk/tests/router/config/router_version_test.py b/sdk/tests/router/config/router_version_test.py index 7bb2f9830..1a9ab026c 100644 --- a/sdk/tests/router/config/router_version_test.py +++ b/sdk/tests/router/config/router_version_test.py @@ -52,14 +52,13 @@ def test_create_version( # Assert the response against an experiment engine object that has its passkey removed expected_experiment_engine = generic_router_version.experiment_engine - if expected_experiment_engine.config is not None and \ - expected_experiment_engine.config.get("client") is not None and \ - expected_experiment_engine.config["client"].get("passkey") != "": + if ( + expected_experiment_engine.config is not None + and expected_experiment_engine.config.get("client") is not None + and expected_experiment_engine.config["client"].get("passkey") != "" + ): expected_experiment_engine.config["client"]["passkey"] = None - assert ( - actual_response.experiment_engine.to_open_api() - == expected_experiment_engine - ) + assert actual_response.experiment_engine.to_open_api() == expected_experiment_engine assert ( actual_response.resource_request.to_open_api() == generic_router_version.resource_request diff --git a/sdk/tests/testdata/api_responses/create_router_0000.json b/sdk/tests/testdata/api_responses/create_router_0000.json index f2789b463..02bbb6f4b 100644 --- a/sdk/tests/testdata/api_responses/create_router_0000.json +++ b/sdk/tests/testdata/api_responses/create_router_0000.json @@ -94,6 +94,12 @@ "value": "abc" } ], + "secrets": [ + { + "mlp_secret_name": "mlp_secret_name", + "env_var_name": "env_var_name" + } + ], "service_account": "" }, "ensembler": { @@ -114,6 +120,12 @@ "timeout": "500ms", "port": 5120, "env": [], + "secrets": [ + { + "mlp_secret_name": "mlp_secret_name", + "env_var_name": "env_var_name" + } + ], "service_account": "" } }, diff --git a/sdk/turing/batch/config/config.py b/sdk/turing/batch/config/config.py index e0e7eb19c..bb6c21a62 100644 --- a/sdk/turing/batch/config/config.py +++ b/sdk/turing/batch/config/config.py @@ -1,10 +1,11 @@ from enum import Enum -from typing import Dict, Optional +from typing import Dict, Optional, List import turing.generated.models from turing.generated.model_utils import OpenApiModel from turing.generated.model.env_var import EnvVar from .source import EnsemblingJobSource, EnsemblingJobPredictionSource from .sink import EnsemblingJobSink +from ...mounted_mlp_secret import MountedMLPSecret ResourceRequest = turing.generated.models.EnsemblingResources @@ -49,6 +50,7 @@ def __init__( service_account: str, resource_request: ResourceRequest = None, env_vars: Dict[str, str] = {}, + secrets: List[MountedMLPSecret] = None, ): """ Create new instance of batch ensembling job configuration @@ -61,6 +63,7 @@ def __init__( :param resource_request: optional resource request for starting the ensembling job. If not given the system default will be used. :param env_vars: optional environment variables in the form of a key value pair in a list. + :param secrets: list of MLP secrets to mount into the ensembling job environment as environment variables """ self._source = source self._predictions = predictions @@ -69,6 +72,7 @@ def __init__( self._service_account = service_account self._resource_request = resource_request self._env_vars = env_vars + self._secrets = secrets @property def source(self) -> "EnsemblingJobSource": @@ -94,6 +98,10 @@ def service_account(self) -> str: def env_vars(self) -> Dict[str, str]: return self._env_vars + @property + def secrets(self) -> Optional[List[MountedMLPSecret]]: + return self._secrets + @property def resource_request(self) -> Optional["ResourceRequest"]: return self._resource_request @@ -121,4 +129,9 @@ def infra_spec(self) -> turing.generated.models.EnsemblerInfraConfig: service_account_name=self.service_account, resources=self.resource_request, env=env_vars, + secrets=( + [secret.to_open_api() for secret in self.secrets] + if self.secrets + else [] + ), ) diff --git a/sdk/turing/ensembler.py b/sdk/turing/ensembler.py index e92442648..7b12a7d05 100644 --- a/sdk/turing/ensembler.py +++ b/sdk/turing/ensembler.py @@ -443,7 +443,7 @@ def delete(cls, ensembler_id: int) -> int: def _process_conda_env( - conda_env: Union[str, Dict[str, Any]] + conda_env: Union[str, Dict[str, Any]], ) -> Tuple[Dict[str, Any], str]: def match_dependency(spec, name): # Using direct match or regex match to match the dependency name, diff --git a/sdk/turing/generated/model/enricher.py b/sdk/turing/generated/model/enricher.py index 728165e9c..da24863db 100644 --- a/sdk/turing/generated/model/enricher.py +++ b/sdk/turing/generated/model/enricher.py @@ -29,9 +29,11 @@ def lazy_import(): from turing.generated.model.autoscaling_policy import AutoscalingPolicy from turing.generated.model.env_var import EnvVar + from turing.generated.model.mounted_mlp_secret import MountedMLPSecret from turing.generated.model.resource_request import ResourceRequest globals()['AutoscalingPolicy'] = AutoscalingPolicy globals()['EnvVar'] = EnvVar + globals()['MountedMLPSecret'] = MountedMLPSecret globals()['ResourceRequest'] = ResourceRequest @@ -92,6 +94,7 @@ def openapi_types(): 'timeout': (str,), # noqa: E501 'port': (int,), # noqa: E501 'env': ([EnvVar],), # noqa: E501 + 'secrets': ([MountedMLPSecret],), # noqa: E501 'id': (int,), # noqa: E501 'autoscaling_policy': (AutoscalingPolicy,), # noqa: E501 'service_account': (str,), # noqa: E501 @@ -111,6 +114,7 @@ def discriminator(): 'timeout': 'timeout', # noqa: E501 'port': 'port', # noqa: E501 'env': 'env', # noqa: E501 + 'secrets': 'secrets', # noqa: E501 'id': 'id', # noqa: E501 'autoscaling_policy': 'autoscaling_policy', # noqa: E501 'service_account': 'service_account', # noqa: E501 @@ -130,7 +134,7 @@ def discriminator(): ]) @convert_js_args_to_python_args - def __init__(self, image, resource_request, endpoint, timeout, port, env, *args, **kwargs): # noqa: E501 + def __init__(self, image, resource_request, endpoint, timeout, port, env, secrets, *args, **kwargs): # noqa: E501 """Enricher - a model defined in OpenAPI Args: @@ -140,6 +144,7 @@ def __init__(self, image, resource_request, endpoint, timeout, port, env, *args, timeout (str): port (int): env ([EnvVar]): + secrets ([MountedMLPSecret]): Keyword Args: _check_type (bool): if True, values for parameters in openapi_types @@ -208,6 +213,7 @@ def __init__(self, image, resource_request, endpoint, timeout, port, env, *args, self.timeout = timeout self.port = port self.env = env + self.secrets = secrets for var_name, var_value in kwargs.items(): if var_name not in self.attribute_map and \ self._configuration is not None and \ diff --git a/sdk/turing/generated/model/ensembler_docker_config.py b/sdk/turing/generated/model/ensembler_docker_config.py index 8bf671415..0667546e5 100644 --- a/sdk/turing/generated/model/ensembler_docker_config.py +++ b/sdk/turing/generated/model/ensembler_docker_config.py @@ -29,9 +29,11 @@ def lazy_import(): from turing.generated.model.autoscaling_policy import AutoscalingPolicy from turing.generated.model.env_var import EnvVar + from turing.generated.model.mounted_mlp_secret import MountedMLPSecret from turing.generated.model.resource_request import ResourceRequest globals()['AutoscalingPolicy'] = AutoscalingPolicy globals()['EnvVar'] = EnvVar + globals()['MountedMLPSecret'] = MountedMLPSecret globals()['ResourceRequest'] = ResourceRequest @@ -97,6 +99,7 @@ def openapi_types(): 'timeout': (str,), # noqa: E501 'port': (int,), # noqa: E501 'env': ([EnvVar],), # noqa: E501 + 'secrets': ([MountedMLPSecret],), # noqa: E501 'autoscaling_policy': (AutoscalingPolicy,), # noqa: E501 'service_account': (str,), # noqa: E501 } @@ -113,6 +116,7 @@ def discriminator(): 'timeout': 'timeout', # noqa: E501 'port': 'port', # noqa: E501 'env': 'env', # noqa: E501 + 'secrets': 'secrets', # noqa: E501 'autoscaling_policy': 'autoscaling_policy', # noqa: E501 'service_account': 'service_account', # noqa: E501 } @@ -129,7 +133,7 @@ def discriminator(): ]) @convert_js_args_to_python_args - def __init__(self, image, resource_request, endpoint, timeout, port, env, *args, **kwargs): # noqa: E501 + def __init__(self, image, resource_request, endpoint, timeout, port, env, secrets, *args, **kwargs): # noqa: E501 """EnsemblerDockerConfig - a model defined in OpenAPI Args: @@ -139,6 +143,7 @@ def __init__(self, image, resource_request, endpoint, timeout, port, env, *args, timeout (str): port (int): env ([EnvVar]): + secrets ([MountedMLPSecret]): Keyword Args: _check_type (bool): if True, values for parameters in openapi_types @@ -204,6 +209,7 @@ def __init__(self, image, resource_request, endpoint, timeout, port, env, *args, self.timeout = timeout self.port = port self.env = env + self.secrets = secrets for var_name, var_value in kwargs.items(): if var_name not in self.attribute_map and \ self._configuration is not None and \ diff --git a/sdk/turing/generated/model/ensembler_infra_config.py b/sdk/turing/generated/model/ensembler_infra_config.py index cdaed0732..c3c5c3b30 100644 --- a/sdk/turing/generated/model/ensembler_infra_config.py +++ b/sdk/turing/generated/model/ensembler_infra_config.py @@ -29,8 +29,10 @@ def lazy_import(): from turing.generated.model.ensembling_resources import EnsemblingResources from turing.generated.model.env_var import EnvVar + from turing.generated.model.mounted_mlp_secret import MountedMLPSecret globals()['EnsemblingResources'] = EnsemblingResources globals()['EnvVar'] = EnvVar + globals()['MountedMLPSecret'] = MountedMLPSecret class EnsemblerInfraConfig(ModelNormal): @@ -82,6 +84,7 @@ def openapi_types(): 'artifact_uri': (str,), # noqa: E501 'ensembler_name': (str,), # noqa: E501 'service_account_name': (str,), # noqa: E501 + 'secrets': ([MountedMLPSecret],), # noqa: E501 'resources': (EnsemblingResources,), # noqa: E501 'run_id': (str,), # noqa: E501 'env': ([EnvVar],), # noqa: E501 @@ -96,6 +99,7 @@ def discriminator(): 'artifact_uri': 'artifact_uri', # noqa: E501 'ensembler_name': 'ensembler_name', # noqa: E501 'service_account_name': 'service_account_name', # noqa: E501 + 'secrets': 'secrets', # noqa: E501 'resources': 'resources', # noqa: E501 'run_id': 'run_id', # noqa: E501 'env': 'env', # noqa: E501 @@ -150,6 +154,7 @@ def __init__(self, *args, **kwargs): # noqa: E501 artifact_uri (str): [optional] # noqa: E501 ensembler_name (str): [optional] # noqa: E501 service_account_name (str): [optional] # noqa: E501 + secrets ([MountedMLPSecret]): [optional] # noqa: E501 resources (EnsemblingResources): [optional] # noqa: E501 run_id (str): [optional] # noqa: E501 env ([EnvVar]): [optional] # noqa: E501 diff --git a/sdk/turing/generated/model/ensembler_pyfunc_config.py b/sdk/turing/generated/model/ensembler_pyfunc_config.py index 8ae8ff246..4828598c9 100644 --- a/sdk/turing/generated/model/ensembler_pyfunc_config.py +++ b/sdk/turing/generated/model/ensembler_pyfunc_config.py @@ -29,9 +29,11 @@ def lazy_import(): from turing.generated.model.autoscaling_policy import AutoscalingPolicy from turing.generated.model.env_var import EnvVar + from turing.generated.model.mounted_mlp_secret import MountedMLPSecret from turing.generated.model.resource_request import ResourceRequest globals()['AutoscalingPolicy'] = AutoscalingPolicy globals()['EnvVar'] = EnvVar + globals()['MountedMLPSecret'] = MountedMLPSecret globals()['ResourceRequest'] = ResourceRequest @@ -90,8 +92,9 @@ def openapi_types(): 'ensembler_id': (int,), # noqa: E501 'resource_request': (ResourceRequest,), # noqa: E501 'timeout': (str,), # noqa: E501 - 'autoscaling_policy': (AutoscalingPolicy,), # noqa: E501 'env': ([EnvVar],), # noqa: E501 + 'secrets': ([MountedMLPSecret],), # noqa: E501 + 'autoscaling_policy': (AutoscalingPolicy,), # noqa: E501 } @cached_property @@ -104,8 +107,9 @@ def discriminator(): 'ensembler_id': 'ensembler_id', # noqa: E501 'resource_request': 'resource_request', # noqa: E501 'timeout': 'timeout', # noqa: E501 - 'autoscaling_policy': 'autoscaling_policy', # noqa: E501 'env': 'env', # noqa: E501 + 'secrets': 'secrets', # noqa: E501 + 'autoscaling_policy': 'autoscaling_policy', # noqa: E501 } _composed_schemas = {} @@ -120,7 +124,7 @@ def discriminator(): ]) @convert_js_args_to_python_args - def __init__(self, project_id, ensembler_id, resource_request, timeout, *args, **kwargs): # noqa: E501 + def __init__(self, project_id, ensembler_id, resource_request, timeout, env, secrets, *args, **kwargs): # noqa: E501 """EnsemblerPyfuncConfig - a model defined in OpenAPI Args: @@ -128,6 +132,8 @@ def __init__(self, project_id, ensembler_id, resource_request, timeout, *args, * ensembler_id (int): resource_request (ResourceRequest): timeout (str): + env ([EnvVar]): + secrets ([MountedMLPSecret]): Keyword Args: _check_type (bool): if True, values for parameters in openapi_types @@ -161,7 +167,6 @@ def __init__(self, project_id, ensembler_id, resource_request, timeout, *args, * through its discriminator because we passed in _visited_composed_classes = (Animal,) autoscaling_policy (AutoscalingPolicy): [optional] # noqa: E501 - env ([EnvVar]): [optional] # noqa: E501 """ _check_type = kwargs.pop('_check_type', True) @@ -191,6 +196,8 @@ def __init__(self, project_id, ensembler_id, resource_request, timeout, *args, * self.ensembler_id = ensembler_id self.resource_request = resource_request self.timeout = timeout + self.env = env + self.secrets = secrets for var_name, var_value in kwargs.items(): if var_name not in self.attribute_map and \ self._configuration is not None and \ diff --git a/sdk/turing/generated/model/mounted_mlp_secret.py b/sdk/turing/generated/model/mounted_mlp_secret.py new file mode 100644 index 000000000..4d855ce9b --- /dev/null +++ b/sdk/turing/generated/model/mounted_mlp_secret.py @@ -0,0 +1,183 @@ +""" + Turing Minimal Openapi Spec for SDK + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) # noqa: E501 + + The version of the OpenAPI document: 0.0.1 + Generated by: https://openapi-generator.tech +""" + + +import re # noqa: F401 +import sys # noqa: F401 + +from turing.generated.model_utils import ( # noqa: F401 + ApiTypeError, + ModelComposed, + ModelNormal, + ModelSimple, + cached_property, + change_keys_js_to_python, + convert_js_args_to_python_args, + date, + datetime, + file_type, + none_type, + validate_get_composed_info, +) + + +class MountedMLPSecret(ModelNormal): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + + Attributes: + allowed_values (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + with a capitalized key describing the allowed value and an allowed + value. These dicts store the allowed enum values. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + discriminator_value_class_map (dict): A dict to go from the discriminator + variable value to the discriminator class name. + validations (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + that stores validations for max_length, min_length, max_items, + min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum, + inclusive_minimum, and regex. + additional_properties_type (tuple): A tuple of classes accepted + as additional properties values. + """ + + allowed_values = { + } + + validations = { + ('mlp_secret_name',): { + 'regex': { + 'pattern': r'^[-._a-zA-Z0-9]+$', # noqa: E501 + }, + }, + ('env_var_name',): { + 'regex': { + 'pattern': r'^[a-zA-Z0-9_]*$', # noqa: E501 + }, + }, + } + + additional_properties_type = None + + _nullable = False + + @cached_property + def openapi_types(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + + Returns + openapi_types (dict): The key is attribute name + and the value is attribute type. + """ + return { + 'mlp_secret_name': (str,), # noqa: E501 + 'env_var_name': (str,), # noqa: E501 + } + + @cached_property + def discriminator(): + return None + + + attribute_map = { + 'mlp_secret_name': 'mlp_secret_name', # noqa: E501 + 'env_var_name': 'env_var_name', # noqa: E501 + } + + _composed_schemas = {} + + required_properties = set([ + '_data_store', + '_check_type', + '_spec_property_naming', + '_path_to_item', + '_configuration', + '_visited_composed_classes', + ]) + + @convert_js_args_to_python_args + def __init__(self, mlp_secret_name, env_var_name, *args, **kwargs): # noqa: E501 + """MountedMLPSecret - a model defined in OpenAPI + + Args: + mlp_secret_name (str): + env_var_name (str): + + Keyword Args: + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + """ + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _path_to_item = kwargs.pop('_path_to_item', ()) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + + self.mlp_secret_name = mlp_secret_name + self.env_var_name = env_var_name + for var_name, var_value in kwargs.items(): + if var_name not in self.attribute_map and \ + self._configuration is not None and \ + self._configuration.discard_unknown_keys and \ + self.additional_properties_type is None: + # discard variable. + continue + setattr(self, var_name, var_value) diff --git a/sdk/turing/generated/models/__init__.py b/sdk/turing/generated/models/__init__.py index 8fba95939..48ce985cd 100644 --- a/sdk/turing/generated/models/__init__.py +++ b/sdk/turing/generated/models/__init__.py @@ -65,6 +65,7 @@ from turing.generated.model.kafka_config import KafkaConfig from turing.generated.model.label import Label from turing.generated.model.log_level import LogLevel +from turing.generated.model.mounted_mlp_secret import MountedMLPSecret from turing.generated.model.pagination_paging import PaginationPaging from turing.generated.model.project import Project from turing.generated.model.protocol import Protocol diff --git a/sdk/turing/mounted_mlp_secret.py b/sdk/turing/mounted_mlp_secret.py new file mode 100644 index 000000000..aea4e4b20 --- /dev/null +++ b/sdk/turing/mounted_mlp_secret.py @@ -0,0 +1,34 @@ +from dataclasses import dataclass, field + +import turing.generated.models +from turing.generated.model_utils import OpenApiModel + + +@dataclass +class MountedMLPSecret: + mlp_secret_name: str + env_var_name: str + + _mlp_secret_name: str = field(init=False, repr=False) + _env_var_name: str = field(init=False, repr=False) + + @property + def mlp_secret_name(self) -> str: + return self._mlp_secret_name + + @mlp_secret_name.setter + def mlp_secret_name(self, mlp_secret_name): + self._mlp_secret_name = mlp_secret_name + + @property + def env_var_name(self) -> str: + return self._env_var_name + + @env_var_name.setter + def env_var_name(self, env_var_name): + self._env_var_name = env_var_name + + def to_open_api(self) -> OpenApiModel: + return turing.generated.models.MountedMLPSecret( + mlp_secret_name=self.mlp_secret_name, env_var_name=self.env_var_name + ) diff --git a/sdk/turing/router/config/autoscaling_policy.py b/sdk/turing/router/config/autoscaling_policy.py index 2f564ea5d..7626681c3 100644 --- a/sdk/turing/router/config/autoscaling_policy.py +++ b/sdk/turing/router/config/autoscaling_policy.py @@ -68,7 +68,10 @@ def target(self) -> str: @target.setter def target(self, target: str): - assert target.isnumeric() + try: + float(target) + except ValueError: + raise AssertionError("Target value must be a number") self._target = target def to_open_api(self) -> OpenApiModel: diff --git a/sdk/turing/router/config/enricher.py b/sdk/turing/router/config/enricher.py index b3ec24adc..b60dbaaff 100644 --- a/sdk/turing/router/config/enricher.py +++ b/sdk/turing/router/config/enricher.py @@ -8,6 +8,7 @@ ) from turing.router.config.resource_request import ResourceRequest from turing.router.config.common.env_var import EnvVar +from turing.mounted_mlp_secret import MountedMLPSecret @dataclass @@ -21,6 +22,7 @@ class Enricher: :param timeout: request timeout which when exceeded, the request to the enricher will be terminated :param port: port number exposed by the container :param env: environment variables required by the container + :param secrets: list of MLP secrets to mount into the enricher environment as environment variables :param id: id of the enricher :param service_account: optional service account for the Docker deployment """ @@ -32,6 +34,7 @@ class Enricher: timeout: str port: int env: List["EnvVar"] + secrets: List["MountedMLPSecret"] id: int = None service_account: str = None @@ -43,6 +46,7 @@ def __init__( timeout: str, port: int, env: List["EnvVar"], + secrets: List["MountedMLPSecret"], id: int = None, service_account: str = None, autoscaling_policy: AutoscalingPolicy = DEFAULT_AUTOSCALING_POLICY, @@ -56,6 +60,7 @@ def __init__( self.timeout = timeout self.port = port self.env = env + self.secrets = secrets self.service_account = service_account @property @@ -144,6 +149,22 @@ def env(self, env: Union[List["EnvVar"], List[Dict[str, str]]]): else: self._env = env + @property + def secrets(self) -> List["MountedMLPSecret"]: + return self._secrets + + @secrets.setter + def secrets(self, secrets: Union[List["MountedMLPSecret"], List[Dict[str, str]]]): + if isinstance(secrets, list): + if all(isinstance(secret, MountedMLPSecret) for secret in secrets): + self._secrets = secrets + elif all(isinstance(secret, dict) for secret in secrets): + self._secrets = [MountedMLPSecret(**secret) for secret in secrets] + else: + self._secrets = secrets + else: + self._secrets = secrets + @property def service_account(self) -> str: return self._service_account @@ -169,5 +190,6 @@ def to_open_api(self) -> OpenApiModel: timeout=self.timeout, port=self.port, env=[env_var.to_open_api() for env_var in self.env], + secrets=[secret.to_open_api() for secret in self.secrets], **kwargs ) diff --git a/sdk/turing/router/config/router_config.py b/sdk/turing/router/config/router_config.py index 0cb0fb139..9d6f82600 100644 --- a/sdk/turing/router/config/router_config.py +++ b/sdk/turing/router/config/router_config.py @@ -217,9 +217,11 @@ def experiment_engine(self, experiment_engine: Union[ExperimentConfig, Dict]): # When the passkey value is not set, the Turing API server is able to automatically retrieve the correct # passkey from the existing router version (assuming it has been configured with the same standard # experiment engine and the same client username). - if self._experiment_engine.config is not None and \ - self._experiment_engine.config.get("client") is not None and \ - self._experiment_engine.config["client"].get("passkey") != "": + if ( + self._experiment_engine.config is not None + and self._experiment_engine.config.get("client") is not None + and self._experiment_engine.config["client"].get("passkey") != "" + ): self._experiment_engine.config["client"]["passkey"] = None else: self._experiment_engine = experiment_engine diff --git a/sdk/turing/router/config/router_ensembler_config.py b/sdk/turing/router/config/router_ensembler_config.py index 30a8084a8..6c7b95b69 100644 --- a/sdk/turing/router/config/router_ensembler_config.py +++ b/sdk/turing/router/config/router_ensembler_config.py @@ -10,6 +10,7 @@ ) from turing.router.config.resource_request import ResourceRequest from turing.router.config.common.env_var import EnvVar +from turing.mounted_mlp_secret import MountedMLPSecret @dataclass @@ -161,7 +162,10 @@ def standard_config(self, standard_config: Union[EnsemblerStandardConfig, Dict]) elif isinstance(standard_config, dict): openapi_standard_config = standard_config.copy() openapi_standard_config["experiment_mappings"] = None - if "experiment_mappings" in standard_config and standard_config["experiment_mappings"] is not None: + if ( + "experiment_mappings" in standard_config + and standard_config["experiment_mappings"] is not None + ): openapi_standard_config["experiment_mappings"] = [ turing.generated.models.EnsemblerStandardConfigExperimentMappings( **mapping @@ -200,6 +204,10 @@ def docker_config( turing.generated.models.EnvVar(**env_var) for env_var in docker_config["env"] ] + openapi_docker_config["secrets"] = [ + turing.generated.models.MountedMLPSecret(**secret) + for secret in docker_config["secrets"] + ] self._docker_config = turing.generated.models.EnsemblerDockerConfig( **openapi_docker_config ) @@ -234,6 +242,10 @@ def pyfunc_config( turing.generated.models.EnvVar(**env_var) for env_var in pyfunc_config["env"] ] + openapi_pyfunc_config["secrets"] = [ + turing.generated.models.MountedMLPSecret(**secret) + for secret in pyfunc_config["secrets"] + ] self._pyfunc_config = turing.generated.models.EnsemblerPyfuncConfig( **openapi_pyfunc_config ) @@ -275,6 +287,7 @@ def __init__( timeout: str, resource_request: ResourceRequest, env: List["EnvVar"], + secrets: List["MountedMLPSecret"], autoscaling_policy: AutoscalingPolicy = DEFAULT_AUTOSCALING_POLICY, ): """ @@ -286,6 +299,7 @@ def __init__( :param autoscaling_policy: AutoscalingPolicy instance containing configs for the deployment autoscaling :param timeout: request timeout which when exceeded, the request to the ensembler will be terminated :param env: environment variables required by the container + :param secrets: list of MLP secrets to mount into the ensembler environment as environment variables """ self.project_id = project_id self.ensembler_id = ensembler_id @@ -293,6 +307,7 @@ def __init__( self.autoscaling_policy = autoscaling_policy self.timeout = timeout self.env = env + self.secrets = secrets super().__init__(type="pyfunc") @property @@ -343,6 +358,14 @@ def env(self) -> List["EnvVar"]: def env(self, env: List["EnvVar"]): self._env = env + @property + def secrets(self) -> List["MountedMLPSecret"]: + return self._secrets + + @secrets.setter + def secrets(self, secrets: List["MountedMLPSecret"]): + self._secrets = secrets + @classmethod def from_config( cls, config: turing.generated.models.EnsemblerPyfuncConfig @@ -357,13 +380,22 @@ def from_config( cpu_request=config.resource_request.cpu_request, memory_request=config.resource_request.memory_request, ), - autoscaling_policy=AutoscalingPolicy( - metric=config.autoscaling_policy.metric, - target=config.autoscaling_policy.target, - ) - if config.autoscaling_policy is not None - else DEFAULT_AUTOSCALING_POLICY, + autoscaling_policy=( + AutoscalingPolicy( + metric=config.autoscaling_policy.metric, + target=config.autoscaling_policy.target, + ) + if config.autoscaling_policy is not None + else DEFAULT_AUTOSCALING_POLICY + ), env=[EnvVar(name=env.name, value=env.value) for env in config.env], + secrets=[ + MountedMLPSecret( + mlp_secret_name=secret.mlp_secret_name, + env_var_name=secret.env_var_name, + ) + for secret in config.secrets + ], ) def to_open_api(self) -> OpenApiModel: @@ -376,6 +408,7 @@ def to_open_api(self) -> OpenApiModel: autoscaling_policy=self.autoscaling_policy.to_open_api(), timeout=self.timeout, env=[env_var.to_open_api() for env_var in self.env], + secrets=[secret.to_open_api() for secret in self.secrets], ) return super().to_open_api() @@ -390,6 +423,7 @@ def __init__( timeout: str, port: int, env: List["EnvVar"], + secrets: List["MountedMLPSecret"], service_account: str = None, autoscaling_policy: AutoscalingPolicy = DEFAULT_AUTOSCALING_POLICY, ): @@ -403,6 +437,7 @@ def __init__( :param timeout: request timeout which when exceeded, the request to the ensembler will be terminated :param port: port number exposed by the container :param env: environment variables required by the container + :param secrets: list of MLP secrets to mount into the ensembler environment as environment variables :param service_account: optional service account for the Docker deployment """ self.image = image @@ -412,6 +447,7 @@ def __init__( self.timeout = timeout self.port = port self.env = env + self.secrets = secrets self.service_account = service_account super().__init__(type="docker") @@ -471,6 +507,14 @@ def env(self) -> List["EnvVar"]: def env(self, env: List["EnvVar"]): self._env = env + @property + def secrets(self) -> List["MountedMLPSecret"]: + return self._secrets + + @secrets.setter + def secrets(self, secrets: List["MountedMLPSecret"]): + self._secrets = secrets + @property def service_account(self) -> str: return self._service_account @@ -491,16 +535,25 @@ def from_config( cpu_request=config.resource_request.cpu_request, memory_request=config.resource_request.memory_request, ), - autoscaling_policy=AutoscalingPolicy( - metric=config.autoscaling_policy.metric, - target=config.autoscaling_policy.target, - ) - if config.autoscaling_policy is not None - else DEFAULT_AUTOSCALING_POLICY, + autoscaling_policy=( + AutoscalingPolicy( + metric=config.autoscaling_policy.metric, + target=config.autoscaling_policy.target, + ) + if config.autoscaling_policy is not None + else DEFAULT_AUTOSCALING_POLICY + ), endpoint=config.endpoint, timeout=config.timeout, port=config.port, env=[EnvVar(name=env.name, value=env.value) for env in config.env], + secrets=[ + MountedMLPSecret( + mlp_secret_name=secret.mlp_secret_name, + env_var_name=secret.env_var_name, + ) + for secret in config.secrets + ], service_account=config["service_account"], ) @@ -519,6 +572,7 @@ def to_open_api(self) -> OpenApiModel: timeout=self.timeout, port=self.port, env=[env_var.to_open_api() for env_var in self.env], + secrets=[secret.to_open_api() for secret in self.secrets], **kwargs, ) return super().to_open_api() @@ -601,9 +655,11 @@ def from_config( cls, config: EnsemblerStandardConfig ) -> "StandardRouterEnsemblerConfig": return cls( - experiment_mappings=[e.to_dict() for e in config.experiment_mappings] - if config.experiment_mappings - else None, + experiment_mappings=( + [e.to_dict() for e in config.experiment_mappings] + if config.experiment_mappings + else None + ), route_name_path=config.route_name_path, fallback_response_route_id=config.fallback_response_route_id, lazy_routing=config.lazy_routing, @@ -611,14 +667,16 @@ def from_config( def to_open_api(self) -> OpenApiModel: self.standard_config = EnsemblerStandardConfig( - experiment_mappings=[ - turing.generated.models.EnsemblerStandardConfigExperimentMappings( - **experiment_mapping - ) - for experiment_mapping in self.experiment_mappings - ] - if self.experiment_mappings - else None, + experiment_mappings=( + [ + turing.generated.models.EnsemblerStandardConfigExperimentMappings( + **experiment_mapping + ) + for experiment_mapping in self.experiment_mappings + ] + if self.experiment_mappings + else None + ), route_name_path=self.route_name_path, fallback_response_route_id=self.fallback_response_route_id, lazy_routing=self.lazy_routing, diff --git a/ui/craco.config.js b/ui/craco.config.js index e409e93f2..7772e6f78 100644 --- a/ui/craco.config.js +++ b/ui/craco.config.js @@ -1,6 +1,6 @@ const { ModuleFederationPlugin } = require("webpack").container; // Remove source code dependency react-lazylog which causes problems with sharing -const { "react-lazylog": undefined, ...sharedDeps } = require("./package.json").dependencies; +const { "react-lazylog-with-emitter": undefined, ...sharedDeps } = require("./package.json").dependencies; module.exports = ({ env }) => ({ plugins: [ diff --git a/ui/package.json b/ui/package.json index 066100e0d..d9fbb96f1 100644 --- a/ui/package.json +++ b/ui/package.json @@ -24,7 +24,10 @@ "react-collapsed": "^4.1.2", "react-diff-viewer": "^3.1.1", "react-dom": "^18.3.1", - "react-lazylog": "git+https://github.com/gojekfarm/react-lazylog#master", + "//": "react-lazylog-with-emitter is just a published version of the forked gojekfarm/react-lazylog, which cannot", + "//": "be installed easily by yarn due to some problem with the `spawn` package on M1 machines used to build the", + "//": "package from git", + "react-lazylog-with-emitter": "^4.5.3", "react-router-dom": "^6.23.1", "react-scroll": "^1.9.0", "react-scroll-to-bottom": "^4.0.0", diff --git a/ui/src/components/pod_logs_viewer/PodLogsViewer.js b/ui/src/components/pod_logs_viewer/PodLogsViewer.js index 1eed3e049..a76915fc4 100644 --- a/ui/src/components/pod_logs_viewer/PodLogsViewer.js +++ b/ui/src/components/pod_logs_viewer/PodLogsViewer.js @@ -8,7 +8,7 @@ import { EuiText, EuiLink } from "@elastic/eui"; -import { LazyLog, ScrollFollow } from "react-lazylog"; +import { LazyLog, ScrollFollow } from "react-lazylog-with-emitter"; import { slugify } from "@caraml-dev/ui-lib"; import isArray from "lodash/isArray"; diff --git a/ui/src/jobs/details/config/configuration_section/ConfigurationConfigSection.js b/ui/src/jobs/details/config/configuration_section/ConfigurationConfigSection.js index 1f8110d8b..66afb2866 100644 --- a/ui/src/jobs/details/config/configuration_section/ConfigurationConfigSection.js +++ b/ui/src/jobs/details/config/configuration_section/ConfigurationConfigSection.js @@ -4,6 +4,7 @@ import { ConfigSectionPanel } from "../../../../components/config_section"; import { ResourceRequestConfigTable } from "./resource_request_table/ResourceRequestConfigTable"; import { ConfigMultiSectionPanel } from "../../../../components/config_multi_section_panel/ConfigMultiSectionPanel"; import { EnvVariablesConfigTable } from "../../../../router/components/configuration/components/docker_config_section/EnvVariablesConfigTable"; +import { SecretsConfigTable } from "../../../../router/components/configuration/components/docker_config_section/SecretsConfigTable"; import { MiscConfigSection } from "./misc_table/MiscConfigSection"; export const ConfigurationConfigSection = ({ job: { infra_config = {} } }) => { @@ -16,6 +17,10 @@ export const ConfigurationConfigSection = ({ job: { infra_config = {} } }) => { title: "Environment Variables", children: , }, + { + title: "Secrets", + children: , + }, ]; return ( diff --git a/ui/src/router/components/configuration/components/docker_config_section/DockerConfigViewGroup.js b/ui/src/router/components/configuration/components/docker_config_section/DockerConfigViewGroup.js index a5248c12a..b3e20d3be 100644 --- a/ui/src/router/components/configuration/components/docker_config_section/DockerConfigViewGroup.js +++ b/ui/src/router/components/configuration/components/docker_config_section/DockerConfigViewGroup.js @@ -2,6 +2,7 @@ import React from "react"; import { EuiFlexGroup, EuiFlexItem } from "@elastic/eui"; import { ContainerConfigTable } from "./ContainerConfigTable"; import { EnvVariablesConfigTable } from "./EnvVariablesConfigTable"; +import { SecretsConfigTable } from "./SecretsConfigTable"; import { ResourcesConfigTable } from "../ResourcesConfigTable"; import { ConfigMultiSectionPanel } from "../../../../../components/config_multi_section_panel/ConfigMultiSectionPanel"; @@ -15,6 +16,10 @@ export const DockerConfigViewGroup = ({ componentName, dockerConfig }) => { title: "Environment Variables", children: , }, + { + title: "Secrets", + children: , + }, ]; return ( diff --git a/ui/src/router/components/configuration/components/docker_config_section/SecretsConfigTable.js b/ui/src/router/components/configuration/components/docker_config_section/SecretsConfigTable.js new file mode 100644 index 000000000..61df41a5a --- /dev/null +++ b/ui/src/router/components/configuration/components/docker_config_section/SecretsConfigTable.js @@ -0,0 +1,36 @@ +import React from "react"; +import { EuiInMemoryTable, EuiText } from "@elastic/eui"; +import { ExpandableContainer } from "../../../../../components/expandable_container/ExpandableContainer"; +import "./SecretsConfigTable.scss" + +export const SecretsConfigTable = ({ variables }) => { + const columns = [ + { + field: "mlp_secret_name", + name: "MLP Secret Name", + width: "30%", + sortable: true + }, + { + field: "env_var_name", + name: "Environment Variable Name", + width: "70%", + sortable: true + } + ]; + + return variables.length ? ( + + + + ) : ( + + None + + ); +}; diff --git a/ui/src/router/components/configuration/components/docker_config_section/SecretsConfigTable.scss b/ui/src/router/components/configuration/components/docker_config_section/SecretsConfigTable.scss new file mode 100644 index 000000000..bb1451108 --- /dev/null +++ b/ui/src/router/components/configuration/components/docker_config_section/SecretsConfigTable.scss @@ -0,0 +1,5 @@ +.euiInMemoryTable--secretsConfigTable { + .euiTableCellContent { + padding: 8px 0px; + } +} diff --git a/ui/src/router/components/configuration/components/pyfunc_config_section/PyFuncConfigViewGroup.js b/ui/src/router/components/configuration/components/pyfunc_config_section/PyFuncConfigViewGroup.js index e1d435ba3..37d96b2bd 100644 --- a/ui/src/router/components/configuration/components/pyfunc_config_section/PyFuncConfigViewGroup.js +++ b/ui/src/router/components/configuration/components/pyfunc_config_section/PyFuncConfigViewGroup.js @@ -5,6 +5,7 @@ import { ResourcesConfigTable } from "../ResourcesConfigTable"; import { ConfigMultiSectionPanel } from "../../../../../components/config_multi_section_panel/ConfigMultiSectionPanel"; import { PyFuncConfigTable } from "./PyFuncConfigTable"; import { EnvVariablesConfigTable } from "../docker_config_section/EnvVariablesConfigTable"; +import { SecretsConfigTable } from "../docker_config_section/SecretsConfigTable"; export const PyFuncConfigViewGroup = ({ componentName, @@ -25,10 +26,16 @@ export const PyFuncConfigViewGroup = ({ }); } - items.push({ - title: "Environment Variables", - children: , - }); + items.push( + { + title: "Environment Variables", + children: , + }, + { + title: "Secrets", + children: , + } + ); return ( diff --git a/ui/src/router/components/form/components/docker_config/DockerConfigFormGroup.js b/ui/src/router/components/form/components/docker_config/DockerConfigFormGroup.js index 96c3ff8ec..b3c4e39c1 100644 --- a/ui/src/router/components/form/components/docker_config/DockerConfigFormGroup.js +++ b/ui/src/router/components/form/components/docker_config/DockerConfigFormGroup.js @@ -5,6 +5,7 @@ import { DockerDeploymentPanel } from "./DockerDeploymentPanel"; import { DockerEnsembler } from "../../../../../services/ensembler"; import { DockerRegistriesContextProvider } from "../../../../../providers/docker/context"; import { EnvVariablesPanel } from "./EnvVariablesPanel"; +import { SecretsPanel } from "./SecretsPanel"; import { ResourcesPanel } from "../ResourcesPanel"; import { SecretsContextProvider } from "../../../../../providers/secrets/context"; import { useOnChangeHandler } from "../../../../../components/form/hooks/useOnChangeHandler"; @@ -30,8 +31,8 @@ export const DockerConfigFormGroup = ({ return ( !!dockerConfig && ( - - + + - - + - - - + + + - - - + + + - - - + + + + + + + + ) ); diff --git a/ui/src/router/components/form/components/docker_config/SecretsPanel.js b/ui/src/router/components/form/components/docker_config/SecretsPanel.js new file mode 100644 index 000000000..916d46ba5 --- /dev/null +++ b/ui/src/router/components/form/components/docker_config/SecretsPanel.js @@ -0,0 +1,119 @@ +import React, { useContext, useEffect, useState } from "react"; +import { EuiButtonIcon, EuiFieldText, EuiSpacer, EuiSuperSelect } from "@elastic/eui"; +import { Panel } from "../Panel"; +import { InMemoryTableForm } from "../../../../../components/form/in_memory_table_form/InMemoryTableForm"; +import { useOnChangeHandler } from "../../../../../components/form/hooks/useOnChangeHandler"; +import SecretsContext from "../../../../../providers/secrets/context"; +import "./SecretsPanel.scss" + +export const SecretsPanel = ({ + variables, + onChangeHandler, + errors = {}, +}) => { + const secrets = useContext(SecretsContext); + const { onChange } = useOnChangeHandler(onChangeHandler); + + const items = [ + ...variables.map((v, idx) => ({ idx, ...v })), + { idx: variables.length }, + ]; + + const onDeleteVariable = (idx) => () => { + variables.splice(idx, 1); + onChangeHandler(variables); + }; + + const getRowProps = (item) => { + const { idx } = item; + const isInvalid = !!errors[idx]; + return { + className: isInvalid ? "euiTableRow--isInvalid" : "", + "data-test-subj": `row-${idx}`, + }; + }; + + const [options, setOptions] = useState([]); + + useEffect(() => { + if (secrets) { + const options = []; + secrets + .sort((a, b) => (a.name > b.name ? -1 : 1)) + .forEach((secret) => { + options.push({ + value: secret.name, + inputDisplay: secret.name, + textwrap: "truncate", + }); + }); + setOptions(options); + } + }, [secrets]); + + const columns = [ + { + name: "MLP Secret Name", + field: "mlp_secret_name", + width: "45%", + textOnly: false, + render: (name, item) => ( + onChange(`${item.idx}.mlp_secret_name`)(e)} + hasDividers + /> + ), + }, + { + name: "Environment Variable Name", + field: "env_var_name", + width: "45%", + render: (value, item) => ( + onChange(`${item.idx}.env_var_name`)(e.target.value)} + /> + ), + }, + { + width: "10%", + actions: [ + { + render: (item) => { + return item.idx < items.length - 1 ? ( + + ) : ( +
+ ); + }, + }, + ], + }, + ]; + + return ( + + + `Row ${parseInt(key) + 1}`} + /> + + ); +}; diff --git a/ui/src/router/components/form/components/docker_config/SecretsPanel.scss b/ui/src/router/components/form/components/docker_config/SecretsPanel.scss new file mode 100644 index 000000000..1bb8992b8 --- /dev/null +++ b/ui/src/router/components/form/components/docker_config/SecretsPanel.scss @@ -0,0 +1,6 @@ +.euiBasicTable--inMemoryFormTable { + .euiTableRowCell > .euiTableCellContent { + display: block; + overflow: hidden; + } +} \ No newline at end of file diff --git a/ui/src/router/components/form/components/pyfunc_config/PyFuncConfigFormGroup.js b/ui/src/router/components/form/components/pyfunc_config/PyFuncConfigFormGroup.js index 31817039f..2f27111f4 100644 --- a/ui/src/router/components/form/components/pyfunc_config/PyFuncConfigFormGroup.js +++ b/ui/src/router/components/form/components/pyfunc_config/PyFuncConfigFormGroup.js @@ -8,6 +8,7 @@ import { PyFuncEnsembler } from "../../../../../services/ensembler"; import { PyFuncDeploymentPanel } from "./PyFuncDeploymentPanel"; import { EnsemblersContextProvider } from "../../../../../providers/ensemblers/context"; import { EnvVariablesPanel } from "../docker_config/EnvVariablesPanel"; +import { SecretsPanel } from "../docker_config/SecretsPanel"; import { AutoscalingPolicyPanel } from "../autoscaling_policy/AutoscalingPolicyPanel"; export const PyFuncConfigFormGroup = ({ @@ -31,8 +32,8 @@ export const PyFuncConfigFormGroup = ({ return ( !!pyfuncConfig && ( - - + + @@ -42,33 +43,41 @@ export const PyFuncConfigFormGroup = ({ errors={errors} /> - - + - - - + + + - - - + + + - - - + + + + + + + + ) ); diff --git a/ui/src/router/components/form/steps/EnricherStep.js b/ui/src/router/components/form/steps/EnricherStep.js index bdd330c46..868131d7a 100644 --- a/ui/src/router/components/form/steps/EnricherStep.js +++ b/ui/src/router/components/form/steps/EnricherStep.js @@ -4,6 +4,7 @@ import React, { Fragment, useContext } from "react"; import { FormContext, FormValidationContext } from "@caraml-dev/ui-lib"; import { AutoscalingPolicyPanel } from "../components/autoscaling_policy/AutoscalingPolicyPanel"; import { EnvVariablesPanel } from "../components/docker_config/EnvVariablesPanel"; +import { SecretsPanel } from "../components/docker_config/SecretsPanel"; import { EnricherTypePanel } from "../components/enricher_config/EnricherTypePanel"; import { DockerDeploymentPanel } from "../components/docker_config/DockerDeploymentPanel"; import { DockerRegistriesContextProvider } from "../../../../providers/docker/context"; @@ -42,8 +43,8 @@ export const EnricherStep = ({ projectId }) => { {enricher.type === "docker" && ( - - + + { errors={get(errors, "config.enricher")} /> - - + - - - + + + - - - + + + - - - + + + + + + + + )} diff --git a/ui/src/router/components/form/validation/schema.js b/ui/src/router/components/form/validation/schema.js index 6f2dbd13b..3f58eefe7 100644 --- a/ui/src/router/components/form/validation/schema.js +++ b/ui/src/router/components/form/validation/schema.js @@ -91,6 +91,7 @@ const routerNameRegex = /^[a-z0-9-]*$/, envVariableNameRegex = /^[a-z0-9_]*$/i, dockerImageRegex = /^([a-z0-9]+(?:[._-][a-z0-9]+)*(?::\d{2,5})?\/)?([a-z0-9]+(?:[._-][a-z0-9]+)*\/)*([a-z0-9]+(?:[._-][a-z0-9]+)*)(?::[a-z0-9]+(?:[._-][a-z0-9]+)*)?$/i, + configMapNameRegex = /^[-._a-zA-Z0-9]+$/, bigqueryTableRegex = /^[a-z][a-z0-9-]+\.\w+([_]?\w)+\.\w+([_]?\w)+$/i, kafkaBrokersRegex = /^([a-z]+:\/\/)?\[?([0-9a-zA-Z\-%._:]*)\]?:([0-9]+)(,([a-z]+:\/\/)?\[?([0-9a-zA-Z\-%._:]*)\]?:([0-9]+))*$/i, @@ -184,11 +185,28 @@ const environmentVariableSchema = yup.object().shape({ .required("Variable name can not be empty") .matches( envVariableNameRegex, - "The name of a variable can contain only alphanumeric character or the underscore" + "The name of an environment variable must contain only alphanumeric characters or '_'" ), value: yup.string(), }); +const secretSchema = yup.object().shape({ + mlp_secret_name: yup + .string() + .required("MLP secret name is required") + .matches( + configMapNameRegex, + "The name of the MLP secret must contain only alphanumeric characters, '-', '_' or '.'" + ), + env_var_name: yup + .string() + .required("Environment variable name is required") + .matches( + envVariableNameRegex, + "The name of an environment variable must contain only alphanumeric characters or '_'" + ), +}); + const resourceRequestSchema = (maxAllowedReplica) => yup.object().shape({ cpu_request: yup @@ -267,6 +285,7 @@ const dockerDeploymentSchema = (maxAllowedReplica) => (_) => .required("Port value is required, e.g. 8080"), timeout: timeoutSchema.required("Timeout is required"), env: yup.array(environmentVariableSchema), + secrets: yup.array(secretSchema), resource_request: resourceRequestSchema(maxAllowedReplica), autoscaling_policy: autoscalingPolicySchema, }); @@ -279,6 +298,7 @@ const pyfuncDeploymentSchema = (maxAllowedReplica) => (_) => resource_request: resourceRequestSchema(maxAllowedReplica), autoscaling_policy: autoscalingPolicySchema, env: yup.array(environmentVariableSchema), + secrets: yup.array(secretSchema), }); const mappingSchema = yup.object().shape({ diff --git a/ui/src/services/ensembler/DockerEnsembler.js b/ui/src/services/ensembler/DockerEnsembler.js index 07971a719..5b8ebb65c 100644 --- a/ui/src/services/ensembler/DockerEnsembler.js +++ b/ui/src/services/ensembler/DockerEnsembler.js @@ -33,6 +33,7 @@ export class DockerEnsembler extends Ensembler { target: "1", }, env: [], + secrets: [], service_account: "", }; } diff --git a/ui/src/services/ensembler/PyFuncEnsembler.js b/ui/src/services/ensembler/PyFuncEnsembler.js index 79ab3cf58..83e0a5512 100644 --- a/ui/src/services/ensembler/PyFuncEnsembler.js +++ b/ui/src/services/ensembler/PyFuncEnsembler.js @@ -34,6 +34,7 @@ export class PyFuncEnsembler extends Ensembler { target: "1", }, env: [], + secrets: [], timeout: "100ms", }; } diff --git a/ui/src/services/router/TuringRouter.js b/ui/src/services/router/TuringRouter.js index e7e55832d..df3c0b003 100644 --- a/ui/src/services/router/TuringRouter.js +++ b/ui/src/services/router/TuringRouter.js @@ -51,6 +51,7 @@ export class TuringRouter { target: "1", }, env: [], + secrets: [], service_account: "", }, ensembler: new NopEnsembler(), diff --git a/ui/src/services/version/RouterVersion.js b/ui/src/services/version/RouterVersion.js index 5b397f05e..317957859 100644 --- a/ui/src/services/version/RouterVersion.js +++ b/ui/src/services/version/RouterVersion.js @@ -108,6 +108,7 @@ export class RouterVersion { endpoint: this.enricher.endpoint, port: this.enricher.port, env: this.enricher.env, + secrets: this.enricher.secrets, service_account: this.enricher.service_account, resource_request: this.enricher.resource_request, autoscaling_policy: { diff --git a/ui/yarn.lock b/ui/yarn.lock index 4a67cf5c2..d8c86aaa2 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -37,6 +37,15 @@ "@babel/highlight" "^7.24.7" picocolors "^1.0.0" +"@babel/code-frame@^7.26.2": + version "7.26.2" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.26.2.tgz#4b5fab97d33338eff916235055f0ebc21e573a85" + integrity sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ== + dependencies: + "@babel/helper-validator-identifier" "^7.25.9" + js-tokens "^4.0.0" + picocolors "^1.0.0" + "@babel/compat-data@^7.22.6", "@babel/compat-data@^7.24.7": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.24.7.tgz#d23bbea508c3883ba8251fb4164982c36ea577ed" @@ -92,6 +101,17 @@ "@jridgewell/trace-mapping" "^0.3.25" jsesc "^2.5.1" +"@babel/generator@^7.26.9": + version "7.26.9" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.26.9.tgz#75a9482ad3d0cc7188a537aa4910bc59db67cbca" + integrity sha512-kEWdzjOAUMW4hAyrzJ0ZaTOu9OmpyDIQicIh0zg0EEcEkYXZb2TjtBhnHi2ViX7PKwZqF4xwqfAm299/QMP3lg== + dependencies: + "@babel/parser" "^7.26.9" + "@babel/types" "^7.26.9" + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" + jsesc "^3.0.2" + "@babel/helper-annotate-as-pure@^7.18.6", "@babel/helper-annotate-as-pure@^7.24.7": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.24.7.tgz#5373c7bc8366b12a033b4be1ac13a206c6656aab" @@ -206,7 +226,15 @@ "@babel/traverse" "^7.24.8" "@babel/types" "^7.24.8" -"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.10.4", "@babel/helper-module-imports@^7.16.7", "@babel/helper-module-imports@^7.24.7": +"@babel/helper-module-imports@^7.0.0": + version "7.25.9" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz#e7f8d20602ebdbf9ebbea0a0751fb0f2a4141715" + integrity sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw== + dependencies: + "@babel/traverse" "^7.25.9" + "@babel/types" "^7.25.9" + +"@babel/helper-module-imports@^7.10.4", "@babel/helper-module-imports@^7.16.7", "@babel/helper-module-imports@^7.24.7": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz#f2f980392de5b84c3328fc71d38bd81bbb83042b" integrity sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA== @@ -288,11 +316,21 @@ resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.24.8.tgz#5b3329c9a58803d5df425e5785865881a81ca48d" integrity sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ== +"@babel/helper-string-parser@^7.25.9": + version "7.25.9" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz#1aabb72ee72ed35789b4bbcad3ca2862ce614e8c" + integrity sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA== + "@babel/helper-validator-identifier@^7.24.7": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz#75b889cfaf9e35c2aaf42cf0d72c8e91719251db" integrity sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w== +"@babel/helper-validator-identifier@^7.25.9": + version "7.25.9" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz#24b64e2c3ec7cd3b3c547729b8d16871f22cbdc7" + integrity sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ== + "@babel/helper-validator-option@^7.24.7": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.24.7.tgz#24c3bb77c7a425d1742eec8fb433b5a1b38e62f6" @@ -336,6 +374,13 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.24.8.tgz#58a4dbbcad7eb1d48930524a3fd93d93e9084c6f" integrity sha512-WzfbgXOkGzZiXXCqk43kKwZjzwx4oulxZi3nq2TYL9mOjQv6kYwul9mz6ID36njuL7Xkp6nJEfok848Zj10j/w== +"@babel/parser@^7.26.9": + version "7.26.9" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.26.9.tgz#d9e78bee6dc80f9efd8f2349dcfbbcdace280fd5" + integrity sha512-81NWa1njQblgZbQHxWHpxxCzNsa3ZwvFqpUg7P+NNUU6f3UU2jBEg4OlF/J6rl8+PQGh1q6/zWScd001YwcA5A== + dependencies: + "@babel/types" "^7.26.9" + "@babel/plugin-bugfix-firefox-class-in-computed-class-key@^7.24.7": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-firefox-class-in-computed-class-key/-/plugin-bugfix-firefox-class-in-computed-class-key-7.24.7.tgz#fd059fd27b184ea2b4c7e646868a9a381bbc3055" @@ -1177,20 +1222,27 @@ integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA== "@babel/runtime-corejs3@^7.15.4": - version "7.24.8" - resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.24.8.tgz#c0ae5a1c380f8442920866d0cc51de8024507e28" - integrity sha512-DXG/BhegtMHhnN7YPIvxWd303/9aXvYFD1TjNL3CD6tUrhI2LVsg3Lck0aql5TRH29n4sj3emcROypkZVUfSuA== + version "7.26.9" + resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.26.9.tgz#8b73bae47453aa3dd2839ac52598581a7dd8332f" + integrity sha512-5EVjbTegqN7RSJle6hMWYxO4voo4rI+9krITk+DWR+diJgGrjZjrIBnJhjrHYYQsFgI7j1w1QnrvV7YSKBfYGg== dependencies: core-js-pure "^3.30.2" regenerator-runtime "^0.14.0" -"@babel/runtime@^7.0.0", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.13", "@babel/runtime@^7.12.5", "@babel/runtime@^7.16.3", "@babel/runtime@^7.18.3", "@babel/runtime@^7.24.1", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.8.7", "@babel/runtime@^7.9.2": +"@babel/runtime@^7.0.0", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.13", "@babel/runtime@^7.12.5", "@babel/runtime@^7.16.3", "@babel/runtime@^7.18.3", "@babel/runtime@^7.24.1", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.24.7.tgz#f4f0d5530e8dbdf59b3451b9b3e594b6ba082e12" integrity sha512-UwgBRMjJP+xv857DCngvqXI3Iq6J4v0wXmwc6sapg+zyhbwmQX67LUEFrkK5tbyJ30jGuG3ZvWpBiB9LCy1kWw== dependencies: regenerator-runtime "^0.14.0" +"@babel/runtime@^7.7.2", "@babel/runtime@^7.8.7": + version "7.26.9" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.26.9.tgz#aa4c6facc65b9cb3f87d75125ffd47781b475433" + integrity sha512-aA63XwOkcl4xxQa3HjPMqOP6LiK0ZDv3mUPYEFXkpHbaFjtGggE1A61FjFzJnB+p7/oy2gA8E+rcBNl/zC1tMg== + dependencies: + regenerator-runtime "^0.14.0" + "@babel/runtime@^7.9.6": version "7.25.0" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.25.0.tgz#3af9a91c1b739c569d5d80cc917280919c544ecb" @@ -1207,6 +1259,15 @@ "@babel/parser" "^7.24.7" "@babel/types" "^7.24.7" +"@babel/template@^7.26.9": + version "7.26.9" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.26.9.tgz#4577ad3ddf43d194528cff4e1fa6b232fa609bb2" + integrity sha512-qyRplbeIpNZhmzOysF/wFMuP9sctmh2cFzRAZOn1YapxBsE1i9bJIY586R/WBLfLcmcBlM8ROBiQURnnNy+zfA== + dependencies: + "@babel/code-frame" "^7.26.2" + "@babel/parser" "^7.26.9" + "@babel/types" "^7.26.9" + "@babel/traverse@^7.24.7", "@babel/traverse@^7.7.2": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.24.7.tgz#de2b900163fa741721ba382163fe46a936c40cf5" @@ -1239,6 +1300,19 @@ debug "^4.3.1" globals "^11.1.0" +"@babel/traverse@^7.25.9": + version "7.26.9" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.26.9.tgz#4398f2394ba66d05d988b2ad13c219a2c857461a" + integrity sha512-ZYW7L+pL8ahU5fXmNbPF+iZFHCv5scFak7MZ9bwaRPLUhHh7QQEMjZUg0HevihoqCM5iSYHN61EyCoZvqC+bxg== + dependencies: + "@babel/code-frame" "^7.26.2" + "@babel/generator" "^7.26.9" + "@babel/parser" "^7.26.9" + "@babel/template" "^7.26.9" + "@babel/types" "^7.26.9" + debug "^4.3.1" + globals "^11.1.0" + "@babel/types@^7.0.0", "@babel/types@^7.12.6", "@babel/types@^7.20.7", "@babel/types@^7.24.7", "@babel/types@^7.3.3", "@babel/types@^7.4.4": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.24.7.tgz#6027fe12bc1aa724cd32ab113fb7f1988f1f66f2" @@ -1257,6 +1331,14 @@ "@babel/helper-validator-identifier" "^7.24.7" to-fast-properties "^2.0.0" +"@babel/types@^7.25.9", "@babel/types@^7.26.9": + version "7.26.9" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.26.9.tgz#08b43dec79ee8e682c2ac631c010bdcac54a21ce" + integrity sha512-Y3IR1cRnOxOCDvMmNiym7XpXQ93iGDDPHx+Zj+NM+rg0fBaShfQLkg+hKPaZCEvg5N/LeCo4+Rj/i3FuJsIQaw== + dependencies: + "@babel/helper-string-parser" "^7.25.9" + "@babel/helper-validator-identifier" "^7.25.9" + "@base2/pretty-print-object@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@base2/pretty-print-object/-/pretty-print-object-1.0.1.tgz#371ba8be66d556812dc7fb169ebc3c08378f69d4" @@ -1476,7 +1558,24 @@ uuid "^8.3.0" vfile "^4.2.0" -"@emotion/babel-plugin@^11.0.0", "@emotion/babel-plugin@^11.11.0": +"@emotion/babel-plugin@^11.0.0": + version "11.13.5" + resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.13.5.tgz#eab8d65dbded74e0ecfd28dc218e75607c4e7bc0" + integrity sha512-pxHCpT2ex+0q+HH91/zsdHkw/lXd468DIN2zvfvLtPKLLMo6gQj7oLObq8PhkrxOZb/gGCq03S3Z7PDhS8pduQ== + dependencies: + "@babel/helper-module-imports" "^7.16.7" + "@babel/runtime" "^7.18.3" + "@emotion/hash" "^0.9.2" + "@emotion/memoize" "^0.9.0" + "@emotion/serialize" "^1.3.3" + babel-plugin-macros "^3.1.0" + convert-source-map "^1.5.0" + escape-string-regexp "^4.0.0" + find-root "^1.1.0" + source-map "^0.5.7" + stylis "4.2.0" + +"@emotion/babel-plugin@^11.11.0": version "11.11.0" resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz#c2d872b6a7767a9d176d007f5b31f7d504bb5d6c" integrity sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ== @@ -1503,7 +1602,18 @@ "@emotion/utils" "0.11.3" "@emotion/weak-memoize" "0.2.5" -"@emotion/cache@^11.1.3", "@emotion/cache@^11.11.0": +"@emotion/cache@^11.1.3": + version "11.14.0" + resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.14.0.tgz#ee44b26986eeb93c8be82bb92f1f7a9b21b2ed76" + integrity sha512-L/B1lc/TViYk4DcpGxtAVbx0ZyiKM5ktoIyafGkH6zg/tj+mA+NE//aPYKG0k8kCHSHVJrpLpcAlOBEXQ3SavA== + dependencies: + "@emotion/memoize" "^0.9.0" + "@emotion/sheet" "^1.4.0" + "@emotion/utils" "^1.4.2" + "@emotion/weak-memoize" "^0.4.0" + stylis "4.2.0" + +"@emotion/cache@^11.11.0": version "11.11.0" resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.11.0.tgz#809b33ee6b1cb1a625fef7a45bc568ccd9b8f3ff" integrity sha512-P34z9ssTCBi3e9EI1ZsWpNHcfY1r09ZO0rZbRO2ob3ZQMnFI35jB536qoXbkdesr5EUhYi22anuEJuyxifaqAQ== @@ -1546,6 +1656,11 @@ resolved "https://registry.yarnpkg.com/@emotion/hash/-/hash-0.9.1.tgz#4ffb0055f7ef676ebc3a5a91fb621393294e2f43" integrity sha512-gJB6HLm5rYwSLI6PQa+X1t5CFGrv1J1TWG+sOyMCeKz2ojaj6Fnl/rZEspogG+cvqbt4AE/2eIyD2QfLKTBNlQ== +"@emotion/hash@^0.9.2": + version "0.9.2" + resolved "https://registry.yarnpkg.com/@emotion/hash/-/hash-0.9.2.tgz#ff9221b9f58b4dfe61e619a7788734bd63f6898b" + integrity sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g== + "@emotion/memoize@0.7.4": version "0.7.4" resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.7.4.tgz#19bf0f5af19149111c40d98bb0cf82119f5d9eeb" @@ -1556,6 +1671,11 @@ resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.8.1.tgz#c1ddb040429c6d21d38cc945fe75c818cfb68e17" integrity sha512-W2P2c/VRW1/1tLox0mVUalvnWXxavmv/Oum2aPsRcoDJuob75FC3Y8FbpfLwUegRcxINtGUMPq0tFCvYNTBXNA== +"@emotion/memoize@^0.9.0": + version "0.9.0" + resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.9.0.tgz#745969d649977776b43fc7648c556aaa462b4102" + integrity sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ== + "@emotion/react@^11.11.4": version "11.11.4" resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.11.4.tgz#3a829cac25c1f00e126408fab7f891f00ecc3c1d" @@ -1581,7 +1701,18 @@ "@emotion/utils" "0.11.3" csstype "^2.5.7" -"@emotion/serialize@^1.0.0", "@emotion/serialize@^1.1.2", "@emotion/serialize@^1.1.3": +"@emotion/serialize@^1.0.0", "@emotion/serialize@^1.3.3": + version "1.3.3" + resolved "https://registry.yarnpkg.com/@emotion/serialize/-/serialize-1.3.3.tgz#d291531005f17d704d0463a032fe679f376509e8" + integrity sha512-EISGqt7sSNWHGI76hC7x1CksiXPahbxEOrC5RjmFRJTqLyEK9/9hZvBbiYn70dw4wuwMKiEMCUlR6ZXTSWQqxA== + dependencies: + "@emotion/hash" "^0.9.2" + "@emotion/memoize" "^0.9.0" + "@emotion/unitless" "^0.10.0" + "@emotion/utils" "^1.4.2" + csstype "^3.0.2" + +"@emotion/serialize@^1.1.2", "@emotion/serialize@^1.1.3": version "1.1.4" resolved "https://registry.yarnpkg.com/@emotion/serialize/-/serialize-1.1.4.tgz#fc8f6d80c492cfa08801d544a05331d1cc7cd451" integrity sha512-RIN04MBT8g+FnDwgvIUi8czvr1LU1alUMI05LekWB5DGyTm8cCBMCRpq3GqaiyEDRptEXOyXnvZ58GZYu4kBxQ== @@ -1597,7 +1728,12 @@ resolved "https://registry.yarnpkg.com/@emotion/sheet/-/sheet-0.9.4.tgz#894374bea39ec30f489bbfc3438192b9774d32e5" integrity sha512-zM9PFmgVSqBw4zL101Q0HrBVTGmpAxFZH/pYx/cjJT5advXguvcgjHFTCaIO3enL/xr89vK2bh0Mfyj9aa0ANA== -"@emotion/sheet@^1.0.0", "@emotion/sheet@^1.2.2": +"@emotion/sheet@^1.0.0", "@emotion/sheet@^1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@emotion/sheet/-/sheet-1.4.0.tgz#c9299c34d248bc26e82563735f78953d2efca83c" + integrity sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg== + +"@emotion/sheet@^1.2.2": version "1.2.2" resolved "https://registry.yarnpkg.com/@emotion/sheet/-/sheet-1.2.2.tgz#d58e788ee27267a14342303e1abb3d508b6d0fec" integrity sha512-0QBtGvaqtWi+nx6doRwDdBIzhNdZrXUppvTM4dtZZWEGTXL/XE/yJxLMGlDT1Gt+UHH5IX1n+jkXyytE/av7OA== @@ -1612,6 +1748,11 @@ resolved "https://registry.yarnpkg.com/@emotion/unitless/-/unitless-0.7.5.tgz#77211291c1900a700b8a78cfafda3160d76949ed" integrity sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg== +"@emotion/unitless@^0.10.0": + version "0.10.0" + resolved "https://registry.yarnpkg.com/@emotion/unitless/-/unitless-0.10.0.tgz#2af2f7c7e5150f497bdabd848ce7b218a27cf745" + integrity sha512-dFoMUuQA20zvtVTuxZww6OHoJYgrzfKM1t52mVySDJnMSEa08ruEvdYQbhvyu6soU+NeLVd3yKfTfT0NeV6qGg== + "@emotion/unitless@^0.8.1": version "0.8.1" resolved "https://registry.yarnpkg.com/@emotion/unitless/-/unitless-0.8.1.tgz#182b5a4704ef8ad91bde93f7a860a88fd92c79a3" @@ -1627,7 +1768,12 @@ resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-0.11.3.tgz#a759863867befa7e583400d322652a3f44820924" integrity sha512-0o4l6pZC+hI88+bzuaX/6BgOvQVhbt2PfmxauVaYOGgbsAw14wdKyvMCZXnsnsHys94iadcF+RG/wZyx6+ZZBw== -"@emotion/utils@^1.0.0", "@emotion/utils@^1.2.1": +"@emotion/utils@^1.0.0", "@emotion/utils@^1.4.2": + version "1.4.2" + resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-1.4.2.tgz#6df6c45881fcb1c412d6688a311a98b7f59c1b52" + integrity sha512-3vLclRofFziIa3J2wDh9jjbkUz9qk5Vi3IZ/FSTKViB0k+ef0fPV7dYrUIugbgupYDx7v9ud/SjrtEP8Y4xLoA== + +"@emotion/utils@^1.2.1": version "1.2.1" resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-1.2.1.tgz#bbab58465738d31ae4cb3dbb6fc00a5991f755e4" integrity sha512-Y2tGf3I+XVnajdItskUCn6LX+VUDmP6lTL4fcqsXAv43dnlbZiuW4MWQW38rW/BVWSE7Q/7+XQocmpnRYILUmg== @@ -1642,6 +1788,11 @@ resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz#d0fce5d07b0620caa282b5131c297bb60f9d87e6" integrity sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww== +"@emotion/weak-memoize@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz#5e13fac887f08c44f76b0ccaf3370eb00fec9bb6" + integrity sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg== + "@eslint-community/eslint-utils@^4.2.0": version "4.4.0" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" @@ -2062,10 +2213,10 @@ resolved "https://registry.yarnpkg.com/@react-oauth/google/-/google-0.12.1.tgz#b76432c3a525e9afe076f787d2ded003fcc1bee9" integrity sha512-qagsy22t+7UdkYAiT5ZhfM4StXi9PPNvw0zuwNmabrWyMKddczMtBIOARflbaIj+wHiQjnMAsZmzsUYuXeyoSg== -"@remix-run/router@1.17.0": - version "1.17.0" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.17.0.tgz#fbb0add487478ef42247d5942e7a5d8a2e20095f" - integrity sha512-2D6XaHEVvkCn682XBnipbJjgZUU7xjLtA4dGJRBVUKpEaDYOZMENZoZjAOSb7qirxt5RupjzZxz4fK2FO+EFPw== +"@remix-run/router@1.22.0": + version "1.22.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.22.0.tgz#dd8096cb055c475a4de6b35322b8d3b118c17b43" + integrity sha512-MBOl8MeOzpK0HQQQshKB7pABXbmyHizdTpqnrIseTbsv0nAepwC2ENZa1aaBExNQcpLoXmWthhak8SABLzvGPw== "@rollup/plugin-babel@^5.2.0": version "5.3.1" @@ -4086,11 +4237,16 @@ core-js-compat@^3.31.0, core-js-compat@^3.36.1: dependencies: browserslist "^4.23.0" -core-js-pure@^3.23.3, core-js-pure@^3.30.2: +core-js-pure@^3.23.3: version "3.37.1" resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.37.1.tgz#2b4b34281f54db06c9a9a5bd60105046900553bd" integrity sha512-J/r5JTHSmzTxbiYYrzXg9w1VpqrYt+gexenBE9pugeyhwPZTAEJddyiReJWsLO6uNQ8xJZFbod6XC7KKwatCiA== +core-js-pure@^3.30.2: + version "3.40.0" + resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.40.0.tgz#d9a019e9160f9b042eeb6abb92242680089d486e" + integrity sha512-AtDzVIgRrmRKQai62yuSIN5vNiQjcJakJb4fbhVw3ehxx7Lohphvw9SGNWKhLFqSxC4ilD0g/L1huAYFQU3Q6A== + core-js@3, core-js@^3.19.2, core-js@^3.6.5: version "3.37.1" resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.37.1.tgz#d21751ddb756518ac5a00e4d66499df981a62db9" @@ -6343,6 +6499,13 @@ is-core-module@^2.13.0, is-core-module@^2.13.1: dependencies: hasown "^2.0.2" +is-core-module@^2.16.0: + version "2.16.1" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.16.1.tgz#2a98801a849f43e2add644fbb6bc6229b19a4ef4" + integrity sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w== + dependencies: + hasown "^2.0.2" + is-data-view@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/is-data-view/-/is-data-view-1.0.1.tgz#4b4d3a511b70f3dc26d42c03ca9ca515d847759f" @@ -7207,6 +7370,11 @@ jsesc@^2.5.1: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== +jsesc@^3.0.2: + version "3.1.0" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-3.1.0.tgz#74d335a234f67ed19907fdadfac7ccf9d409825d" + integrity sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA== + jsesc@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" @@ -9143,9 +9311,10 @@ react-is@^18.0.0: resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.3.1.tgz#e83557dc12eae63a99e003a46388b1dcbb44db7e" integrity sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg== -"react-lazylog@git+https://github.com/gojekfarm/react-lazylog#master": +react-lazylog-with-emitter@^4.5.3: version "4.5.3" - resolved "git+https://github.com/gojekfarm/react-lazylog#e3a7f026983df0dc59d25843fe87ce7e37e24e82" + resolved "https://registry.yarnpkg.com/react-lazylog-with-emitter/-/react-lazylog-with-emitter-4.5.3.tgz#ee4bf810b7d0df063b876aa7afcdacb80e177a41" + integrity sha512-jfo3wl8w13juTaKlBO+FxjAWKUtWW1u4FRfKWp6vWRWMwf/OBx9l++kvLG/L9e8Nndg9EDTY5BoCiFknFSHIzg== dependencies: "@mattiasbuelens/web-streams-polyfill" "^0.2.0" fetch-readablestream "^0.2.0" @@ -9199,19 +9368,19 @@ react-remove-scroll@^2.5.7: use-sidecar "^1.1.2" react-router-dom@^6.23.1: - version "6.24.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.24.0.tgz#ec49dc38c49bb9bd25b310a8ae849268d3085e1d" - integrity sha512-960sKuau6/yEwS8e+NVEidYQb1hNjAYM327gjEyXlc6r3Skf2vtwuJ2l7lssdegD2YjoKG5l8MsVyeTDlVeY8g== + version "6.29.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.29.0.tgz#2ffb56b03ef3d6d6daafcfad9f3922132d2ced94" + integrity sha512-pkEbJPATRJ2iotK+wUwHfy0xs2T59YPEN8BQxVCPeBZvK7kfPESRc/nyxzdcxR17hXgUPYx2whMwl+eo9cUdnQ== dependencies: - "@remix-run/router" "1.17.0" - react-router "6.24.0" + "@remix-run/router" "1.22.0" + react-router "6.29.0" -react-router@6.24.0: - version "6.24.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.24.0.tgz#aa46648f26b6525e07f908ad3e1ad2e68d131155" - integrity sha512-sQrgJ5bXk7vbcC4BxQxeNa5UmboFm35we1AFK0VvQaz9g0LzxEIuLOhHIoZ8rnu9BO21ishGeL9no1WB76W/eg== +react-router@6.29.0: + version "6.29.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.29.0.tgz#14a329ca838b4de048fc5cca82874b727ee546b7" + integrity sha512-DXZJoE0q+KyeVw75Ck6GkPxFak63C4fGqZGNijnWgzB/HzSP1ZfTlBj5COaGWwhrMQ/R8bXiq5Ooy4KG+ReyjQ== dependencies: - "@remix-run/router" "1.17.0" + "@remix-run/router" "1.22.0" react-scripts@^5.0.1: version "5.0.1" @@ -9332,9 +9501,9 @@ react-virtualized-auto-sizer@^1.0.20: integrity sha512-3kCn7N9NEb3FlvJrSHWGQ4iVl+ydQObq2fHMn12i5wbtm74zHOPhz/i64OL3c1S1vi9i2GXtZqNqUJTQ+BnNfg== react-virtualized@^9.21.0: - version "9.22.5" - resolved "https://registry.yarnpkg.com/react-virtualized/-/react-virtualized-9.22.5.tgz#bfb96fed519de378b50d8c0064b92994b3b91620" - integrity sha512-YqQMRzlVANBv1L/7r63OHa2b0ZsAaDp1UhVNEdUaXI8A5u6hTpA5NYtUueLH2rFuY/27mTGIBl7ZhqFKzw18YQ== + version "9.22.6" + resolved "https://registry.yarnpkg.com/react-virtualized/-/react-virtualized-9.22.6.tgz#3ae2aa69eca61cf3af332e2f9d6b4aa5638786d5" + integrity sha512-U5j7KuUQt3AaMatlMJ0UJddqSiX+Km0YJxSqbAzIiGw5EmNz0khMyqP2hzgu4+QUtm+QPIrxzUX4raJxmVJnHg== dependencies: "@babel/runtime" "^7.7.2" clsx "^1.0.4" @@ -9650,7 +9819,7 @@ resolve.exports@^1.1.0: resolved "https://registry.yarnpkg.com/resolve.exports/-/resolve.exports-1.1.1.tgz#05cfd5b3edf641571fd46fa608b610dda9ead999" integrity sha512-/NtpHNDN7jWhAaQ9BvBUYZ6YTXsRBgfqWFWP7BZBaoMJO/I3G5OFzvTuWNlZC3aPjins1F+TNrLKsGbH4rfsRQ== -resolve@^1.1.7, resolve@^1.12.0, resolve@^1.14.2, resolve@^1.19.0, resolve@^1.20.0, resolve@^1.22.2, resolve@^1.22.4: +resolve@^1.1.7, resolve@^1.14.2, resolve@^1.19.0, resolve@^1.20.0, resolve@^1.22.2, resolve@^1.22.4: version "1.22.8" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== @@ -9659,6 +9828,15 @@ resolve@^1.1.7, resolve@^1.12.0, resolve@^1.14.2, resolve@^1.19.0, resolve@^1.20 path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" +resolve@^1.12.0: + version "1.22.10" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.10.tgz#b663e83ffb09bbf2386944736baae803029b8b39" + integrity sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w== + dependencies: + is-core-module "^2.16.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + resolve@^2.0.0-next.5: version "2.0.0-next.5" resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.5.tgz#6b0ec3107e671e52b68cd068ef327173b90dc03c"