From c7617d3fb2d90d11b8eabb42012215ca837f3db3 Mon Sep 17 00:00:00 2001 From: adamschmidt Date: Tue, 28 Mar 2023 21:44:03 +1100 Subject: [PATCH 1/3] fix: Bytewax materializer security context Signed-off-by: adamschmidt --- .../bytewax/bytewax_materialization_engine.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py index 991eafa641c..32c523b0985 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py @@ -58,6 +58,8 @@ class BytewaxMaterializationEngineConfig(FeastConfigBaseModel): annotations: dict = {} """ (optional) Annotations to apply to the job container. Useful for linking the service account to IAM roles, operational metadata, etc """ + include_security_context_capabilities: bool = True + """ (optional) Include security context capabilities in the init and job container spec """ class BytewaxMaterializationEngine(BatchMaterializationEngine): def __init__( @@ -198,6 +200,9 @@ def _create_configuration_map(self, job_id, paths, feature_view, namespace): "apiVersion": "v1", "metadata": { "name": f"feast-{job_id}", + "labels": { + "feast-bytewax-materializer": "configmap", + }, }, "data": { "feature_store.yaml": feature_store_configuration, @@ -253,6 +258,9 @@ def _create_job_definition(self, job_id, namespace, pods, env): "metadata": { "name": f"dataflow-{job_id}", "namespace": namespace, + "labels": { + "feast-bytewax-materializer": "job", + }, }, "spec": { "ttlSecondsAfterFinished": 3600, @@ -262,6 +270,9 @@ def _create_job_definition(self, job_id, namespace, pods, env): "template": { "metadata": { "annotations": self.batch_engine_config.annotations, + "labels": { + "feast-bytewax-materializer": "pod", + }, }, "spec": { "restartPolicy": "Never", @@ -282,10 +293,7 @@ def _create_job_definition(self, job_id, namespace, pods, env): "resources": {}, "securityContext": { "allowPrivilegeEscalation": False, - "capabilities": { - "add": ["NET_BIND_SERVICE"], - "drop": ["ALL"], - }, + "capabilities": { "add": ["NET_BIND_SERVICE"], "drop": ["ALL"], } if self.batch_engine_config.include_security_context_capabilities else None, "readOnlyRootFilesystem": True, }, "terminationMessagePath": "/dev/termination-log", @@ -320,10 +328,7 @@ def _create_job_definition(self, job_id, namespace, pods, env): "resources": self.batch_engine_config.resources, "securityContext": { "allowPrivilegeEscalation": False, - "capabilities": { - "add": ["NET_BIND_SERVICE"], - "drop": ["ALL"], - }, + "capabilities": { "add": ["NET_BIND_SERVICE"], "drop": ["ALL"], } if self.batch_engine_config.include_security_context_capabilities else None, "readOnlyRootFilesystem": False, }, "terminationMessagePath": "/dev/termination-log", From 5931aeff9ea93d53a8b469087b8e2bd868188628 Mon Sep 17 00:00:00 2001 From: adamschmidt Date: Tue, 28 Mar 2023 21:56:58 +1100 Subject: [PATCH 2/3] fix: lint Signed-off-by: adamschmidt --- .../bytewax/bytewax_materialization_engine.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py index 32c523b0985..b222128bbbe 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py @@ -61,6 +61,7 @@ class BytewaxMaterializationEngineConfig(FeastConfigBaseModel): include_security_context_capabilities: bool = True """ (optional) Include security context capabilities in the init and job container spec """ + class BytewaxMaterializationEngine(BatchMaterializationEngine): def __init__( self, @@ -252,6 +253,13 @@ def _create_job_definition(self, job_id, namespace, pods, env): # Add any Feast configured environment variables job_env.extend(env) + securityContextCapabilities = None + if self.batch_engine_config.include_security_context_capabilities: + securityContextCapabilities = { + "add": ["NET_BIND_SERVICE"], + "drop": ["ALL"], + } + job_definition = { "apiVersion": "batch/v1", "kind": "Job", @@ -293,7 +301,7 @@ def _create_job_definition(self, job_id, namespace, pods, env): "resources": {}, "securityContext": { "allowPrivilegeEscalation": False, - "capabilities": { "add": ["NET_BIND_SERVICE"], "drop": ["ALL"], } if self.batch_engine_config.include_security_context_capabilities else None, + "capabilities": securityContextCapabilities, "readOnlyRootFilesystem": True, }, "terminationMessagePath": "/dev/termination-log", @@ -328,7 +336,7 @@ def _create_job_definition(self, job_id, namespace, pods, env): "resources": self.batch_engine_config.resources, "securityContext": { "allowPrivilegeEscalation": False, - "capabilities": { "add": ["NET_BIND_SERVICE"], "drop": ["ALL"], } if self.batch_engine_config.include_security_context_capabilities else None, + "capabilities": securityContextCapabilities, "readOnlyRootFilesystem": False, }, "terminationMessagePath": "/dev/termination-log", From bb84fc8f4049c430b2250706df1e1bd8a2554aa9 Mon Sep 17 00:00:00 2001 From: adamschmidt Date: Tue, 28 Mar 2023 22:07:13 +1100 Subject: [PATCH 3/3] chore: docs Signed-off-by: adamschmidt --- docs/reference/batch-materialization/bytewax.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/reference/batch-materialization/bytewax.md b/docs/reference/batch-materialization/bytewax.md index 2e28937f50e..6a97bd391db 100644 --- a/docs/reference/batch-materialization/bytewax.md +++ b/docs/reference/batch-materialization/bytewax.md @@ -58,6 +58,7 @@ batch_engine: image_pull_secrets: - my_container_secret service_account_name: my-k8s-service-account + include_security_context_capabilities: false annotations: # example annotation you might include if running on AWS EKS iam.amazonaws.com/role: arn:aws:iam:::role/MyBytewaxPlatformRole @@ -73,8 +74,9 @@ batch_engine: **Notes:** * The `namespace` configuration directive specifies which Kubernetes [namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) jobs, services and configuration maps will be created in. -* The `image_pull_secrets` configuration directive specifies the pre-configured secret to use when pulling the image container from your registry -* The `service_account_name` specifies which Kubernetes service account to run the job under +* The `image_pull_secrets` configuration directive specifies the pre-configured secret to use when pulling the image container from your registry. +* The `service_account_name` specifies which Kubernetes service account to run the job under. +* The `include_security_context_capabilities` flag indicates whether or not `"add": ["NET_BIND_SERVICE"]` and `"drop": ["ALL"]` are included in the job & pod security context capabilities. * `annotations` allows you to include additional Kubernetes annotations to the job. This is particularly useful for IAM roles which grant the running pod access to cloud platform resources (for example). * The `resources` configuration directive sets the standard Kubernetes [resource requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for the job containers to utilise when materializing data.