From d26fd2106faa41d04d4b1db201f939463e570346 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Tue, 1 Nov 2022 21:44:35 +0000 Subject: [PATCH] YARN-11364. Docker Container to accept docker Image name with sha256 digest (#5092) Co-authored-by: Ashutosh Gupta Reviewed-by: slfan1989 <55643692+slfan1989@users.noreply.github.com> Signed-off-by: Chris Nauroth (cherry picked from commit 83acb559817a97c14c4e3fd846dcc16ab615093e) (cherry picked from commit 0961014262c5fffe69fcec6b9874c607e1e8ce77) --- .../runtime/DockerLinuxContainerRuntime.java | 17 ++++++++-- .../runtime/TestDockerContainerRuntime.java | 34 ++++++++++++------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 0010e49aeaceb..4d95599becfd6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -213,6 +213,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { "|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" + ":[0-9]+:[0-9]+$"; private static final int HOST_NAME_LENGTH = 64; + + private static final Pattern DOCKER_DIGEST_PATTERN = Pattern.compile("^sha256:[a-z0-9]{12,64}$"); + private static final String DEFAULT_PROCFS = "/proc"; @InterfaceAudience.Private @@ -1264,9 +1267,17 @@ public static void validateImageName(String imageName) throw new ContainerExecutionException( ENV_DOCKER_CONTAINER_IMAGE + " not set!"); } - if (!dockerImagePattern.matcher(imageName).matches()) { - throw new ContainerExecutionException("Image name '" + imageName - + "' doesn't match docker image name pattern"); + // check if digest is part of imageName, extract and validate it. + String digest = null; + if (imageName.contains("@sha256")) { + String[] digestParts = imageName.split("@"); + digest = digestParts[1]; + imageName = digestParts[0]; + } + if (!dockerImagePattern.matcher(imageName).matches() || (digest != null + && !DOCKER_DIGEST_PATTERN.matcher(digest).matches())) { + throw new ContainerExecutionException( + "Image name '" + imageName + "' doesn't match docker image name pattern"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index ed530caa36aac..1aeed7e3a1b1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -1950,19 +1950,27 @@ public static Configuration enableMockContainerExecutor(Configuration conf) { @Test public void testDockerImageNamePattern() throws Exception { - String[] validNames = - { "ubuntu", "fedora/httpd:version1.0", - "fedora/httpd:version1.0.test", - "fedora/httpd:version1.0.TEST", - "myregistryhost:5000/ubuntu", - "myregistryhost:5000/fedora/httpd:version1.0", - "myregistryhost:5000/fedora/httpd:version1.0.test", - "myregistryhost:5000/fedora/httpd:version1.0.TEST"}; - - String[] invalidNames = { "Ubuntu", "ubuntu || fedora", "ubuntu#", - "myregistryhost:50AB0/ubuntu", "myregistry#host:50AB0/ubuntu", - ":8080/ubuntu" - }; + String[] validNames = {"ubuntu", "fedora/httpd:version1.0", "fedora/httpd:version1.0.test", + "fedora/httpd:version1.0.TEST", "myregistryhost:5000/ubuntu", + "myregistryhost:5000/fedora/httpd:version1.0", + "myregistryhost:5000/fedora/httpd:version1.0.test", + "myregistryhost:5000/fedora/httpd:version1.0.TEST", + "123456789123.dkr.ecr.us-east-1.amazonaws.com/emr-docker-examples:pyspark-example" + + "@sha256:f1d4ae3f7261a72e98c6ebefe9985cf10a0ea5bd762585a43e0700ed99863807"}; + + String[] invalidNames = {"Ubuntu", "ubuntu || fedora", "ubuntu#", "myregistryhost:50AB0/ubuntu", + "myregistry#host:50AB0/ubuntu", ":8080/ubuntu", + + // Invalid: contains "@sha256" but doesn't really contain a digest. + "123456789123.dkr.ecr.us-east-1.amazonaws.com/emr-docker-examples:pyspark-example@sha256", + + // Invalid: digest is too short. + "123456789123.dkr.ecr.us-east-1.amazonaws.com/emr-docker-examples:pyspark-example" + + "@sha256:f1d4", + + // Invalid: digest is too long + "123456789123.dkr.ecr.us-east-1.amazonaws.com/emr-docker-examples:pyspark-example" + + "@sha256:f1d4ae3f7261a72e98c6ebefe9985cf10a0ea5bd762585a43e0700ed99863807f"}; for (String name : validNames) { DockerLinuxContainerRuntime.validateImageName(name);