diff --git a/README.md b/README.md index 935bfe2b0..54396f7b7 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,20 @@ adequate communication from Jenkins to the Kubernetes cluster, as seen below ![image](images/cloud-configuration.png) + +### Garbage collection (beta) + +In some exceptional cases, agent pods can be left behind, with no declared Jenkins agent in the controller. They will try to reconnect over and over, until something deletes them. + +The plugin provides a garbage collection mechanism to clean up these pods. As it has been introduced recently, +and generates extra load on the Kubernetes API server, it is disabled by default. + +Feel free to enable it and provide feedback about this functionality. + +![image](images/garbage-collection.png) + +## Static pod templates + In addition to that, in the **Kubernetes Pod Template** section, we need to configure the image that will be used to spin up the agent pod. We do not recommend overriding the `jnlp` container except under unusual circumstances. For your agent, you can use the default Jenkins agent image available in [Docker Hub](https://hub.docker.com). In the diff --git a/images/garbage-collection.png b/images/garbage-collection.png new file mode 100644 index 000000000..778f3c0fe Binary files /dev/null and b/images/garbage-collection.png differ diff --git a/pom.xml b/pom.xml index b9fa29bd8..3795ccb6d 100644 --- a/pom.xml +++ b/pom.xml @@ -295,6 +295,7 @@ 0 3000 + 5 60000 ${connectorHost} diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/GarbageCollection.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/GarbageCollection.java new file mode 100644 index 000000000..3d7c6e98d --- /dev/null +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/GarbageCollection.java @@ -0,0 +1,216 @@ +package org.csanchez.jenkins.plugins.kubernetes; + +import static org.csanchez.jenkins.plugins.kubernetes.PodTemplateBuilder.LABEL_KUBERNETES_CONTROLLER; +import static org.csanchez.jenkins.plugins.kubernetes.PodTemplateUtils.sanitizeLabel; + +import edu.umd.cs.findbugs.annotations.NonNull; +import hudson.Extension; +import hudson.Main; +import hudson.Util; +import hudson.model.AbstractDescribableImpl; +import hudson.model.AsyncPeriodicWork; +import hudson.model.Descriptor; +import hudson.model.TaskListener; +import hudson.util.FormValidation; +import io.fabric8.kubernetes.api.model.Pod; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import jenkins.model.Jenkins; +import jenkins.util.SystemProperties; +import org.jenkinsci.plugins.kubernetes.auth.KubernetesAuthException; +import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.DataBoundSetter; +import org.kohsuke.stapler.QueryParameter; + +/** + * Manages garbage collection of orphaned pods. + */ +public class GarbageCollection extends AbstractDescribableImpl { + public static final String ANNOTATION_LAST_REFRESH = "kubernetes.jenkins.io/last-refresh"; + private static final Logger LOGGER = Logger.getLogger(GarbageCollection.class.getName()); + + public static final int MINIMUM_GC_TIMEOUT = 120; + + private String namespaces; + private transient Set namespaceSet; + private int timeout; + + private static Long RECURRENCE_PERIOD = SystemProperties.getLong( + GarbageCollection.class.getName() + ".recurrencePeriod", TimeUnit.MINUTES.toSeconds(1)); + + @DataBoundConstructor + public GarbageCollection() {} + + public String getNamespaces() { + return namespaces; + } + + @DataBoundSetter + public void setNamespaces(String namespaces) { + this.namespaces = Util.fixEmptyAndTrim(namespaces); + if (this.namespaces == null) { + this.namespaceSet = Set.of(); + } else { + this.namespaceSet = Set.of(this.namespaces.split("\n")); + } + } + + public int getTimeout() { + return timeout; + } + + protected Object readResolve() { + if (namespaces != null) { + setNamespaces(namespaces); + } + return this; + } + + @DataBoundSetter + public void setTimeout(int timeout) { + if (Main.isUnitTest) { + this.timeout = timeout; + } else { + this.timeout = Math.max(timeout, MINIMUM_GC_TIMEOUT); + } + } + + public Duration getDurationTimeout() { + return Duration.ofSeconds(timeout); + } + + @NonNull + public Set getNamespaceSet() { + return namespaceSet == null ? Set.of() : namespaceSet; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GarbageCollection that = (GarbageCollection) o; + return timeout == that.timeout && Objects.equals(namespaces, that.namespaces); + } + + @Override + public int hashCode() { + return Objects.hash(namespaces, timeout); + } + + @Override + public String toString() { + return "GarbageCollection{" + "namespaces='" + namespaces + '\'' + ", timeout=" + timeout + '}'; + } + + @Extension + public static class DescriptorImpl extends Descriptor { + @SuppressWarnings("unused") // stapler + public FormValidation doCheckTimeout(@QueryParameter String value) { + return FormValidation.validateIntegerInRange(value, MINIMUM_GC_TIMEOUT, Integer.MAX_VALUE); + } + } + + /** + * Annotate pods owned by live Kubernetes agents to help with garbage collection. + */ + @Extension + public static final class PeriodicGarbageCollection extends AsyncPeriodicWork { + public PeriodicGarbageCollection() { + super("Garbage collection of orphaned Kubernetes pods"); + } + + @Override + protected void execute(TaskListener listener) throws IOException, InterruptedException { + annotateLiveAgents(listener); + garbageCollect(); + } + + private static void annotateLiveAgents(TaskListener listener) { + Arrays.stream(Jenkins.get().getComputers()) + .filter(KubernetesComputer.class::isInstance) + .map(KubernetesComputer.class::cast) + .forEach(kc -> kc.annotateTtl(listener)); + } + + private static void garbageCollect() { + for (var cloud : Jenkins.get().clouds.getAll(KubernetesCloud.class)) { + Optional.ofNullable(cloud.getGarbageCollection()).ifPresent(gc -> { + try { + var client = cloud.connect(); + var namespaces = new HashSet(); + namespaces.add(client.getNamespace()); + namespaces.addAll(gc.getNamespaceSet()); + for (var ns : namespaces) { + client + .pods() + .inNamespace(ns) + // Only look at pods created by this controller + .withLabel(LABEL_KUBERNETES_CONTROLLER, sanitizeLabel(cloud.getJenkinsUrlOrNull())) + .list() + .getItems() + .stream() + .filter(pod -> { + var lastRefresh = pod.getMetadata() + .getAnnotations() + .get(ANNOTATION_LAST_REFRESH); + if (lastRefresh != null) { + try { + var refreshTime = Long.parseLong(lastRefresh); + var now = Instant.now(); + LOGGER.log( + Level.FINE, + () -> getQualifiedName(pod) + " refresh diff = " + + (now.toEpochMilli() - refreshTime) + ", timeout is " + + gc.getDurationTimeout() + .toMillis()); + return Duration.between(Instant.ofEpochMilli(refreshTime), now) + .compareTo(gc.getDurationTimeout()) + > 0; + } catch (NumberFormatException e) { + LOGGER.log( + Level.WARNING, + e, + () -> "Unable to parse last refresh for pod " + + getQualifiedName(pod) + ", ignoring"); + return false; + } + } else { + LOGGER.log( + Level.FINE, () -> "Ignoring legacy pod " + getQualifiedName(pod)); + return false; + } + }) + .forEach(pod -> { + LOGGER.log(Level.INFO, () -> "Deleting orphan pod " + getQualifiedName(pod)); + client.resource(pod).delete(); + }); + } + } catch (KubernetesAuthException e) { + LOGGER.log(Level.WARNING, "Error authenticating to Kubernetes", e); + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Error while getting Kubernetes client", e); + } + }); + } + } + + private static String getQualifiedName(@NonNull Pod pod) { + var metadata = pod.getMetadata(); + return metadata.getNamespace() + "/" + metadata.getName(); + } + + @Override + public long getRecurrencePeriod() { + return TimeUnit.SECONDS.toMillis(RECURRENCE_PERIOD); + } + } +} diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java index 135e3d20e..6e417504c 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java @@ -147,6 +147,9 @@ public class KubernetesCloud extends Cloud implements PodTemplateGroup { @CheckForNull private PodRetention podRetention = PodRetention.getKubernetesCloudDefault(); + @CheckForNull + private GarbageCollection garbageCollection; + @DataBoundConstructor public KubernetesCloud(String name) { super(name); @@ -334,6 +337,15 @@ public boolean isCapOnlyOnAlivePods() { return capOnlyOnAlivePods; } + public GarbageCollection getGarbageCollection() { + return garbageCollection; + } + + @DataBoundSetter + public void setGarbageCollection(GarbageCollection garbageCollection) { + this.garbageCollection = garbageCollection; + } + /** * @return same as {@link #getJenkinsUrlOrNull}, if set * @throws IllegalStateException if no Jenkins URL could be computed. @@ -767,6 +779,7 @@ public boolean equals(Object o) { && Objects.equals(getPodLabels(), that.getPodLabels()) && Objects.equals(podRetention, that.podRetention) && Objects.equals(waitForPodSec, that.waitForPodSec) + && Objects.equals(garbageCollection, that.garbageCollection) && useJenkinsProxy == that.useJenkinsProxy; } @@ -794,7 +807,8 @@ public int hashCode() { usageRestricted, maxRequestsPerHost, podRetention, - useJenkinsProxy); + useJenkinsProxy, + garbageCollection); } public Integer getWaitForPodSec() { @@ -1068,7 +1082,8 @@ public String toString() { + waitForPodSec + ", podRetention=" + podRetention + ", useJenkinsProxy=" + useJenkinsProxy + ", templates=" - + templates + '}'; + + templates + ", garbageCollection=" + + garbageCollection + '}'; } private Object readResolve() { diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesComputer.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesComputer.java index 2355dbc22..26fb11521 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesComputer.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesComputer.java @@ -4,6 +4,7 @@ import hudson.model.Computer; import hudson.model.Executor; import hudson.model.Queue; +import hudson.model.TaskListener; import hudson.security.ACL; import hudson.security.Permission; import hudson.slaves.AbstractCloudComputer; @@ -18,6 +19,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; import jenkins.model.Jenkins; @@ -165,6 +167,10 @@ public ACL getACL() { return new KubernetesComputerACL(base); } + public void annotateTtl(TaskListener listener) { + Optional.ofNullable(getNode()).ifPresent(ks -> ks.annotateTtl(listener)); + } + /** * Simple static inner class to be used by {@link #getACL()}. * It replaces an anonymous inner class in order to fix diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java index cf59602b5..bfc09685a 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java @@ -31,6 +31,7 @@ import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.kubernetes.client.utils.Serialization; import java.io.IOException; +import java.time.Instant; import java.util.HashSet; import java.util.Locale; import java.util.Objects; @@ -273,7 +274,7 @@ public Cloud getCloud() { } public Optional getPod() { - return pod == null ? Optional.empty() : Optional.of(pod); + return Optional.ofNullable(pod); } /** @@ -538,6 +539,39 @@ public static Builder builder() { return new Builder(); } + public void annotateTtl(TaskListener listener) { + try { + var kubernetesCloud = getKubernetesCloud(); + Optional.ofNullable(kubernetesCloud.getGarbageCollection()).ifPresent(gc -> { + var ns = getNamespace(); + var name = getPodName(); + var l = Instant.now(); + try { + kubernetesCloud + .connect() + .pods() + .inNamespace(ns) + .withName(name) + .patch("{\"metadata\":{\"annotations\":{\"" + GarbageCollection.ANNOTATION_LAST_REFRESH + + "\":\"" + l.toEpochMilli() + "\"}}}"); + } catch (KubernetesAuthException e) { + e.printStackTrace(listener.error("Failed to authenticate to Kubernetes cluster")); + } catch (IOException e) { + e.printStackTrace(listener.error("Failed to connect to Kubernetes cluster")); + } + listener.getLogger().println("Annotated agent pod " + ns + "/" + name + " with TTL"); + LOGGER.log(Level.FINE, () -> "Annotated agent pod " + ns + "/" + name + " with TTL"); + try { + save(); + } catch (IOException e) { + LOGGER.log(Level.WARNING, e, () -> "Failed to save"); + } + }); + } catch (RuntimeException e) { + e.printStackTrace(listener.error("Failed to annotate agent pod with TTL")); + } + } + /** * Builds a {@link KubernetesSlave} instance. */ diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplateBuilder.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplateBuilder.java index e2c669a7c..c27b30359 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplateBuilder.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplateBuilder.java @@ -27,6 +27,7 @@ import static org.csanchez.jenkins.plugins.kubernetes.KubernetesCloud.JNLP_NAME; import static org.csanchez.jenkins.plugins.kubernetes.PodTemplateUtils.combine; import static org.csanchez.jenkins.plugins.kubernetes.PodTemplateUtils.isNullOrEmpty; +import static org.csanchez.jenkins.plugins.kubernetes.PodTemplateUtils.sanitizeLabel; import static org.csanchez.jenkins.plugins.kubernetes.PodTemplateUtils.substituteEnv; import edu.umd.cs.findbugs.annotations.CheckForNull; @@ -98,6 +99,8 @@ public class PodTemplateBuilder { private static final String WORKSPACE_VOLUME_NAME = "workspace-volume"; public static final Pattern FROM_DIRECTIVE = Pattern.compile("^FROM (.*)$"); + public static final String LABEL_KUBERNETES_CONTROLLER = "kubernetes.jenkins.io/controller"; + @SuppressFBWarnings(value = "MS_SHOULD_BE_FINAL", justification = "tests") @Restricted(NoExternalUse.class) static String DEFAULT_JNLP_DOCKER_REGISTRY_PREFIX = @@ -230,6 +233,9 @@ public Pod build() { if (!labels.isEmpty()) { metadataBuilder.withLabels(labels); } + if (cloud != null) { + metadataBuilder.addToLabels(LABEL_KUBERNETES_CONTROLLER, sanitizeLabel(cloud.getJenkinsUrlOrNull())); + } Map annotations = getAnnotationsMap(template.getAnnotations()); if (!annotations.isEmpty()) { @@ -618,6 +624,7 @@ private Map getAnnotationsMap(List annotations) { builder.put(podAnnotation.getKey(), substituteEnv(podAnnotation.getValue())); } } + builder.put(GarbageCollection.ANNOTATION_LAST_REFRESH, String.valueOf(System.currentTimeMillis())); return Collections.unmodifiableMap(builder); } diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplateUtils.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplateUtils.java index b12b84ed9..3f15a344f 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplateUtils.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplateUtils.java @@ -767,7 +767,10 @@ public static boolean validateImage(String image) { * @return the sanitized and validated label * @throws AssertionError if the generated label is not valid */ - public static String sanitizeLabel(String input) { + public static String sanitizeLabel(@CheckForNull String input) { + if (input == null) { + return null; + } int max = /* Kubernetes limit */ 63 - /* hyphen */ 1 - /* suffix */ 5; String label; if (input.length() > max) { diff --git a/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/GarbageCollection/config.jelly b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/GarbageCollection/config.jelly new file mode 100644 index 000000000..d958f89ff --- /dev/null +++ b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/GarbageCollection/config.jelly @@ -0,0 +1,30 @@ + + + + + + + + + + + + diff --git a/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/GarbageCollection/help-namespaces.html b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/GarbageCollection/help-namespaces.html new file mode 100644 index 000000000..c9639c424 --- /dev/null +++ b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/GarbageCollection/help-namespaces.html @@ -0,0 +1,2 @@ +Namespaces to look at for garbage collection, in addition to the default namespace defined for the cloud. +One namespace per line. diff --git a/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/config.jelly b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/config.jelly index f9551e250..a7310009b 100644 --- a/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/config.jelly +++ b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/config.jelly @@ -119,5 +119,5 @@ THE SOFTWARE. - + diff --git a/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/help-garbageCollection.html b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/help-garbageCollection.html new file mode 100644 index 000000000..45f5ce9ea --- /dev/null +++ b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/help-garbageCollection.html @@ -0,0 +1,5 @@ +

+ Enables garbage collection of orphan pods for this Kubernetes cloud.
+ + When enabled, Jenkins will periodically check for orphan pods that have not been touched for the given timeout period and delete them. +

diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloudTest.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloudTest.java index 2afc3af02..9b3d7e78f 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloudTest.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloudTest.java @@ -11,6 +11,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import jenkins.model.JenkinsLocationConfiguration; @@ -208,8 +209,8 @@ public void copyConstructor() throws Exception { pt.setName("podTemplate"); KubernetesCloud cloud = new KubernetesCloud("name"); - ArrayList objectProperties = - new ArrayList<>(Arrays.asList("templates", "podRetention", "podLabels", "labels", "serverCertificate")); + var objectProperties = + Set.of("templates", "podRetention", "podLabels", "labels", "serverCertificate", "garbageCollection"); for (String property : PropertyUtils.describe(cloud).keySet()) { if (PropertyUtils.isWriteable(cloud, property)) { Class propertyType = PropertyUtils.getPropertyType(cloud, property); diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java index af65d0edb..101933c8b 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java @@ -41,7 +41,8 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.*; +import static org.junit.Assume.assumeNoException; +import static org.junit.Assume.assumeNotNull; import hudson.model.Computer; import hudson.model.Label; @@ -59,11 +60,14 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import jenkins.metrics.api.Metrics; import jenkins.model.Jenkins; +import org.csanchez.jenkins.plugins.kubernetes.GarbageCollection; +import org.csanchez.jenkins.plugins.kubernetes.KubernetesComputer; import org.csanchez.jenkins.plugins.kubernetes.KubernetesSlave; import org.csanchez.jenkins.plugins.kubernetes.MetricNames; import org.csanchez.jenkins.plugins.kubernetes.PodAnnotation; @@ -81,6 +85,7 @@ import org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep; import org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; @@ -852,4 +857,33 @@ public void cancelOnlyRelevantQueueItem() throws Exception { r.assertBuildStatus(Result.ABORTED, r.waitForCompletion(b)); r.assertLogContains("ran on special agent", b); } + + @Test + public void garbageCollection() throws Exception { + // Pod exists, need to kill the build, delete the agent without deleting the pod. + // Wait for the timeout to expire and check that the pod is deleted. + var garbageCollection = new GarbageCollection(); + // Considering org.csanchez.jenkins.plugins.kubernetes.GarbageCollection.recurrencePeriod=5, this leaves 3 ticks + garbageCollection.setTimeout(15); + cloud.setGarbageCollection(garbageCollection); + r.jenkins.save(); + r.waitForMessage("Running on remote agent", b); + Pod pod = null; + for (var c : r.jenkins.getComputers()) { + if (c instanceof KubernetesComputer) { + var node = (KubernetesSlave) c.getNode(); + pod = node.getPod().get(); + Assert.assertNotNull(pod); + b.doKill(); + r.jenkins.removeNode(node); + break; + } + } + r.assertBuildStatus(Result.ABORTED, r.waitForCompletion(b)); + final var finalPod = pod; + var client = cloud.connect(); + assertNotNull(client.resource(finalPod).get()); + await().timeout(1, TimeUnit.MINUTES) + .until(() -> client.resource(finalPod).get() == null); + } } diff --git a/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/garbageCollection.groovy b/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/garbageCollection.groovy new file mode 100644 index 000000000..854b77f23 --- /dev/null +++ b/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/garbageCollection.groovy @@ -0,0 +1,6 @@ +podTemplate { + node(POD_LABEL) { + echo 'Running on remote agent' + sh 'sleep 600' + } +}