Skip to content

Commit

Permalink
Kubernetes annotations (#1240)
Browse files Browse the repository at this point in the history 
* Add kubernetes annotations support to nextflow

Signed-off-by: Ólafur Haukur Flygenring <olafurh@wuxinextcode.com>
  • Loading branch information
@pditommaso
Ólafur Haukur Flygenring authored and pditommaso committed Jul 23, 2019
1 parent 11caac6 commit 8eeb7f9
Show file tree
Hide file tree
Showing 8 changed files with 116 additions and 6 deletions.
1 change: 1 addition & 0 deletions docs/process.rst
View file
Expand Up @@ -1769,6 +1769,7 @@ The ``pod`` directive allows the definition of the following options:

================================================= =================================================
``label: <K>, value: <V>`` Defines a pod label with key ``K`` and value ``V``.
``annotation: <K>, value: <V>`` Defines a pod annotation with key ``K`` and value ``V``.
``env: <E>, value: <V>`` Defines an environment variable with name ``E`` and whose value is given by the ``V`` string.
``env: <E>, config: <C/K>`` Defines an environment variable with name ``E`` and whose value is given by the entry associated to the key with name ``K`` in the `ConfigMap <https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/>`_ with name ``C``.
``env: <E>, secret: <S/K>`` Defines an environment variable with name ``E`` and whose value is given by the entry associated to the key with name ``K`` in the `Secret <https://kubernetes.io/docs/concepts/configuration/secret/>`_ with name ``S``.
Expand Down
4 changes: 4 additions & 0 deletions modules/nextflow/src/main/groovy/nextflow/k8s/K8sConfig.groovy
View file
Expand Up @@ -84,6 +84,10 @@ class K8sConfig implements Map<String,Object> {
podOptions.getLabels()
}

Map<String,String> getAnnotations() {
podOptions.getAnnotations()
}

K8sDebug getDebug() {
new K8sDebug( (Map<String,Object>)get('debug') )
}
Expand Down
10 changes: 10 additions & 0 deletions modules/nextflow/src/main/groovy/nextflow/k8s/K8sTaskHandler.groovy
View file
Expand Up @@ -165,6 +165,7 @@ class K8sTaskHandler extends TaskHandler {
.withNamespace(clientConfig.namespace)
.withServiceAccount(clientConfig.serviceAccount)
.withLabels(getLabels(task))
.withAnnotations(getAnnotations())
.withPodOptions(getPodOptions())

// note: task environment is managed by the task bash wrapper
Expand Down Expand Up @@ -214,6 +215,15 @@ class K8sTaskHandler extends TaskHandler {
return result
}

protected Map getAnnotations() {
Map result = [:]
def annotations = k8sConfig.getAnnotations()
if( annotations ) {
annotations.each { k,v -> result.put(k,sanitize0(v)) }
}
return result
}

/**
* Valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.',
* and must start and end with an alphanumeric character.
Expand Down
11 changes: 11 additions & 0 deletions modules/nextflow/src/main/groovy/nextflow/k8s/model/PodOptions.groovy
View file
Expand Up @@ -46,6 +46,8 @@ class PodOptions {

private Map<String,String> labels = [:]

private Map<String,String> annotations = [:]

private PodNodeSelector nodeSelector

private PodSecurityContext securityContext
Expand Down Expand Up @@ -103,6 +105,9 @@ class PodOptions {
else if( entry.nodeSelector ) {
this.nodeSelector = new PodNodeSelector(entry.nodeSelector)
}
else if( entry.annotation && entry.value ) {
this.annotations.put(entry.annotation as String, entry.value as String)
}
else
throw new IllegalArgumentException("Unknown pod options: $entry")
}
Expand All @@ -118,6 +123,8 @@ class PodOptions {

Map<String,String> getLabels() { labels }

Map<String,String> getAnnotations() { annotations }

PodSecurityContext getSecurityContext() { securityContext }

PodNodeSelector getNodeSelector() { nodeSelector }
Expand Down Expand Up @@ -189,6 +196,10 @@ class PodOptions {
result.labels.putAll(labels)
result.labels.putAll(other.labels)

// annotations
result.annotations.putAll(annotations)
result.annotations.putAll(other.annotations)

return result
}
}
19 changes: 19 additions & 0 deletions modules/nextflow/src/main/groovy/nextflow/k8s/model/PodSpecBuilder.groovy
View file
Expand Up @@ -46,6 +46,8 @@ class PodSpecBuilder {

Map<String,String> labels = [:]

Map<String,String> annotations = [:]

String namespace

String restart
Expand Down Expand Up @@ -153,6 +155,16 @@ class PodSpecBuilder {
return this
}

PodSpecBuilder withAnnotation( String name, String value ) {
this.annotations.put(name, value)
return this
}

PodSpecBuilder withAnnotations(Map annotations) {
this.annotations.putAll(annotations)
return this
}


PodSpecBuilder withEnv( PodEnv var ) {
envVars.add(var)
Expand Down Expand Up @@ -229,6 +241,10 @@ class PodSpecBuilder {
if( 'runName' in keys ) throw new IllegalArgumentException("Invalid pod label -- `runName` is a reserved label")
labels.putAll( opts.labels )
}
// - annotations
if( opts.annotations ) {
annotations.putAll( opts.annotations )
}
// -- security context
if( opts.securityContext )
securityContext = opts.securityContext
Expand Down Expand Up @@ -302,6 +318,9 @@ class PodSpecBuilder {
if( labels )
metadata.labels = labels

if( annotations)
metadata.annotations = annotations

final pod = [
apiVersion: 'v1',
kind: 'Pod',
Expand Down
13 changes: 11 additions & 2 deletions modules/nextflow/src/test/groovy/nextflow/k8s/K8sTaskHandlerTest.groovy
View file
Expand Up @@ -66,6 +66,7 @@ class K8sTaskHandlerTest extends Specification {
1 * handler.getPodOptions() >> new PodOptions()
1 * handler.getSyntheticPodName(task) >> 'nf-123'
1 * handler.getLabels(task) >> [foo: 'bar', hello: 'world']
1 * handler.getAnnotations() >> [fooz: 'barz', ciao: 'mondo']
1 * handler.getContainerMounts() >> []
1 * task.getContainer() >> 'debian:latest'
1 * task.getWorkDir() >> WORK_DIR
Expand All @@ -78,7 +79,9 @@ class K8sTaskHandlerTest extends Specification {
metadata: [
name:'nf-123',
namespace:'default',
labels:[ foo:'bar', hello: 'world'] ],
labels:[ foo:'bar', hello: 'world'],
annotations:[ fooz:'barz', ciao: 'mondo']
],
spec: [
restartPolicy:'Never',
containers:[
Expand All @@ -95,6 +98,7 @@ class K8sTaskHandlerTest extends Specification {
then:
1 * handler.getSyntheticPodName(task) >> 'nf-foo'
1 * handler.getLabels(task) >> [sessionId:'xxx']
1 * handler.getAnnotations() >> [evict: 'false']
1 * handler.getPodOptions() >> new PodOptions()
1 * handler.getContainerMounts() >> []
1 * builder.fixOwnership() >> true
Expand All @@ -107,7 +111,7 @@ class K8sTaskHandlerTest extends Specification {
1 * client.getConfig() >> new ClientConfig()
result == [ apiVersion: 'v1',
kind: 'Pod',
metadata: [name:'nf-foo', namespace:'default', labels: [sessionId: 'xxx']],
metadata: [name:'nf-foo', namespace:'default', labels: [sessionId: 'xxx'], annotations: [evict: 'false']],
spec: [
restartPolicy:'Never',
containers:[
Expand All @@ -128,6 +132,7 @@ class K8sTaskHandlerTest extends Specification {
then:
1 * handler.getSyntheticPodName(task) >> 'nf-abc'
1 * handler.getLabels(task) >> [:]
1 * handler.getAnnotations() >> [:]
1 * handler.getPodOptions() >> new PodOptions()
1 * handler.getContainerMounts() >> []
1 * task.getContainer() >> 'user/alpine:1.0'
Expand Down Expand Up @@ -173,6 +178,7 @@ class K8sTaskHandlerTest extends Specification {
1 * handler.getSyntheticPodName(task) >> 'nf-123'
1 * handler.getPodOptions() >> new PodOptions()
1 * handler.getLabels(task) >> [:]
1 * handler.getAnnotations() >> [:]
1 * handler.getContainerMounts() >> []
1 * task.getContainer() >> 'debian:latest'
1 * task.getWorkDir() >> WORK_DIR
Expand Down Expand Up @@ -218,6 +224,7 @@ class K8sTaskHandlerTest extends Specification {
1 * client.getConfig() >> new ClientConfig()
1 * handler.getSyntheticPodName(task) >> 'nf-123'
1 * handler.getLabels(task) >> [:]
1 * handler.getAnnotations() >> [:]
1 * handler.getPodOptions() >> podOptions
1 * handler.getContainerMounts() >> []
1 * task.getContainer() >> 'debian:latest'
Expand Down Expand Up @@ -273,6 +280,7 @@ class K8sTaskHandlerTest extends Specification {
1 * handler.getSyntheticPodName(task) >> 'nf-123'
1 * handler.getContainerMounts() >> []
1 * handler.getLabels(task) >> [:]
1 * handler.getAnnotations() >> [:]
1 * handler.getPodOptions() >> podOptions
1 * task.getContainer() >> 'debian:latest'
1 * task.getWorkDir() >> WORK_DIR
Expand Down Expand Up @@ -309,6 +317,7 @@ class K8sTaskHandlerTest extends Specification {
1 * handler.getSyntheticPodName(task) >> 'nf-123'
1 * handler.getContainerMounts() >> ['/tmp', '/data']
1 * handler.getLabels(task) >> [:]
1 * handler.getAnnotations() >> [:]
1 * handler.getPodOptions() >> new PodOptions()
1 * task.getContainer() >> 'debian:latest'
1 * task.getWorkDir() >> WORK_DIR
Expand Down
39 changes: 39 additions & 0 deletions modules/nextflow/src/test/groovy/nextflow/k8s/model/PodOptionsTest.groovy
View file
Expand Up @@ -361,6 +361,45 @@ class PodOptionsTest extends Specification {

}

def 'should copy pod annotations' (){
given:
def data = [
[annotation: "ANNOTATION", value: 'VALUE']
]

when:
def opts = new PodOptions() + new PodOptions(data)
then:
opts.annotations == ["ANNOTATION": "VALUE"]

when:
opts = new PodOptions(data) + new PodOptions()
then:
opts.annotations == ["ANNOTATION": "VALUE"]

when:
opts = new PodOptions([[annotation:"FOO", value:'one']]) + new PodOptions([[annotation:"BAR", value:'two']])
then:
opts.annotations == [FOO: 'one', BAR: 'two']
}

def 'should create pod annotations' () {

given:
def options = [
[annotation: 'ALPHA', value: 'aaa'],
[annotation: 'DELTA', value: 'bbb'],
[annotation: 'DELTA', value: 'ddd']
]

when:
def opts = new PodOptions(options)
then:
opts.annotations.size() == 2
opts.annotations == [ALPHA: 'aaa', DELTA: 'ddd']

}

def 'should create user security context' () {
when:
def opts = new PodOptions([ [runAsUser: 1000] ])
Expand Down
25 changes: 21 additions & 4 deletions modules/nextflow/src/test/groovy/nextflow/k8s/model/PodSpecBuilderTest.groovy
View file
Expand Up @@ -60,7 +60,7 @@ class PodSpecBuilderTest extends Specification {
}


def 'should set namespace and labels' () {
def 'should set namespace, labels and annotations' () {

when:
def spec = new PodSpecBuilder()
Expand All @@ -71,12 +71,26 @@ class PodSpecBuilderTest extends Specification {
.withNamespace('xyz')
.withLabel('app','myApp')
.withLabel('runName','something')
.withAnnotation("anno1", "value1")
.withAnnotations([anno2: "value2", anno3: "value3"])
.build()

then:
spec == [ apiVersion: 'v1',
kind: 'Pod',
metadata: [name:'foo', namespace:'xyz', labels:[app: 'myApp', runName: 'something']],
metadata: [
name:'foo',
namespace:'xyz',
labels: [
app: 'myApp',
runName: 'something'
],
annotations: [
anno1: "value1",
anno2: "value2",
anno3: "value3"
]
],
spec: [
restartPolicy:'Never',
containers:[
Expand Down Expand Up @@ -425,7 +439,7 @@ class PodSpecBuilderTest extends Specification {

given:
def opts = Mock(PodOptions)
def builder = new PodSpecBuilder(podName: 'foo', imageName: 'image', command: ['echo'], labels: [runName: 'crazy_john'])
def builder = new PodSpecBuilder(podName: 'foo', imageName: 'image', command: ['echo'], labels: [runName: 'crazy_john'], annotations: [evict: 'false'])

when:
def spec = builder.withPodOptions(opts).build()
Expand All @@ -437,6 +451,7 @@ class PodSpecBuilderTest extends Specification {
2 * opts.getMountSecrets() >> [ new PodMountSecret('blah', '/etc/secret.txt') ]
2 * opts.getEnvVars() >> [ PodEnv.value('HELLO','WORLD') ]
_ * opts.getLabels() >> [ALPHA: 'xxx', GAMMA: 'yyy']
_ * opts.getAnnotations() >> [OMEGA:'zzz', SIGMA:'www']
_ * opts.getSecurityContext() >> new PodSecurityContext(1000)
_ * opts.getNodeSelector() >> new PodNodeSelector(gpu:true, queue: 'fast')

Expand All @@ -446,7 +461,9 @@ class PodSpecBuilderTest extends Specification {
metadata: [
name:'foo',
namespace:'default',
labels:[runName:'crazy_john', ALPHA:'xxx', GAMMA:'yyy'] ],
labels:[runName:'crazy_john', ALPHA:'xxx', GAMMA:'yyy'],
annotations: [evict: 'false', OMEGA:'zzz', SIGMA:'www']
],
spec: [
restartPolicy:'Never',
securityContext: [ runAsUser: 1000 ],
Expand Down

0 comments on commit 8eeb7f9

Please sign in to comment.