Skip to content

Commit c3364d0

Browse files
pditommasopditommaso
committed
Fix K8s cluster token when using serviceAccount
Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
1 parent a94fa9c commit c3364d0

File tree

6 files changed

+212
-88
lines changed

6 files changed

+212
-88
lines changed

‎modules/nextflow/src/main/groovy/nextflow/k8s/K8sConfig.groovy

Lines changed: 35 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package nextflow.k8s
1919

20+
import javax.annotation.Nullable
21+
2022
import groovy.transform.CompileStatic
2123
import groovy.transform.Memoized
2224
import groovy.transform.PackageScope
@@ -208,18 +210,10 @@ class K8sConfig implements Map<String,Object> {
208210
ClientConfig getClient() {
209211

210212
final result = ( target.client instanceof Map
211-
? clientFromMap(target.client as Map)
212-
: clientDiscovery(target.context as String)
213+
? clientFromNextflow(target.client as Map, target.namespace as String, target.serviceAccount as String)
214+
: clientDiscovery(target.context as String, target.namespace as String, target.serviceAccount as String)
213215
)
214216

215-
if( target.namespace ) {
216-
result.namespace = target.namespace as String
217-
}
218-
219-
if( target.serviceAccount ) {
220-
result.serviceAccount = target.serviceAccount as String
221-
}
222-
223217
if( target.httpConnectTimeout )
224218
result.httpConnectTimeout = target.httpConnectTimeout as Duration
225219

@@ -232,12 +226,39 @@ class K8sConfig implements Map<String,Object> {
232226
return result
233227
}
234228

235-
@PackageScope ClientConfig clientFromMap( Map map ) {
236-
ClientConfig.fromMap(map)
229+
/**
230+
* Get the K8s client config from the declaration made in the Nextflow config file
231+
*
232+
* @param map
233+
* A map representing the clint configuration options define in the nextflow
234+
* config file
235+
* @param namespace
236+
* The K8s namespace to be used. If omitted {@code default} is used.
237+
* @param serviceAccount
238+
* The K8s service account to be used. If omitted {@code default} is used.
239+
* @return
240+
* The Kubernetes {@link ClientConfig} object
241+
*/
242+
@PackageScope ClientConfig clientFromNextflow(Map map, @Nullable String namespace, @Nullable String serviceAccount ) {
243+
ClientConfig.fromNextflowConfig(map,namespace,serviceAccount)
237244
}
238245

239-
@PackageScope ClientConfig clientDiscovery( String ctx ) {
240-
ClientConfig.discover(ctx)
246+
/**
247+
* Discover the K8s client config from the execution environment
248+
* that can be either a `.kube/config` file or service meta file
249+
* when running in a pod.
250+
*
251+
* @param contextName
252+
* The name of the configuration context to be used
253+
* @param namespace
254+
* The Kubernetes namespace to be used
255+
* @param serviceAccount
256+
* The Kubernetes serviceAccount to be used
257+
* @return
258+
* The discovered Kube {@link ClientConfig} object
259+
*/
260+
@PackageScope ClientConfig clientDiscovery(String contextName, String namespace, String serviceAccount) {
261+
ClientConfig.discover(contextName, namespace, serviceAccount)
241262
}
242263

243264
void checkStorageAndPaths(K8sClient client) {

‎modules/nextflow/src/main/groovy/nextflow/k8s/client/ClientConfig.groovy

Lines changed: 31 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ class ClientConfig {
8282
}
8383

8484
String toString() {
85-
"${this.class.getSimpleName()}[ server=$server, namespace=$namespace, token=${cut(token)}, sslCert=${cut(sslCert)}, clientCert=${cut(clientCert)}, clientKey=${cut(clientKey)}, verifySsl=$verifySsl, fromFile=$isFromCluster, httpReadTimeout=$httpReadTimeout, httpConnectTimeout=$httpConnectTimeout, maxErrorRetry=$maxErrorRetry ]"
85+
"${this.class.getSimpleName()}[ server=$server, namespace=$namespace, serviceAccount=$serviceAccount, token=${cut(token)}, sslCert=${cut(sslCert)}, clientCert=${cut(clientCert)}, clientKey=${cut(clientKey)}, verifySsl=$verifySsl, fromFile=$isFromCluster, httpReadTimeout=$httpReadTimeout, httpConnectTimeout=$httpConnectTimeout, maxErrorRetry=$maxErrorRetry ]"
8686
}
8787

8888
private String cut(String str) {
@@ -95,43 +95,45 @@ class ClientConfig {
9595
cut(bytes.encodeBase64().toString())
9696
}
9797

98-
static ClientConfig discover(String context=null) {
99-
new ConfigDiscovery(context: context).discover()
98+
static ClientConfig discover(String context, String namespace, String serviceAccount) {
99+
new ConfigDiscovery().discover(context, namespace, serviceAccount)
100100
}
101101

102-
static ClientConfig fromMap(Map map) {
103-
def result = new ClientConfig()
104-
if( map.server )
105-
result.server = map.server
102+
static ClientConfig fromNextflowConfig(Map opts, String namespace, String serviceAccount) {
103+
final result = new ClientConfig()
104+
105+
if( opts.server )
106+
result.server = opts.server
107+
108+
if( opts.token )
109+
result.token = opts.token
110+
else if( opts.tokenFile )
111+
result.token = Paths.get(opts.tokenFile.toString()).getText('UTF-8')
106112

107-
if( map.token )
108-
result.token = map.token
109-
else if( map.tokenFile )
110-
result.token = Paths.get(map.tokenFile.toString()).getText('UTF-8')
113+
result.namespace = namespace ?: opts.namespace ?: 'default'
111114

112-
if( map.namespace )
113-
result.namespace = map.namespace
115+
result.serviceAccount = serviceAccount ?: 'default'
114116

115-
if( map.verifySsl )
116-
result.verifySsl = map.verifySsl as boolean
117+
if( opts.verifySsl )
118+
result.verifySsl = opts.verifySsl as boolean
117119

118-
if( map.sslCert )
119-
result.sslCert = map.sslCert.toString().decodeBase64()
120-
else if( map.sslCertFile )
121-
result.sslCert = Paths.get(map.sslCertFile.toString()).bytes
120+
if( opts.sslCert )
121+
result.sslCert = opts.sslCert.toString().decodeBase64()
122+
else if( opts.sslCertFile )
123+
result.sslCert = Paths.get(opts.sslCertFile.toString()).bytes
122124

123-
if( map.clientCert )
124-
result.clientCert = map.clientCert.toString().decodeBase64()
125-
else if( map.clientCertFile )
126-
result.clientCert = Paths.get(map.clientCertFile.toString()).bytes
125+
if( opts.clientCert )
126+
result.clientCert = opts.clientCert.toString().decodeBase64()
127+
else if( opts.clientCertFile )
128+
result.clientCert = Paths.get(opts.clientCertFile.toString()).bytes
127129

128-
if( map.clientKey )
129-
result.clientKey = map.clientKey.toString().decodeBase64()
130-
else if( map.clientKeyFile )
131-
result.clientKey = Paths.get(map.clientKeyFile.toString()).bytes
130+
if( opts.clientKey )
131+
result.clientKey = opts.clientKey.toString().decodeBase64()
132+
else if( opts.clientKeyFile )
133+
result.clientKey = Paths.get(opts.clientKeyFile.toString()).bytes
132134

133-
if( map.maxErrorRetry )
134-
result.maxErrorRetry = map.maxErrorRetry as Integer
135+
if( opts.maxErrorRetry )
136+
result.maxErrorRetry = opts.maxErrorRetry as Integer
135137

136138
return result
137139
}

‎modules/nextflow/src/main/groovy/nextflow/k8s/client/ConfigDiscovery.groovy

Lines changed: 42 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -36,32 +36,37 @@ class ConfigDiscovery {
3636

3737
private Map<String,String> env = System.getenv()
3838

39-
private ClientConfig config
40-
41-
private String context
39+
ConfigDiscovery() { }
4240

4341
/**
44-
* Discover Kubernetes service from current environment settings
42+
* Discover Kubernetes client configuration from current environment using
43+
* either the .kube/config file or the pod service service account virtual
44+
* file system when running in a pod.
45+
*
46+
* @param contextName The K8s config context name.
47+
* @param namespace The K8s cluster namespace
48+
* @param serviceAccount The K8s cluster service account
49+
* @return The e
4550
*/
46-
ClientConfig discover() {
47-
48-
config = new ClientConfig()
51+
ClientConfig discover(String contextName, String namespace, String serviceAccount) {
4952

5053
// Note: System.getProperty('user.home') may not report the correct home path when
5154
// running in a container. Use env HOME instead.
5255
def home = System.getenv('HOME')
5356
def kubeConfig = env.get('KUBECONFIG') ? env.get('KUBECONFIG') : "$home/.kube/config"
5457
def configFile = Paths.get(kubeConfig)
5558

59+
// determine the Kubernetes client configuration via the `.kube/config` file
5660
if( configFile.exists() ) {
57-
return fromConfig(configFile)
61+
return fromKubeConfig(configFile, contextName, namespace, serviceAccount)
5862
}
5963
else {
6064
log.debug "K8s config file does not exist: $configFile"
6165
}
6266

67+
// determine the Kubernetes client configuration via the pod environment
6368
if( env.get('KUBERNETES_SERVICE_HOST') ) {
64-
return fromCluster(env)
69+
return fromCluster(env, namespace, serviceAccount)
6570
}
6671
else {
6772
log.debug "K8s env variable KUBERNETES_SERVICE_HOST is not defined"
@@ -70,7 +75,7 @@ class ConfigDiscovery {
7075
throw new IllegalStateException("Unable to lookup Kubernetes cluster configuration")
7176
}
7277

73-
protected ClientConfig fromCluster(Map<String,String> env) {
78+
protected ClientConfig fromCluster(Map<String,String> env, String cfgNamespace, String serviceAccount) {
7479

7580
// See https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod
7681

@@ -82,17 +87,21 @@ class ConfigDiscovery {
8287
final token = path('/var/run/secrets/kubernetes.io/serviceaccount/token').text
8388
final namespace = path('/var/run/secrets/kubernetes.io/serviceaccount/namespace').text
8489

85-
new ClientConfig( server: server, token: token, namespace: namespace, sslCert: cert, isFromCluster: true )
90+
if( namespace && namespace != cfgNamespace )
91+
log.warn ("K8s namespace provided in the nextflow configuration does not match with pod namespace")
92+
93+
new ClientConfig( server: server, token: token, namespace: namespace, serviceAccount: serviceAccount, sslCert: cert, isFromCluster: true )
8694
}
8795

8896
protected Path path(String path) {
8997
Paths.get(path)
9098
}
9199

92-
protected ClientConfig fromConfig(Path path) {
100+
protected ClientConfig fromKubeConfig(Path path, String contextName, String namespace, String serviceAccount) {
93101
def yaml = (Map)new Yaml().load(Files.newInputStream(path))
94102

95-
final contextName = context ?: yaml."current-context" as String
103+
contextName ?= yaml."current-context" as String
104+
96105
final allContext = yaml.contexts as List
97106
final allClusters = yaml.clusters as List
98107
final allUsers = yaml.users as List
@@ -104,17 +113,18 @@ class ConfigDiscovery {
104113
final user = allUsers.find{ Map it -> it.name == userName } ?.user ?: [:]
105114
final cluster = allClusters.find{ Map it -> it.name == clusterName } ?.cluster ?: [:]
106115

107-
def config = ClientConfig.fromUserAndCluster(user, cluster, path)
116+
final config = ClientConfig.fromUserAndCluster(user, cluster, path)
108117

109-
if( context?.namespace ) {
110-
config.namespace = context?.namespace
111-
}
118+
// the namespace provided should have priority over the context current namespace
119+
config.namespace = namespace ?: context?.namespace ?: 'default'
120+
121+
config.serviceAccount = serviceAccount ?: 'default'
112122

113123
if( config.clientCert && config.clientKey ) {
114124
config.keyManagers = createKeyManagers(config.clientCert, config.clientKey)
115125
}
116126
else if( !config.token ) {
117-
config.token = discoverAuthToken()
127+
config.token = discoverAuthToken(config.namespace, serviceAccount)
118128
}
119129

120130
return config
@@ -151,20 +161,27 @@ class ConfigDiscovery {
151161
return kmf.getKeyManagers();
152162
}
153163

154-
protected String discoverAuthToken() {
155-
def cmd = /echo $(kubectl describe secret $(kubectl get secrets | grep default | cut -f1 -d ' ') | grep -E '^token' | cut -f2 -d':' | tr -d '\t')/
156-
def proc = new ProcessBuilder('bash','-o','pipefail','-c', cmd).redirectErrorStream(true).start()
157-
def status = proc.waitFor()
164+
String discoverAuthToken(String namespace, String serviceAccount) {
165+
namespace ?= 'default'
166+
serviceAccount ?= 'default'
167+
168+
final cmd = "kubectl -n ${namespace} get secret `kubectl -n ${namespace} get serviceaccount ${serviceAccount} -o jsonpath='{.secrets[0].name}'` -o jsonpath='{.data.token}'"
169+
final proc = new ProcessBuilder('bash','-o','pipefail','-c', cmd).redirectErrorStream(true).start()
170+
final status = proc.waitFor()
158171
if( status==0 ) {
159-
return proc.text.trim()
172+
try {
173+
return new String(proc.text.trim().decodeBase64())
174+
}
175+
catch( Exception e ) {
176+
log.warn "Unable to read K8s cluster auth token -- cause: ${e.message}"
177+
}
160178
}
161179
else {
162180
final msg = proc.text
163181
final cause = msg ? "-- cause:\n${msg.indent(' ')}" : ''
164182
log.debug "[K8s] unable to fetch auth token ${cause}"
165-
return null
166183
}
184+
return null
167185
}
168186

169-
170187
}

‎modules/nextflow/src/test/groovy/nextflow/k8s/K8sConfigTest.groovy

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ import nextflow.k8s.model.PodSecurityContext
2424
import nextflow.k8s.model.PodVolumeClaim
2525
import nextflow.util.Duration
2626
import spock.lang.Specification
27+
import spock.lang.Unroll
28+
2729
/**
2830
*
2931
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
@@ -180,19 +182,26 @@ class K8sConfigTest extends Specification {
180182

181183
}
182184

185+
@Unroll
183186
def 'should create client config with discovery' () {
184187

185188
given:
186-
def CONTEXT = 'pizza'
187-
def CONFIG = [context: CONTEXT]
189+
def CONFIG = [context: CONTEXT, namespace: NAMESPACE, serviceAccount: SERVICE_ACCOUNT]
188190
K8sConfig config = Spy(K8sConfig, constructorArgs: [ CONFIG ])
189191

190192
when:
191193
def client = config.getClient()
192194
then:
193-
1 * config.clientDiscovery(CONTEXT) >> new ClientConfig(namespace: 'foo', server: 'bar')
194-
client.server == 'bar'
195-
client.namespace == 'foo'
195+
1 * config.clientDiscovery(CONTEXT, NAMESPACE, SERVICE_ACCOUNT) >> new ClientConfig(namespace: NAMESPACE, server: SERVER)
196+
and:
197+
client.server == SERVER
198+
client.namespace == NAMESPACE ?: 'default'
199+
client.serviceAccount == SERVICE_ACCOUNT ?: 'default'
200+
201+
where:
202+
CONTEXT | SERVER | NAMESPACE | SERVICE_ACCOUNT
203+
'foo' | 'host.com'| null | null
204+
'bar' | 'this.com'| 'ns1' | 'sa2'
196205

197206
}
198207

‎modules/nextflow/src/test/groovy/nextflow/k8s/client/ClientConfigTest.groovy

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,25 @@ class ClientConfigTest extends Specification {
5656
clientKey: 'world'.bytes.encodeBase64().toString() ]
5757

5858
when:
59-
def result = ClientConfig.fromMap(MAP)
59+
def result = ClientConfig.fromNextflowConfig(MAP, null, null)
6060

6161
then:
6262
result.server == 'foo.com'
6363
result.token == 'blah-blah'
6464
result.namespace == 'my-namespace'
65+
result.serviceAccount == 'default'
66+
result.verifySsl
67+
result.clientCert == 'hello'.bytes
68+
result.clientKey == 'world'.bytes
69+
result.sslCert == 'fizzbuzz'.bytes
70+
71+
when:
72+
result = ClientConfig.fromNextflowConfig(MAP, 'ns1', 'sa2')
73+
then:
74+
result.server == 'foo.com'
75+
result.token == 'blah-blah'
76+
result.namespace == 'ns1'
77+
result.serviceAccount == 'sa2'
6578
result.verifySsl
6679
result.clientCert == 'hello'.bytes
6780
result.clientKey == 'world'.bytes
@@ -89,12 +102,13 @@ class ClientConfigTest extends Specification {
89102
clientKeyFile: file3 ]
90103

91104
when:
92-
def result = ClientConfig.fromMap(MAP)
105+
def result = ClientConfig.fromNextflowConfig(MAP, null, null)
93106

94107
then:
95108
result.server == 'foo.com'
96109
result.token == 'blah-blah'
97110
result.namespace == 'my-namespace'
111+
result.serviceAccount == 'default'
98112
!result.verifySsl
99113
result.sslCert == file1.text.bytes
100114
result.clientCert == file2.text.bytes

0 commit comments

Comments
 (0)