From 16904444b467e78d50be4396996605b8d2e22b5c Mon Sep 17 00:00:00 2001 From: Rohith Date: Mon, 13 Aug 2018 14:10:52 +0100 Subject: [PATCH 1/6] Node Authorizer Directory - creating the directory incase it's not there, is fixes an issue on a rolling update --- .../addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template b/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template index 8262eef0c9788..8bfa3ed6be22b 100644 --- a/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template +++ b/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template @@ -147,7 +147,7 @@ spec: - name: config hostPath: path: /srv/kubernetes/node-authorizer - type: Directory + type: DirectoryOrCreate containers: - name: {{ $name }} image: {{ $na.Image }} From 8c11ecf1082fb412bc0a7bb52e343cc6275fdfb8 Mon Sep 17 00:00:00 2001 From: Rohith Date: Tue, 18 Sep 2018 10:59:04 +0100 Subject: [PATCH 2/6] - adding the waitForCertificates method to wait for the certificates to arrive (this fixes the rollout on a in-place cluster) --- node-authorizer/cmd/node-authorizer/server.go | 49 +++++++++++++++++++ node-authorizer/pkg/utils/misc.go | 10 ++++ .../k8s-1.10.yaml.template | 24 ++++----- 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/node-authorizer/cmd/node-authorizer/server.go b/node-authorizer/cmd/node-authorizer/server.go index b6ac4ca8b50b9..83f68c8060770 100644 --- a/node-authorizer/cmd/node-authorizer/server.go +++ b/node-authorizer/cmd/node-authorizer/server.go @@ -25,6 +25,7 @@ import ( "k8s.io/kops/node-authorizer/pkg/authorizers/aws" "k8s.io/kops/node-authorizer/pkg/server" + "github.com/gambol99/aws-sso/pkg/utils" "github.com/urfave/cli" ) @@ -90,6 +91,12 @@ func addServerCommand() cli.Command { EnvVar: "CLIENT_COMMON_NAME", Value: "node-authorizer-client", }, + cli.DurationFlag{ + Name: "certificate-ttl", + Usage: "check the certificates exist and if not wait for x period `DURATION`", + EnvVar: "CERTIFICATE_TTL", + Value: 10 * time.Minute, + }, cli.DurationFlag{ Name: "authorization-timeout", Usage: "max time permitted for a authorization `DURATION`", @@ -122,6 +129,16 @@ func actionServerCommand(ctx *cli.Context) error { if ctx.String("authorizer") == "" { return errors.New("no authorizer specified") } + + // @step: should we wait for the certificates to appear + if ctx.Duration("certificate-ttl") > 0 { + var files = []string{ctx.String("tls-cert"), ctx.String("tls-client-ca"), ctx.String("tls-private-key")} + var timeout = ctx.Duration("certificate-ttl") + if err := waitForCertificates(files, timeout); err != nil { + return err + } + } + // @step: create the authorizers auth, err := createAuthorizer(ctx.String("authorizer"), config) if err != nil { @@ -136,6 +153,38 @@ func actionServerCommand(ctx *cli.Context) error { return svc.Run() } +// waitForCertificates is responisble for waiting for the certificates to appear +func waitForCertificates(files []string, timeout time.Duration) error { + doneCh := make(chan struct{}, 0) + + go func() { + expires := time.Now().Add(timeout) + + // @step: iterate the file we are looking for + for _, x := range files { + if x == "" { + continue + } + // @step: iterate until we find the file + for { + if utils.FileExists(x) { + break + } + fmt.Printf("waiting for file: %s to appear, timeouts in %s", x, expires.Sub(time.Now())) + time.Sleep(5 * time.Second) + } + } + doneCh <- struct{}{} + }() + + select { + case <-doneCh: + return nil + case <-time.After(timeout): + return fmt.Errorf("unable to find the certificates after %s timeout", timeout) + } +} + // createAuthorizer creates and returns a authorizer func createAuthorizer(name string, config *server.Config) (server.Authorizer, error) { switch name { diff --git a/node-authorizer/pkg/utils/misc.go b/node-authorizer/pkg/utils/misc.go index e3c8fdb7594c5..68c6335a22bda 100644 --- a/node-authorizer/pkg/utils/misc.go +++ b/node-authorizer/pkg/utils/misc.go @@ -19,6 +19,7 @@ package utils import ( crypto_rand "crypto/rand" "encoding/hex" + "os" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -34,6 +35,15 @@ func GetKubernetesClient() (kubernetes.Interface, error) { return kubernetes.NewForConfig(config) } +// FileExists checks if the file exists +func FileExists(filename string) bool { + if _, err := os.Stat(filename); !os.IsNotExist(err) { + return true + } + + return false +} + // RandomBytes generates some random bytes func RandomBytes(length int) (string, error) { b := make([]byte, length) diff --git a/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template b/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template index 8bfa3ed6be22b..1efb3dbf7cbe9 100644 --- a/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template +++ b/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template @@ -152,18 +152,18 @@ spec: - name: {{ $name }} image: {{ $na.Image }} args: - - server - - --authorization-timeout={{ $na.Timeout.Duration }} - - --authorizer={{ $na.Authorizer }} - - --cluster-name={{ ClusterName }} - {{- range $na.Features }} - - --feature={{ . }} - {{- end }} - - --listen=0.0.0.0:{{ $na.Port }} - - --tls-cert=/config/tls.pem - - --tls-client-ca=/config/ca.pem - - --tls-private-key=/config/tls-key.pem - - --token-ttl={{ $na.TokenTTL.Duration }} + - server + - --authorization-timeout={{ $na.Timeout.Duration }} + - --authorizer={{ $na.Authorizer }} + - --cluster-name={{ ClusterName }} + {{- range $na.Features }} + - --feature={{ . }} + {{- end }} + - --listen=0.0.0.0:{{ $na.Port }} + - --tls-cert=/config/tls.pem + - --tls-client-ca=/config/ca.pem + - --tls-private-key=/config/tls-key.pem + - --token-ttl={{ $na.TokenTTL.Duration }} resources: limits: cpu: 100m From d784db094ad43d7076d692d966869dcf65304800 Mon Sep 17 00:00:00 2001 From: Rohith Date: Wed, 26 Sep 2018 23:01:26 +0100 Subject: [PATCH 3/6] - removing the liveness and readiness probes for now until i figure a better way of performing the rollout while not getting hit by the cluster validation code. perhaps we could add a label no the master and control the deployment of the daemonset via the label. --- node-authorizer/cmd/node-authorizer/server.go | 2 +- .../k8s-1.10.yaml.template | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/node-authorizer/cmd/node-authorizer/server.go b/node-authorizer/cmd/node-authorizer/server.go index 83f68c8060770..ef406afa36bd7 100644 --- a/node-authorizer/cmd/node-authorizer/server.go +++ b/node-authorizer/cmd/node-authorizer/server.go @@ -170,7 +170,7 @@ func waitForCertificates(files []string, timeout time.Duration) error { if utils.FileExists(x) { break } - fmt.Printf("waiting for file: %s to appear, timeouts in %s", x, expires.Sub(time.Now())) + fmt.Printf("waiting for file: %s to appear, timeouts in %s\n", x, expires.Sub(time.Now())) time.Sleep(5 * time.Second) } } diff --git a/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template b/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template index 1efb3dbf7cbe9..665df2a27d2ee 100644 --- a/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template +++ b/upup/models/cloudup/resources/addons/node-authorizer.addons.k8s.io/k8s-1.10.yaml.template @@ -171,20 +171,6 @@ spec: requests: cpu: 10m memory: 10Mi - livenessProbe: - httpGet: - path: /health - port: {{ $na.Port }} - scheme: HTTPS - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /health - port: {{ $na.Port }} - scheme: HTTPS - periodSeconds: 10 volumeMounts: - mountPath: /config readOnly: true From 26942eb6016635894bbb15138eb7e7430de3b139 Mon Sep 17 00:00:00 2001 From: Rohith Date: Thu, 27 Sep 2018 09:59:23 +0100 Subject: [PATCH 4/6] - updating to the fix rollout image for node authorizer --- pkg/model/components/node-authorizer/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/model/components/node-authorizer/options.go b/pkg/model/components/node-authorizer/options.go index b739c648fe8b6..a065bd944ec4a 100644 --- a/pkg/model/components/node-authorizer/options.go +++ b/pkg/model/components/node-authorizer/options.go @@ -100,5 +100,5 @@ func GetNodeAuthorizerImage() string { return v } - return "quay.io/gambol99/node-authorizer:v0.0.1@sha256:3ff243f5af76a73b6faaa6a0b0be8e3882dd1e7ffea6bacda9bede2273446059" + return "quay.io/gambol99/node-authorizer:v0.0.2@sha256:4a9f17072cb937ccbe3042add3e7da8152a7f35ffbdf8b96ae49306f6b59c080" } From 8401273b06102a934b5d7a6668b5df64d91c2934 Mon Sep 17 00:00:00 2001 From: Rohith Date: Thu, 27 Sep 2018 10:17:30 +0100 Subject: [PATCH 5/6] - fixing the reference to the import, goimports made an error - updating the version of the node-authorizer manifest --- node-authorizer/cmd/node-authorizer/server.go | 2 +- node-authorizer/pkg/utils/misc.go | 6 +++--- pkg/model/components/node-authorizer/options.go | 2 +- upup/pkg/fi/cloudup/bootstrapchannelbuilder.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/node-authorizer/cmd/node-authorizer/server.go b/node-authorizer/cmd/node-authorizer/server.go index ef406afa36bd7..d6ddb1c3eb7fa 100644 --- a/node-authorizer/cmd/node-authorizer/server.go +++ b/node-authorizer/cmd/node-authorizer/server.go @@ -24,8 +24,8 @@ import ( "k8s.io/kops/node-authorizer/pkg/authorizers/alwaysallow" "k8s.io/kops/node-authorizer/pkg/authorizers/aws" "k8s.io/kops/node-authorizer/pkg/server" + "k8s.io/kops/node-authorizer/pkg/utils" - "github.com/gambol99/aws-sso/pkg/utils" "github.com/urfave/cli" ) diff --git a/node-authorizer/pkg/utils/misc.go b/node-authorizer/pkg/utils/misc.go index 68c6335a22bda..26227b8674d82 100644 --- a/node-authorizer/pkg/utils/misc.go +++ b/node-authorizer/pkg/utils/misc.go @@ -37,11 +37,11 @@ func GetKubernetesClient() (kubernetes.Interface, error) { // FileExists checks if the file exists func FileExists(filename string) bool { - if _, err := os.Stat(filename); !os.IsNotExist(err) { - return true + if _, err := os.Stat(filename); err != nil { + return false } - return false + return true } // RandomBytes generates some random bytes diff --git a/pkg/model/components/node-authorizer/options.go b/pkg/model/components/node-authorizer/options.go index a065bd944ec4a..34fcd7ba2c059 100644 --- a/pkg/model/components/node-authorizer/options.go +++ b/pkg/model/components/node-authorizer/options.go @@ -100,5 +100,5 @@ func GetNodeAuthorizerImage() string { return v } - return "quay.io/gambol99/node-authorizer:v0.0.2@sha256:4a9f17072cb937ccbe3042add3e7da8152a7f35ffbdf8b96ae49306f6b59c080" + return "quay.io/gambol99/node-authorizer:v0.0.2@sha256:78c20c69187d3098e196e2b645d0571aeef377adc5cbd89684023ec668306268" } diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 661d5a527c996..34430f27b4ed0 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -150,7 +150,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri if b.cluster.Spec.NodeAuthorization != nil { { key := "node-authorizer.addons.k8s.io" - version := "v0.0.1" + version := "v0.0.2" { location := key + "/k8s-1.10.yaml" From 90c48a76c327c2dc5b09727e08efebf755a1310b Mon Sep 17 00:00:00 2001 From: Rohith Date: Thu, 27 Sep 2018 10:28:11 +0100 Subject: [PATCH 6/6] - fixing the bazel issue (missing out the file) - fixing the spelling mistake --- node-authorizer/cmd/node-authorizer/BUILD.bazel | 1 + node-authorizer/cmd/node-authorizer/server.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/node-authorizer/cmd/node-authorizer/BUILD.bazel b/node-authorizer/cmd/node-authorizer/BUILD.bazel index bf3dd9b3b4b3f..873056353bbca 100644 --- a/node-authorizer/cmd/node-authorizer/BUILD.bazel +++ b/node-authorizer/cmd/node-authorizer/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//node-authorizer/pkg/authorizers/aws:go_default_library", "//node-authorizer/pkg/client:go_default_library", "//node-authorizer/pkg/server:go_default_library", + "//node-authorizer/pkg/utils:go_default_library", "//vendor/github.com/urfave/cli:go_default_library", ], ) diff --git a/node-authorizer/cmd/node-authorizer/server.go b/node-authorizer/cmd/node-authorizer/server.go index d6ddb1c3eb7fa..a90817c0ba70d 100644 --- a/node-authorizer/cmd/node-authorizer/server.go +++ b/node-authorizer/cmd/node-authorizer/server.go @@ -153,7 +153,7 @@ func actionServerCommand(ctx *cli.Context) error { return svc.Run() } -// waitForCertificates is responisble for waiting for the certificates to appear +// waitForCertificates is responsible for waiting for the certificates to appear func waitForCertificates(files []string, timeout time.Duration) error { doneCh := make(chan struct{}, 0)