Refactor API server

This change accomplishes 2 primary things:
1. It generalizes work to enable the EventRateLimit admission plugin.
2. It restructures the anchor so that during an upgrade an "old" anchor
   does not try to coordinate the injection of "new" data from
   configmaps/secrets.

It also includes these ancillary changes:
* Clean up apiserver argument specification in the chart.
* De-duplicate and realign apiserver arguments in bootstrapping templates.

It has the side effects of:
* Adding a new field, ".apiserver.arguments" to the Genesis config,
  which will be the preferred way to configure bootstrapping apiservers
  going forward (in lieu of command_prefix).

Change-Id: I33cfe80ee8e29cd79e479a7985e3c098a2288fda

charts/apiserver/templates/bin/_anchor.tpl

@@ -15,26 +15,54 @@
 
 set -x
 
-compare_copy_files() {
+snapshot_files() {
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.const.files_to_copy }}
+    mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
+    cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
+    {{- end }}
+    {{ range $key, $val := .Values.conf }}
+    cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
+    {{- end }}
+}
 
-    {{range .Values.anchor.files_to_copy}}
-    if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
-        mkdir -p $(dirname /host{{ .dest }})
-        cp {{ .source }} /host{{ .dest }}
-        chmod go-rwx /host{{ .dest }}
+compare_copy_files() {
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.const.files_to_copy }}
+    SRC="${SNAPSHOT_DIR}{{ $dest }}"
+    DEST="/host{{ $dest }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
     fi
-    {{end}}
+    {{- end}}
+    {{ range $key, $val := .Values.conf }}
+    SRC="${SNAPSHOT_DIR}/etc/kubernetes/apiserver/{{ $val.file }}"
+    DEST="/host/etc/kubernetes/apiserver/{{ $val.file }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
+    fi
+    {{- end }}
 }
 
 cleanup() {
-
-    {{range .Values.anchor.files_to_copy}}
-    rm -f /host{{ .dest }}
-    {{end}}
+    {{- range $dest, $source := .Values.const.files_to_copy }}
+    rm -f "/host{{ $dest }}"
+    {{- end }}
+    {{  range $key, $val := .Values.conf }}
+    rm -f "/host/{{ $val.file }}"
+    {{- end }}
 }
 
-while true; do
 
+SNAPSHOT_DIR=$(mktemp -d)
+
+snapshot_files "${SNAPSHOT_DIR}"
+
+while true; do
     if [ -e /tmp/stop ]; then
         echo Stopping
         cleanup
@@ -43,7 +71,7 @@ while true; do
 
     # Compare and replace files on Genesis host if needed
     # Copy files to other master nodes
-    compare_copy_files
+    compare_copy_files "${SNAPSHOT_DIR}"
 
     sleep {{ .Values.anchor.period }}
 done

charts/apiserver/templates/configmap-etc.yaml

@@ -17,34 +17,19 @@ limitations under the License.
 {{- if .Values.manifests.configmap_etc }}
 {{- $envAll := . }}
 
-{{/* This slightly involved merge of AC config files into the anchor
-     files uses HTK merge, as straighforward appends result in duplicates. */}}
-{{- $_ := set .Values "_ac_files_to_copy" list }}
-{{- range $key, $val := .Values.conf.admission_controllers }}
-  {{- $source := printf "/tmp/etc/%s" $key }}
-  {{- $dest := printf "/etc/kubernetes/apiserver/%s" $key }}
-  {{- $file_to_copy := dict "source" $source "dest" $dest }}
-  {{- $ac_files_to_copy := append $.Values._ac_files_to_copy $file_to_copy }}
-  {{- $_ := set $.Values "_ac_files_to_copy" $ac_files_to_copy }}
-{{- end }}
-{{ $all_files_to_copy := dict }}
-{{ $_ := set $all_files_to_copy "values" (tuple .Values.anchor.files_to_copy .Values._ac_files_to_copy) }}
-{{ $_ := $all_files_to_copy | include "helm-toolkit.utils.merge" }}
-{{ $_ := set .Values.anchor "files_to_copy" $all_files_to_copy.result }}
-
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ .Values.service.name }}-etc
 data:
-  kubernetes-apiserver.yaml: |+
+  kubernetes-apiserver.yaml: |
 {{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  kubeconfig.yaml: |+
+  kubeconfig.yaml: |
 {{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{/* Dynamically add config files for admission controllers */}}
-{{ range $key, $val := .Values.conf.admission_controllers }}
-  {{ $key }}: |+
-{{ toYaml $val | indent 4 }}
-{{ end }}
+{{/* Dynamically added config files */}}
+{{- range $key, $val := .Values.conf }}
+  {{ $val.file }}: |
+{{ toYaml $val.content | indent 4 }}
+{{- end }}
 {{- end }}

charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl

@@ -42,30 +42,25 @@ spec:
               fieldPath: spec.nodeName
         - name: KUBECONFIG
           value: /etc/kubernetes/apiserver/kubeconfig.yaml
+        - name: APISERVER_PORT
+          value: {{ .Values.network.kubernetes_apiserver.port | quote }}
+        - name: ETCD_ENDPOINTS
+          value: {{ .Values.apiserver.etcd.endpoints | quote }}
 
       command:
-        {{- range .Values.command_prefix }}
+        {{- range .Values.const.command_prefix }}
         - {{ . }}
         {{- end }}
-        - --advertise-address=$(POD_IP)
-        - --anonymous-auth=false
-        - --bind-address=0.0.0.0
-        - --secure-port={{ .Values.network.kubernetes_apiserver.port }}
-        - --insecure-port=0
-        - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
-        - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
-        - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-        - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
-        - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
-        - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
-        - --etcd-servers={{ .Values.apiserver.etcd.endpoints }}
-        - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
-        - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
-        - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
-        - --allow-privileged=true
-        - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
-        - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
+        {{- range .Values.apiserver.arguments }}
+        - {{ . }}
+        {{- end }}
+        {{- range $key, $val := .Values.conf }}
+        {{- if hasKey $val "command_options" }}
+        {{- range $val.command_options }}
+        - {{ . }}
+        {{- end }}
+        {{- end }}
+        {{- end }}
 
       ports:
         - containerPort: {{ .Values.network.kubernetes_apiserver.port }}

charts/apiserver/values.yaml

@@ -14,6 +14,45 @@
 
 release_group: null
 
+# NOTE(mark-burnett): These values are not really configurable -- they live
+# here to keep the templates cleaner.
+const:
+  command_prefix:
+    - /apiserver
+    - --advertise-address=$(POD_IP)
+    - --allow-privileged=true
+    - --anonymous-auth=false
+    - --bind-address=0.0.0.0
+    - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+    - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
+    - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
+    - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
+    - --etcd-servers=$(ETCD_ENDPOINTS)
+    - --insecure-port=0
+    - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+    - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
+    - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
+    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+    - --secure-port=$(APISERVER_PORT)
+    - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
+    - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
+    - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
+
+  files_to_copy:
+    # NOTE(mark-burnett): These are (host dest): (container source) pairs
+    /etc/kubernetes/apiserver/kubeconfig.yaml: /tmp/etc/kubeconfig.yaml
+    /etc/kubernetes/apiserver/pki/apiserver-key.pem: /keys/apiserver-key.pem
+    /etc/kubernetes/apiserver/pki/apiserver.pem: /certs/apiserver.pem
+    /etc/kubernetes/apiserver/pki/cluster-ca.pem: /certs/cluster-ca.pem
+    /etc/kubernetes/apiserver/pki/etcd-client-ca.pem: /certs/etcd-client-ca.pem
+    /etc/kubernetes/apiserver/pki/etcd-client-key.pem: /keys/etcd-client-key.pem
+    /etc/kubernetes/apiserver/pki/etcd-client.pem: /certs/etcd-client.pem
+    /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem: /certs/kubelet-client-ca.pem
+    /etc/kubernetes/apiserver/pki/kubelet-client-key.pem: /keys/kubelet-client-key.pem
+    /etc/kubernetes/apiserver/pki/kubelet-client.pem: /certs/kubelet-client.pem
+    /etc/kubernetes/apiserver/pki/service-account.pub: /certs/service-account.pub
+    /etc/kubernetes/manifests/kubernetes-apiserver.yaml: /tmp/etc/kubernetes-apiserver.yaml
+
 images:
   tags:
     anchor: gcr.io/google_containers/hyperkube-amd64:v1.10.11
@@ -30,65 +69,58 @@ anchor:
   kubelet:
     manifest_path: /etc/kubernetes/manifests
   period: 15
-  files_to_copy:
-    - source: /certs/apiserver.pem
-      dest: /etc/kubernetes/apiserver/pki/apiserver.pem
-    - source: /certs/kubelet-client.pem
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client.pem
-    - source: /certs/kubelet-client-ca.pem
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
-    - source: /certs/cluster-ca.pem
-      dest: /etc/kubernetes/apiserver/pki/cluster-ca.pem
-    - source: /certs/etcd-client-ca.pem
-      dest: /etc/kubernetes/apiserver/pki/etcd-client-ca.pem
-    - source: /certs/etcd-client.pem
-      dest: /etc/kubernetes/apiserver/pki/etcd-client.pem
-    - source: /certs/service-account.pub
-      dest: /etc/kubernetes/apiserver/pki/service-account.pub
-    - source: /keys/apiserver-key.pem
-      dest: /etc/kubernetes/apiserver/pki/apiserver-key.pem
-    - source: /keys/kubelet-client-key.pem
-      dest: /etc/kubernetes/apiserver/pki/kubelet-client-key.pem
-    - source: /keys/etcd-client-key.pem
-      dest: /etc/kubernetes/apiserver/pki/etcd-client-key.pem
-    - source: /tmp/etc/kubernetes-apiserver.yaml
-      dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml
-    - source: /tmp/etc/kubeconfig.yaml
-      dest: /etc/kubernetes/apiserver/kubeconfig.yaml
-    # Note: config files for admission controllers are added to this dynamically
 
-command_prefix:
-  - /apiserver
-  - --authorization-mode=Node,RBAC
-  - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
-  - --service-cluster-ip-range=10.96.0.0/16
-  - --endpoint-reconciler-type=lease
-  # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
-  - --repair-malformed-updates=false
+conf:
+# Uncomment any of the below to enable the file placement and associated apiserver
+# command line options
+#
+#  acconfig:
+#    file: acconfig.yaml
+#    command_options:
+#      - '--admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml'
+#      - '--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit'
+#    content:
+#      kind: AdmissionConfiguration
+#      apiVersion: apiserver.k8s.io/v1alpha1
+#      plugins:
+#        - name: EventRateLimit
+#          path: eventconfig.yaml
+#  eventconfig:
+#    file: eventconfig.yaml
+#    command_options:
+#      - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
+#    content:
+#      kind: Configuration
+#      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+#      limits:
+#        - type: Server
+#          qps: 1000
+#          burst: 10000
+#  encryption_provider:
+#    file: encryption_provider.yaml
+#    command_option: ''
+#    content:
+#      kind: EncryptionConfig
+#      apiVersion: v1
+#      resources:
+#        - resources:
+#            - 'secrets'
+#          providers:
+#            - identity: {}
 
 apiserver:
-  host_etc_path: /etc/kubernetes/apiserver
+  arguments:
+    - --authorization-mode=Node,RBAC
+    - --service-cluster-ip-range=10.96.0.0/16
+    - --endpoint-reconciler-type=lease
+    - --feature-gates=PodShareProcessNamespace=true
+    # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
+    - --repair-malformed-updates=false
+    - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction
+    - --v=3
   etcd:
     endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local
-
-conf:
-  # Admission controllers config files are generated dynamically based on the
-  # config below, as they are specific to particular ACs that may be
-  # configured by the operator (or added by k8s in the future).
-  admission_controllers:
-    eventconfig.yaml:
-      kind: Configuration
-      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
-      limits:
-      - type: Server
-        qps: 100
-        burst: 1000
-    acconfig.yaml:
-      kind: AdmissionConfiguration
-      apiVersion: apiserver.k8s.io/v1alpha1
-      plugins:
-      - name: EventRateLimit
-        path: eventconfig.yaml
+  host_etc_path: /etc/kubernetes/apiserver
 
 network:
   kubernetes_apiserver:
@@ -130,7 +162,6 @@ secrets:
       cert: null
       key: null
 
-
 # typically overriden by environmental
 # values, but should include all endpoints
 # required by this chart
@@ -170,7 +201,7 @@ pod:
     upgrades:
       daemonsets:
         pod_replacement_strategy: RollingUpdate
-        kubernetes_apiserver:
+        kubernetes-apiserver-anchor:
           enabled: false
           min_ready_seconds: 0
           max_unavailable: 1

examples/basic/Genesis.yaml

@@ -11,15 +11,16 @@ data:
   hostname: n0
   ip: 192.168.77.10
   apiserver:
-    command_prefix:
-      - /apiserver
+    arguments:
       - --authorization-mode=Node,RBAC
-      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
       - --service-cluster-ip-range=10.96.0.0/16
       - --endpoint-reconciler-type=lease
       - --feature-gates=PodShareProcessNamespace=true
       # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
       - --repair-malformed-updates=false
+      - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
+      - --v=3
   armada:
     target_manifest: cluster-bootstrap
   labels:
@@ -45,4 +46,22 @@ data:
     - path: /var/lib/anchor/calico-etcd-bootstrap
       content: "# placeholder for triggering calico etcd bootstrapping"
       mode: 0644
+    # NOTE(mark-burnett): These are referenced by the apiserver arguments above.
+    - path: /etc/genesis/apiserver/acconfig.yaml
+      mode: 0444
+      content: |
+        kind: AdmissionConfiguration
+        apiVersion: apiserver.k8s.io/v1alpha1
+        plugins:
+          - name: EventRateLimit
+            path: eventconfig.yaml
+    - path: /etc/genesis/apiserver/eventconfig.yaml
+      mode: 0444
+      content: |
+        kind: Configuration
+        apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+        limits:
+          - type: Server
+            qps: 1000
+            burst: 10000
 ...

examples/basic/armada-resources.yaml

@@ -719,15 +719,6 @@ data:
   upgrade:
     no_hooks: true
   values:
-    command_prefix:
-      - /apiserver
-      - --authorization-mode=Node,RBAC
-      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
-      - --service-cluster-ip-range=10.96.0.0/16
-      - --endpoint-reconciler-type=lease
-      - --feature-gates=PodShareProcessNamespace=true
-      # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
-      - --repair-malformed-updates=false
     apiserver:
       etcd:
         endpoints: https://127.0.0.1:2378