[WIP] Feature awx mesh ingress

PROJECT

@@ -1,3 +1,7 @@
+# Code generated by tool. DO NOT EDIT.
+# This file is used to track the info used to scaffold your project
+# and allow the plugins properly work.
+# More info: https://book.kubebuilder.io/reference/project-config.html
 domain: ansible.com
 layout:
 - ansible.sdk.operatorframework.io/v1
@@ -27,4 +31,11 @@ resources:
   group: awx
   kind: AWXRestore
   version: v1beta1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: ansible.com
+  group: awx
+  kind: AWXMeshIngress
+  version: v1alpha1
 version: "3"

config/crd/bases/awx.ansible.com_awxmeshingresses.yaml

@@ -0,0 +1,50 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: awxmeshingresses.awx.ansible.com
+spec:
+  group: awx.ansible.com
+  names:
+    kind: AWXMeshIngress
+    listKind: AWXMeshIngressList
+    plural: awxmeshingresses
+    singular: awxmeshingress
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: AWXMeshIngress is the Schema for the awxmeshingresses API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec defines the desired state of AWXMeshIngress
+            type: object
+            x-kubernetes-preserve-unknown-fields: true
+            required:
+            - deployment_name
+            properties:
+              deployment_name:
+                description: Name of the AWX deployment to create the Mesh Ingress for.
+                type: string
+          status:
+            description: Status defines the observed state of AWXMeshIngress
+            type: object
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/crd/kustomization.yaml

@@ -5,4 +5,5 @@ resources:
 - bases/awx.ansible.com_awxs.yaml
 - bases/awx.ansible.com_awxbackups.yaml
 - bases/awx.ansible.com_awxrestores.yaml
+- bases/awx.ansible.com_awxmeshingresses.yaml
 #+kubebuilder:scaffold:crdkustomizeresource

config/rbac/awxmeshingress_editor_role.yaml

@@ -0,0 +1,31 @@
+# permissions for end users to edit awxmeshingresses.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: clusterrole
+    app.kubernetes.io/instance: awxmeshingress-editor-role
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: awx-operator
+    app.kubernetes.io/part-of: awx-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: awxmeshingress-editor-role
+rules:
+- apiGroups:
+  - awx.ansible.com
+  resources:
+  - awxmeshingresses
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - awx.ansible.com
+  resources:
+  - awxmeshingresses/status
+  verbs:
+  - get

config/rbac/awxmeshingress_viewer_role.yaml

@@ -0,0 +1,27 @@
+# permissions for end users to view awxmeshingresses.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: clusterrole
+    app.kubernetes.io/instance: awxmeshingress-viewer-role
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: awx-operator
+    app.kubernetes.io/part-of: awx-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: awxmeshingress-viewer-role
+rules:
+- apiGroups:
+  - awx.ansible.com
+  resources:
+  - awxmeshingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - awx.ansible.com
+  resources:
+  - awxmeshingresses/status
+  verbs:
+  - get

config/samples/awx_v1alpha1_awxmeshingress.yaml

@@ -0,0 +1,12 @@
+apiVersion: awx.ansible.com/v1alpha1
+kind: AWXMeshIngress
+metadata:
+  labels:
+    app.kubernetes.io/name: awxmeshingress
+    app.kubernetes.io/instance: awxmeshingress-sample
+    app.kubernetes.io/part-of: awx-operator
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: awx-operator
+  name: awxmeshingress-sample
+spec:
+  # TODO(user): Add fields here

config/samples/kustomization.yaml

@@ -3,4 +3,5 @@ resources:
 - awx_v1beta1_awx.yaml
 - awx_v1beta1_awxbackup.yaml
 - awx_v1beta1_awxrestore.yaml
+- awx_v1alpha1_awxmeshingress.yaml
 #+kubebuilder:scaffold:manifestskustomizesamples

molecule/default/tasks/awxmeshingress_test.yml

@@ -0,0 +1,18 @@
+---
+- name: Create the awx.ansible.com/v1alpha1.AWXMeshIngress
+  k8s:
+    state: present
+    namespace: '{{ namespace }}'
+    definition: "{{ lookup('template', '/'.join([samples_dir, cr_file])) | from_yaml }}"
+    wait: yes
+    wait_timeout: 300
+    wait_condition:
+      type: Successful
+      status: "True"
+  vars:
+    cr_file: 'awx_v1alpha1_awxmeshingress.yaml'
+
+- name: Add assertions here
+  assert:
+    that: false
+    fail_msg: FIXME Add real assertions for your operator

roles/mesh_ingress/defaults/main.yml

@@ -0,0 +1,4 @@
+deployment_type: awx
+
+_control_plane_ee_image: quay.io/ansible/awx-ee:latest
+_image_pull_policy: Always

roles/mesh_ingress/tasks/main.yml

@@ -0,0 +1,90 @@
+---
+- name: Check for presence of AWX instance that we will use to create the Mesh Ingress for.
+  k8s_info:
+    api_version: awx.ansible.com/v1beta1
+    kind: AWX
+    name: "{{ deployment_name }}"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+  register: awx_instance
+
+- name: Fail if awx_deployment does not exist in the same namespace
+  fail:
+    msg: "AWX instance {{ deployment_name }} does not exist in the same namespace as the AWXMeshIngress instance."
+  when: awx_instance.resources | length == 0
+
+- name: Set awx_spec
+  set_fact:
+    awx_spec: "{{ awx_instance.resources[0].spec }}"
+
+- name: Set user provided control plane ee image
+  set_fact:
+    _custom_control_plane_ee_image: "{{ awx_spec.control_plane_ee_image }}"
+  when:
+    - awx_spec.control_plane_ee_image | default([]) | length
+
+- name: Set Control Plane EE image URL
+  set_fact:
+    _control_plane_ee_image: "{{ _custom_control_plane_ee_image | default(lookup('env', 'RELATED_IMAGE_CONTROL_PLANE_EE')) | default(_control_plane_ee_image, true) }}"
+
+- name: Set Image Pull Policy
+  set_fact:
+    _image_pull_policy: "{{ awx_spec.image_pull_policy | default(_image_pull_policy, true) }}"
+
+- name: Apply Route resource
+  k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'route.yml.j2') }}"
+    wait: yes
+    wait_timeout: "120"
+  register: route
+
+- name: Set external_hostname
+  set_fact:
+    external_hostname: "{{ route.result.status.ingress[0].host }}"
+
+- name: Create other resources
+  k8s:
+    apply: yes
+    definition: "{{ lookup('template', '{{ item }}.yml.j2') }}"
+    wait: yes
+    wait_timeout: "120"
+  loop:
+  - service_account
+  - receptor_conf.configmap
+  - service
+  - statefulset
+
+- name: Get the current resource task pod information.
+  k8s_info:
+    api_version: v1
+    kind: Pod
+    namespace: '{{ ansible_operator_meta.namespace }}'
+    label_selectors:
+      - "app.kubernetes.io/name={{ deployment_name }}-task"
+      - "app.kubernetes.io/managed-by={{ deployment_type }}-operator"
+      - "app.kubernetes.io/component={{ deployment_type }}"
+    field_selectors:
+      - status.phase=Running
+  register: awx_task_pod
+
+- name: Set the resource pod as a variable.
+  set_fact:
+    awx_task_pod: >-
+      {{ awx_task_pod['resources']
+         | rejectattr('metadata.deletionTimestamp', 'defined')
+         | sort(attribute='metadata.creationTimestamp')
+         | first | default({}) }}
+
+- name: Set the resource pod name as a variable.
+  set_fact:
+    awx_task_pod_name: "{{ awx_task_pod['metadata']['name'] | default('') }}"
+
+## TODO: awx-manage provision_instance does not currently support peer from control nodes
+## !!!dependent on API/CLI changes!!!
+# - name: Add new instance to AWX
+#   k8s_exec:
+#     namespace: "{{ ansible_operator_meta.namespace }}"
+#     pod: "{{ awx_task_pod_name }}"
+#     container: "{{ deployment_name }}-task"
+#     command: awx-manage provision_instance ...
+#   register: result

roles/mesh_ingress/templates/receptor_conf.configmap.yml.j2

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ ansible_operator_meta.name }}-receptor-config
+  namespace: {{ ansible_operator_meta.namespace }}
+data:
+  receptor_conf: |
+    ---
+    - node: 
+        id: {{ ansible_operator_meta.name }}
+    - log-level: debug
+    - ws-listener:
+        port: 27199
+        tls: tlsserver
+    - tls-server:
+        cert: /etc/receptor/tls/receptor.crt
+        key: /etc/receptor/tls/receptor.key
+        name: tlsserver
+        clientcas: /etc/receptor/tls/ca/mesh-CA.crt
+        requireclientcert: true
+        mintls13: false

roles/mesh_ingress/templates/route.yml.j2

@@ -0,0 +1,19 @@
+---
+apiVersion: route.openshift.io/v1
+kind: Route
+metadata:
+  annotations:
+    openshift.io/host.generated: "true"
+  name: {{ ansible_operator_meta.name }}
+  namespace: {{ ansible_operator_meta.namespace }}
+spec:
+  port:
+    targetPort: ws
+  tls:
+    insecureEdgeTerminationPolicy: None
+    termination: passthrough
+  to:
+    kind: Service
+    name: {{ ansible_operator_meta.name }}
+    weight: 100
+  wildcardPolicy: None

roles/mesh_ingress/templates/service.yml.j2

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ ansible_operator_meta.name }}
+  namespace: '{{ ansible_operator_meta.namespace }}'
+spec:
+  type: ClusterIP
+  ports:
+  - name: ws
+    port: 27199
+    targetPort: 27199
+  selector:
+    statefulset.kubernetes.io/pod-name: {{ ansible_operator_meta.name }}-0

roles/mesh_ingress/templates/service_account.yml.j2

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: '{{ ansible_operator_meta.name }}'
+  namespace: '{{ ansible_operator_meta.namespace }}'

roles/mesh_ingress/templates/statefulset.yml.j2

@@ -0,0 +1,57 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ ansible_operator_meta.name }}
+  namespace: {{ ansible_operator_meta.namespace }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ ansible_operator_meta.name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ ansible_operator_meta.name }}
+    spec:
+      containers:
+      - args:
+        - /bin/sh
+        - -c
+        - |
+          internal_hostname={{ ansible_operator_meta.name }}
+          external_hostname={{ external_hostname }}
+          receptor --cert-makereq bits=2048 commonname=$internal_hostname dnsname=$internal_hostname dnsname=$external_hostname nodeid=$internal_hostname outreq=/etc/receptor/tls/receptor.req outkey=/etc/receptor/tls/receptor.key
+          receptor --cert-signreq req=/etc/receptor/tls/receptor.req cacert=/etc/receptor/tls/ca/mesh-CA.crt cakey=/etc/receptor/tls/ca/mesh-CA.key outcert=/etc/receptor/tls/receptor.crt verify=yes
+          exec receptor --config /etc/receptor/receptor.conf
+        image: '{{ _control_plane_ee_image }}'
+        imagePullPolicy: '{{ _image_pull_policy }}'
+        name: {{ ansible_operator_meta.name }}-mesh-ingress
+        volumeMounts:
+        - mountPath: /etc/receptor/receptor.conf
+          name: {{ ansible_operator_meta.name }}-receptor-config
+          subPath: receptor.conf
+        - mountPath: /etc/receptor/tls/ca/mesh-CA.crt
+          name: {{ ansible_operator_meta.name }}-receptor-ca
+          readOnly: true
+          subPath: tls.crt
+        - mountPath: /etc/receptor/tls/ca/mesh-CA.key
+          name: {{ ansible_operator_meta.name }}-receptor-ca
+          readOnly: true
+          subPath: tls.key
+        - mountPath: /etc/receptor/tls/
+          name: {{ ansible_operator_meta.name }}-receptor-tls
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      serviceAccount: {{ ansible_operator_meta.name }}
+      volumes:
+      - name: {{ ansible_operator_meta.name }}-receptor-tls
+      - name: {{ ansible_operator_meta.name }}-receptor-ca
+        secret:
+          defaultMode: 420
+          secretName: {{ deployment_name }}-receptor-ca
+      - configMap:
+          defaultMode: 420
+          items:
+          - key: receptor_conf
+            path: receptor.conf
+          name: {{ ansible_operator_meta.name }}-receptor-config
+        name: {{ ansible_operator_meta.name }}-receptor-config