dch-webhooks: Enable host provisioning feature

The *dch-webhooks* server now has a _POST /host/online_ hook that can
be triggered by a new machine when it first comes online. This hook
starts an automatic provisioning process by creating a Kubernetes Job
to run Ansible and publishing information about the host to provision
via AMQP.  Thus, the server now needs access to the Kubernetes API in
order to create the Job and access to RabbitMQ in order to publish the
task parameters.

dch-webhooks/ansible-job.yaml

@@ -0,0 +1,115 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  generateName: host-provision-
+  labels: &labels
+    app.kubernetes.io/name: host-provisioner
+    app.kubernetes.io/component: host-provisioner
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels: *labels
+    spec:
+      restartPolicy: Never
+      initContainers:
+      - name: ssh-agent
+        image: &image git.pyrocufflink.net/infra/host-provisioner
+        imagePullPolicy: Always
+        command:
+        - tini
+        - ssh-agent
+        - --
+        - -D
+        - -a
+        - /run/ssh/agent.sock
+        restartPolicy: Always
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /run/ssh
+          name: tmp
+          subPath: run/ssh
+      - name: ssh-add
+        image: *image
+        command:
+        - ssh-add
+        - -t
+        - 30m
+        - /run/secrets/ssh/host-provisioner.key
+        env:
+        - name: SSH_AUTH_SOCK
+          value: /run/ssh/agent.sock
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /run/ssh
+          name: tmp
+          subPath: run/ssh
+        - mountPath: /run/secrets/ssh
+          name: provisioner-key
+          readOnly: true
+      containers:
+      - name: host-provisioner
+        image: *image
+        env:
+        - name: SSH_AUTH_SOCK
+          value: /run/ssh/agent.sock
+        - name: AMQP_HOST
+          value: rabbitmq.pyrocufflink.blue
+        - name: AMQP_PORT
+          value: '5671'
+        - name: AMQP_CA_CERT
+          value: /run/dch-ca/dch-root-ca.crt
+        - name: AMQP_CLIENT_CERT
+          value: /run/secrets/host-provisioner/rabbitmq/tls.crt
+        - name: AMQP_CLIENT_KEY
+          value: /run/secrets/host-provisioner/rabbitmq/tls.key
+        - name: AMQP_EXTERNAL_CREDENTIALS
+          value: '1'
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /etc/ssh/ssh_known_hosts
+          name: ssh-known-hosts
+          subPath: ssh_known_hosts
+          readOnly: true
+        - mountPath: /home/jenkins
+          name: workspace
+        - mountPath: /run/dch-ca
+          name: dch-root-ca
+          readOnly: true
+        - mountPath: /run/ssh
+          name: tmp
+          subPath: run/ssh
+        - mountPath: /run/secrets/host-provisioner/rabbitmq
+          name: rabbitmq-cert
+          readOnly: true
+        - mountPath: /tmp
+          name: tmp
+          subPath: tmp
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      volumes:
+      - name: dch-root-ca
+        configMap:
+          name: dch-root-ca
+      - name: provisioner-key
+        secret:
+          secretName: provisioner-ssh-key
+          defaultMode: 0440
+      - name: ssh-known-hosts
+        configMap:
+          name: ssh-known-hosts
+      - name: rabbitmq-cert
+        secret:
+          secretName: rabbitmq-cert
+          defaultMode: 0440
+      - name: tmp
+        emptyDir:
+          medium: Memory
+      - name: workspace
+        emptyDir: {}

dch-webhooks/certificate.yaml

@@ -0,0 +1,14 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: rabbitmq
+spec:
+  secretName: rabbitmq-cert
+  commonName: dch-webhooks
+  issuerRef:
+    group: cert-manager.io
+    kind: ClusterIssuer
+    name: rabbitmq-ca
+  privateKey:
+    algorithm: ECDSA
+    rotationPolicy: Always

dch-webhooks/dch-webhooks.env

@@ -7,3 +7,10 @@ STEP_CA_URL=https://ca.pyrocufflink.blue:32599
 STEP_ROOT=/run/dch-root-ca.crt
 STEP_PROVISIONER=host-bootstrap
 STEP_PROVISIONER_PASSWORD_FILE=/run/secrets/du5t1n.me/step-ca/provisioner.password
+
+AMQP_HOST=rabbitmq.pyrocufflink.blue
+AMQP_PORT=5671
+AMQP_EXTERNAL_CREDENTIALS=1
+AMQP_CA_CERT=/run/dch-root-ca.crt
+AMQP_CLIENT_CERT=/run/secrets/du5t1n.me/rabbitmq/tls.crt
+AMQP_CLIENT_KEY=/run/secrets/du5t1n.me/rabbitmq/tls.key

dch-webhooks/dch-webhooks.yaml

@@ -1,4 +1,14 @@
 apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dch-webhooks
+  labels:
+    app.kubernetes.io/name: dch-webhooks
+    app.kubernetes.io/component: dch-webhooks
+    app.kubernetes.io/part-of: dch-webhooks
+
+---
+apiVersion: v1
 kind: Service
 metadata:
   labels:
@@ -42,12 +52,14 @@ spec:
     spec:
       containers:
       - name: dch-webhooks
-        image: git.pyrocufflink.net/containerimages/dch-webhooks
+        image: git.pyrocufflink.net/infra/dch-webhooks
         env:
         - name: UVICORN_HOST
           value: 0.0.0.0
         - name: UVICORN_LOG_LEVEL
           value: debug
+        - name: ANSIBLE_JOB_YAML
+          value: /etc/dch-webhooks/ansible-job.yaml
         envFrom:
         - configMapRef:
             name: dch-webhooks
@@ -76,22 +88,37 @@ spec:
           name: firefly-token
         - mountPath: /run/secrets/du5t1n.me/paperless
           name: paperless-token
+        - mountPath: /run/secrets/du5t1n.me/rabbitmq
+          name: rabbitmq-cert
+          readOnly: true
         - mountPath: /run/secrets/du5t1n.me/step-ca
           name: step-ca-password
         - mountPath: /tmp
           name: tmp
           subPath: tmp
+        - mountPath: /etc/dch-webhooks
+          name: host-provisioner
+          readOnly: true
       securityContext:
         runAsNonRoot: true
+      serviceAccountName: dch-webhooks
       volumes:
       - name: firefly-token
         secret:
           secretName: firefly-token
           optional: true
+      - name: host-provisioner
+        configMap:
+          name: host-provisioner
+          optional: true
       - name: paperless-token
         secret:
           secretName: paperless-token
           optional: true
+      - name: rabbitmq-cert
+        secret:
+          secretName: rabbitmq-cert
+          optional: true
       - name: root-ca
         configMap:
           name: dch-root-ca

dch-webhooks/kustomization.yaml

@@ -1,15 +1,29 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+labels:
+- pairs:
+    app.kubernetes.io/instance: dch-webhooks
+  includeSelectors: true
+  includeTemplates: true
+- pairs:
+    app.kubernetes.io/part-of: dch-webhooks
+
 resources:
 - ../dch-root-ca
 - dch-webhooks.yaml
+- certificate.yaml
 - ingress.yaml
 
 configMapGenerator:
 - name: dch-webhooks
   envs:
   - dch-webhooks.env
+- name: host-provisioner
+  files:
+  - ansible-job.yaml
+  options:
+    disableNameSuffixHash: true
 
 secretGenerator:
 - name: firefly-token