dch-webhooks: Enable host provisioning feature

The *dch-webhooks* server now has a _POST /host/online_ hook that can
be triggered by a new machine when it first comes online. This hook
starts an automatic provisioning process by creating a Kubernetes Job
to run Ansible and publishing information about the host to provision
via AMQP.  Thus, the server now needs access to the Kubernetes API in
order to create the Job and access to RabbitMQ in order to publish the
task parameters.

ansible/.gitignore

@@ -1 +1,2 @@
 ara/.secrets.toml
+host-provisioner.key

ansible/host-provisioner.key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICoOO/ZYMxRgmyvqZwGN3NM5pHyh3NBdC7iZrXIopt93 Host Provisioner

ansible/kustomization.yaml

@@ -13,6 +13,8 @@ namespace: ansible
 
 resources:
 - ../dch-root-ca
+- ../ssh-host-keys
+- rbac.yaml
 - secrets.yaml
 - namespace.yaml
 - ara.yaml

ansible/rbac.yaml

@@ -0,0 +1,25 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: dch-webhooks
+rules:
+  - apiGroups:
+    - batch
+    resources:
+    - jobs
+    verbs:
+    - create
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: dch-webhooks
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: dch-webhooks
+subjects:
+- kind: ServiceAccount
+  name: dch-webhooks
+  namespace: default

ansible/secrets.yaml

@@ -17,3 +17,21 @@ spec:
       labels:
         app.kubernetes.io/name: ara
         app.kubernetes.io/component: ara
+
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: provisioner-ssh-key
+  namespace: ansible
+  labels: &labels
+    app.kubernetes.io/name: provisioner-ssh-key
+    app.kubernetes.io/component: host-provisioner
+spec:
+  encryptedData:
+    host-provisioner.key: AgCiBQEtPmzQO9GHoVxZNQmjFn9GjeKWlHmG2lz4uqeva77QByqnejUg8CbpQnoT61ct9o39tVtrrHgC8WjseXKGKkd+YELRaWENXBNBFlv4YN6id2iPzf4xHrgfowXLjzly7s4s4Fg3QntXFglQxwG009Z7K5HQmAvgCFzGm6y+fyLoNd2v2eNc9pGurynnm7wKQFuBBpBtoDVfYMNSS+tp9D3MFVMwEG8kU/kszv5OEEwVkipG4v2LeY86HXIQnHyeE3vITz0TFPoQjpjusBNg7MxBVHcz/ZruqF43DZ+uz2aRFL6D5udm86hsxZGSi3yVq8PSCUANtWoTicIpSc5yxpB+5FLsc3Q380vQPrRYvx8luAZwMGQPDv2NpGoSRUmutb/+4vpQbCBwaP9s8WHaYNDByUeoQKAX2o+RYp+/csfxmSy1/pK93RUjMnsWGYiyI7jpNdTqWkakDN+cM0Lrje9+VqcZge3FYp/4y78AI75pWEiEMy1VSXeqE2pgu5vZzyRdw9zORC2sAWhnTeu+Obbly1UdptpXPmGclmRbpwAPM1F7m7pDsCQ9SYAhoGg251Yu0sF3Nnc0uOElmP0KSn1bt/Jca9M0syg9DMntHo41dn40+Aihujej0ll4h3GXVZ0auUuzSLZEMe0dHQY0YCaq3JdeOa6AJuNyo63/0BmvmWP2T06INtVw4EqBsmvNl/YJIlZiRGhIOGaRyJllu21L6QBtToYjxI7fhTxZaCG6fyPbvCd1hu+IchoMr86rl7W0+k6d/lkamx9dg+PtWjjThOGyeLYwSRhMcsvQPsSdTl8BEmB9TRgRIk42txVRQkb+ei1+1y9nBYCOV9P540lDXQVbqgyb0j2cccWOa/AG7l3P3s8haDs2OujUaVkBJWJ489nsLsC33972wwAQbOk+uTKmXKL6nWSNL31rivW9R8ea+vfdcyN2/tLsI0mE9XTR/kRmS12qdLmDbzeX/scqSKaWFQZwnLHak5Xaqkd25ZGHqB3zrnTBIryDaXSzgC1CPCv7QNqbKvd6vmH+16j1DWqLycbUeorx5O9nwpEZ+GzQm8q62PJEO8xdvqr3nu5OvFgGrnX/Y6giV8Cv8ogyAxbmNrq0cC5VYxDlt4VvzlOpmWUlfxD/wd+gaHQoibhhHRbGtsJCX9AFclVQrY3kmduTi/iJlTTJjMQKm9JJRXKbNDxH+B25Zj/hUIC1/NNZOSDn654Pi/yOGXITp86j9unfnIXdJPg=
+  template:
+    metadata:
+      name: provisioner-ssh-key
+      namespace: ansible
+      labels: *labels

dch-webhooks/ansible-job.yaml

@@ -0,0 +1,115 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  generateName: host-provision-
+  labels: &labels
+    app.kubernetes.io/name: host-provisioner
+    app.kubernetes.io/component: host-provisioner
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels: *labels
+    spec:
+      restartPolicy: Never
+      initContainers:
+      - name: ssh-agent
+        image: &image git.pyrocufflink.net/infra/host-provisioner
+        imagePullPolicy: Always
+        command:
+        - tini
+        - ssh-agent
+        - --
+        - -D
+        - -a
+        - /run/ssh/agent.sock
+        restartPolicy: Always
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /run/ssh
+          name: tmp
+          subPath: run/ssh
+      - name: ssh-add
+        image: *image
+        command:
+        - ssh-add
+        - -t
+        - 30m
+        - /run/secrets/ssh/host-provisioner.key
+        env:
+        - name: SSH_AUTH_SOCK
+          value: /run/ssh/agent.sock
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /run/ssh
+          name: tmp
+          subPath: run/ssh
+        - mountPath: /run/secrets/ssh
+          name: provisioner-key
+          readOnly: true
+      containers:
+      - name: host-provisioner
+        image: *image
+        env:
+        - name: SSH_AUTH_SOCK
+          value: /run/ssh/agent.sock
+        - name: AMQP_HOST
+          value: rabbitmq.pyrocufflink.blue
+        - name: AMQP_PORT
+          value: '5671'
+        - name: AMQP_CA_CERT
+          value: /run/dch-ca/dch-root-ca.crt
+        - name: AMQP_CLIENT_CERT
+          value: /run/secrets/host-provisioner/rabbitmq/tls.crt
+        - name: AMQP_CLIENT_KEY
+          value: /run/secrets/host-provisioner/rabbitmq/tls.key
+        - name: AMQP_EXTERNAL_CREDENTIALS
+          value: '1'
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /etc/ssh/ssh_known_hosts
+          name: ssh-known-hosts
+          subPath: ssh_known_hosts
+          readOnly: true
+        - mountPath: /home/jenkins
+          name: workspace
+        - mountPath: /run/dch-ca
+          name: dch-root-ca
+          readOnly: true
+        - mountPath: /run/ssh
+          name: tmp
+          subPath: run/ssh
+        - mountPath: /run/secrets/host-provisioner/rabbitmq
+          name: rabbitmq-cert
+          readOnly: true
+        - mountPath: /tmp
+          name: tmp
+          subPath: tmp
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      volumes:
+      - name: dch-root-ca
+        configMap:
+          name: dch-root-ca
+      - name: provisioner-key
+        secret:
+          secretName: provisioner-ssh-key
+          defaultMode: 0440
+      - name: ssh-known-hosts
+        configMap:
+          name: ssh-known-hosts
+      - name: rabbitmq-cert
+        secret:
+          secretName: rabbitmq-cert
+          defaultMode: 0440
+      - name: tmp
+        emptyDir:
+          medium: Memory
+      - name: workspace
+        emptyDir: {}

dch-webhooks/certificate.yaml

@@ -0,0 +1,14 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: rabbitmq
+spec:
+  secretName: rabbitmq-cert
+  commonName: dch-webhooks
+  issuerRef:
+    group: cert-manager.io
+    kind: ClusterIssuer
+    name: rabbitmq-ca
+  privateKey:
+    algorithm: ECDSA
+    rotationPolicy: Always

dch-webhooks/dch-webhooks.env

@@ -7,3 +7,10 @@ STEP_CA_URL=https://ca.pyrocufflink.blue:32599
 STEP_ROOT=/run/dch-root-ca.crt
 STEP_PROVISIONER=host-bootstrap
 STEP_PROVISIONER_PASSWORD_FILE=/run/secrets/du5t1n.me/step-ca/provisioner.password
+
+AMQP_HOST=rabbitmq.pyrocufflink.blue
+AMQP_PORT=5671
+AMQP_EXTERNAL_CREDENTIALS=1
+AMQP_CA_CERT=/run/dch-root-ca.crt
+AMQP_CLIENT_CERT=/run/secrets/du5t1n.me/rabbitmq/tls.crt
+AMQP_CLIENT_KEY=/run/secrets/du5t1n.me/rabbitmq/tls.key

dch-webhooks/dch-webhooks.yaml

@@ -1,4 +1,14 @@
 apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dch-webhooks
+  labels:
+    app.kubernetes.io/name: dch-webhooks
+    app.kubernetes.io/component: dch-webhooks
+    app.kubernetes.io/part-of: dch-webhooks
+
+---
+apiVersion: v1
 kind: Service
 metadata:
   labels:
@@ -42,12 +52,14 @@ spec:
     spec:
       containers:
       - name: dch-webhooks
-        image: git.pyrocufflink.net/containerimages/dch-webhooks
+        image: git.pyrocufflink.net/infra/dch-webhooks
         env:
         - name: UVICORN_HOST
           value: 0.0.0.0
         - name: UVICORN_LOG_LEVEL
           value: debug
+        - name: ANSIBLE_JOB_YAML
+          value: /etc/dch-webhooks/ansible-job.yaml
         envFrom:
         - configMapRef:
             name: dch-webhooks
@@ -76,22 +88,37 @@ spec:
           name: firefly-token
         - mountPath: /run/secrets/du5t1n.me/paperless
           name: paperless-token
+        - mountPath: /run/secrets/du5t1n.me/rabbitmq
+          name: rabbitmq-cert
+          readOnly: true
         - mountPath: /run/secrets/du5t1n.me/step-ca
           name: step-ca-password
         - mountPath: /tmp
           name: tmp
           subPath: tmp
+        - mountPath: /etc/dch-webhooks
+          name: host-provisioner
+          readOnly: true
       securityContext:
         runAsNonRoot: true
+      serviceAccountName: dch-webhooks
       volumes:
       - name: firefly-token
         secret:
           secretName: firefly-token
           optional: true
+      - name: host-provisioner
+        configMap:
+          name: host-provisioner
+          optional: true
       - name: paperless-token
         secret:
           secretName: paperless-token
           optional: true
+      - name: rabbitmq-cert
+        secret:
+          secretName: rabbitmq-cert
+          optional: true
       - name: root-ca
         configMap:
           name: dch-root-ca

dch-webhooks/kustomization.yaml

@@ -1,15 +1,29 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+labels:
+- pairs:
+    app.kubernetes.io/instance: dch-webhooks
+  includeSelectors: true
+  includeTemplates: true
+- pairs:
+    app.kubernetes.io/part-of: dch-webhooks
+
 resources:
 - ../dch-root-ca
 - dch-webhooks.yaml
+- certificate.yaml
 - ingress.yaml
 
 configMapGenerator:
 - name: dch-webhooks
   envs:
   - dch-webhooks.env
+- name: host-provisioner
+  files:
+  - ansible-job.yaml
+  options:
+    disableNameSuffixHash: true
 
 secretGenerator:
 - name: firefly-token