ci lib: Add applyConfigPolicy pipeline function

The Jenkins pipeline definition files are highly redundant.  Each one
implements almost the same stages, with only a few variations.  Whenever
a new pipeline is added, it's copied from the most recent file and
modified.  If any improvements are made to it, they do not usually get
implemented in any of the existing pipelines.

To address this, the `applyConfigPolicy` pipeline library function is
now available.  This function generates the full pipeline for a
particular application, including stages for setup, each individual
playbook, and cleanup.  Using this function, pipeline files can be as
simple as:

    @Library('cfgpol')_

    applyConfigPolicy(
        'gitea',
        [
            'Gitea': [
                'gitea.yml',
            ],
        ]
    )

This will create a pipeline that mounts the root filesystem read-write
on all hosts in the "gitea" group (any Ansible host pattern is allowed),
applies the `gitea.yml` playbook (in a stage named "Gitea"), and then
remounts the filesystems read-only.

Since this "library" is so simple, containing only a single function in
a single file, and since it will not be used by any pipelines outside
this repository, it makes sense to keep it in this repository, instead
of a separate repository as is customary for Jenkins pipeline shared
libraries.

vars/applyConfigPolicy.groovy

@@ -0,0 +1,104 @@
+import groovy.transform.Field
+
+@Field
+def DOCKER_ARGS = '''\
+-v /etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro
+'''
+
+def call(rw_limit, stages) {
+    properties([
+        pipelineTriggers([cron('H H * * *')])
+    ])
+
+    timeout(time: 1, unit: 'HOURS') {
+        lock('cfgpol') {
+            node {
+                checkout scm
+                docker.build("configpolicy", 'ci').inside(DOCKER_ARGS) {
+                    withEnv(["KRB5CCNAME=${WORKSPACE}/.krb5cc"]) {
+                        try {
+                            stageKinit()
+                            stageRemountRW(rw_limit)
+                            generateStages(stages)
+                            stageRemountRO(rw_limit)
+                        } catch (err) {
+                            postFailure(err)
+                        } finally {
+                            postCleanup()
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+def stageKinit() {
+    stage('kinit') {
+        withCredentials([file(
+            credentialsId: 'keytab-jenkins@pyrocufflink.blue',
+            variable: 'KEYTAB'
+        )]) {
+            sh 'kinit -kt "${KEYTAB}" jenkins@PYROCUFFLINK.BLUE'
+        }
+        withCredentials([file(
+            credentialsId: 'vault-jenkins@pyrocufflink.blue',
+            variable: 'SUDO_PASS_FILE'
+        )]) {
+            sh 'cp "${SUDO_PASS_FILE}" group_vars/pyrocufflink/sudo-pass'
+        }
+    }
+}
+
+
+def stageRemountRW(limit) {
+    stage('Remount R/W') {
+        ansiblePlaybook \
+            playbook: 'remount.yml',
+            limit: limit,
+            become: true,
+            vaultCredentialsId: 'ansible-vault',
+            extraVars: [
+                remount_state: 'rw',
+            ]
+    }
+}
+
+def generateStages(stages) {
+    stages.each { name, playbooks ->
+        stage(name) {
+            playbooks.each { playbook -> 
+                ansiblePlaybook \
+                    playbook: playbook,
+                    become: true,
+                    vaultCredentialsId: 'ansible-vault',
+                    extras: '--diff'
+            }
+        }
+    }
+}
+
+def stageRemountRO(limit) {
+    stage('Remount R/O') {
+        ansiblePlaybook \
+            playbook: 'remount.yml',
+            limit: limit,
+            become: true,
+            vaultCredentialsId: 'ansible-vault',
+            extras: '--diff'
+    }
+}
+
+def postCleanup() {
+    sh 'kdestroy'
+    sh 'find . -name sudo-pass -delete'
+}
+
+def postFailure(err) {
+    currentBuild.result = 'FAILURE'
+    emailext \
+        to: 'gyrfalcon@ebonfire.com',
+        subject: '$DEFAULT_SUBJECT',
+        body: '$DEFAULT_CONTENT'
+    error "${err}"
+}