tests: Begin integration tests

Cargo uses the sources in the `tests` directory to build and run
integration tests.  For each `tests/foo.rs` or `tests/foo/main.rs`, it
creates an executable that runs the test functions therein.  These
executables are separate crates from the main package, and thus do not
have access to its private members.  Integration tests are expected to
test only the public functionality of the package.

Application crates do not have any public members; their public
interface is the command line.  Integration tests would typically run
the command (e.g. using `std::process::Command`) and test its output.

Since *dynk8s-provisioner* is not really a command-line tool, testing it
this way would be difficult; each test would need to start the server,
make requests to it, and then stop it.  This would be slow and
cumbersome.

In order to avoid this tedium and be able to use Rocket's built-in test
client, I have converted *dynk8s-provisioner* into a library crate that
also includes an executable.  The library makes the `rocket` function
public, which allows the integration tests to import it and pass it to
the Rocket test client.

The point of integration tests, of course, is to validate the
functionality of the application as a whole.  This necessarily requires
allowing it to communicate with the Kubernetes API.  In the Jenkins CI
environment, the application will need the appropriate credentials, and
will need to use a separate Kubernetes namespace from the production
deployment.  The `setup.yaml` manifest in the `tests` directory defines
the resources necessary to run integration tests, and the
`genkubeconfig.sh` script can be used to create the appropriate
kubeconfig file containing the credentials.  The kubeconfig is exposed
to the tests via the `KUBECONFIG` environment variable, which is
populated from a Jenkins secret file credential.

Note: The `data` directory moved from `test` to `tests` to avoid
duplication and confusing names.

ci/Jenkinsfile

@@ -20,7 +20,12 @@ pipeline {
         stage('Test') {
             steps {
                 container('build') {
-                    sh '. ci/test.sh'
+                    withCredentials([file(
+                        credentialsId: 'dynk8s-test-kubeconfig',
+                        variable: 'KUBECONFIG'
+                    )]) {
+                        sh '. ci/test.sh'
+                    }
                 }
             }
         }

src/lib.rs

@@ -0,0 +1,25 @@
+mod error;
+mod events;
+mod k8s;
+mod model;
+mod routes;
+mod sns;
+
+#[doc(hidden)]
+pub fn rocket() -> rocket::Rocket<rocket::Build> {
+    rocket::build().mount(
+        "/",
+        rocket::routes![
+            routes::health::get_health,
+            routes::kubeadm::get_node_kubeconfig,
+            routes::kubeadm::post_node_kubeconfig,
+            routes::kubeadm::patch_node_kubeconfig,
+            routes::kubeadm::put_node_kubeconfig,
+            routes::wireguard::get_node_wireguard,
+            routes::sns::post_sns_notify,
+            routes::sns::get_sns_notify,
+            routes::sns::put_sns_notify,
+            routes::sns::patch_sns_notify,
+        ],
+    )
+}

src/main.rs

@@ -1,26 +1,6 @@
-mod error;
-mod events;
-mod k8s;
-mod model;
-mod routes;
-mod sns;
+use dynk8s_provisioner::rocket;
 
-#[doc(hidden)]
-#[rocket::launch]
-fn rocket() -> _ {
-    rocket::build().mount(
-        "/",
-        rocket::routes![
-            routes::health::get_health,
-            routes::kubeadm::get_node_kubeconfig,
-            routes::kubeadm::post_node_kubeconfig,
-            routes::kubeadm::patch_node_kubeconfig,
-            routes::kubeadm::put_node_kubeconfig,
-            routes::wireguard::get_node_wireguard,
-            routes::sns::post_sns_notify,
-            routes::sns::get_sns_notify,
-            routes::sns::put_sns_notify,
-            routes::sns::patch_sns_notify,
-        ],
-    )
+#[rocket::main]
+async fn main() {
+    let _ = rocket().launch().await;
 }

src/routes/sns.rs

@@ -50,7 +50,7 @@ mod test {
     fn test_sub_conf_msg() {
         let client = Client::tracked(rocket()).unwrap();
         let data = std::fs::read_to_string(
-            "test/data/sns/subscriptionconfirmation.json",
+            "tests/data/sns/subscriptionconfirmation.json",
         )
         .unwrap();
         let res = client.post(uri!(post_sns_notify)).body(&data).dispatch();
@@ -68,7 +68,7 @@ mod test {
     fn test_sub_conf_msg_bad() {
         let client = Client::tracked(rocket()).unwrap();
         let data = std::fs::read_to_string(
-            "test/data/sns/subscriptionconfirmation-bad.json",
+            "tests/data/sns/subscriptionconfirmation-bad.json",
         )
         .unwrap();
         let res = client.post(uri!(post_sns_notify)).body(&data).dispatch();
@@ -88,7 +88,7 @@ mod test {
     fn test_sub_conf_msg_bad_cert_url() {
         let client = Client::tracked(rocket()).unwrap();
         let data = std::fs::read_to_string(
-            "test/data/sns/subscriptionconfirmation-bad-url.json",
+            "tests/data/sns/subscriptionconfirmation-bad-url.json",
         )
         .unwrap();
         let res = client.post(uri!(post_sns_notify)).body(&data).dispatch();

tests/README.md

@@ -0,0 +1,21 @@
+# Integration Tests
+
+## Prerequisites
+
+Create the test resources in Kubernetes:
+
+```sh
+kubectl apply -f setup.yaml
+```
+
+Generate a kubeconfig for the test service account:
+
+```sh
+sh genkubeconfig.sh > kubeconfig
+```
+
+## Run the Tests
+
+```sh
+KUBECONFIG=${PWD}/kubeconfig cargo test
+```

tests/data/sns/notification-running.json

@@ -0,0 +1,11 @@
+{
+  "Type": "Notification",
+  "MessageId": "b87b7cca-d698-552b-bf91-2764a57ca6a8",
+  "TopicArn": "arn:aws:sns:us-east-2:566967686773:ec2-events",
+  "Message": "{\"version\":\"0\",\"id\":\"ae9cb41b-2d06-c688-8d20-82e90720cfa4\",\"detail-type\":\"EC2 Instance State-change Notification\",\"source\":\"aws.ec2\",\"account\":\"566967686773\",\"time\":\"2022-09-28T16:09:08Z\",\"region\":\"us-east-2\",\"resources\":[\"arn:aws:ec2:us-east-2:566967686773:instance/i-0e50d560c8bf9f0f8\"],\"detail\":{\"instance-id\":\"i-0e50d560c8bf9f0f8\",\"state\":\"running\"}}",
+  "Subject": null,
+  "Timestamp": "2022-09-28T16:09:08.795Z",
+  "SignatureVersion": "1",
+  "Signature": "LBaVKayZhbAzQa8k78oKNUgo62KqFU/GZJ8GCkNmMJJSeAyZNFOPY7Rcy7x3tmvXnds9ns/xWnzgrK2SdD/Q0Zniu9st2o1lDKfQnbUHU1Wv7g65jOuXTJlBu6teuziJ/bpsFTv4z9yw4fPm7gNYZ3xF5yjwXn0j0IHv92YgzEJtewV+MKRgMtp0vq9+TvRhpOiYZC8DCWZdqQoHm7B9VruPPPG9yHSZ2eF9H8cOQgjRA0IweanGpU+qej6Ts0IMeMTPPcPzhSynCdQJ8UfBSNPPfnlkkbq714XUjcms01UcMFuAQyMHrrr9CsTOtOJn4R/h9qccjZ7DhzXSkGVocw==",
+  "SigningCertURL": "https://sns.us-east-2.amazonaws.com/SimpleNotificationService-56e67fcb41f6fec09b0196692625d385.pem"
+}

tests/data/sns/notification-terminated.json

@@ -0,0 +1,11 @@
+{
+  "Type": "Notification",
+  "MessageId": "7d38c409-0edc-5cd0-87cd-9fa77aff18bc",
+  "TopicArn": "arn:aws:sns:us-east-2:566967686773:ec2-events",
+  "Message": "{\"version\":\"0\",\"id\":\"49c93129-5cb6-c94f-54c9-3b0c4fc07b03\",\"detail-type\":\"EC2 Instance State-change Notification\",\"source\":\"aws.ec2\",\"account\":\"566967686773\",\"time\":\"2022-09-28T16:10:11Z\",\"region\":\"us-east-2\",\"resources\":[\"arn:aws:ec2:us-east-2:566967686773:instance/i-0e50d560c8bf9f0f8\"],\"detail\":{\"instance-id\":\"i-0e50d560c8bf9f0f8\",\"state\":\"terminated\"}}",
+  "Subject": null,
+  "Timestamp": "2022-09-28T16:10:11.889Z",
+  "SignatureVersion": "1",
+  "Signature": "YVeRPXDW4zpPhBow/Le73zN1RJ754a/o9IHBAG8wSDNC2lLushh0ztknfp4G1qNA5ZenrKgTKyta0avLECh2qiU9lhp+M5qeY9CRIW2SH9xAL25MIN7psxkhJgSaqblIBENUE8gjge0pzDuKepgY5zD9f68Uf/0voMWTVWjcI2IdtOoS3LMqx6XC9K8LYMPgPf+0wGdH7AUXumQJTNBwX5jjMql2uIccN/xQxmUmIRHubWvDX82m78PZL+2mZlbGvsRRpkSPBPT/xu1fBwWkLQ0ziOf8sX7FyyKMKn8x4ohLkQjwU/21nSY8ei4n749ggJRNB0CxfQFPnNW5/iA4Qw==",
+  "SigningCertURL": "https://sns.us-east-2.amazonaws.com/SimpleNotificationService-56e67fcb41f6fec09b0196692625d385.pem"
+}

tests/genkubeconfig.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+server=$(
+    kubectl config view --minify --raw \
+        -o jsonpath='{.clusters[].cluster.server}'
+)
+token=$(
+    kubectl get secret \
+        -n dynk8s-test dynk8s-provisioner \
+        -o jsonpath='{.data.token}' \
+    | base64 -d
+)
+ca=$(
+    kubectl get secret \
+        -n dynk8s-test dynk8s-provisioner \
+        -o jsonpath='{.data.ca\.crt}'
+)
+
+cat <<EOF
+apiVersion: v1
+kind: Config
+clusters:
+- name: kubernetes
+  cluster:
+    certificate-authority-data: ${ca}
+    server: ${server}
+contexts:
+- name: dynk8s-test@kubernetes
+  context:
+    cluster: kubernetes
+    namespace: dynk8s-test
+    user: dynk8s-test
+current-context: dynk8s-test@kubernetes
+users:
+- name: dynk8s-test
+  user:
+    token: ${token}
+EOF

tests/integration/main.rs

@@ -0,0 +1,46 @@
+use std::collections::btree_map::BTreeMap;
+
+use k8s_openapi::api::core::v1::Secret;
+use kube::core::params::ListParams;
+
+mod sns;
+
+const WIREGUARD_CONFIG: &str = "\
+[Interface]
+Address = 10.11.12.13/14
+PrivateKey = UEdAkIaF80zhlOpgacOYL2UckrfCAWXfsDDSAAzNH3g=
+
+[Peer]
+PublicKey = zbeTpUFA014kvTezIEGBt4yi3BVocST9j1dBElp9liI=
+PreSharedKey = V6hAm01dxv2ib8AML2dSyX68hlPZm8En+IXfsknK3Zc=
+AllowedIPs = 0.0.0.0/0
+Endpoint = wireguard.example.org:24680
+";
+
+async fn setup() {
+    let client = kube::Client::try_default().await.unwrap();
+    let secrets: kube::Api<Secret> = kube::Api::default_namespaced(client);
+    let lp =
+        ListParams::default().fields("type=dynk8s.du5t1n.me/wireguard-config");
+    secrets
+        .delete_collection(&Default::default(), &lp)
+        .await
+        .unwrap();
+
+    let mut labels = BTreeMap::<String, String>::new();
+    labels.insert(
+        "app.kubernetes.io/part-of".into(),
+        "dynk8s-provisioner".into(),
+    );
+    labels.insert("dynk8s.du5t1n.me/ec2-instance-id".into(), "".into());
+    let mut data = BTreeMap::<String, String>::new();
+    data.insert("wireguard-config".into(), WIREGUARD_CONFIG.into());
+    let mut secret = Secret::default();
+    secret.type_ = Some("dynk8s.du5t1n.me/wireguard-config".into());
+    secret.immutable = Some(true);
+    secret.metadata.generate_name = Some("wireguard-config-".into());
+    secret.metadata.labels = Some(labels);
+    secret.string_data = Some(data);
+    secrets.create(&Default::default(), &secret).await.unwrap();
+}
+

tests/integration/sns.rs

@@ -0,0 +1,57 @@
+use rocket::http::Status;
+use rocket::local::asynchronous::Client;
+
+use dynk8s_provisioner::rocket;
+
+use crate::{setup, WIREGUARD_CONFIG};
+
+#[rocket::async_test]
+async fn test_sns_ec2_lifecycle() {
+    setup().await;
+    let client = Client::tracked(rocket()).await.unwrap();
+
+    // Simulate an instance state-change event indicating an instance has
+    // started. This should generate a bootstrap token and assign the WireGuard
+    // config to the instance.
+    let data =
+        std::fs::read_to_string("tests/data/sns/notification-running.json")
+            .unwrap();
+    let res = client.post("/sns/notify").body(&data).dispatch().await;
+    assert_eq!(res.status(), Status::NoContent);
+
+    // Ensure the bootstrap token was generated
+    let res = client
+        .get("/kubeadm/kubeconfig/i-0e50d560c8bf9f0f8")
+        .dispatch()
+        .await;
+    assert_eq!(res.status(), Status::Ok);
+    // Ensure the WireGuard config was assigned
+    let res = client
+        .get("/wireguard/config/i-0e50d560c8bf9f0f8")
+        .dispatch()
+        .await;
+    assert_eq!(res.status(), Status::Ok);
+    assert_eq!(res.into_string().await, Some(WIREGUARD_CONFIG.into()));
+
+    // Simulate an instance state-change event indicating the instance has
+    // terminated.  This should delete the bootstrap token and unassing the
+    // WireGuard config.
+    let data =
+        std::fs::read_to_string("tests/data/sns/notification-terminated.json")
+            .unwrap();
+    let res = client.post("/sns/notify").body(&data).dispatch().await;
+    assert_eq!(res.status(), Status::NoContent);
+
+    // Ensure the bootstrap token was deleted
+    let res = client
+        .get("/kubeadm/kubeconfig/i-0e50d560c8bf9f0f8")
+        .dispatch()
+        .await;
+    assert_eq!(res.status(), Status::NotFound);
+    // Ensure the WireGuard config was deleted
+    let res = client
+        .get("/wireguard/config/i-0e50d560c8bf9f0f8")
+        .dispatch()
+        .await;
+    assert_eq!(res.status(), Status::NotFound);
+}

tests/setup.yaml

@@ -0,0 +1,76 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dynk8s-test
+  labels:
+    kubernetes.io/metadata.name: dynk8s
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s-test
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+automountServiceAccountToken: true
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s-test
+  annotations:
+    kubernetes.io/service-account.name: dynk8s-provisioner
+type: kubernetes.io/service-account-token
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s-test
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - secrets
+  verbs:
+  - '*'
+- apiGroups:
+  - ''
+  resources:
+  - configmaps
+  resourceNames:
+  - cluster-info
+  verbs:
+  - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s-test
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: dynk8s-provisioner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: dynk8s-provisioner
+subjects:
+- kind: ServiceAccount
+  name: dynk8s-provisioner
+  namespace: dynk8s-test