tests: Begin integration tests

Cargo uses the sources in the `tests` directory to build and run
integration tests.  For each `tests/foo.rs` or `tests/foo/main.rs`, it
creates an executable that runs the test functions therein.  These
executables are separate crates from the main package, and thus do not
have access to its private members.  Integration tests are expected to
test only the public functionality of the package.

Application crates do not have any public members; their public
interface is the command line.  Integration tests would typically run
the command (e.g. using `std::process::Command`) and test its output.

Since *dynk8s-provisioner* is not really a command-line tool, testing it
this way would be difficult; each test would need to start the server,
make requests to it, and then stop it.  This would be slow and
cumbersome.

In order to avoid this tedium and be able to use Rocket's built-in test
client, I have converted *dynk8s-provisioner* into a library crate that
also includes an executable.  The library makes the `rocket` function
public, which allows the integration tests to import it and pass it to
the Rocket test client.

The point of integration tests, of course, is to validate the
functionality of the application as a whole.  This necessarily requires
allowing it to communicate with the Kubernetes API.  In the Jenkins CI
environment, the application will need the appropriate credentials, and
will need to use a separate Kubernetes namespace from the production
deployment.  The `setup.yaml` manifest in the `tests` directory defines
the resources necessary to run integration tests, and the
`genkubeconfig.sh` script can be used to create the appropriate
kubeconfig file containing the credentials.  The kubeconfig is exposed
to the tests via the `KUBECONFIG` environment variable, which is
populated from a Jenkins secret file credential.

Note: The `data` directory moved from `test` to `tests` to avoid
duplication and confusing names.

Cargo.lock

@@ -450,6 +450,7 @@ dependencies = [
  "rsa",
  "serde",
  "serde_json",
+ "serde_yaml 0.9.13",
  "sha1",
  "x509-parser",
 ]
@@ -957,7 +958,7 @@ dependencies = [
  "secrecy",
  "serde",
  "serde_json",
- "serde_yaml",
+ "serde_yaml 0.8.26",
  "thiserror",
  "tokio",
  "tokio-util",
@@ -1863,6 +1864,19 @@ dependencies = [
  "yaml-rust",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.9.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8613d593412a0deb7bbd8de9d908efff5a0cb9ccd8f62c641e7b2ed2f57291d1"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.2"
@@ -2361,6 +2375,12 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1e5fa573d8ac5f1a856f8d7be41d390ee973daf97c806b2c1a465e4e1406e68"
+
 [[package]]
 name = "url"
 version = "2.2.2"

Cargo.toml

@@ -15,6 +15,7 @@ rocket = { version = "0.5.0-rc.2", features = ["json"] }
 rsa = "0.6.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.85"
+serde_yaml = "0.9.13"
 sha1 = "0.10.2"
 x509-parser = "0.14.0"
 

ci/Jenkinsfile

@@ -20,7 +20,12 @@ pipeline {
         stage('Test') {
             steps {
                 container('build') {
-                    sh '. ci/test.sh'
+                    withCredentials([file(
+                        credentialsId: 'dynk8s-test-kubeconfig',
+                        variable: 'KUBECONFIG'
+                    )]) {
+                        sh '. ci/test.sh'
+                    }
                 }
             }
         }

src/events.rs

@@ -6,7 +6,8 @@
 use log::{debug, error};
 
 use crate::k8s::{
-    assign_wireguard_key, create_bootstrap_token, unassign_wireguard_key,
+    assign_wireguard_config, create_bootstrap_token, delete_bootstrap_tokens,
+    unassign_wireguard_config,
 };
 use crate::model::events::*;
 
@@ -16,18 +17,19 @@ use crate::model::events::*;
 /// associated with ephemeral nodes running as EC2 instances.
 ///
 /// When an instance starts:
-/// 1. A WireGuard key is assigned to the instance
+/// 1. A WireGuard config is assigned to the instance
 /// 2. A Kubernetes bootstrap token is generated, to be used by `kubeadm` to
 ///    add the node to the cluster.
 ///
 /// When an instance is terminated:
-/// 1. Any WireGuard keys assigned to the instance are unassigned
+/// 1. Any WireGuard configs assigned to the instance are unassigned
+/// 2. All bootstrap tokens for the instance are deleted
 pub async fn on_ec2_instance_state_change(evt: Ec2InstanceStateChange) {
     debug!("EC2 instance {} is now {}", &evt.instance_id, &evt.state);
     if evt.state == "running" {
-        if let Err(e) = assign_wireguard_key(&evt.instance_id).await {
+        if let Err(e) = assign_wireguard_config(&evt.instance_id).await {
             error!(
-                "Failed to assign WireGuard key to instnce {}: {}",
+                "Failed to assign WireGuard config to instnce {}: {}",
                 &evt.instance_id, e
             );
             return;
@@ -39,9 +41,15 @@ pub async fn on_ec2_instance_state_change(evt: Ec2InstanceStateChange) {
             )
         };
     } else if evt.state == "terminated" {
-        if let Err(e) = unassign_wireguard_key(&evt.instance_id).await {
+        if let Err(e) = unassign_wireguard_config(&evt.instance_id).await {
             error!(
-                "Failed to unassign WireGuard key from instance: {}: {}",
+                "Failed to unassign WireGuard config from instance: {}: {}",
+                &evt.instance_id, e
+            );
+        }
+        if let Err(e) = delete_bootstrap_tokens(&evt.instance_id).await {
+            error!(
+                "Failed to delete bootstrap tokens for instance {}: {}",
                 &evt.instance_id, e
             );
         }

src/k8s.rs

@@ -3,13 +3,15 @@ use std::collections::btree_map::BTreeMap;
 
 use chrono::offset::Utc;
 use chrono::{DateTime, Duration};
-use k8s_openapi::api::core::v1::Secret;
+use k8s_openapi::api::core::v1::{ConfigMap, Secret};
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta;
 use kube::core::params::{ListParams, Patch, PatchParams, PostParams};
 use kube::{Api, Client};
-use log::{debug, error, info};
+use log::{debug, error, info, warn};
 use rand::seq::SliceRandom;
 
+use crate::model::k8s::*;
+
 /// The set of characters allowed to appear in bootstrap tokens
 const TOKEN_CHARS: [char; 36] = [
     'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
@@ -157,40 +159,48 @@ impl Into<Secret> for BootstrapToken {
     }
 }
 
-/// Assign an existing WireGuard key to the specified EC2 instance
+/// Assign an existing WireGuard configuration to the specified EC2 instance
 ///
-/// This function finds the first unused WireGuard key, stored as a Kubernetes
-/// Secret resource, and assigns it to the specified EC2 instance.  Keys are
-/// assigned by setting the `dynk8s.du5t1n.me/ec2-instance-id` label in the
-/// Secret resource's metadata.
+/// This function finds the first unused WireGuard client configuration, stored
+/// as a Kubernetes Secret resource, and assigns it to the specified EC2
+/// instance.  Configs are assigned by setting the
+/// `dynk8s.du5t1n.me/ec2-instance-id` label in the Secret resource's metadata.
 ///
-/// Secret resources for WireGuard keys have a *type* of
-/// `dynk8s.du5t1n.me/wireguard-key`.  They must be created ahead of time and
-/// must refer to working keys already configured on the WireGuard server.
-pub async fn assign_wireguard_key(
+/// Secret resources for WireGuard configuration have a *type* of
+/// `dynk8s.du5t1n.me/wireguard-config`.  The Secret's `data` field must
+/// contain a `wireguard-config` property, which contains the WireGuard client
+/// configuration the node should use.  Configs must be created ahead of time
+/// and must refer to working keys already configured on the WireGuard server.
+pub async fn assign_wireguard_config(
     instance_id: &str,
 ) -> Result<(), kube::Error> {
     let client = Client::try_default().await?;
     let secrets: Api<Secret> = Api::default_namespaced(client);
     debug!(
-        "Checking for WireGuard keys already assigned to instance {}",
+        "Checking for WireGuard configs already assigned to instance {}",
         instance_id
     );
     let lp = ListParams::default()
-        .fields("type=dynk8s.du5t1n.me/wireguard-key")
+        .fields("type=dynk8s.du5t1n.me/wireguard-config")
         .labels(&format!("dynk8s.du5t1n.me/ec2-instance-id={}", instance_id));
     let res = secrets.list(&lp).await?;
     if !res.items.is_empty() {
-        info!("WireGuard key already assigned to instance {}", instance_id);
+        info!(
+            "WireGuard config already assigned to instance {}",
+            instance_id
+        );
         return Ok(());
     }
-    debug!("Looking for available WireGuard keys");
+    debug!("Looking for available WireGuard configs");
     let lp = ListParams::default()
-        .fields("type=dynk8s.du5t1n.me/wireguard-key")
+        .fields("type=dynk8s.du5t1n.me/wireguard-config")
         .labels("dynk8s.du5t1n.me/ec2-instance-id=");
     let res = secrets.list(&lp).await?;
     if res.items.is_empty() {
-        error!("No WireGuard keys available for instance {}", &instance_id);
+        error!(
+            "No WireGuard config available for instance {}",
+            &instance_id
+        );
     } else {
         if let Some(name) = &res.items[0].metadata.name {
             let mut labels = BTreeMap::<String, String>::new();
@@ -204,7 +214,7 @@ pub async fn assign_wireguard_key(
             let pp = PatchParams::apply(env!("CARGO_PKG_NAME")).force();
             secrets.patch(&name, &pp, &patch).await?;
             info!(
-                "Assigned WireGuard key {} to instance {}",
+                "Assigned WireGuard config {} to instance {}",
                 name, &instance_id
             );
         }
@@ -212,21 +222,24 @@ pub async fn assign_wireguard_key(
     Ok(())
 }
 
-/// Unassign all WireGuard keys from the specified EC2 instance
+/// Unassign all WireGuard configs from the specified EC2 instance
 ///
-/// This function finds all WireGuard keys, stored as Kubernetes Secret
+/// This function finds all WireGuard configs, stored as Kubernetes Secret
 /// resources, associated with the specified EC2 instance and unassigns them.
-/// Unassigned keys have the `dynk8s.du5t1n.me/ec2-instance-id` label set to
+/// Unassigned configs have the `dynk8s.du5t1n.me/ec2-instance-id` label set to
 /// the empty string.
-pub async fn unassign_wireguard_key(
+pub async fn unassign_wireguard_config(
     instance_id: &str,
 ) -> Result<(), kube::Error> {
     let client = Client::try_default().await?;
     let secrets: Api<Secret> = Api::default_namespaced(client);
     let lp = ListParams::default()
-        .fields("type=dynk8s.du5t1n.me/wireguard-key")
+        .fields("type=dynk8s.du5t1n.me/wireguard-config")
         .labels(&format!("dynk8s.du5t1n.me/ec2-instance-id={}", instance_id));
-    info!("Unassigning WireGuard keys from instance {}", instance_id);
+    info!(
+        "Unassigning WireGuard configs from instance {}",
+        instance_id
+    );
     for s in secrets.list(&lp).await? {
         if let Some(name) = &s.metadata.name {
             let mut labels = BTreeMap::<String, String>::new();
@@ -238,7 +251,7 @@ pub async fn unassign_wireguard_key(
             let pp = PatchParams::apply(env!("CARGO_PKG_NAME")).force();
             secrets.patch(&name, &pp, &patch).await?;
             info!(
-                "Unassigned WireGuard key {} from instance {}",
+                "Unassigned WireGuard config {} from instance {}",
                 name, &instance_id
             );
         }
@@ -246,6 +259,42 @@ pub async fn unassign_wireguard_key(
     Ok(())
 }
 
+/// Retrieve the WireGuard config assigned to the specified EC2 instance
+///
+/// This function finds the first WireGuard client configuration, stored as a
+/// Kubernetes Secret resource, associated with the specified EC2 instance.
+///
+/// If multiple WireGuard configs are assigned to an EC2 instance, only the
+/// first one returned by the Kubernetes list query is returned.
+pub async fn get_wireguard_config(
+    instance_id: &str,
+) -> Result<Option<String>, kube::Error> {
+    let client = Client::try_default().await?;
+    let secrets: Api<Secret> = Api::default_namespaced(client);
+    let lp = ListParams::default()
+        .fields("type=dynk8s.du5t1n.me/wireguard-config")
+        .labels(&format!("dynk8s.du5t1n.me/ec2-instance-id={}", instance_id));
+    for s in secrets.list(&lp).await? {
+        if let Some(data) = s.data {
+            match data.get("wireguard-config") {
+                Some(s) => match String::from_utf8(s.0.clone()) {
+                    Ok(s) => return Ok(Some(s)),
+                    Err(e) => {
+                        error!("Invalid WireGuard configuration: {}", e);
+                    }
+                },
+                None => {
+                    error!(concat!(
+                        "Invalid WireGuard configuration: ",
+                        "missing wireguard-config property"
+                    ));
+                }
+            };
+        }
+    }
+    Ok(None)
+}
+
 /// Generate and store a bootstrap token for the specified EC2 instance
 ///
 /// This function generates a new bootstrap token and stores it as a Kubernetes
@@ -265,9 +314,153 @@ pub async fn create_bootstrap_token<I: AsRef<str>>(
     Ok(())
 }
 
+/// Delete bootstrap tokens associated withthe specified EC2 instance
+pub async fn delete_bootstrap_tokens<I: AsRef<str>>(
+    instance_id: I,
+) -> Result<(), kube::Error> {
+    let instance_id = instance_id.as_ref();
+    info!("Deleting bootstrap tokens for instance {}", &instance_id);
+    let client = Client::try_default().await?;
+    let secrets: Api<Secret> = Api::namespaced(client, "kube-system");
+    let lp = ListParams::default()
+        .fields("type=bootstrap.kubernetes.io/token")
+        .labels(&format!("dynk8s.du5t1n.me/ec2-instance-id={}", &instance_id));
+    secrets.delete_collection(&Default::default(), &lp).await?;
+    Ok(())
+}
+
+/// Get the `kubeadm join` configuration for the specified EC2 instance
+///
+/// This function creates a kubeconfig file that can be passed to `kubeadm
+/// join` to add the specified EC2 instance to the Kubernetes cluster as a
+/// worker node.  The cluster configuration is read from the `cluster-info`
+/// ConfigMap in the *kube-public* namespace.  The bootstrap token assigned to
+/// the instance is included for client authentication.
+pub async fn get_kubeconfig<I: AsRef<str>>(
+    instance_id: I,
+) -> Result<Option<KubeConfig>, kube::Error> {
+    let instance_id = instance_id.as_ref();
+    let token = match get_bootstrap_token(&instance_id).await {
+        Ok(Some(t)) => t,
+        Ok(None) => {
+            warn!("No bootstrap token assigned to instance {}", &instance_id);
+            return Ok(None);
+        }
+        Err(e) => {
+            error!(
+                "Could not get bootstrap token for instance {}: {}",
+                &instance_id, e
+            );
+            return Ok(None);
+        }
+    };
+    match get_cluster_info().await? {
+        Some(config) => {
+            let cluster = Cluster {
+                name: "kubernetes".into(),
+                cluster: config.clusters[0].cluster.clone(),
+            };
+            let context = Context {
+                name: "kubeadm".into(),
+                context: ContextInfo {
+                    cluster: "kubernetes".into(),
+                    user: "kubeadm".into(),
+                },
+            };
+            let user = User {
+                name: "kubeadm".into(),
+                user: UserInfo { token: token },
+            };
+            let mut kubeconfig = KubeConfig::default();
+            kubeconfig.clusters = vec![cluster];
+            kubeconfig.contexts = Some(vec![context]);
+            kubeconfig.current_context = "kubeadm".into();
+            kubeconfig.users = Some(vec![user]);
+            Ok(Some(kubeconfig))
+        }
+        None => {
+            warn!("No kubeconfig loaded from cluster-info");
+            Ok(None)
+        }
+    }
+}
+
+/// Retrieve the bootstrap token assigned to an EC2 instance
+async fn get_bootstrap_token<I: AsRef<str>>(
+    instance_id: I,
+) -> Result<Option<String>, kube::Error> {
+    let instance_id = instance_id.as_ref();
+    let client = Client::try_default().await?;
+    let secrets: Api<Secret> = Api::namespaced(client, "kube-system");
+    let lp = ListParams::default()
+        .fields("type=bootstrap.kubernetes.io/token")
+        .labels(&format!(
+            "dynk8s.du5t1n.me/ec2-instance-id={}",
+            &instance_id
+        ));
+    for s in secrets.list(&lp).await? {
+        match token_string(&s) {
+            Ok(t) => return Ok(Some(t)),
+            Err(e) => {
+                error!("Invalid bootstrap token: {}", e);
+            }
+        }
+    }
+    Ok(None)
+}
+
+/// Get cluster information from the ConfigMap
+async fn get_cluster_info() -> Result<Option<KubeConfig>, kube::Error> {
+    let client = Client::try_default().await?;
+    let configmaps: Api<ConfigMap> = Api::namespaced(client, "kube-public");
+    let cluster_info = configmaps.get("cluster-info").await?;
+    if let Some(data) = cluster_info.data {
+        if let Some(config) = data.get("kubeconfig") {
+            match serde_yaml::from_str::<KubeConfig>(config) {
+                Ok(c) => return Ok(Some(c)),
+                Err(e) => {
+                    error!(
+                        "Could not load kubeconfig from cluster-info: {}",
+                        e
+                    );
+                }
+            };
+        } else {
+            error!("No kubeconfig property found in cluster-info ConfigMap");
+        }
+    } else {
+        error!("No data property found in cluster-info ConfigMap");
+    }
+    Ok(None)
+}
+
+/// Get the string representation of a bootstrap token from a Secret
+fn token_string(secret: &Secret) -> Result<String, String> {
+    let data = match &secret.data {
+        Some(d) => d,
+        None => return Err("Missing data property".into()),
+    };
+    let token_id = match data.get("token-id") {
+        Some(s) => match String::from_utf8(s.0.clone()) {
+            Ok(s) => s,
+            Err(e) => return Err(e.to_string()),
+        },
+        None => return Err("Missing token-id".into()),
+    };
+    let secret = match data.get("token-secret") {
+        Some(s) => match String::from_utf8(s.0.clone()) {
+            Ok(s) => s,
+            Err(e) => return Err(e.to_string()),
+        },
+        None => return Err("Missing token-secret".into()),
+    };
+    Ok(format!("{}.{}", token_id, secret))
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
+    use k8s_openapi::ByteString;
     use regex::Regex;
 
     #[test]
@@ -330,4 +523,17 @@ mod test {
             "i-0a1b2c3d4e5f6f7f8"
         );
     }
+
+    #[test]
+    fn test_token_string() {
+        let token = BootstrapToken::new();
+        let mut data = BTreeMap::new();
+        data.insert("token-id".into(), ByteString(token.token_id().into()));
+        data.insert("token-secret".into(), ByteString(token.secret().into()));
+        let secret = Secret {
+            data: Some(data),
+            ..Default::default()
+        };
+        assert_eq!(token.token(), token_string(&secret).unwrap());
+    }
 }

src/lib.rs

@@ -0,0 +1,25 @@
+mod error;
+mod events;
+mod k8s;
+mod model;
+mod routes;
+mod sns;
+
+#[doc(hidden)]
+pub fn rocket() -> rocket::Rocket<rocket::Build> {
+    rocket::build().mount(
+        "/",
+        rocket::routes![
+            routes::health::get_health,
+            routes::kubeadm::get_node_kubeconfig,
+            routes::kubeadm::post_node_kubeconfig,
+            routes::kubeadm::patch_node_kubeconfig,
+            routes::kubeadm::put_node_kubeconfig,
+            routes::wireguard::get_node_wireguard,
+            routes::sns::post_sns_notify,
+            routes::sns::get_sns_notify,
+            routes::sns::put_sns_notify,
+            routes::sns::patch_sns_notify,
+        ],
+    )
+}

src/main.rs

@@ -1,21 +1,6 @@
-mod error;
-mod events;
-mod k8s;
-mod model;
-mod routes;
-mod sns;
+use dynk8s_provisioner::rocket;
 
-#[doc(hidden)]
-#[rocket::launch]
-fn rocket() -> _ {
-    rocket::build().mount(
-        "/",
-        rocket::routes![
-            routes::health::get_health,
-            routes::sns::post_sns_notify,
-            routes::sns::get_sns_notify,
-            routes::sns::put_sns_notify,
-            routes::sns::patch_sns_notify,
-        ],
-    )
+#[rocket::main]
+async fn main() {
+    let _ = rocket().launch().await;
 }

src/model/k8s.rs

@@ -0,0 +1,91 @@
+//! kubeadm configuration data types
+//!
+//! The Kubernetes API reference does not include a specification for the
+//! `Config` resource, and as such there is no model for it in [`k8s_openapi`].
+//! Since *dynk8s* needs to read and write objects of this type, to provide
+//! configuration for `kubeadm` on dynamic nodes, a subset of the required
+//! model is defined here.
+use serde::{Deserialize, Serialize};
+
+/// Cluster information
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct ClusterInfo {
+    /// X.509 certificate of the Kubernetes certificate authority
+    pub certificate_authority_data: String,
+    /// URL of the Kubernetes API server
+    pub server: String,
+}
+
+/// Cluster definition
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Cluster {
+    /// Cluster information
+    pub cluster: ClusterInfo,
+    /// Cluster name
+    pub name: String,
+}
+
+/// kubeconfig context information
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ContextInfo {
+    /// The cluster to use
+    pub cluster: String,
+    /// The user to use
+    pub user: String,
+}
+
+/// kubeconfig context definition
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Context {
+    /// Context information
+    pub context: ContextInfo,
+    /// Context name
+    pub name: String,
+}
+
+/// User information
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct UserInfo {
+    /// Bootstrap token for authentication
+    pub token: String,
+}
+
+/// User definition
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct User {
+    /// User name
+    pub name: String,
+    /// User information
+    pub user: UserInfo,
+}
+
+/// kubeconfig
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct KubeConfig {
+    #[serde(rename = "apiVersion")]
+    pub api_version: String,
+    pub kind: String,
+    /// List of defined clusters
+    pub clusters: Vec<Cluster>,
+    /// List of defined contexts (user–cluster associations)
+    pub contexts: Option<Vec<Context>>,
+    #[serde(rename = "current-context")]
+    /// Current context
+    pub current_context: String,
+    /// List of defined users
+    pub users: Option<Vec<User>>,
+}
+
+impl Default for KubeConfig {
+    fn default() -> Self {
+        Self {
+            api_version: "v1".into(),
+            kind: "Config".into(),
+            clusters: vec![],
+            contexts: None,
+            current_context: "".into(),
+            users: None,
+        }
+    }
+}

src/model/mod.rs

@@ -1,3 +1,4 @@
 //! The dynk8s provisioner data model
 pub mod events;
+pub mod k8s;
 pub mod sns;

src/routes/kubeadm.rs

@@ -0,0 +1,72 @@
+use rocket::http::Status;
+
+use crate::k8s::get_kubeconfig;
+
+#[rocket::get("/kubeadm/kubeconfig/<instance_id>")]
+pub async fn get_node_kubeconfig(instance_id: String) -> Option<String> {
+    if let Ok(Some(kubeconfig)) = get_kubeconfig(&instance_id).await {
+        Some(serde_yaml::to_string(&kubeconfig).unwrap())
+    } else {
+        None
+    }
+}
+
+#[rocket::post("/kubeadm/kubeconfig/<_instance_id>")]
+pub async fn post_node_kubeconfig(_instance_id: String) -> Status {
+    Status::MethodNotAllowed
+}
+
+#[rocket::patch("/kubeadm/kubeconfig/<_instance_id>")]
+pub async fn patch_node_kubeconfig(_instance_id: String) -> Status {
+    Status::MethodNotAllowed
+}
+
+#[rocket::put("/kubeadm/kubeconfig/<_instance_id>")]
+pub async fn put_node_kubeconfig(_instance_id: String) -> Status {
+    Status::MethodNotAllowed
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::rocket;
+    use rocket::local::blocking::Client;
+    use rocket::uri;
+
+    #[test]
+    fn test_get_node_token_404() {
+        let client = Client::tracked(rocket()).unwrap();
+        let res = client
+            .get(uri!(get_node_kubeconfig(
+                instance_id = "i-0a1b2c3d4e5f6f7f8"
+            )))
+            .dispatch();
+        assert_eq!(res.status(), Status::NotFound);
+    }
+
+    #[test]
+    fn test_kubeconfig_msg_wrong_method() {
+        let client = Client::tracked(rocket()).unwrap();
+
+        let res = client
+            .post(uri!(get_node_kubeconfig(
+                instance_id = "i-0a1b2c3d4e5f6f7f8"
+            )))
+            .dispatch();
+        assert_eq!(res.status(), Status::MethodNotAllowed);
+
+        let res = client
+            .patch(uri!(get_node_kubeconfig(
+                instance_id = "i-0a1b2c3d4e5f6f7f8"
+            )))
+            .dispatch();
+        assert_eq!(res.status(), Status::MethodNotAllowed);
+
+        let res = client
+            .put(uri!(get_node_kubeconfig(
+                instance_id = "i-0a1b2c3d4e5f6f7f8"
+            )))
+            .dispatch();
+        assert_eq!(res.status(), Status::MethodNotAllowed);
+    }
+}

src/routes/mod.rs

@@ -1,3 +1,5 @@
 //! Rocket route handlers
 pub mod health;
+pub mod kubeadm;
 pub mod sns;
+pub mod wireguard;

src/routes/sns.rs

@@ -50,7 +50,7 @@ mod test {
     fn test_sub_conf_msg() {
         let client = Client::tracked(rocket()).unwrap();
         let data = std::fs::read_to_string(
-            "test/data/sns/subscriptionconfirmation.json",
+            "tests/data/sns/subscriptionconfirmation.json",
         )
         .unwrap();
         let res = client.post(uri!(post_sns_notify)).body(&data).dispatch();
@@ -68,7 +68,7 @@ mod test {
     fn test_sub_conf_msg_bad() {
         let client = Client::tracked(rocket()).unwrap();
         let data = std::fs::read_to_string(
-            "test/data/sns/subscriptionconfirmation-bad.json",
+            "tests/data/sns/subscriptionconfirmation-bad.json",
         )
         .unwrap();
         let res = client.post(uri!(post_sns_notify)).body(&data).dispatch();
@@ -88,7 +88,7 @@ mod test {
     fn test_sub_conf_msg_bad_cert_url() {
         let client = Client::tracked(rocket()).unwrap();
         let data = std::fs::read_to_string(
-            "test/data/sns/subscriptionconfirmation-bad-url.json",
+            "tests/data/sns/subscriptionconfirmation-bad-url.json",
         )
         .unwrap();
         let res = client.post(uri!(post_sns_notify)).body(&data).dispatch();

src/routes/wireguard.rs

@@ -0,0 +1,30 @@
+use crate::k8s::get_wireguard_config;
+
+#[rocket::get("/wireguard/config/<instance_id>")]
+pub async fn get_node_wireguard(instance_id: String) -> Option<String> {
+    if let Ok(Some(token)) = get_wireguard_config(&instance_id).await {
+        Some(token)
+    } else {
+        None
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::rocket;
+    use rocket::http::Status;
+    use rocket::local::blocking::Client;
+    use rocket::uri;
+
+    #[test]
+    fn test_get_node_wireguard_404() {
+        let client = Client::tracked(rocket()).unwrap();
+        let res = client
+            .get(uri!(get_node_wireguard(
+                instance_id = "i-0a1b2c3d4e5f6f7f8"
+            )))
+            .dispatch();
+        assert_eq!(res.status(), Status::NotFound);
+    }
+}

tests/README.md

@@ -0,0 +1,21 @@
+# Integration Tests
+
+## Prerequisites
+
+Create the test resources in Kubernetes:
+
+```sh
+kubectl apply -f setup.yaml
+```
+
+Generate a kubeconfig for the test service account:
+
+```sh
+sh genkubeconfig.sh > kubeconfig
+```
+
+## Run the Tests
+
+```sh
+KUBECONFIG=${PWD}/kubeconfig cargo test
+```

tests/data/sns/notification-running.json

@@ -0,0 +1,11 @@
+{
+  "Type": "Notification",
+  "MessageId": "b87b7cca-d698-552b-bf91-2764a57ca6a8",
+  "TopicArn": "arn:aws:sns:us-east-2:566967686773:ec2-events",
+  "Message": "{\"version\":\"0\",\"id\":\"ae9cb41b-2d06-c688-8d20-82e90720cfa4\",\"detail-type\":\"EC2 Instance State-change Notification\",\"source\":\"aws.ec2\",\"account\":\"566967686773\",\"time\":\"2022-09-28T16:09:08Z\",\"region\":\"us-east-2\",\"resources\":[\"arn:aws:ec2:us-east-2:566967686773:instance/i-0e50d560c8bf9f0f8\"],\"detail\":{\"instance-id\":\"i-0e50d560c8bf9f0f8\",\"state\":\"running\"}}",
+  "Subject": null,
+  "Timestamp": "2022-09-28T16:09:08.795Z",
+  "SignatureVersion": "1",
+  "Signature": "LBaVKayZhbAzQa8k78oKNUgo62KqFU/GZJ8GCkNmMJJSeAyZNFOPY7Rcy7x3tmvXnds9ns/xWnzgrK2SdD/Q0Zniu9st2o1lDKfQnbUHU1Wv7g65jOuXTJlBu6teuziJ/bpsFTv4z9yw4fPm7gNYZ3xF5yjwXn0j0IHv92YgzEJtewV+MKRgMtp0vq9+TvRhpOiYZC8DCWZdqQoHm7B9VruPPPG9yHSZ2eF9H8cOQgjRA0IweanGpU+qej6Ts0IMeMTPPcPzhSynCdQJ8UfBSNPPfnlkkbq714XUjcms01UcMFuAQyMHrrr9CsTOtOJn4R/h9qccjZ7DhzXSkGVocw==",
+  "SigningCertURL": "https://sns.us-east-2.amazonaws.com/SimpleNotificationService-56e67fcb41f6fec09b0196692625d385.pem"
+}

tests/data/sns/notification-terminated.json

@@ -0,0 +1,11 @@
+{
+  "Type": "Notification",
+  "MessageId": "7d38c409-0edc-5cd0-87cd-9fa77aff18bc",
+  "TopicArn": "arn:aws:sns:us-east-2:566967686773:ec2-events",
+  "Message": "{\"version\":\"0\",\"id\":\"49c93129-5cb6-c94f-54c9-3b0c4fc07b03\",\"detail-type\":\"EC2 Instance State-change Notification\",\"source\":\"aws.ec2\",\"account\":\"566967686773\",\"time\":\"2022-09-28T16:10:11Z\",\"region\":\"us-east-2\",\"resources\":[\"arn:aws:ec2:us-east-2:566967686773:instance/i-0e50d560c8bf9f0f8\"],\"detail\":{\"instance-id\":\"i-0e50d560c8bf9f0f8\",\"state\":\"terminated\"}}",
+  "Subject": null,
+  "Timestamp": "2022-09-28T16:10:11.889Z",
+  "SignatureVersion": "1",
+  "Signature": "YVeRPXDW4zpPhBow/Le73zN1RJ754a/o9IHBAG8wSDNC2lLushh0ztknfp4G1qNA5ZenrKgTKyta0avLECh2qiU9lhp+M5qeY9CRIW2SH9xAL25MIN7psxkhJgSaqblIBENUE8gjge0pzDuKepgY5zD9f68Uf/0voMWTVWjcI2IdtOoS3LMqx6XC9K8LYMPgPf+0wGdH7AUXumQJTNBwX5jjMql2uIccN/xQxmUmIRHubWvDX82m78PZL+2mZlbGvsRRpkSPBPT/xu1fBwWkLQ0ziOf8sX7FyyKMKn8x4ohLkQjwU/21nSY8ei4n749ggJRNB0CxfQFPnNW5/iA4Qw==",
+  "SigningCertURL": "https://sns.us-east-2.amazonaws.com/SimpleNotificationService-56e67fcb41f6fec09b0196692625d385.pem"
+}

tests/genkubeconfig.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+server=$(
+    kubectl config view --minify --raw \
+        -o jsonpath='{.clusters[].cluster.server}'
+)
+token=$(
+    kubectl get secret \
+        -n dynk8s-test dynk8s-provisioner \
+        -o jsonpath='{.data.token}' \
+    | base64 -d
+)
+ca=$(
+    kubectl get secret \
+        -n dynk8s-test dynk8s-provisioner \
+        -o jsonpath='{.data.ca\.crt}'
+)
+
+cat <<EOF
+apiVersion: v1
+kind: Config
+clusters:
+- name: kubernetes
+  cluster:
+    certificate-authority-data: ${ca}
+    server: ${server}
+contexts:
+- name: dynk8s-test@kubernetes
+  context:
+    cluster: kubernetes
+    namespace: dynk8s-test
+    user: dynk8s-test
+current-context: dynk8s-test@kubernetes
+users:
+- name: dynk8s-test
+  user:
+    token: ${token}
+EOF

tests/integration/main.rs

@@ -0,0 +1,46 @@
+use std::collections::btree_map::BTreeMap;
+
+use k8s_openapi::api::core::v1::Secret;
+use kube::core::params::ListParams;
+
+mod sns;
+
+const WIREGUARD_CONFIG: &str = "\
+[Interface]
+Address = 10.11.12.13/14
+PrivateKey = UEdAkIaF80zhlOpgacOYL2UckrfCAWXfsDDSAAzNH3g=
+
+[Peer]
+PublicKey = zbeTpUFA014kvTezIEGBt4yi3BVocST9j1dBElp9liI=
+PreSharedKey = V6hAm01dxv2ib8AML2dSyX68hlPZm8En+IXfsknK3Zc=
+AllowedIPs = 0.0.0.0/0
+Endpoint = wireguard.example.org:24680
+";
+
+async fn setup() {
+    let client = kube::Client::try_default().await.unwrap();
+    let secrets: kube::Api<Secret> = kube::Api::default_namespaced(client);
+    let lp =
+        ListParams::default().fields("type=dynk8s.du5t1n.me/wireguard-config");
+    secrets
+        .delete_collection(&Default::default(), &lp)
+        .await
+        .unwrap();
+
+    let mut labels = BTreeMap::<String, String>::new();
+    labels.insert(
+        "app.kubernetes.io/part-of".into(),
+        "dynk8s-provisioner".into(),
+    );
+    labels.insert("dynk8s.du5t1n.me/ec2-instance-id".into(), "".into());
+    let mut data = BTreeMap::<String, String>::new();
+    data.insert("wireguard-config".into(), WIREGUARD_CONFIG.into());
+    let mut secret = Secret::default();
+    secret.type_ = Some("dynk8s.du5t1n.me/wireguard-config".into());
+    secret.immutable = Some(true);
+    secret.metadata.generate_name = Some("wireguard-config-".into());
+    secret.metadata.labels = Some(labels);
+    secret.string_data = Some(data);
+    secrets.create(&Default::default(), &secret).await.unwrap();
+}
+

tests/integration/sns.rs

@@ -0,0 +1,57 @@
+use rocket::http::Status;
+use rocket::local::asynchronous::Client;
+
+use dynk8s_provisioner::rocket;
+
+use crate::{setup, WIREGUARD_CONFIG};
+
+#[rocket::async_test]
+async fn test_sns_ec2_lifecycle() {
+    setup().await;
+    let client = Client::tracked(rocket()).await.unwrap();
+
+    // Simulate an instance state-change event indicating an instance has
+    // started. This should generate a bootstrap token and assign the WireGuard
+    // config to the instance.
+    let data =
+        std::fs::read_to_string("tests/data/sns/notification-running.json")
+            .unwrap();
+    let res = client.post("/sns/notify").body(&data).dispatch().await;
+    assert_eq!(res.status(), Status::NoContent);
+
+    // Ensure the bootstrap token was generated
+    let res = client
+        .get("/kubeadm/kubeconfig/i-0e50d560c8bf9f0f8")
+        .dispatch()
+        .await;
+    assert_eq!(res.status(), Status::Ok);
+    // Ensure the WireGuard config was assigned
+    let res = client
+        .get("/wireguard/config/i-0e50d560c8bf9f0f8")
+        .dispatch()
+        .await;
+    assert_eq!(res.status(), Status::Ok);
+    assert_eq!(res.into_string().await, Some(WIREGUARD_CONFIG.into()));
+
+    // Simulate an instance state-change event indicating the instance has
+    // terminated.  This should delete the bootstrap token and unassing the
+    // WireGuard config.
+    let data =
+        std::fs::read_to_string("tests/data/sns/notification-terminated.json")
+            .unwrap();
+    let res = client.post("/sns/notify").body(&data).dispatch().await;
+    assert_eq!(res.status(), Status::NoContent);
+
+    // Ensure the bootstrap token was deleted
+    let res = client
+        .get("/kubeadm/kubeconfig/i-0e50d560c8bf9f0f8")
+        .dispatch()
+        .await;
+    assert_eq!(res.status(), Status::NotFound);
+    // Ensure the WireGuard config was deleted
+    let res = client
+        .get("/wireguard/config/i-0e50d560c8bf9f0f8")
+        .dispatch()
+        .await;
+    assert_eq!(res.status(), Status::NotFound);
+}

tests/setup.yaml

@@ -0,0 +1,76 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dynk8s-test
+  labels:
+    kubernetes.io/metadata.name: dynk8s
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s-test
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+automountServiceAccountToken: true
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s-test
+  annotations:
+    kubernetes.io/service-account.name: dynk8s-provisioner
+type: kubernetes.io/service-account-token
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s-test
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - secrets
+  verbs:
+  - '*'
+- apiGroups:
+  - ''
+  resources:
+  - configmaps
+  resourceNames:
+  - cluster-info
+  verbs:
+  - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s-test
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: dynk8s-provisioner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: dynk8s-provisioner
+subjects:
+- kind: ServiceAccount
+  name: dynk8s-provisioner
+  namespace: dynk8s-test