From d85f314a8b0144163e68bd7cd227a13d366c43a4 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Fri, 7 Oct 2022 06:49:58 -0500 Subject: [PATCH] tests: Begin integration tests Cargo uses the sources in the `tests` directory to build and run integration tests. For each `tests/foo.rs` or `tests/foo/main.rs`, it creates an executable that runs the test functions therein. These executables are separate crates from the main package, and thus do not have access to its private members. Integration tests are expected to test only the public functionality of the package. Application crates do not have any public members; their public interface is the command line. Integration tests would typically run the command (e.g. using `std::process::Command`) and test its output. Since *dynk8s-provisioner* is not really a command-line tool, testing it this way would be difficult; each test would need to start the server, make requests to it, and then stop it. This would be slow and cumbersome. In order to avoid this tedium and be able to use Rocket's built-in test client, I have converted *dynk8s-provisioner* into a library crate that also includes an executable. The library makes the `rocket` function public, which allows the integration tests to import it and pass it to the Rocket test client. The point of integration tests, of course, is to validate the functionality of the application as a whole. This necessarily requires allowing it to communicate with the Kubernetes API. In the Jenkins CI environment, the application will need the appropriate credentials, and will need to use a separate Kubernetes namespace from the production deployment. The `setup.yaml` manifest in the `tests` directory defines the resources necessary to run integration tests, and the `genkubeconfig.sh` script can be used to create the appropriate kubeconfig file containing the credentials. The kubeconfig is exposed to the tests via the `KUBECONFIG` environment variable, which is populated from a Jenkins secret file credential. Note: The `data` directory moved from `test` to `tests` to avoid duplication and confusing names. --- ci/Jenkinsfile | 7 +- src/lib.rs | 25 ++++++ src/main.rs | 28 +------ src/routes/sns.rs | 6 +- tests/data/sns/notification-running.json | 11 +++ tests/data/sns/notification-terminated.json | 11 +++ .../sns/subscriptionconfirmation-bad-url.json | 0 .../sns/subscriptionconfirmation-bad.json | 0 .../data/sns/subscriptionconfirmation.json | 0 tests/genkubeconfig.sh | 38 ++++++++++ tests/integration/main.rs | 65 ++++++++++++++++ tests/integration/sns.rs | 57 ++++++++++++++ tests/setup.yaml | 76 +++++++++++++++++++ 13 files changed, 296 insertions(+), 28 deletions(-) create mode 100644 src/lib.rs create mode 100644 tests/data/sns/notification-running.json create mode 100644 tests/data/sns/notification-terminated.json rename {test => tests}/data/sns/subscriptionconfirmation-bad-url.json (100%) rename {test => tests}/data/sns/subscriptionconfirmation-bad.json (100%) rename {test => tests}/data/sns/subscriptionconfirmation.json (100%) create mode 100644 tests/genkubeconfig.sh create mode 100644 tests/integration/main.rs create mode 100644 tests/integration/sns.rs create mode 100644 tests/setup.yaml diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index afa4ada..2143109 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -20,7 +20,12 @@ pipeline { stage('Test') { steps { container('build') { - sh '. ci/test.sh' + withCredentials([file( + credentialsId: 'dynk8s-test-kubeconfig', + variable: 'KUBECONFIG' + )]) { + sh '. ci/test.sh' + } } } } diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..920d9f7 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,25 @@ +mod error; +mod events; +mod k8s; +mod model; +mod routes; +mod sns; + +#[doc(hidden)] +pub fn rocket() -> rocket::Rocket { + rocket::build().mount( + "/", + rocket::routes![ + routes::health::get_health, + routes::kubeadm::get_node_kubeconfig, + routes::kubeadm::post_node_kubeconfig, + routes::kubeadm::patch_node_kubeconfig, + routes::kubeadm::put_node_kubeconfig, + routes::wireguard::get_node_wireguard, + routes::sns::post_sns_notify, + routes::sns::get_sns_notify, + routes::sns::put_sns_notify, + routes::sns::patch_sns_notify, + ], + ) +} diff --git a/src/main.rs b/src/main.rs index de3af1f..eba3270 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,26 +1,6 @@ -mod error; -mod events; -mod k8s; -mod model; -mod routes; -mod sns; +use dynk8s_provisioner::rocket; -#[doc(hidden)] -#[rocket::launch] -fn rocket() -> _ { - rocket::build().mount( - "/", - rocket::routes![ - routes::health::get_health, - routes::kubeadm::get_node_kubeconfig, - routes::kubeadm::post_node_kubeconfig, - routes::kubeadm::patch_node_kubeconfig, - routes::kubeadm::put_node_kubeconfig, - routes::wireguard::get_node_wireguard, - routes::sns::post_sns_notify, - routes::sns::get_sns_notify, - routes::sns::put_sns_notify, - routes::sns::patch_sns_notify, - ], - ) +#[rocket::main] +async fn main() { + let _ = rocket().launch().await; } diff --git a/src/routes/sns.rs b/src/routes/sns.rs index 2435491..10511fe 100644 --- a/src/routes/sns.rs +++ b/src/routes/sns.rs @@ -50,7 +50,7 @@ mod test { fn test_sub_conf_msg() { let client = Client::tracked(rocket()).unwrap(); let data = std::fs::read_to_string( - "test/data/sns/subscriptionconfirmation.json", + "tests/data/sns/subscriptionconfirmation.json", ) .unwrap(); let res = client.post(uri!(post_sns_notify)).body(&data).dispatch(); @@ -68,7 +68,7 @@ mod test { fn test_sub_conf_msg_bad() { let client = Client::tracked(rocket()).unwrap(); let data = std::fs::read_to_string( - "test/data/sns/subscriptionconfirmation-bad.json", + "tests/data/sns/subscriptionconfirmation-bad.json", ) .unwrap(); let res = client.post(uri!(post_sns_notify)).body(&data).dispatch(); @@ -88,7 +88,7 @@ mod test { fn test_sub_conf_msg_bad_cert_url() { let client = Client::tracked(rocket()).unwrap(); let data = std::fs::read_to_string( - "test/data/sns/subscriptionconfirmation-bad-url.json", + "tests/data/sns/subscriptionconfirmation-bad-url.json", ) .unwrap(); let res = client.post(uri!(post_sns_notify)).body(&data).dispatch(); diff --git a/tests/data/sns/notification-running.json b/tests/data/sns/notification-running.json new file mode 100644 index 0000000..6347150 --- /dev/null +++ b/tests/data/sns/notification-running.json @@ -0,0 +1,11 @@ +{ + "Type": "Notification", + "MessageId": "b87b7cca-d698-552b-bf91-2764a57ca6a8", + "TopicArn": "arn:aws:sns:us-east-2:566967686773:ec2-events", + "Message": "{\"version\":\"0\",\"id\":\"ae9cb41b-2d06-c688-8d20-82e90720cfa4\",\"detail-type\":\"EC2 Instance State-change Notification\",\"source\":\"aws.ec2\",\"account\":\"566967686773\",\"time\":\"2022-09-28T16:09:08Z\",\"region\":\"us-east-2\",\"resources\":[\"arn:aws:ec2:us-east-2:566967686773:instance/i-0e50d560c8bf9f0f8\"],\"detail\":{\"instance-id\":\"i-0e50d560c8bf9f0f8\",\"state\":\"running\"}}", + "Subject": null, + "Timestamp": "2022-09-28T16:09:08.795Z", + "SignatureVersion": "1", + "Signature": "LBaVKayZhbAzQa8k78oKNUgo62KqFU/GZJ8GCkNmMJJSeAyZNFOPY7Rcy7x3tmvXnds9ns/xWnzgrK2SdD/Q0Zniu9st2o1lDKfQnbUHU1Wv7g65jOuXTJlBu6teuziJ/bpsFTv4z9yw4fPm7gNYZ3xF5yjwXn0j0IHv92YgzEJtewV+MKRgMtp0vq9+TvRhpOiYZC8DCWZdqQoHm7B9VruPPPG9yHSZ2eF9H8cOQgjRA0IweanGpU+qej6Ts0IMeMTPPcPzhSynCdQJ8UfBSNPPfnlkkbq714XUjcms01UcMFuAQyMHrrr9CsTOtOJn4R/h9qccjZ7DhzXSkGVocw==", + "SigningCertURL": "https://sns.us-east-2.amazonaws.com/SimpleNotificationService-56e67fcb41f6fec09b0196692625d385.pem" +} diff --git a/tests/data/sns/notification-terminated.json b/tests/data/sns/notification-terminated.json new file mode 100644 index 0000000..042c20b --- /dev/null +++ b/tests/data/sns/notification-terminated.json @@ -0,0 +1,11 @@ +{ + "Type": "Notification", + "MessageId": "7d38c409-0edc-5cd0-87cd-9fa77aff18bc", + "TopicArn": "arn:aws:sns:us-east-2:566967686773:ec2-events", + "Message": "{\"version\":\"0\",\"id\":\"49c93129-5cb6-c94f-54c9-3b0c4fc07b03\",\"detail-type\":\"EC2 Instance State-change Notification\",\"source\":\"aws.ec2\",\"account\":\"566967686773\",\"time\":\"2022-09-28T16:10:11Z\",\"region\":\"us-east-2\",\"resources\":[\"arn:aws:ec2:us-east-2:566967686773:instance/i-0e50d560c8bf9f0f8\"],\"detail\":{\"instance-id\":\"i-0e50d560c8bf9f0f8\",\"state\":\"terminated\"}}", + "Subject": null, + "Timestamp": "2022-09-28T16:10:11.889Z", + "SignatureVersion": "1", + "Signature": "YVeRPXDW4zpPhBow/Le73zN1RJ754a/o9IHBAG8wSDNC2lLushh0ztknfp4G1qNA5ZenrKgTKyta0avLECh2qiU9lhp+M5qeY9CRIW2SH9xAL25MIN7psxkhJgSaqblIBENUE8gjge0pzDuKepgY5zD9f68Uf/0voMWTVWjcI2IdtOoS3LMqx6XC9K8LYMPgPf+0wGdH7AUXumQJTNBwX5jjMql2uIccN/xQxmUmIRHubWvDX82m78PZL+2mZlbGvsRRpkSPBPT/xu1fBwWkLQ0ziOf8sX7FyyKMKn8x4ohLkQjwU/21nSY8ei4n749ggJRNB0CxfQFPnNW5/iA4Qw==", + "SigningCertURL": "https://sns.us-east-2.amazonaws.com/SimpleNotificationService-56e67fcb41f6fec09b0196692625d385.pem" +} diff --git a/test/data/sns/subscriptionconfirmation-bad-url.json b/tests/data/sns/subscriptionconfirmation-bad-url.json similarity index 100% rename from test/data/sns/subscriptionconfirmation-bad-url.json rename to tests/data/sns/subscriptionconfirmation-bad-url.json diff --git a/test/data/sns/subscriptionconfirmation-bad.json b/tests/data/sns/subscriptionconfirmation-bad.json similarity index 100% rename from test/data/sns/subscriptionconfirmation-bad.json rename to tests/data/sns/subscriptionconfirmation-bad.json diff --git a/test/data/sns/subscriptionconfirmation.json b/tests/data/sns/subscriptionconfirmation.json similarity index 100% rename from test/data/sns/subscriptionconfirmation.json rename to tests/data/sns/subscriptionconfirmation.json diff --git a/tests/genkubeconfig.sh b/tests/genkubeconfig.sh new file mode 100644 index 0000000..2fd17ed --- /dev/null +++ b/tests/genkubeconfig.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +server=$( + kubectl config view --minify --raw \ + -o jsonpath='{.clusters[].cluster.server}' +) +token=$( + kubectl get secret \ + -n dynk8s-test dynk8s-provisioner \ + -o jsonpath='{.data.token}' \ + | base64 -d +) +ca=$( + kubectl get secret \ + -n dynk8s-test dynk8s-provisioner \ + -o jsonpath='{.data.ca\.crt}' +) + +cat < kubeconfig +//! ``` +//! +//! Run the tests: +//! +//! ```sh +//! KUBECONFIG=${PWD}/kubeconfig cargo test +//! ``` +use std::collections::btree_map::BTreeMap; + +use k8s_openapi::api::core::v1::Secret; +use kube::core::params::ListParams; + +mod sns; + +const WIREGUARD_CONFIG: &str = "\ +[Interface] +Address = 10.11.12.13/14 +PrivateKey = UEdAkIaF80zhlOpgacOYL2UckrfCAWXfsDDSAAzNH3g= + +[Peer] +PublicKey = zbeTpUFA014kvTezIEGBt4yi3BVocST9j1dBElp9liI= +PreSharedKey = V6hAm01dxv2ib8AML2dSyX68hlPZm8En+IXfsknK3Zc= +AllowedIPs = 0.0.0.0/0 +Endpoint = wireguard.example.org:24680 +"; + +async fn setup() { + let client = kube::Client::try_default().await.unwrap(); + let secrets: kube::Api = kube::Api::default_namespaced(client); + let lp = + ListParams::default().fields("type=dynk8s.du5t1n.me/wireguard-config"); + secrets + .delete_collection(&Default::default(), &lp) + .await + .unwrap(); + + let mut labels = BTreeMap::::new(); + labels.insert( + "app.kubernetes.io/part-of".into(), + "dynk8s-provisioner".into(), + ); + labels.insert("dynk8s.du5t1n.me/ec2-instance-id".into(), "".into()); + let mut data = BTreeMap::::new(); + data.insert("wireguard-config".into(), WIREGUARD_CONFIG.into()); + let mut secret = Secret::default(); + secret.type_ = Some("dynk8s.du5t1n.me/wireguard-config".into()); + secret.immutable = Some(true); + secret.metadata.generate_name = Some("wireguard-config-".into()); + secret.metadata.labels = Some(labels); + secret.string_data = Some(data); + secrets.create(&Default::default(), &secret).await.unwrap(); +} + diff --git a/tests/integration/sns.rs b/tests/integration/sns.rs new file mode 100644 index 0000000..e1feab6 --- /dev/null +++ b/tests/integration/sns.rs @@ -0,0 +1,57 @@ +use rocket::http::Status; +use rocket::local::asynchronous::Client; + +use dynk8s_provisioner::rocket; + +use crate::{setup, WIREGUARD_CONFIG}; + +#[rocket::async_test] +async fn test_sns_ec2_lifecycle() { + setup().await; + let client = Client::tracked(rocket()).await.unwrap(); + + // Simulate an instance state-change event indicating an instance has + // started. This should generate a bootstrap token and assign the WireGuard + // config to the instance. + let data = + std::fs::read_to_string("tests/data/sns/notification-running.json") + .unwrap(); + let res = client.post("/sns/notify").body(&data).dispatch().await; + assert_eq!(res.status(), Status::NoContent); + + // Ensure the bootstrap token was generated + let res = client + .get("/kubeadm/kubeconfig/i-0e50d560c8bf9f0f8") + .dispatch() + .await; + assert_eq!(res.status(), Status::Ok); + // Ensure the WireGuard config was assigned + let res = client + .get("/wireguard/config/i-0e50d560c8bf9f0f8") + .dispatch() + .await; + assert_eq!(res.status(), Status::Ok); + assert_eq!(res.into_string().await, Some(WIREGUARD_CONFIG.into())); + + // Simulate an instance state-change event indicating the instance has + // terminated. This should delete the bootstrap token and unassing the + // WireGuard config. + let data = + std::fs::read_to_string("tests/data/sns/notification-terminated.json") + .unwrap(); + let res = client.post("/sns/notify").body(&data).dispatch().await; + assert_eq!(res.status(), Status::NoContent); + + // Ensure the bootstrap token was deleted + let res = client + .get("/kubeadm/kubeconfig/i-0e50d560c8bf9f0f8") + .dispatch() + .await; + assert_eq!(res.status(), Status::NotFound); + // Ensure the WireGuard config was deleted + let res = client + .get("/wireguard/config/i-0e50d560c8bf9f0f8") + .dispatch() + .await; + assert_eq!(res.status(), Status::NotFound); +} diff --git a/tests/setup.yaml b/tests/setup.yaml new file mode 100644 index 0000000..7b58f86 --- /dev/null +++ b/tests/setup.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: dynk8s-test + labels: + kubernetes.io/metadata.name: dynk8s + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dynk8s-provisioner + namespace: dynk8s-test + labels: + app.kubernetes.io/name: dynk8s-provisioner + app.kubernetes.io/instance: default + app.kubernetes.io/component: http-api + app.kubernetes.io/part-of: dynk8s-provisioner +automountServiceAccountToken: true + +--- +apiVersion: v1 +kind: Secret +metadata: + name: dynk8s-provisioner + namespace: dynk8s-test + annotations: + kubernetes.io/service-account.name: dynk8s-provisioner +type: kubernetes.io/service-account-token + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dynk8s-provisioner + namespace: dynk8s-test + labels: + app.kubernetes.io/name: dynk8s-provisioner + app.kubernetes.io/instance: default + app.kubernetes.io/component: http-api + app.kubernetes.io/part-of: dynk8s-provisioner +rules: +- apiGroups: + - '' + resources: + - secrets + verbs: + - '*' +- apiGroups: + - '' + resources: + - configmaps + resourceNames: + - cluster-info + verbs: + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: dynk8s-provisioner + namespace: dynk8s-test + labels: + app.kubernetes.io/name: dynk8s-provisioner + app.kubernetes.io/instance: default + app.kubernetes.io/part-of: dynk8s-provisioner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: dynk8s-provisioner +subjects: +- kind: ServiceAccount + name: dynk8s-provisioner + namespace: dynk8s-test