diff --git a/src/events.rs b/src/events.rs index 72d869d..6286e7b 100644 --- a/src/events.rs +++ b/src/events.rs @@ -7,7 +7,7 @@ use log::{debug, error}; use crate::k8s::{ assign_wireguard_config, create_bootstrap_token, delete_bootstrap_tokens, - unassign_wireguard_config, + delete_node, unassign_wireguard_config, }; use crate::model::events::*; @@ -24,6 +24,7 @@ use crate::model::events::*; /// When an instance is terminated: /// 1. Any WireGuard configs assigned to the instance are unassigned /// 2. All bootstrap tokens for the instance are deleted +/// 3. The Kubernetes Node resource for the instance is deleted pub async fn on_ec2_instance_state_change(evt: Ec2InstanceStateChange) { debug!("EC2 instance {} is now {}", &evt.instance_id, &evt.state); if evt.state == "running" { @@ -53,5 +54,11 @@ pub async fn on_ec2_instance_state_change(evt: Ec2InstanceStateChange) { &evt.instance_id, e ); } + if let Err(e) = delete_node(&evt.instance_id).await { + error!( + "Failed to delete node for instance {}: {}", + &evt.instance_id, e + ); + } } } diff --git a/src/k8s.rs b/src/k8s.rs index ed1bf5f..189cb27 100644 --- a/src/k8s.rs +++ b/src/k8s.rs @@ -3,7 +3,7 @@ use std::collections::btree_map::BTreeMap; use chrono::offset::Utc; use chrono::{DateTime, Duration}; -use k8s_openapi::api::core::v1::{ConfigMap, Secret}; +use k8s_openapi::api::core::v1::{ConfigMap, Node, Secret}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::core::params::{ListParams, Patch, PatchParams, PostParams}; use kube::{Api, Client}; @@ -385,6 +385,31 @@ pub async fn get_kubeconfig>( } } +/// Delete the node representing an EC2 instance +/// +/// When an EC2 node is terminated, it is permanently offline. If the instance +/// was a member of the cluster, it may have a Node resource still present in +/// Kubernetes. This object needs to be deleted; neither the Cluster +/// Autoscaler nor Kubernetes itself will do this. +pub async fn delete_node>( + instance_id: I, +) -> Result<(), kube::Error> { + let instance_id = instance_id.as_ref(); + let client = Client::try_default().await?; + let nodes: Api = Api::all(client); + for node in nodes.list(&Default::default()).await? { + if let (Some(name), Some(spec)) = (node.metadata.name, node.spec) { + if let Some(pid) = spec.provider_id { + if pid.starts_with("aws:///") && pid.ends_with(instance_id) { + info!("Deleting node {}", &name); + nodes.delete(&name, &Default::default()).await?; + } + } + } + } + Ok(()) +} + /// Retrieve the bootstrap token assigned to an EC2 instance async fn get_bootstrap_token>( instance_id: I,