tf/userdata: Taint node for Jenkins only

If a Jenkins job runs for a while, Kubernetes may schedule other Pods
on it eventually.  If a long-running Pod gets assigned to the ephemeral
node, the Cluster Autoscaler won't be able to scale down the ASG.  To
prevent this, we apply a taint to the node so normal Pods will not get
assigned to it.  We have to apply the corresponding toleration to Pods
for Jenkins jobs.

terraform/asg.tf

@@ -52,7 +52,7 @@ resource "aws_launch_template" "k8s-aarch64" {
 
   update_default_version = true
   image_id               = "${data.aws_ami.latest-fedora.id}"
-  instance_type          = "t4g.medium"
+  instance_type          = "c7gd.xlarge"
   security_group_names   = [aws_security_group.k8s-node.name]
   key_name               = "dustin@rosalina"
 
@@ -65,6 +65,14 @@ resource "aws_launch_template" "k8s-aarch64" {
   private_dns_name_options {
     hostname_type = "resource-name"
   }
+
+  block_device_mappings {
+    device_name = "/dev/sda1"
+
+    ebs {
+      volume_size = 64
+    }
+  }
 }
 
 resource "aws_autoscaling_group" "k8s-aarch64" {

terraform/terraform.tfstate

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.6.2",
-  "serial": 98,
+  "serial": 108,
   "lineage": "a100be74-c98e-0769-2d6a-bf6a2c5f3ebf",
   "outputs": {},
   "resources": [
@@ -107,9 +107,9 @@
           "schema_version": 0,
           "attributes": {
             "account_id": "566967686773",
-            "arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1705246977054689837",
+            "arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1707258546409158274",
             "id": "566967686773",
-            "user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1705246977054689837"
+            "user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1707258546409158274"
           },
           "sensitive_attributes": []
         }
@@ -218,7 +218,7 @@
             "context": "",
             "default_cooldown": 300,
             "default_instance_warmup": 0,
-            "desired_capacity": 0,
+            "desired_capacity": 1,
             "enabled_metrics": [],
             "force_delete": false,
             "force_delete_warm_pool": false,
@@ -358,11 +358,29 @@
           "schema_version": 0,
           "attributes": {
             "arn": "arn:aws:ec2:us-east-2:566967686773:launch-template/lt-0789a3800bdaec215",
-            "block_device_mappings": [],
+            "block_device_mappings": [
+              {
+                "device_name": "/dev/sda1",
+                "ebs": [
+                  {
+                    "delete_on_termination": "",
+                    "encrypted": "",
+                    "iops": 0,
+                    "kms_key_id": "",
+                    "snapshot_id": "",
+                    "throughput": 0,
+                    "volume_size": 64,
+                    "volume_type": "gp2"
+                  }
+                ],
+                "no_device": "",
+                "virtual_name": ""
+              }
+            ],
             "capacity_reservation_specification": [],
             "cpu_options": [],
             "credit_specification": [],
-            "default_version": 21,
+            "default_version": 27,
             "description": "",
             "disable_api_stop": false,
             "disable_api_termination": false,
@@ -382,10 +400,10 @@
               }
             ],
             "instance_requirements": [],
-            "instance_type": "t4g.medium",
+            "instance_type": "c7gd.xlarge",
             "kernel_id": "",
             "key_name": "dustin@rosalina",
-            "latest_version": 21,
+            "latest_version": 27,
             "license_specification": [],
             "maintenance_options": [],
             "metadata_options": [],
@@ -409,7 +427,7 @@
             "tags": {},
             "tags_all": {},
             "update_default_version": true,
-            "user_data": "I2Nsb3VkLWNvbmZpZwpib290Y21kOgotIFsgbG4sIC1zZiwgL3J1bi9zeXN0ZW1kL3Jlc29sdmUvc3R1Yi1yZXNvbHYuY29uZiwgL2V0Yy9yZXNvbHYuY29uZiBdCgpwYWNrYWdlczoKLSBjcmktbwotIGNyaS10b29scwotIGV0aHRvb2wKLSBpcHRhYmxlcy1uZnQKLSBpc2NzaS1pbml0aWF0b3ItdXRpbHMKLSBrdWJlcm5ldGVzLWt1YmVhZG0KLSBrdWJlcm5ldGVzLW5vZGUKLSBydW5jCi0gd2lyZWd1YXJkLXRvb2xzCgp3cml0ZV9maWxlczoKLSBwYXRoOiAvZXRjL2RuZi9kbmYuY29uZgogIGNvbnRlbnQ6IHwrCiAgICBpbnN0YWxsX3dlYWtfZGVwcz1GYWxzZQogIGFwcGVuZDogdHJ1ZQotIHBhdGg6IC9ldGMvbW9kdWxlcy1sb2FkLmQvazhzLmNvbmYKICBjb250ZW50OiB8KwogICAgYnJfbmV0ZmlsdGVyCi0gcGF0aDogL2V0Yy9zeXNjdGwuZC9rOHMuY29uZgogIGNvbnRlbnQ6IHwrCiAgICBuZXQuYnJpZGdlLmJyaWRnZS1uZi1jYWxsLWlwdGFibGVzID0gMQogICAgbmV0LmJyaWRnZS5icmlkZ2UtbmYtY2FsbC1pcDZ0YWJsZXMgPSAxCiAgICBuZXQuaXB2NC5pcF9mb3J3YXJkID0gMQotIHBhdGg6IC92YXIvbGliL2Nsb3VkL3NjcmlwdHMvcGVyLWluc3RhbmNlL2t1YmVhZG0tam9pbgogIHBlcm1pc3Npb25zOiAnMDc1NScKICBjb250ZW50OiB8KwogICAgIyEvYmluL3NoCgogICAgQkFTRV9VUkw9aHR0cHM6Ly9keW5rOHMtcHJvdmlzaW9uZXIucHlyb2N1ZmZsaW5rLm5ldAoKICAgIGluc3RhbmNlX2lkPSQoY3VybCAtcyAxNjkuMjU0LjE2OS4yNTQvbGF0ZXN0L21ldGEtZGF0YS9pbnN0YW5jZS1pZCkKICAgIGF6PSQoY3VybCAtcyAxNjkuMjU0LjE2OS4yNTQvbGF0ZXN0L21ldGEtZGF0YS9wbGFjZW1lbnQvYXZhaWxhYmlsaXR5LXpvbmUpCgogICAgY3VybCAtZnMgIiR7QkFTRV9VUkx9Ii93aXJlZ3VhcmQvY29uZmlnLyR7aW5zdGFuY2VfaWR9IFwKICAgICAgICAtbyAvZXRjL3dpcmVndWFyZC93ZzAuY29uZiB8fCBleGl0CiAgICBzeXN0ZW1jdGwgZW5hYmxlIC0tbm93IHdnLXF1aWNrQHdnMCB8fCBleGl0CgogICAgcmVzb2x2ZWN0bCByZXZlcnQgZXRoMAoKICAgIG1vZHByb2JlIGJyX25ldGZpbHRlciB8fCBleGl0CiAgICBzeXNjdGwgLXcgLWYgL2V0Yy9zeXNjdGwuZC9rOHMuY29uZiB8fCBleGl0CgogICAgc3dhcG9mZiAtYSB8fCBleGl0CiAgICB0b3VjaCAvZXRjL3N5c3RlbWQvenJhbS1nZW5lcmF0b3IuY29uZiB8fCBleGl0CiAgICBzeXN0ZW1jdGwgZGFlbW9uLXJlbG9hZCB8fCBleGl0CiAgICBzeXN0ZW1jdGwgc3RvcCAnc3lzdGVtZC16cmFtLXNldHVwQConIHx8IGV4aXQKCiAgICBzeXN0ZW1jdGwgZW5hYmxlIGNyaW8gaXNjc2lkIGt1YmVsZXQgfHwgZXhpdAogICAgc3lzdGVtY3RsIHN0YXJ0IGNyaW8gaXNjc2lkIHx8IGV4aXQKCiAgICBpbnRlcm5hbF9pcD0kKAogICAgICBpcCBhZGRyZXNzIHNob3cgZGV2IHdnMCBwcmltYXJ5IHwgXAogICAgICBzZWQgLXJuICdzLy4qaW5ldCAoWzAtOS5dKykuKi9cMS9wJwogICAgKQoKICAgIGNhdCA+IGxvbmdob3JuLWlzc3VlNDk4OC5jaWwgPDxFT0YKICAgIChhbGxvdyBpc2NzaWRfdCBzZWxmIChjYXBhYmlsaXR5IChkYWNfb3ZlcnJpZGUpKSkKICAgIEVPRgogICAgc2Vtb2R1bGUgLWkgbG9uZ2hvcm4taXNzdWU0OTg4LmNpbAoKICAgIGNhdCA+IC9ydW4vam9pbmNvbmZpZ3VyYXRpb24gPDxFT0YKICAgIGFwaVZlcnNpb246IGt1YmVhZG0uazhzLmlvL3YxYmV0YTMKICAgIGtpbmQ6IEpvaW5Db25maWd1cmF0aW9uCiAgICBub2RlUmVnaXN0cmF0aW9uOgogICAgICBrdWJlbGV0RXh0cmFBcmdzOgogICAgICAgIHByb3ZpZGVyLWlkOiBhd3M6Ly8vJHthen0vJHtpbnN0YW5jZV9pZH0KICAgICAgICBub2RlLWlwOiAke2ludGVybmFsX2lwfQogICAgICAgIGNvbmZpZzogL3Zhci9saWIva3ViZWxldC9jb25maWcueWFtbAogICAgZGlzY292ZXJ5OgogICAgICBmaWxlOgogICAgICAgIGt1YmVDb25maWdQYXRoOiAke0JBU0VfVVJMfS9rdWJlYWRtL2t1YmVjb25maWcvJHtpbnN0YW5jZV9pZH0KICAgIEVPRgogICAga3ViZWFkbSBqb2luIC0tY29uZmlnPS9ydW4vam9pbmNvbmZpZ3VyYXRpb24KCnJ1bmNtZDoKLSBbIGRuZiwgcmVtb3ZlLCAteSwgenJhbS1nZW5lcmF0b3IgXQo=",
+            "user_data": "I2Nsb3VkLWNvbmZpZwpib290Y21kOgotIFsgbG4sIC1zZiwgL3J1bi9zeXN0ZW1kL3Jlc29sdmUvc3R1Yi1yZXNvbHYuY29uZiwgL2V0Yy9yZXNvbHYuY29uZiBdCgpwYWNrYWdlczoKLSBjcmktbwotIGNyaS10b29scwotIGV0aHRvb2wKLSBpcHRhYmxlcy1uZnQKLSBpc2NzaS1pbml0aWF0b3ItdXRpbHMKLSBrdWJlcm5ldGVzLWt1YmVhZG0KLSBrdWJlcm5ldGVzLW5vZGUKLSBydW5jCi0gd2lyZWd1YXJkLXRvb2xzCgp3cml0ZV9maWxlczoKLSBwYXRoOiAvZXRjL2RuZi9kbmYuY29uZgogIGNvbnRlbnQ6IHwrCiAgICBpbnN0YWxsX3dlYWtfZGVwcz1GYWxzZQogIGFwcGVuZDogdHJ1ZQotIHBhdGg6IC9ldGMvbW9kdWxlcy1sb2FkLmQvazhzLmNvbmYKICBjb250ZW50OiB8KwogICAgYnJfbmV0ZmlsdGVyCi0gcGF0aDogL2V0Yy9zeXNjdGwuZC9rOHMuY29uZgogIGNvbnRlbnQ6IHwrCiAgICBuZXQuYnJpZGdlLmJyaWRnZS1uZi1jYWxsLWlwdGFibGVzID0gMQogICAgbmV0LmJyaWRnZS5icmlkZ2UtbmYtY2FsbC1pcDZ0YWJsZXMgPSAxCiAgICBuZXQuaXB2NC5pcF9mb3J3YXJkID0gMQotIHBhdGg6IC92YXIvbGliL2Nsb3VkL3NjcmlwdHMvcGVyLWluc3RhbmNlL2t1YmVhZG0tam9pbgogIHBlcm1pc3Npb25zOiAnMDc1NScKICBjb250ZW50OiB8KwogICAgIyEvYmluL3NoCgogICAgQkFTRV9VUkw9aHR0cHM6Ly9keW5rOHMtcHJvdmlzaW9uZXIucHlyb2N1ZmZsaW5rLm5ldAoKICAgIGluc3RhbmNlX2lkPSQoY3VybCAtcyAxNjkuMjU0LjE2OS4yNTQvbGF0ZXN0L21ldGEtZGF0YS9pbnN0YW5jZS1pZCkKICAgIGF6PSQoY3VybCAtcyAxNjkuMjU0LjE2OS4yNTQvbGF0ZXN0L21ldGEtZGF0YS9wbGFjZW1lbnQvYXZhaWxhYmlsaXR5LXpvbmUpCgogICAgY3VybCAtZnMgIiR7QkFTRV9VUkx9Ii93aXJlZ3VhcmQvY29uZmlnLyR7aW5zdGFuY2VfaWR9IFwKICAgICAgICAtbyAvZXRjL3dpcmVndWFyZC93ZzAuY29uZiB8fCBleGl0CiAgICBzeXN0ZW1jdGwgZW5hYmxlIC0tbm93IHdnLXF1aWNrQHdnMCB8fCBleGl0CgogICAgcmVzb2x2ZWN0bCByZXZlcnQgZXRoMAoKICAgIG1vZHByb2JlIGJyX25ldGZpbHRlciB8fCBleGl0CiAgICBzeXNjdGwgLXcgLWYgL2V0Yy9zeXNjdGwuZC9rOHMuY29uZiB8fCBleGl0CgogICAgc3dhcG9mZiAtYSB8fCBleGl0CiAgICB0b3VjaCAvZXRjL3N5c3RlbWQvenJhbS1nZW5lcmF0b3IuY29uZiB8fCBleGl0CiAgICBzeXN0ZW1jdGwgZGFlbW9uLXJlbG9hZCB8fCBleGl0CiAgICBzeXN0ZW1jdGwgc3RvcCAnc3lzdGVtZC16cmFtLXNldHVwQConIHx8IGV4aXQKCiAgICBpZiBbIC1iIC9kZXYvbnZtZTFuMSBdOyB0aGVuCiAgICAgIHByaW50ZiAnJXMgJXMgJXMgJXMgMCAwXG4nIFwKICAgICAgICAvZGV2L252bWUxbjEgXAogICAgICAgIC92YXIvbGliL2t1YmVsZXQgXAogICAgICAgIGV4dDQgXAogICAgICAgIG5vYXRpbWUseC1zeXN0ZW1kLm1ha2Vmcyxub2ZhaWwgXAogICAgICAgID4+IC9ldGMvZnN0YWIKICAgICAgc3lzdGVtY3RsIGRhZW1vbi1yZWxvYWQKICAgICAgc3lzdGVtY3RsIHN0YXJ0IHZhci1saWIta3ViZWxldC5tb3VudAogICAgZmkKCiAgICBzeXN0ZW1jdGwgZW5hYmxlIGNyaW8gaXNjc2lkIGt1YmVsZXQgfHwgZXhpdAogICAgc3lzdGVtY3RsIHN0YXJ0IGNyaW8gaXNjc2lkIHx8IGV4aXQKCiAgICBpbnRlcm5hbF9pcD0kKAogICAgICBpcCBhZGRyZXNzIHNob3cgZGV2IHdnMCBwcmltYXJ5IHwgXAogICAgICBzZWQgLXJuICdzLy4qaW5ldCAoWzAtOS5dKykuKi9cMS9wJwogICAgKQoKICAgIGNhdCA+IGxvbmdob3JuLWlzc3VlNDk4OC5jaWwgPDxFT0YKICAgIChhbGxvdyBpc2NzaWRfdCBzZWxmIChjYXBhYmlsaXR5IChkYWNfb3ZlcnJpZGUpKSkKICAgIEVPRgogICAgc2Vtb2R1bGUgLWkgbG9uZ2hvcm4taXNzdWU0OTg4LmNpbAoKICAgIHJtIC1mIC9ldGMvY25pL25ldC5kLzEwMC1jcmlvLWJyaWRnZS5jb25mbGlzdAoKICAgIGNhdCA+IC9ydW4vam9pbmNvbmZpZ3VyYXRpb24gPDxFT0YKICAgIGFwaVZlcnNpb246IGt1YmVhZG0uazhzLmlvL3YxYmV0YTMKICAgIGtpbmQ6IEpvaW5Db25maWd1cmF0aW9uCiAgICBub2RlUmVnaXN0cmF0aW9uOgogICAgICB0YWludHM6CiAgICAgIC0ga2V5OiBkdTV0MW4ubWUvamVua2lucwogICAgICAgIGVmZmVjdDogTm9TY2hlZHVsZQogICAgICBrdWJlbGV0RXh0cmFBcmdzOgogICAgICAgIHByb3ZpZGVyLWlkOiBhd3M6Ly8vJHthen0vJHtpbnN0YW5jZV9pZH0KICAgICAgICBub2RlLWlwOiAke2ludGVybmFsX2lwfQogICAgICAgIGNvbmZpZzogL3Zhci9saWIva3ViZWxldC9jb25maWcueWFtbAogICAgZGlzY292ZXJ5OgogICAgICBmaWxlOgogICAgICAgIGt1YmVDb25maWdQYXRoOiAke0JBU0VfVVJMfS9rdWJlYWRtL2t1YmVjb25maWcvJHtpbnN0YW5jZV9pZH0KICAgIEVPRgogICAga3ViZWFkbSBqb2luIC0tY29uZmlnPS9ydW4vam9pbmNvbmZpZ3VyYXRpb24KCnJ1bmNtZDoKLSBbIGRuZiwgcmVtb3ZlLCAteSwgenJhbS1nZW5lcmF0b3IgXQo=",
             "vpc_security_group_ids": []
           },
           "sensitive_attributes": [],

terraform/userdata.yml

@@ -50,6 +50,17 @@ write_files:
     systemctl daemon-reload || exit
     systemctl stop 'systemd-zram-setup@*' || exit
 
+    if [ -b /dev/nvme1n1 ]; then
+      printf '%s %s %s %s 0 0\n' \
+        /dev/nvme1n1 \
+        /var/lib/kubelet \
+        ext4 \
+        noatime,x-systemd.makefs,nofail \
+        >> /etc/fstab
+      systemctl daemon-reload
+      systemctl start var-lib-kubelet.mount
+    fi
+
     systemctl enable crio iscsid kubelet || exit
     systemctl start crio iscsid || exit
 
@@ -63,10 +74,15 @@ write_files:
     EOF
     semodule -i longhorn-issue4988.cil
 
+    rm -f /etc/cni/net.d/100-crio-bridge.conflist
+
     cat > /run/joinconfiguration <<EOF
     apiVersion: kubeadm.k8s.io/v1beta3
     kind: JoinConfiguration
     nodeRegistration:
+      taints:
+      - key: du5t1n.me/jenkins
+        effect: NoSchedule
       kubeletExtraArgs:
         provider-id: aws:///${az}/${instance_id}
         node-ip: ${internal_ip}