terraform: Add config for auto-scaling group

The Cluser Autoscaler uses EC2 Auto-Scaling Groups to configure the instances it launches when it determines additional worker nodes are necessary. Auto-Scaling Groups have an associated Launch Template, which describes the properties of the instances, such as AMI ID, instance type, security groups, etc. When instances are first launched, they need to be configured to join the on-premises Kubernetes cluster. This is handled by *cloud-init* using the configuration in the instance user data. The configuration supplied here specifies the Fedora packages that need to be installed on a Kubernetes worker node, plus some additional configuration required by `kubeadm`, `kubelet`, and/or `cri-o`. It also includes a script that fetches the WireGuard client configuration and connects to the VPN, finalizes the setup process, and joins the cluster.
2022-10-11 21:40:42 -05:00 · 2022-10-11 21:40:42 -05:00 · e11f98b430
parent c48076b8f0
commit e11f98b430
5 changed files with 430 additions and 28 deletions
--- a/.editorconfig
+++ b/.editorconfig
@ -13,3 +13,6 @@ max_line_length = 79
 max_line_length = 79
 indent_style = space
 indent_size = 4
+
+[**.tf]
+indent_size = 2
--- a/terraform/asg.tf
+++ b/terraform/asg.tf
@ -0,0 +1,71 @@
+resource "aws_security_group" "k8s-node" {
+  name        = "k8s-node"
+  description = "Kubernetes Node"
+
+  egress {
+    from_port        = 19998
+    to_port          = 19998
+    protocol         = "udp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 443
+    to_port          = 443
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 80
+    to_port          = 80
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+}
+
+resource "aws_launch_template" "k8s-aarch64" {
+  name = "k8s-aarch64"
+
+  update_default_version = true
+  image_id               = "ami-000ec96ccb51eb679"
+  instance_type          = "t4g.medium"
+  security_group_names   = [aws_security_group.k8s-node.name]
+
+  user_data = filebase64("${path.module}/userdata.yml")
+
+  instance_market_options {
+    market_type = "spot"
+  }
+
+  private_dns_name_options {
+    hostname_type = "resource-name"
+  }
+}
+
+resource "aws_autoscaling_group" "k8s-aarch64" {
+  name = "k8s-aarch64"
+
+  availability_zones = ["us-east-2a", "us-east-2b", "us-east-2c"]
+  min_size           = 0
+  max_size           = 1
+
+  launch_template {
+    id      = aws_launch_template.k8s-aarch64.id
+    version = "$Latest"
+  }
+
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/enabled"
+    value               = "true"
+    propagate_at_launch = true
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/kubernetes"
+    value               = "owned"
+    propagate_at_launch = true
+  }
+}
--- a/terraform/iam-policy.json
+++ b/terraform/iam-policy.json
@ -5,45 +5,75 @@
            "Sid": "VisualEditor0",
            "Effect": "Allow",
            "Action": [
-                "sns:ListTagsForResource",
-                "events:DescribeRule",
-                "sns:GetTopicAttributes",
-                "events:EnableRule",
-                "sns:DeleteTopic",
-                "events:PutRule",
-                "sns:CreateTopic",
-                "sns:SetTopicAttributes",
-                "events:DeleteRule",
-                "events:PutTargets",
-                "events:ListTagsForResource",
-                "sns:Subscribe",
-                "events:RemoveTargets",
-                "events:ListTargetsByRule",
-                "events:DisableRule"
+                "ec2:DescribeLaunchTemplates",
+                "autoscaling:DescribeAutoScalingGroups",
+                "ec2:DescribeLaunchTemplateVersions",
+                "autoscaling:DescribeTags",
+                "sns:Unsubscribe",
+                "sns:GetSubscriptionAttributes",
+                "ec2:DescribeSecurityGroups"
            ],
-            "Resource": [
-                "arn:aws:sns:*:566967686773:*",
-                "arn:aws:events:*:566967686773:rule/*"
-            ]
+            "Resource": "*"
        },
        {
            "Sid": "VisualEditor1",
            "Effect": "Allow",
            "Action": [
-                "events:DescribeEventBus",
+                "autoscaling:DeleteTags",
+                "sns:ListTagsForResource",
+                "ec2:UpdateSecurityGroupRuleDescriptionsEgress",
+                "sns:GetTopicAttributes",
+                "ec2:DeleteTags",
                "events:CreateEventBus",
-                "events:DeleteEventBus"
+                "sns:DeleteTopic",
+                "ec2:CreateTags",
+                "sns:CreateTopic",
+                "sns:SetTopicAttributes",
+                "ec2:ModifySecurityGroupRules",
+                "ec2:UpdateSecurityGroupRuleDescriptionsIngress",
+                "events:DescribeEventBus",
+                "ec2:RevokeSecurityGroupIngress",
+                "autoscaling:CreateOrUpdateTags",
+                "ec2:CreateSecurityGroup",
+                "ec2:RevokeSecurityGroupEgress",
+                "ec2:DeleteSecurityGroup",
+                "events:DeleteEventBus",
+                "autoscaling:UpdateAutoScalingGroup",
+                "sns:Subscribe",
+                "autoscaling:DeleteAutoScalingGroup",
+                "autoscaling:CreateAutoScalingGroup"
            ],
-            "Resource": "arn:aws:events:*:566967686773:event-bus/*"
+            "Resource": [
+                "arn:aws:events:*:566967686773:event-bus/*",
+                "arn:aws:autoscaling:*:566967686773:autoScalingGroup:*:autoScalingGroupName/*",
+                "arn:aws:sns:*:566967686773:*",
+                "arn:aws:ec2:*:566967686773:security-group/*",
+                "arn:aws:ec2:*:566967686773:security-group-rule/*"
+            ]
        },
        {
            "Sid": "VisualEditor2",
            "Effect": "Allow",
            "Action": [
-                "sns:Unsubscribe",
-                "sns:GetSubscriptionAttributes"
+                "events:DescribeRule",
+                "ec2:DeleteLaunchTemplate",
+                "events:EnableRule",
+                "events:PutRule",
+                "ec2:CreateLaunchTemplateVersion",
+                "events:DeleteRule",
+                "events:PutTargets",
+                "ec2:CreateLaunchTemplate",
+                "events:ListTagsForResource",
+                "events:RemoveTargets",
+                "ec2:ModifyLaunchTemplate",
+                "ec2:DeleteLaunchTemplateVersions",
+                "events:ListTargetsByRule",
+                "events:DisableRule"
            ],
-            "Resource": "*"
+            "Resource": [
+                "arn:aws:events:*:566967686773:rule/*",
+                "arn:aws:ec2:*:566967686773:launch-template/*"
+            ]
        }
    ]
 }
--- a/terraform/terraform.tfstate
+++ b/terraform/terraform.tfstate
@ -1,7 +1,7 @@
 {
  "version": 4,
  "terraform_version": "1.2.9",
-  "serial": 49,
+  "serial": 78,
  "lineage": "a100be74-c98e-0769-2d6a-bf6a2c5f3ebf",
  "outputs": {},
  "resources": [
@ -15,9 +15,9 @@
          "schema_version": 0,
          "attributes": {
            "account_id": "566967686773",
-            "arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1664301518318294107",
+            "arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1665542385873038019",
            "id": "566967686773",
-            "user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1664301518318294107"
+            "user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1665542385873038019"
          },
          "sensitive_attributes": []
        }
@ -107,6 +107,85 @@
        }
      ]
    },
+    {
+      "mode": "managed",
+      "type": "aws_autoscaling_group",
+      "name": "k8s-aarch64",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "arn": "arn:aws:autoscaling:us-east-2:566967686773:autoScalingGroup:e5c074b6-9c58-478c-92ec-f0685465ca3d:autoScalingGroupName/k8s-aarch64",
+            "availability_zones": [
+              "us-east-2a",
+              "us-east-2b",
+              "us-east-2c"
+            ],
+            "capacity_rebalance": false,
+            "context": "",
+            "default_cooldown": 300,
+            "default_instance_warmup": 0,
+            "desired_capacity": 0,
+            "enabled_metrics": [],
+            "force_delete": false,
+            "force_delete_warm_pool": false,
+            "health_check_grace_period": 300,
+            "health_check_type": "EC2",
+            "id": "k8s-aarch64",
+            "initial_lifecycle_hook": [],
+            "instance_refresh": [],
+            "launch_configuration": "",
+            "launch_template": [
+              {
+                "id": "lt-0789a3800bdaec215",
+                "name": "k8s-aarch64",
+                "version": "$Latest"
+              }
+            ],
+            "load_balancers": [],
+            "max_instance_lifetime": 0,
+            "max_size": 1,
+            "metrics_granularity": "1Minute",
+            "min_elb_capacity": null,
+            "min_size": 0,
+            "mixed_instances_policy": [],
+            "name": "k8s-aarch64",
+            "name_prefix": "",
+            "placement_group": "",
+            "protect_from_scale_in": false,
+            "service_linked_role_arn": "arn:aws:iam::566967686773:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling",
+            "suspended_processes": [],
+            "tag": [
+              {
+                "key": "k8s.io/cluster-autoscaler/enabled",
+                "propagate_at_launch": true,
+                "value": "true"
+              },
+              {
+                "key": "k8s.io/cluster-autoscaler/kubernetes",
+                "propagate_at_launch": true,
+                "value": "owned"
+              }
+            ],
+            "tags": null,
+            "target_group_arns": [],
+            "termination_policies": [],
+            "timeouts": null,
+            "vpc_zone_identifier": [],
+            "wait_for_capacity_timeout": "10m",
+            "wait_for_elb_capacity": null,
+            "warm_pool": []
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjo2MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfX0=",
+          "dependencies": [
+            "aws_launch_template.k8s-aarch64",
+            "aws_security_group.k8s-node"
+          ]
+        }
+      ]
+    },
    {
      "mode": "managed",
      "type": "aws_cloudwatch_event_rule",
@ -171,6 +250,152 @@
        }
      ]
    },
+    {
+      "mode": "managed",
+      "type": "aws_launch_template",
+      "name": "k8s-aarch64",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "arn": "arn:aws:ec2:us-east-2:566967686773:launch-template/lt-0789a3800bdaec215",
+            "block_device_mappings": [],
+            "capacity_reservation_specification": [],
+            "cpu_options": [],
+            "credit_specification": [],
+            "default_version": 6,
+            "description": "",
+            "disable_api_stop": false,
+            "disable_api_termination": false,
+            "ebs_optimized": "",
+            "elastic_gpu_specifications": [],
+            "elastic_inference_accelerator": [],
+            "enclave_options": [],
+            "hibernation_options": [],
+            "iam_instance_profile": [],
+            "id": "lt-0789a3800bdaec215",
+            "image_id": "ami-000ec96ccb51eb679",
+            "instance_initiated_shutdown_behavior": "",
+            "instance_market_options": [
+              {
+                "market_type": "spot",
+                "spot_options": []
+              }
+            ],
+            "instance_requirements": [],
+            "instance_type": "t4g.medium",
+            "kernel_id": "",
+            "key_name": "",
+            "latest_version": 6,
+            "license_specification": [],
+            "maintenance_options": [],
+            "metadata_options": [],
+            "monitoring": [],
+            "name": "k8s-aarch64",
+            "name_prefix": "",
+            "network_interfaces": [],
+            "placement": [],
+            "private_dns_name_options": [
+              {
+                "enable_resource_name_dns_a_record": false,
+                "enable_resource_name_dns_aaaa_record": false,
+                "hostname_type": "resource-name"
+              }
+            ],
+            "ram_disk_id": "",
+            "security_group_names": [
+              "k8s-node"
+            ],
+            "tag_specifications": [],
+            "tags": {},
+            "tags_all": {},
+            "update_default_version": true,
+            "user_data": "I2Nsb3VkLWNvbmZpZwpib290Y21kOgotIFsgZG5mLCBtb2R1bGUsIGVuYWJsZSwgJ2NyaS1vOjEuMjInLCAteSBdCi0gWyBsbiwgLXNmLCAvcnVuL3N5c3RlbWQvcmVzb2x2ZS9zdHViLXJlc29sdi5jb25mLCAvZXRjL3Jlc29sdi5jb25mIF0KCnBhY2thZ2VzOgotIGNyaS1vCi0gY3JpLXRvb2xzCi0gZXRodG9vbAotIGlwdGFibGVzLW5mdAotIGlzY3NpLWluaXRpYXRvci11dGlscwotIGt1YmVybmV0ZXMta3ViZWFkbQotIGt1YmVybmV0ZXMtbm9kZQotIHdpcmVndWFyZC10b29scwoKd3JpdGVfZmlsZXM6Ci0gcGF0aDogL2V0Yy9kbmYvZG5mLmNvbmYKICBjb250ZW50OiB8KwogICAgaW5zdGFsbF93ZWFrX2RlcHM9RmFsc2UKICBhcHBlbmQ6IHRydWUKLSBwYXRoOiAvZXRjL21vZHVsZXMtbG9hZC5kL2s4cy5jb25mCiAgY29udGVudDogfCsKICAgIGJyX25ldGZpbHRlcgotIHBhdGg6IC9ldGMvc3lzY3RsLmQvazhzLmNvbmYKICBjb250ZW50OiB8KwogICAgbmV0LmJyaWRnZS5icmlkZ2UtbmYtY2FsbC1pcHRhYmxlcyA9IDEKICAgIG5ldC5icmlkZ2UuYnJpZGdlLW5mLWNhbGwtaXA2dGFibGVzID0gMQogICAgbmV0LmlwdjQuaXBfZm9yd2FyZCA9IDEKLSBwYXRoOiAvdmFyL2xpYi9jbG91ZC9zY3JpcHRzL3Blci1pbnN0YW5jZS9rdWJlYWRtLWpvaW4KICBwZXJtaXNzaW9uczogJzA3NTUnCiAgY29udGVudDogfCsKICAgICMhL2Jpbi9zaAoKICAgIEJBU0VfVVJMPWh0dHBzOi8vZHluazhzLXByb3Zpc2lvbmVyLnB5cm9jdWZmbGluay5uZXQKCiAgICBpbnN0YW5jZV9pZD0kKGN1cmwgLXMgMTY5LjI1NC4xNjkuMjU0L2xhdGVzdC9tZXRhLWRhdGEvaW5zdGFuY2UtaWQpCiAgICBhej0kKGN1cmwgLXMgMTY5LjI1NC4xNjkuMjU0L2xhdGVzdC9tZXRhLWRhdGEvcGxhY2VtZW50L2F2YWlsYWJpbGl0eS16b25lKQoKICAgIGN1cmwgLWZzICIke0JBU0VfVVJMfSIvd2lyZWd1YXJkL2NvbmZpZy8ke2luc3RhbmNlX2lkfSBcCiAgICAgICAgLW8gL2V0Yy93aXJlZ3VhcmQvd2cwLmNvbmYgfHwgZXhpdAogICAgc3lzdGVtY3RsIGVuYWJsZSAtLW5vdyB3Zy1xdWlja0B3ZzAgfHwgZXhpdAoKICAgIG1vZHByb2JlIGJyX25ldGZpbHRlciB8fCBleGl0CiAgICBzeXNjdGwgLXcgLWYgL2V0Yy9zeXNjdGwuZC9rOHMuY29uZiB8fCBleGl0CgogICAgc3dhcG9mZiAtYSB8fCBleGl0CiAgICB0b3VjaCAvZXRjL3N5c3RlbWQvenJhbS1nZW5lcmF0b3IuY29uZiB8fCBleGl0CiAgICBzeXN0ZW1jdGwgZGFlbW9uLXJlbG9hZCB8fCBleGl0CiAgICBzeXN0ZW1jdGwgc3RvcCAnc3lzdGVtZC16cmFtLXNldHVwQConIHx8IGV4aXQKCiAgICBzeXN0ZW1jdGwgZW5hYmxlIGNyaW8gaXNjc2lkIGt1YmVsZXQgfHwgZXhpdAogICAgc3lzdGVtY3RsIHN0YXJ0IGNyaW8gaXNjc2lkIHx8IGV4aXQKCiAgICBpbnRlcm5hbF9pcD0kKAogICAgICBpcCBhZGRyZXNzIHNob3cgZGV2IHdnMCBwcmltYXJ5IHwgXAogICAgICBzZWQgLXJuICdzLy4qaW5ldCAoWzAtOS5dKykuKi9cMS9wJwogICAgKQoKICAgIGNhdCA+IC9ydW4vam9pbmNvbmZpZ3VyYXRpb24gPDxFT0YKICAgIGFwaVZlcnNpb246IGt1YmVhZG0uazhzLmlvL3YxYmV0YTMKICAgIGtpbmQ6IEpvaW5Db25maWd1cmF0aW9uCiAgICBub2RlUmVnaXN0cmF0aW9uOgogICAgICBrdWJlbGV0RXh0cmFBcmdzOgogICAgICAgIHByb3ZpZGVyLWlkOiBhd3M6Ly8vJHthen0vJHtpbnN0YW5jZV9pZH0KICAgICAgICBub2RlLWlwOiAke2ludGVybmFsX2lwfQogICAgZGlzY292ZXJ5OgogICAgICBmaWxlOgogICAgICAgIGt1YmVDb25maWdQYXRoOiAke0JBU0VfVVJMfS9rdWJlYWRtL2t1YmVjb25maWcvJHtpbnN0YW5jZV9pZH0KICAgIEVPRgogICAga3ViZWFkbSBqb2luIC0tY29uZmlnPS9ydW4vam9pbmNvbmZpZ3VyYXRpb24KCnJ1bmNtZDoKLSBbIGRuZiwgcmVtb3ZlLCAteSwgenJhbS1nZW5lcmF0b3IgXQo=",
+            "vpc_security_group_ids": []
+          },
+          "sensitive_attributes": [],
+          "private": "bnVsbA==",
+          "dependencies": [
+            "aws_security_group.k8s-node"
+          ]
+        }
+      ]
+    },
+    {
+      "mode": "managed",
+      "type": "aws_security_group",
+      "name": "k8s-node",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ec2:us-east-2:566967686773:security-group/sg-05258c3ff1812e83b",
+            "description": "Kubernetes Node",
+            "egress": [
+              {
+                "cidr_blocks": [
+                  "0.0.0.0/0"
+                ],
+                "description": "",
+                "from_port": 19998,
+                "ipv6_cidr_blocks": [
+                  "::/0"
+                ],
+                "prefix_list_ids": [],
+                "protocol": "udp",
+                "security_groups": [],
+                "self": false,
+                "to_port": 19998
+              },
+              {
+                "cidr_blocks": [
+                  "0.0.0.0/0"
+                ],
+                "description": "",
+                "from_port": 443,
+                "ipv6_cidr_blocks": [
+                  "::/0"
+                ],
+                "prefix_list_ids": [],
+                "protocol": "tcp",
+                "security_groups": [],
+                "self": false,
+                "to_port": 443
+              },
+              {
+                "cidr_blocks": [
+                  "0.0.0.0/0"
+                ],
+                "description": "",
+                "from_port": 80,
+                "ipv6_cidr_blocks": [
+                  "::/0"
+                ],
+                "prefix_list_ids": [],
+                "protocol": "tcp",
+                "security_groups": [],
+                "self": false,
+                "to_port": 80
+              }
+            ],
+            "id": "sg-05258c3ff1812e83b",
+            "ingress": [],
+            "name": "k8s-node",
+            "name_prefix": "",
+            "owner_id": "566967686773",
+            "revoke_rules_on_delete": false,
+            "tags": {},
+            "tags_all": {},
+            "timeouts": null,
+            "vpc_id": "vpc-2483044d"
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
+        }
+      ]
+    },
    {
      "mode": "managed",
      "type": "aws_sns_topic",
--- a/terraform/userdata.yml
+++ b/terraform/userdata.yml
@ -0,0 +1,73 @@
+#cloud-config
+bootcmd:
+- [ dnf, module, enable, 'cri-o:1.22', -y ]
+- [ ln, -sf, /run/systemd/resolve/stub-resolv.conf, /etc/resolv.conf ]
+
+packages:
+- cri-o
+- cri-tools
+- ethtool
+- iptables-nft
+- iscsi-initiator-utils
+- kubernetes-kubeadm
+- kubernetes-node
+- wireguard-tools
+
+write_files:
+- path: /etc/dnf/dnf.conf
+  content: |+
+    install_weak_deps=False
+  append: true
+- path: /etc/modules-load.d/k8s.conf
+  content: |+
+    br_netfilter
+- path: /etc/sysctl.d/k8s.conf
+  content: |+
+    net.bridge.bridge-nf-call-iptables = 1
+    net.bridge.bridge-nf-call-ip6tables = 1
+    net.ipv4.ip_forward = 1
+- path: /var/lib/cloud/scripts/per-instance/kubeadm-join
+  permissions: '0755'
+  content: |+
+    #!/bin/sh
+
+    BASE_URL=https://dynk8s-provisioner.pyrocufflink.net
+
+    instance_id=$(curl -s 169.254.169.254/latest/meta-data/instance-id)
+    az=$(curl -s 169.254.169.254/latest/meta-data/placement/availability-zone)
+
+    curl -fs "${BASE_URL}"/wireguard/config/${instance_id} \
+        -o /etc/wireguard/wg0.conf || exit
+    systemctl enable --now wg-quick@wg0 || exit
+
+    modprobe br_netfilter || exit
+    sysctl -w -f /etc/sysctl.d/k8s.conf || exit
+
+    swapoff -a || exit
+    touch /etc/systemd/zram-generator.conf || exit
+    systemctl daemon-reload || exit
+    systemctl stop 'systemd-zram-setup@*' || exit
+
+    systemctl enable crio iscsid kubelet || exit
+    systemctl start crio iscsid || exit
+
+    internal_ip=$(
+      ip address show dev wg0 primary | \
+      sed -rn 's/.*inet ([0-9.]+).*/\1/p'
+    )
+
+    cat > /run/joinconfiguration <<EOF
+    apiVersion: kubeadm.k8s.io/v1beta3
+    kind: JoinConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        provider-id: aws:///${az}/${instance_id}
+        node-ip: ${internal_ip}
+    discovery:
+      file:
+        kubeConfigPath: ${BASE_URL}/kubeadm/kubeconfig/${instance_id}
+    EOF
+    kubeadm join --config=/run/joinconfiguration
+
+runcmd:
+- [ dnf, remove, -y, zram-generator ]