dustin
/
dynk8s-provisioner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

terraform: Add config for auto-scaling group 

The Cluser Autoscaler uses EC2 Auto-Scaling Groups to configure the
instances it launches when it determines additional worker nodes are
necessary.  Auto-Scaling Groups have an associated Launch Template,
which describes the properties of the instances, such as AMI ID,
instance type, security groups, etc.

When instances are first launched, they need to be configured to join
the on-premises Kubernetes cluster.  This is handled by *cloud-init*
using the configuration in the instance user data.  The configuration
supplied here specifies the Fedora packages that need to be installed on
a Kubernetes worker node, plus some additional configuration required by
`kubeadm`, `kubelet`, and/or `cri-o`.  It also includes a script that
fetches the WireGuard client configuration and connects to the VPN,
finalizes the setup process, and joins the cluster.
master
Dustin 2022-10-11 21:40:42 -05:00
parent c48076b8f0
commit e11f98b430
5 changed files with 430 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.editorconfig
View File

@ -13,3 +13,6 @@ max_line_length = 79
max_line_length = 79
indent_style = space
indent_size = 4
[**.tf]
indent_size = 2

71
terraform/asg.tf Normal file
View File

@ -0,0 +1,71 @@
resource "aws_security_group" "k8s-node" {
name = "k8s-node"
description = "Kubernetes Node"
egress {
from_port = 19998
to_port = 19998
protocol = "udp"
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
}
egress {
from_port = 443
to_port = 443
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
}
egress {
from_port = 80
to_port = 80
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
}
}
resource "aws_launch_template" "k8s-aarch64" {
name = "k8s-aarch64"
update_default_version = true
image_id = "ami-000ec96ccb51eb679"
instance_type = "t4g.medium"
security_group_names = [aws_security_group.k8s-node.name]
user_data = filebase64("${path.module}/userdata.yml")
instance_market_options {
market_type = "spot"
}
private_dns_name_options {
hostname_type = "resource-name"
}
}
resource "aws_autoscaling_group" "k8s-aarch64" {
name = "k8s-aarch64"
availability_zones = ["us-east-2a", "us-east-2b", "us-east-2c"]
min_size = 0
max_size = 1
launch_template {
id = aws_launch_template.k8s-aarch64.id
version = "$Latest"
}
tag {
key = "k8s.io/cluster-autoscaler/enabled"
value = "true"
propagate_at_launch = true
}
tag {
key = "k8s.io/cluster-autoscaler/kubernetes"
value = "owned"
propagate_at_launch = true
}
}

80
terraform/iam-policy.json
View File

@ -5,45 +5,75 @@
"Sid": "VisualEditor0",
"Effect": "Allow",
"Action": [
"sns:ListTagsForResource",
"events:DescribeRule",
"sns:GetTopicAttributes",
"events:EnableRule",
"sns:DeleteTopic",
"events:PutRule",
"sns:CreateTopic",
"sns:SetTopicAttributes",
"events:DeleteRule",
"events:PutTargets",
"events:ListTagsForResource",
"sns:Subscribe",
"events:RemoveTargets",
"events:ListTargetsByRule",
"events:DisableRule"
"ec2:DescribeLaunchTemplates",
"autoscaling:DescribeAutoScalingGroups",
"ec2:DescribeLaunchTemplateVersions",
"autoscaling:DescribeTags",
"sns:Unsubscribe",
"sns:GetSubscriptionAttributes",
"ec2:DescribeSecurityGroups"
],
"Resource": [
"arn:aws:sns:*:566967686773:*",
"arn:aws:events:*:566967686773:rule/*"
]
"Resource": "*"
},
{
"Sid": "VisualEditor1",
"Effect": "Allow",
"Action": [
"events:DescribeEventBus",
"autoscaling:DeleteTags",
"sns:ListTagsForResource",
"ec2:UpdateSecurityGroupRuleDescriptionsEgress",
"sns:GetTopicAttributes",
"ec2:DeleteTags",
"events:CreateEventBus",
"events:DeleteEventBus"
"sns:DeleteTopic",
"ec2:CreateTags",
"sns:CreateTopic",
"sns:SetTopicAttributes",
"ec2:ModifySecurityGroupRules",
"ec2:UpdateSecurityGroupRuleDescriptionsIngress",
"events:DescribeEventBus",
"ec2:RevokeSecurityGroupIngress",
"autoscaling:CreateOrUpdateTags",
"ec2:CreateSecurityGroup",
"ec2:RevokeSecurityGroupEgress",
"ec2:DeleteSecurityGroup",
"events:DeleteEventBus",
"autoscaling:UpdateAutoScalingGroup",
"sns:Subscribe",
"autoscaling:DeleteAutoScalingGroup",
"autoscaling:CreateAutoScalingGroup"
],
"Resource": "arn:aws:events:*:566967686773:event-bus/*"
"Resource": [
"arn:aws:events:*:566967686773:event-bus/*",
"arn:aws:autoscaling:*:566967686773:autoScalingGroup:*:autoScalingGroupName/*",
"arn:aws:sns:*:566967686773:*",
"arn:aws:ec2:*:566967686773:security-group/*",
"arn:aws:ec2:*:566967686773:security-group-rule/*"
]
},
{
"Sid": "VisualEditor2",
"Effect": "Allow",
"Action": [
"sns:Unsubscribe",
"sns:GetSubscriptionAttributes"
"events:DescribeRule",
"ec2:DeleteLaunchTemplate",
"events:EnableRule",
"events:PutRule",
"ec2:CreateLaunchTemplateVersion",
"events:DeleteRule",
"events:PutTargets",
"ec2:CreateLaunchTemplate",
"events:ListTagsForResource",
"events:RemoveTargets",
"ec2:ModifyLaunchTemplate",
"ec2:DeleteLaunchTemplateVersions",
"events:ListTargetsByRule",
"events:DisableRule"
],
"Resource": "*"
"Resource": [
"arn:aws:events:*:566967686773:rule/*",
"arn:aws:ec2:*:566967686773:launch-template/*"
]
}
]
}

231
terraform/terraform.tfstate
View File

@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.2.9",
"serial": 49,
"serial": 78,
"lineage": "a100be74-c98e-0769-2d6a-bf6a2c5f3ebf",
"outputs": {},
"resources": [
@ -15,9 +15,9 @@
"schema_version": 0,
"attributes": {
"account_id": "566967686773",
"arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1664301518318294107",
"arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1665542385873038019",
"id": "566967686773",
"user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1664301518318294107"
"user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1665542385873038019"
},
"sensitive_attributes": []
}
@ -107,6 +107,85 @@
}
]
},
{
"mode": "managed",
"type": "aws_autoscaling_group",
"name": "k8s-aarch64",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:autoscaling:us-east-2:566967686773:autoScalingGroup:e5c074b6-9c58-478c-92ec-f0685465ca3d:autoScalingGroupName/k8s-aarch64",
"availability_zones": [
"us-east-2a",
"us-east-2b",
"us-east-2c"
],
"capacity_rebalance": false,
"context": "",
"default_cooldown": 300,
"default_instance_warmup": 0,
"desired_capacity": 0,
"enabled_metrics": [],
"force_delete": false,
"force_delete_warm_pool": false,
"health_check_grace_period": 300,
"health_check_type": "EC2",
"id": "k8s-aarch64",
"initial_lifecycle_hook": [],
"instance_refresh": [],
"launch_configuration": "",
"launch_template": [
{
"id": "lt-0789a3800bdaec215",
"name": "k8s-aarch64",
"version": "$Latest"
}
],
"load_balancers": [],
"max_instance_lifetime": 0,
"max_size": 1,
"metrics_granularity": "1Minute",
"min_elb_capacity": null,
"min_size": 0,
"mixed_instances_policy": [],
"name": "k8s-aarch64",
"name_prefix": "",
"placement_group": "",
"protect_from_scale_in": false,
"service_linked_role_arn": "arn:aws:iam::566967686773:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling",
"suspended_processes": [],
"tag": [
{
"key": "k8s.io/cluster-autoscaler/enabled",
"propagate_at_launch": true,
"value": "true"
},
{
"key": "k8s.io/cluster-autoscaler/kubernetes",
"propagate_at_launch": true,
"value": "owned"
}
],
"tags": null,
"target_group_arns": [],
"termination_policies": [],
"timeouts": null,
"vpc_zone_identifier": [],
"wait_for_capacity_timeout": "10m",
"wait_for_elb_capacity": null,
"warm_pool": []
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjo2MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfX0=",
"dependencies": [
"aws_launch_template.k8s-aarch64",
"aws_security_group.k8s-node"
]
}
]
},
{
"mode": "managed",
"type": "aws_cloudwatch_event_rule",
@ -171,6 +250,152 @@
}
]
},
{
"mode": "managed",
"type": "aws_launch_template",
"name": "k8s-aarch64",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:ec2:us-east-2:566967686773:launch-template/lt-0789a3800bdaec215",
"block_device_mappings": [],
"capacity_reservation_specification": [],
"cpu_options": [],
"credit_specification": [],
"default_version": 6,
"description": "",
"disable_api_stop": false,
"disable_api_termination": false,
"ebs_optimized": "",
"elastic_gpu_specifications": [],
"elastic_inference_accelerator": [],
"enclave_options": [],
"hibernation_options": [],
"iam_instance_profile": [],
"id": "lt-0789a3800bdaec215",
"image_id": "ami-000ec96ccb51eb679",
"instance_initiated_shutdown_behavior": "",
"instance_market_options": [
{
"market_type": "spot",
"spot_options": []
}
],
"instance_requirements": [],
"instance_type": "t4g.medium",
"kernel_id": "",
"key_name": "",
"latest_version": 6,
"license_specification": [],
"maintenance_options": [],
"metadata_options": [],
"monitoring": [],
"name": "k8s-aarch64",
"name_prefix": "",
"network_interfaces": [],
"placement": [],
"private_dns_name_options": [
{
"enable_resource_name_dns_a_record": false,
"enable_resource_name_dns_aaaa_record": false,
"hostname_type": "resource-name"
}
],
"ram_disk_id": "",
"security_group_names": [
"k8s-node"
],
"tag_specifications": [],
"tags": {},
"tags_all": {},
"update_default_version": true,
"user_data": "I2Nsb3VkLWNvbmZpZwpib290Y21kOgotIFsgZG5mLCBtb2R1bGUsIGVuYWJsZSwgJ2NyaS1vOjEuMjInLCAteSBdCi0gWyBsbiwgLXNmLCAvcnVuL3N5c3RlbWQvcmVzb2x2ZS9zdHViLXJlc29sdi5jb25mLCAvZXRjL3Jlc29sdi5jb25mIF0KCnBhY2thZ2VzOgotIGNyaS1vCi0gY3JpLXRvb2xzCi0gZXRodG9vbAotIGlwdGFibGVzLW5mdAotIGlzY3NpLWluaXRpYXRvci11dGlscwotIGt1YmVybmV0ZXMta3ViZWFkbQotIGt1YmVybmV0ZXMtbm9kZQotIHdpcmVndWFyZC10b29scwoKd3JpdGVfZmlsZXM6Ci0gcGF0aDogL2V0Yy9kbmYvZG5mLmNvbmYKICBjb250ZW50OiB8KwogICAgaW5zdGFsbF93ZWFrX2RlcHM9RmFsc2UKICBhcHBlbmQ6IHRydWUKLSBwYXRoOiAvZXRjL21vZHVsZXMtbG9hZC5kL2s4cy5jb25mCiAgY29udGVudDogfCsKICAgIGJyX25ldGZpbHRlcgotIHBhdGg6IC9ldGMvc3lzY3RsLmQvazhzLmNvbmYKICBjb250ZW50OiB8KwogICAgbmV0LmJyaWRnZS5icmlkZ2UtbmYtY2FsbC1pcHRhYmxlcyA9IDEKICAgIG5ldC5icmlkZ2UuYnJpZGdlLW5mLWNhbGwtaXA2dGFibGVzID0gMQogICAgbmV0LmlwdjQuaXBfZm9yd2FyZCA9IDEKLSBwYXRoOiAvdmFyL2xpYi9jbG91ZC9zY3JpcHRzL3Blci1pbnN0YW5jZS9rdWJlYWRtLWpvaW4KICBwZXJtaXNzaW9uczogJzA3NTUnCiAgY29udGVudDogfCsKICAgICMhL2Jpbi9zaAoKICAgIEJBU0VfVVJMPWh0dHBzOi8vZHluazhzLXByb3Zpc2lvbmVyLnB5cm9jdWZmbGluay5uZXQKCiAgICBpbnN0YW5jZV9pZD0kKGN1cmwgLXMgMTY5LjI1NC4xNjkuMjU0L2xhdGVzdC9tZXRhLWRhdGEvaW5zdGFuY2UtaWQpCiAgICBhej0kKGN1cmwgLXMgMTY5LjI1NC4xNjkuMjU0L2xhdGVzdC9tZXRhLWRhdGEvcGxhY2VtZW50L2F2YWlsYWJpbGl0eS16b25lKQoKICAgIGN1cmwgLWZzICIke0JBU0VfVVJMfSIvd2lyZWd1YXJkL2NvbmZpZy8ke2luc3RhbmNlX2lkfSBcCiAgICAgICAgLW8gL2V0Yy93aXJlZ3VhcmQvd2cwLmNvbmYgfHwgZXhpdAogICAgc3lzdGVtY3RsIGVuYWJsZSAtLW5vdyB3Zy1xdWlja0B3ZzAgfHwgZXhpdAoKICAgIG1vZHByb2JlIGJyX25ldGZpbHRlciB8fCBleGl0CiAgICBzeXNjdGwgLXcgLWYgL2V0Yy9zeXNjdGwuZC9rOHMuY29uZiB8fCBleGl0CgogICAgc3dhcG9mZiAtYSB8fCBleGl0CiAgICB0b3VjaCAvZXRjL3N5c3RlbWQvenJhbS1nZW5lcmF0b3IuY29uZiB8fCBleGl0CiAgICBzeXN0ZW1jdGwgZGFlbW9uLXJlbG9hZCB8fCBleGl0CiAgICBzeXN0ZW1jdGwgc3RvcCAnc3lzdGVtZC16cmFtLXNldHVwQConIHx8IGV4aXQKCiAgICBzeXN0ZW1jdGwgZW5hYmxlIGNyaW8gaXNjc2lkIGt1YmVsZXQgfHwgZXhpdAogICAgc3lzdGVtY3RsIHN0YXJ0IGNyaW8gaXNjc2lkIHx8IGV4aXQKCiAgICBpbnRlcm5hbF9pcD0kKAogICAgICBpcCBhZGRyZXNzIHNob3cgZGV2IHdnMCBwcmltYXJ5IHwgXAogICAgICBzZWQgLXJuICdzLy4qaW5ldCAoWzAtOS5dKykuKi9cMS9wJwogICAgKQoKICAgIGNhdCA+IC9ydW4vam9pbmNvbmZpZ3VyYXRpb24gPDxFT0YKICAgIGFwaVZlcnNpb246IGt1YmVhZG0uazhzLmlvL3YxYmV0YTMKICAgIGtpbmQ6IEpvaW5Db25maWd1cmF0aW9uCiAgICBub2RlUmVnaXN0cmF0aW9uOgogICAgICBrdWJlbGV0RXh0cmFBcmdzOgogICAgICAgIHByb3ZpZGVyLWlkOiBhd3M6Ly8vJHthen0vJHtpbnN0YW5jZV9pZH0KICAgICAgICBub2RlLWlwOiAke2ludGVybmFsX2lwfQogICAgZGlzY292ZXJ5OgogICAgICBmaWxlOgogICAgICAgIGt1YmVDb25maWdQYXRoOiAke0JBU0VfVVJMfS9rdWJlYWRtL2t1YmVjb25maWcvJHtpbnN0YW5jZV9pZH0KICAgIEVPRgogICAga3ViZWFkbSBqb2luIC0tY29uZmlnPS9ydW4vam9pbmNvbmZpZ3VyYXRpb24KCnJ1bmNtZDoKLSBbIGRuZiwgcmVtb3ZlLCAteSwgenJhbS1nZW5lcmF0b3IgXQo=",
"vpc_security_group_ids": []
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_security_group.k8s-node"
]
}
]
},
{
"mode": "managed",
"type": "aws_security_group",
"name": "k8s-node",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ec2:us-east-2:566967686773:security-group/sg-05258c3ff1812e83b",
"description": "Kubernetes Node",
"egress": [
{
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "",
"from_port": 19998,
"ipv6_cidr_blocks": [
"::/0"
],
"prefix_list_ids": [],
"protocol": "udp",
"security_groups": [],
"self": false,
"to_port": 19998
},
{
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "",
"from_port": 443,
"ipv6_cidr_blocks": [
"::/0"
],
"prefix_list_ids": [],
"protocol": "tcp",
"security_groups": [],
"self": false,
"to_port": 443
},
{
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "",
"from_port": 80,
"ipv6_cidr_blocks": [
"::/0"
],
"prefix_list_ids": [],
"protocol": "tcp",
"security_groups": [],
"self": false,
"to_port": 80
}
],
"id": "sg-05258c3ff1812e83b",
"ingress": [],
"name": "k8s-node",
"name_prefix": "",
"owner_id": "566967686773",
"revoke_rules_on_delete": false,
"tags": {},
"tags_all": {},
"timeouts": null,
"vpc_id": "vpc-2483044d"
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
}
]
},
{
"mode": "managed",
"type": "aws_sns_topic",

73
terraform/userdata.yml Normal file
View File

@ -0,0 +1,73 @@
#cloud-config
bootcmd:
- [ dnf, module, enable, 'cri-o:1.22', -y ]
- [ ln, -sf, /run/systemd/resolve/stub-resolv.conf, /etc/resolv.conf ]
packages:
- cri-o
- cri-tools
- ethtool
- iptables-nft
- iscsi-initiator-utils
- kubernetes-kubeadm
- kubernetes-node
- wireguard-tools
write_files:
- path: /etc/dnf/dnf.conf
content: |+
install_weak_deps=False
append: true
- path: /etc/modules-load.d/k8s.conf
content: |+
br_netfilter
- path: /etc/sysctl.d/k8s.conf
content: |+
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
- path: /var/lib/cloud/scripts/per-instance/kubeadm-join
permissions: '0755'
content: |+
#!/bin/sh
BASE_URL=https://dynk8s-provisioner.pyrocufflink.net
instance_id=$(curl -s 169.254.169.254/latest/meta-data/instance-id)
az=$(curl -s 169.254.169.254/latest/meta-data/placement/availability-zone)
curl -fs "${BASE_URL}"/wireguard/config/${instance_id} \
-o /etc/wireguard/wg0.conf || exit
systemctl enable --now wg-quick@wg0 || exit
modprobe br_netfilter || exit
sysctl -w -f /etc/sysctl.d/k8s.conf || exit
swapoff -a || exit
touch /etc/systemd/zram-generator.conf || exit
systemctl daemon-reload || exit
systemctl stop 'systemd-zram-setup@*' || exit
systemctl enable crio iscsid kubelet || exit
systemctl start crio iscsid || exit
internal_ip=$(
ip address show dev wg0 primary | \
sed -rn 's/.*inet ([0-9.]+).*/\1/p'
)
cat > /run/joinconfiguration <<EOF
apiVersion: kubeadm.k8s.io/v1beta3
kind: JoinConfiguration
nodeRegistration:
kubeletExtraArgs:
provider-id: aws:///${az}/${instance_id}
node-ip: ${internal_ip}
discovery:
file:
kubeConfigPath: ${BASE_URL}/kubeadm/kubeconfig/${instance_id}
EOF
kubeadm join --config=/run/joinconfiguration
runcmd:
- [ dnf, remove, -y, zram-generator ]