examples: Add Kubernetes manifest

The `dynk8s-provisioner.yaml` file contains an example of how to deploy
the *dynk8s-provisioner* in Kubernetes using `kubectl`.

.editorconfig

@@ -13,3 +13,6 @@ max_line_length = 79
 max_line_length = 79
 indent_style = space
 indent_size = 4
+
+[**.tf]
+indent_size = 2

examples/dynk8s-provisioner.yaml

@@ -0,0 +1,311 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dynk8s
+  labels:
+    kubernetes.io/metadata.name: dynk8s
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+automountServiceAccountToken: true
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - secrets
+  verbs:
+  - '*'
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: dynk8s-provisioner
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - secrets
+  verbs:
+  - '*'
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: dynk8s-provisioner
+  namespace: kube-public
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - configmaps
+  resourceNames:
+  - cluster-info
+  verbs:
+  - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: dynk8s-provisioner
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - nodes
+  verbs:
+  - list
+  - get
+  - delete
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: dynk8s-provisioner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: dynk8s-provisioner
+subjects:
+- kind: ServiceAccount
+  name: dynk8s-provisioner
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: dynk8s-provisioner
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: dynk8s-provisioner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: dynk8s-provisioner
+subjects:
+- kind: ServiceAccount
+  name: dynk8s-provisioner
+  namespace: dynk8s
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: dynk8s-provisioner
+  namespace: kube-public
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: dynk8s-provisioner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: dynk8s-provisioner
+subjects:
+- kind: ServiceAccount
+  name: dynk8s-provisioner
+  namespace: dynk8s
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: dynk8s-provisioner
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: dynk8s-provisioner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: dynk8s-provisioner
+subjects:
+- kind: ServiceAccount
+  name: dynk8s-provisioner
+  namespace: dynk8s
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: dynk8s-provisioner-pvc
+  namespace: dynk8s
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner-pvc
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: storage
+    app.kubernetes.io/part-of: dynk8s-provisioner
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+spec:
+  serviceName: dynk8s-provisioner
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: dynk8s-provisioner
+      app.kubernetes.io/instance: default
+      app.kubernetes.io/component: http-api
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: dynk8s-provisioner
+        app.kubernetes.io/instance: default
+        app.kubernetes.io/component: http-api
+    spec:
+      containers:
+      - env:
+        - name: ROCKET_ADDRESS
+          value: 0.0.0.0
+        - name: ROCKET_LOG_LEVEL
+          value: normal
+        image: git.pyrocufflink.net/packages/dynk8s-provisioner:master
+        imagePullPolicy: Always
+        name: dynk8s-provisioner
+        ports:
+        - containerPort: 8000
+          name: http
+        startupProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /
+            port: 8000
+          initialDelaySeconds: 1
+          periodSeconds: 2
+          successThreshold: 1
+          timeoutSeconds: 1
+        volumeMounts:
+        - mountPath: /data
+          name: dynk8s-provisioner
+        workingDir: /data
+      serviceAccountName: dynk8s-provisioner
+      volumes:
+      - name: dynk8s-provisioner
+        persistentVolumeClaim:
+          claimName: dynk8s-provisioner-pvc
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+spec:
+  selector:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+  ports:
+  - port: 8000
+    name: http
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: dynk8s-provisioner
+  namespace: dynk8s
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+spec:
+  ingressClassName: nginx
+  tls:
+  - hosts:
+    - dynk8s-provisioner.pyrocufflink.net
+  defaultBackend:
+    service:
+      name: dynk8s-provisioner
+      port:
+        number: 8000
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: wireguard-config-0
+  namespace: dynk8s
+  labels:
+    app.kubernetes.io/part-of: dynk8s-provisioner
+    dynk8s.du5t1n.me/ec2-instance-id: ''
+type: dynk8s.du5t1n.me/wireguard-config
+stringData:
+  wireguard-config: |+
+    [Interface]
+    Address = 10.11.12.13/14
+    PrivateKey = UEdAkIaF80zhlOpgacOYL2UckrfCAWXfsDDSAAzNH3g=
+
+    [Peer]
+    PublicKey = zbeTpUFA014kvTezIEGBt4yi3BVocST9j1dBElp9liI=
+    PreSharedKey = V6hAm01dxv2ib8AML2dSyX68hlPZm8En+IXfsknK3Zc=
+    AllowedIPs = 0.0.0.0/0
+    Endpoint = wireguard.example.org:24680

terraform/asg.tf

@@ -0,0 +1,71 @@
+resource "aws_security_group" "k8s-node" {
+  name        = "k8s-node"
+  description = "Kubernetes Node"
+
+  egress {
+    from_port        = 19998
+    to_port          = 19998
+    protocol         = "udp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 443
+    to_port          = 443
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 80
+    to_port          = 80
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+}
+
+resource "aws_launch_template" "k8s-aarch64" {
+  name = "k8s-aarch64"
+
+  update_default_version = true
+  image_id               = "ami-000ec96ccb51eb679"
+  instance_type          = "t4g.medium"
+  security_group_names   = [aws_security_group.k8s-node.name]
+
+  user_data = filebase64("${path.module}/userdata.yml")
+
+  instance_market_options {
+    market_type = "spot"
+  }
+
+  private_dns_name_options {
+    hostname_type = "resource-name"
+  }
+}
+
+resource "aws_autoscaling_group" "k8s-aarch64" {
+  name = "k8s-aarch64"
+
+  availability_zones = ["us-east-2a", "us-east-2b", "us-east-2c"]
+  min_size           = 0
+  max_size           = 1
+
+  launch_template {
+    id      = aws_launch_template.k8s-aarch64.id
+    version = "$Latest"
+  }
+
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/enabled"
+    value               = "true"
+    propagate_at_launch = true
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/kubernetes"
+    value               = "owned"
+    propagate_at_launch = true
+  }
+}

terraform/iam-policy.json

@@ -5,45 +5,75 @@
             "Sid": "VisualEditor0",
             "Effect": "Allow",
             "Action": [
-                "sns:ListTagsForResource",
+                "ec2:DescribeLaunchTemplates",
-                "events:DescribeRule",
+                "autoscaling:DescribeAutoScalingGroups",
-                "sns:GetTopicAttributes",
+                "ec2:DescribeLaunchTemplateVersions",
-                "events:EnableRule",
+                "autoscaling:DescribeTags",
-                "sns:DeleteTopic",
+                "sns:Unsubscribe",
-                "events:PutRule",
+                "sns:GetSubscriptionAttributes",
-                "sns:CreateTopic",
+                "ec2:DescribeSecurityGroups"
-                "sns:SetTopicAttributes",
-                "events:DeleteRule",
-                "events:PutTargets",
-                "events:ListTagsForResource",
-                "sns:Subscribe",
-                "events:RemoveTargets",
-                "events:ListTargetsByRule",
-                "events:DisableRule"
             ],
-            "Resource": [
+            "Resource": "*"
-                "arn:aws:sns:*:566967686773:*",
-                "arn:aws:events:*:566967686773:rule/*"
-            ]
         },
         {
             "Sid": "VisualEditor1",
             "Effect": "Allow",
             "Action": [
-                "events:DescribeEventBus",
+                "autoscaling:DeleteTags",
+                "sns:ListTagsForResource",
+                "ec2:UpdateSecurityGroupRuleDescriptionsEgress",
+                "sns:GetTopicAttributes",
+                "ec2:DeleteTags",
                 "events:CreateEventBus",
-                "events:DeleteEventBus"
+                "sns:DeleteTopic",
+                "ec2:CreateTags",
+                "sns:CreateTopic",
+                "sns:SetTopicAttributes",
+                "ec2:ModifySecurityGroupRules",
+                "ec2:UpdateSecurityGroupRuleDescriptionsIngress",
+                "events:DescribeEventBus",
+                "ec2:RevokeSecurityGroupIngress",
+                "autoscaling:CreateOrUpdateTags",
+                "ec2:CreateSecurityGroup",
+                "ec2:RevokeSecurityGroupEgress",
+                "ec2:DeleteSecurityGroup",
+                "events:DeleteEventBus",
+                "autoscaling:UpdateAutoScalingGroup",
+                "sns:Subscribe",
+                "autoscaling:DeleteAutoScalingGroup",
+                "autoscaling:CreateAutoScalingGroup"
             ],
-            "Resource": "arn:aws:events:*:566967686773:event-bus/*"
+            "Resource": [
+                "arn:aws:events:*:566967686773:event-bus/*",
+                "arn:aws:autoscaling:*:566967686773:autoScalingGroup:*:autoScalingGroupName/*",
+                "arn:aws:sns:*:566967686773:*",
+                "arn:aws:ec2:*:566967686773:security-group/*",
+                "arn:aws:ec2:*:566967686773:security-group-rule/*"
+            ]
         },
         {
             "Sid": "VisualEditor2",
             "Effect": "Allow",
             "Action": [
-                "sns:Unsubscribe",
+                "events:DescribeRule",
-                "sns:GetSubscriptionAttributes"
+                "ec2:DeleteLaunchTemplate",
+                "events:EnableRule",
+                "events:PutRule",
+                "ec2:CreateLaunchTemplateVersion",
+                "events:DeleteRule",
+                "events:PutTargets",
+                "ec2:CreateLaunchTemplate",
+                "events:ListTagsForResource",
+                "events:RemoveTargets",
+                "ec2:ModifyLaunchTemplate",
+                "ec2:DeleteLaunchTemplateVersions",
+                "events:ListTargetsByRule",
+                "events:DisableRule"
             ],
-            "Resource": "*"
+            "Resource": [
+                "arn:aws:events:*:566967686773:rule/*",
+                "arn:aws:ec2:*:566967686773:launch-template/*"
+            ]
         }
     ]
 }

terraform/terraform.tfstate

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.2.9",
-  "serial": 49,
+  "serial": 78,
   "lineage": "a100be74-c98e-0769-2d6a-bf6a2c5f3ebf",
   "outputs": {},
   "resources": [
@@ -15,9 +15,9 @@
           "schema_version": 0,
           "attributes": {
             "account_id": "566967686773",
-            "arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1664301518318294107",
+            "arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1665542385873038019",
             "id": "566967686773",
-            "user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1664301518318294107"
+            "user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1665542385873038019"
           },
           "sensitive_attributes": []
         }
@@ -107,6 +107,85 @@
         }
       ]
     },
+    {
+      "mode": "managed",
+      "type": "aws_autoscaling_group",
+      "name": "k8s-aarch64",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "arn": "arn:aws:autoscaling:us-east-2:566967686773:autoScalingGroup:e5c074b6-9c58-478c-92ec-f0685465ca3d:autoScalingGroupName/k8s-aarch64",
+            "availability_zones": [
+              "us-east-2a",
+              "us-east-2b",
+              "us-east-2c"
+            ],
+            "capacity_rebalance": false,
+            "context": "",
+            "default_cooldown": 300,
+            "default_instance_warmup": 0,
+            "desired_capacity": 0,
+            "enabled_metrics": [],
+            "force_delete": false,
+            "force_delete_warm_pool": false,
+            "health_check_grace_period": 300,
+            "health_check_type": "EC2",
+            "id": "k8s-aarch64",
+            "initial_lifecycle_hook": [],
+            "instance_refresh": [],
+            "launch_configuration": "",
+            "launch_template": [
+              {
+                "id": "lt-0789a3800bdaec215",
+                "name": "k8s-aarch64",
+                "version": "$Latest"
+              }
+            ],
+            "load_balancers": [],
+            "max_instance_lifetime": 0,
+            "max_size": 1,
+            "metrics_granularity": "1Minute",
+            "min_elb_capacity": null,
+            "min_size": 0,
+            "mixed_instances_policy": [],
+            "name": "k8s-aarch64",
+            "name_prefix": "",
+            "placement_group": "",
+            "protect_from_scale_in": false,
+            "service_linked_role_arn": "arn:aws:iam::566967686773:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling",
+            "suspended_processes": [],
+            "tag": [
+              {
+                "key": "k8s.io/cluster-autoscaler/enabled",
+                "propagate_at_launch": true,
+                "value": "true"
+              },
+              {
+                "key": "k8s.io/cluster-autoscaler/kubernetes",
+                "propagate_at_launch": true,
+                "value": "owned"
+              }
+            ],
+            "tags": null,
+            "target_group_arns": [],
+            "termination_policies": [],
+            "timeouts": null,
+            "vpc_zone_identifier": [],
+            "wait_for_capacity_timeout": "10m",
+            "wait_for_elb_capacity": null,
+            "warm_pool": []
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjo2MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfX0=",
+          "dependencies": [
+            "aws_launch_template.k8s-aarch64",
+            "aws_security_group.k8s-node"
+          ]
+        }
+      ]
+    },
     {
       "mode": "managed",
       "type": "aws_cloudwatch_event_rule",
@@ -171,6 +250,152 @@
         }
       ]
     },
+    {
+      "mode": "managed",
+      "type": "aws_launch_template",
+      "name": "k8s-aarch64",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "arn": "arn:aws:ec2:us-east-2:566967686773:launch-template/lt-0789a3800bdaec215",
+            "block_device_mappings": [],
+            "capacity_reservation_specification": [],
+            "cpu_options": [],
+            "credit_specification": [],
+            "default_version": 6,
+            "description": "",
+            "disable_api_stop": false,
+            "disable_api_termination": false,
+            "ebs_optimized": "",
+            "elastic_gpu_specifications": [],
+            "elastic_inference_accelerator": [],
+            "enclave_options": [],
+            "hibernation_options": [],
+            "iam_instance_profile": [],
+            "id": "lt-0789a3800bdaec215",
+            "image_id": "ami-000ec96ccb51eb679",
+            "instance_initiated_shutdown_behavior": "",
+            "instance_market_options": [
+              {
+                "market_type": "spot",
+                "spot_options": []
+              }
+            ],
+            "instance_requirements": [],
+            "instance_type": "t4g.medium",
+            "kernel_id": "",
+            "key_name": "",
+            "latest_version": 6,
+            "license_specification": [],
+            "maintenance_options": [],
+            "metadata_options": [],
+            "monitoring": [],
+            "name": "k8s-aarch64",
+            "name_prefix": "",
+            "network_interfaces": [],
+            "placement": [],
+            "private_dns_name_options": [
+              {
+                "enable_resource_name_dns_a_record": false,
+                "enable_resource_name_dns_aaaa_record": false,
+                "hostname_type": "resource-name"
+              }
+            ],
+            "ram_disk_id": "",
+            "security_group_names": [
+              "k8s-node"
+            ],
+            "tag_specifications": [],
+            "tags": {},
+            "tags_all": {},
+            "update_default_version": true,
+            "user_data": "I2Nsb3VkLWNvbmZpZwpib290Y21kOgotIFsgZG5mLCBtb2R1bGUsIGVuYWJsZSwgJ2NyaS1vOjEuMjInLCAteSBdCi0gWyBsbiwgLXNmLCAvcnVuL3N5c3RlbWQvcmVzb2x2ZS9zdHViLXJlc29sdi5jb25mLCAvZXRjL3Jlc29sdi5jb25mIF0KCnBhY2thZ2VzOgotIGNyaS1vCi0gY3JpLXRvb2xzCi0gZXRodG9vbAotIGlwdGFibGVzLW5mdAotIGlzY3NpLWluaXRpYXRvci11dGlscwotIGt1YmVybmV0ZXMta3ViZWFkbQotIGt1YmVybmV0ZXMtbm9kZQotIHdpcmVndWFyZC10b29scwoKd3JpdGVfZmlsZXM6Ci0gcGF0aDogL2V0Yy9kbmYvZG5mLmNvbmYKICBjb250ZW50OiB8KwogICAgaW5zdGFsbF93ZWFrX2RlcHM9RmFsc2UKICBhcHBlbmQ6IHRydWUKLSBwYXRoOiAvZXRjL21vZHVsZXMtbG9hZC5kL2s4cy5jb25mCiAgY29udGVudDogfCsKICAgIGJyX25ldGZpbHRlcgotIHBhdGg6IC9ldGMvc3lzY3RsLmQvazhzLmNvbmYKICBjb250ZW50OiB8KwogICAgbmV0LmJyaWRnZS5icmlkZ2UtbmYtY2FsbC1pcHRhYmxlcyA9IDEKICAgIG5ldC5icmlkZ2UuYnJpZGdlLW5mLWNhbGwtaXA2dGFibGVzID0gMQogICAgbmV0LmlwdjQuaXBfZm9yd2FyZCA9IDEKLSBwYXRoOiAvdmFyL2xpYi9jbG91ZC9zY3JpcHRzL3Blci1pbnN0YW5jZS9rdWJlYWRtLWpvaW4KICBwZXJtaXNzaW9uczogJzA3NTUnCiAgY29udGVudDogfCsKICAgICMhL2Jpbi9zaAoKICAgIEJBU0VfVVJMPWh0dHBzOi8vZHluazhzLXByb3Zpc2lvbmVyLnB5cm9jdWZmbGluay5uZXQKCiAgICBpbnN0YW5jZV9pZD0kKGN1cmwgLXMgMTY5LjI1NC4xNjkuMjU0L2xhdGVzdC9tZXRhLWRhdGEvaW5zdGFuY2UtaWQpCiAgICBhej0kKGN1cmwgLXMgMTY5LjI1NC4xNjkuMjU0L2xhdGVzdC9tZXRhLWRhdGEvcGxhY2VtZW50L2F2YWlsYWJpbGl0eS16b25lKQoKICAgIGN1cmwgLWZzICIke0JBU0VfVVJMfSIvd2lyZWd1YXJkL2NvbmZpZy8ke2luc3RhbmNlX2lkfSBcCiAgICAgICAgLW8gL2V0Yy93aXJlZ3VhcmQvd2cwLmNvbmYgfHwgZXhpdAogICAgc3lzdGVtY3RsIGVuYWJsZSAtLW5vdyB3Zy1xdWlja0B3ZzAgfHwgZXhpdAoKICAgIG1vZHByb2JlIGJyX25ldGZpbHRlciB8fCBleGl0CiAgICBzeXNjdGwgLXcgLWYgL2V0Yy9zeXNjdGwuZC9rOHMuY29uZiB8fCBleGl0CgogICAgc3dhcG9mZiAtYSB8fCBleGl0CiAgICB0b3VjaCAvZXRjL3N5c3RlbWQvenJhbS1nZW5lcmF0b3IuY29uZiB8fCBleGl0CiAgICBzeXN0ZW1jdGwgZGFlbW9uLXJlbG9hZCB8fCBleGl0CiAgICBzeXN0ZW1jdGwgc3RvcCAnc3lzdGVtZC16cmFtLXNldHVwQConIHx8IGV4aXQKCiAgICBzeXN0ZW1jdGwgZW5hYmxlIGNyaW8gaXNjc2lkIGt1YmVsZXQgfHwgZXhpdAogICAgc3lzdGVtY3RsIHN0YXJ0IGNyaW8gaXNjc2lkIHx8IGV4aXQKCiAgICBpbnRlcm5hbF9pcD0kKAogICAgICBpcCBhZGRyZXNzIHNob3cgZGV2IHdnMCBwcmltYXJ5IHwgXAogICAgICBzZWQgLXJuICdzLy4qaW5ldCAoWzAtOS5dKykuKi9cMS9wJwogICAgKQoKICAgIGNhdCA+IC9ydW4vam9pbmNvbmZpZ3VyYXRpb24gPDxFT0YKICAgIGFwaVZlcnNpb246IGt1YmVhZG0uazhzLmlvL3YxYmV0YTMKICAgIGtpbmQ6IEpvaW5Db25maWd1cmF0aW9uCiAgICBub2RlUmVnaXN0cmF0aW9uOgogICAgICBrdWJlbGV0RXh0cmFBcmdzOgogICAgICAgIHByb3ZpZGVyLWlkOiBhd3M6Ly8vJHthen0vJHtpbnN0YW5jZV9pZH0KICAgICAgICBub2RlLWlwOiAke2ludGVybmFsX2lwfQogICAgZGlzY292ZXJ5OgogICAgICBmaWxlOgogICAgICAgIGt1YmVDb25maWdQYXRoOiAke0JBU0VfVVJMfS9rdWJlYWRtL2t1YmVjb25maWcvJHtpbnN0YW5jZV9pZH0KICAgIEVPRgogICAga3ViZWFkbSBqb2luIC0tY29uZmlnPS9ydW4vam9pbmNvbmZpZ3VyYXRpb24KCnJ1bmNtZDoKLSBbIGRuZiwgcmVtb3ZlLCAteSwgenJhbS1nZW5lcmF0b3IgXQo=",
+            "vpc_security_group_ids": []
+          },
+          "sensitive_attributes": [],
+          "private": "bnVsbA==",
+          "dependencies": [
+            "aws_security_group.k8s-node"
+          ]
+        }
+      ]
+    },
+    {
+      "mode": "managed",
+      "type": "aws_security_group",
+      "name": "k8s-node",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ec2:us-east-2:566967686773:security-group/sg-05258c3ff1812e83b",
+            "description": "Kubernetes Node",
+            "egress": [
+              {
+                "cidr_blocks": [
+                  "0.0.0.0/0"
+                ],
+                "description": "",
+                "from_port": 19998,
+                "ipv6_cidr_blocks": [
+                  "::/0"
+                ],
+                "prefix_list_ids": [],
+                "protocol": "udp",
+                "security_groups": [],
+                "self": false,
+                "to_port": 19998
+              },
+              {
+                "cidr_blocks": [
+                  "0.0.0.0/0"
+                ],
+                "description": "",
+                "from_port": 443,
+                "ipv6_cidr_blocks": [
+                  "::/0"
+                ],
+                "prefix_list_ids": [],
+                "protocol": "tcp",
+                "security_groups": [],
+                "self": false,
+                "to_port": 443
+              },
+              {
+                "cidr_blocks": [
+                  "0.0.0.0/0"
+                ],
+                "description": "",
+                "from_port": 80,
+                "ipv6_cidr_blocks": [
+                  "::/0"
+                ],
+                "prefix_list_ids": [],
+                "protocol": "tcp",
+                "security_groups": [],
+                "self": false,
+                "to_port": 80
+              }
+            ],
+            "id": "sg-05258c3ff1812e83b",
+            "ingress": [],
+            "name": "k8s-node",
+            "name_prefix": "",
+            "owner_id": "566967686773",
+            "revoke_rules_on_delete": false,
+            "tags": {},
+            "tags_all": {},
+            "timeouts": null,
+            "vpc_id": "vpc-2483044d"
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
+        }
+      ]
+    },
     {
       "mode": "managed",
       "type": "aws_sns_topic",

terraform/userdata.yml

@@ -0,0 +1,73 @@
+#cloud-config
+bootcmd:
+- [ dnf, module, enable, 'cri-o:1.22', -y ]
+- [ ln, -sf, /run/systemd/resolve/stub-resolv.conf, /etc/resolv.conf ]
+
+packages:
+- cri-o
+- cri-tools
+- ethtool
+- iptables-nft
+- iscsi-initiator-utils
+- kubernetes-kubeadm
+- kubernetes-node
+- wireguard-tools
+
+write_files:
+- path: /etc/dnf/dnf.conf
+  content: |+
+    install_weak_deps=False
+  append: true
+- path: /etc/modules-load.d/k8s.conf
+  content: |+
+    br_netfilter
+- path: /etc/sysctl.d/k8s.conf
+  content: |+
+    net.bridge.bridge-nf-call-iptables = 1
+    net.bridge.bridge-nf-call-ip6tables = 1
+    net.ipv4.ip_forward = 1
+- path: /var/lib/cloud/scripts/per-instance/kubeadm-join
+  permissions: '0755'
+  content: |+
+    #!/bin/sh
+
+    BASE_URL=https://dynk8s-provisioner.pyrocufflink.net
+
+    instance_id=$(curl -s 169.254.169.254/latest/meta-data/instance-id)
+    az=$(curl -s 169.254.169.254/latest/meta-data/placement/availability-zone)
+
+    curl -fs "${BASE_URL}"/wireguard/config/${instance_id} \
+        -o /etc/wireguard/wg0.conf || exit
+    systemctl enable --now wg-quick@wg0 || exit
+
+    modprobe br_netfilter || exit
+    sysctl -w -f /etc/sysctl.d/k8s.conf || exit
+
+    swapoff -a || exit
+    touch /etc/systemd/zram-generator.conf || exit
+    systemctl daemon-reload || exit
+    systemctl stop 'systemd-zram-setup@*' || exit
+
+    systemctl enable crio iscsid kubelet || exit
+    systemctl start crio iscsid || exit
+
+    internal_ip=$(
+      ip address show dev wg0 primary | \
+      sed -rn 's/.*inet ([0-9.]+).*/\1/p'
+    )
+
+    cat > /run/joinconfiguration <<EOF
+    apiVersion: kubeadm.k8s.io/v1beta3
+    kind: JoinConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        provider-id: aws:///${az}/${instance_id}
+        node-ip: ${internal_ip}
+    discovery:
+      file:
+        kubeConfigPath: ${BASE_URL}/kubeadm/kubeconfig/${instance_id}
+    EOF
+    kubeadm join --config=/run/joinconfiguration
+
+runcmd:
+- [ dnf, remove, -y, zram-generator ]

tests/setup.yaml

@@ -14,7 +14,7 @@ metadata:
   namespace: dynk8s-test
   labels:
     app.kubernetes.io/name: dynk8s-provisioner
-    app.kubernetes.io/instance: default
+    app.kubernetes.io/instance: integration-test
     app.kubernetes.io/component: http-api
     app.kubernetes.io/part-of: dynk8s-provisioner
 automountServiceAccountToken: true
@@ -31,13 +31,13 @@ type: kubernetes.io/service-account-token
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
 metadata:
   name: dynk8s-provisioner
   namespace: dynk8s-test
   labels:
     app.kubernetes.io/name: dynk8s-provisioner
-    app.kubernetes.io/instance: default
+    app.kubernetes.io/instance: integration-test
     app.kubernetes.io/component: http-api
     app.kubernetes.io/part-of: dynk8s-provisioner
 rules:
@@ -47,6 +47,38 @@ rules:
   - secrets
   verbs:
   - '*'
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: dynk8s-provisioner-test
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: integration-test
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - secrets
+  verbs:
+  - '*'
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: dynk8s-provisioner-test
+  namespace: kube-public
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: integration-test
+    app.kubernetes.io/component: http-api
+    app.kubernetes.io/part-of: dynk8s-provisioner
+rules:
 - apiGroups:
   - ''
   resources:
@@ -58,17 +90,54 @@ rules:
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
   name: dynk8s-provisioner
   namespace: dynk8s-test
   labels:
     app.kubernetes.io/name: dynk8s-provisioner
-    app.kubernetes.io/instance: default
+    app.kubernetes.io/instance: integration-test
     app.kubernetes.io/part-of: dynk8s-provisioner
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
+  kind: Role
+  name: dynk8s-provisioner
+subjects:
+- kind: ServiceAccount
+  name: dynk8s-provisioner
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: dynk8s-provisioner-test
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: integration-test
+    app.kubernetes.io/part-of: dynk8s-provisioner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: dynk8s-provisioner
+subjects:
+- kind: ServiceAccount
+  name: dynk8s-provisioner
+  namespace: dynk8s-test
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: dynk8s-provisioner-test
+  namespace: kube-public
+  labels:
+    app.kubernetes.io/name: dynk8s-provisioner
+    app.kubernetes.io/instance: integration-test
+    app.kubernetes.io/part-of: dynk8s-provisioner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
   name: dynk8s-provisioner
 subjects:
 - kind: ServiceAccount