dustin
/
dynk8s-provisioner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tf/asg: Add CA resource tag for FUSE device plugin
dustin/dynk8s-provisioner/pipeline/head This commit looks good Details 

Jenkins jobs that build container images in user namespaces need access
to `/dev/fuse`, which is provided by the [fuse-device-plugin][0].  This
plugin runs as a DaemonSet, which updates the status of the node it's
running on when it starts to indicate that the FUSE device is available.
When scaling up from zero nodes, Cluster Autoscaler has no way to know
that this will occur, and therefore cannot determine that scaling up the
ASG will create a node with the required resources.  Thus, the ASG needs
a tag to inform CA that the nodes it creates will indeed have the
resources and scaling it up will allow the pod to be scheduled.

Although this feature of CA was added in 1.14, it apparently got broken
at some point and no longer works in 1.22.  It works again in 1.26,
though.

[0]: https://github.com/kuberenetes-learning-group/fuse-device-plugin/tree/master
master
Dustin 2024-01-14 11:42:46 -06:00
parent 5a79680b22
commit f6910f04df
2 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
terraform/asg.tf
View File

@ -89,4 +89,9 @@ resource "aws_autoscaling_group" "k8s-aarch64" {
value = "owned" value = "owned"
propagate_at_launch = true propagate_at_launch = true
} }
tag {
key = "k8s.io/cluster-autoscaler/node-template/resources/github.com/fuse"
value = "1"
propagate_at_launch = false
}
} }

13
terraform/terraform.tfstate
View File

@ -1,7 +1,7 @@
{ {
"version": 4, "version": 4,
"terraform_version": "1.6.2", "terraform_version": "1.6.2",
"serial": 96, "serial": 98,
"lineage": "a100be74-c98e-0769-2d6a-bf6a2c5f3ebf", "lineage": "a100be74-c98e-0769-2d6a-bf6a2c5f3ebf",
"outputs": {}, "outputs": {},
"resources": [ "resources": [
@ -107,9 +107,9 @@
"schema_version": 0, "schema_version": 0,
"attributes": { "attributes": {
"account_id": "566967686773", "account_id": "566967686773",
"arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1705162223321505341", "arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1705246977054689837",
"id": "566967686773", "id": "566967686773",
"user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1705162223321505341" "user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1705246977054689837"
}, },
"sensitive_attributes": [] "sensitive_attributes": []
} }
@ -218,7 +218,7 @@
"context": "", "context": "",
"default_cooldown": 300, "default_cooldown": 300,
"default_instance_warmup": 0, "default_instance_warmup": 0,
"desired_capacity": 1, "desired_capacity": 0,
"enabled_metrics": [], "enabled_metrics": [],
"force_delete": false, "force_delete": false,
"force_delete_warm_pool": false, "force_delete_warm_pool": false,
@ -258,6 +258,11 @@
"key": "k8s.io/cluster-autoscaler/kubernetes", "key": "k8s.io/cluster-autoscaler/kubernetes",
"propagate_at_launch": true, "propagate_at_launch": true,
"value": "owned" "value": "owned"
},
{
"key": "k8s.io/cluster-autoscaler/node-template/resources/github.com/fuse",
"propagate_at_launch": false,
"value": "1"
} }
], ],
"tags": null, "tags": null,