dustin
/
dynk8s-provisioner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tf/userdata: Remove default DNS configuration 

Lately, cloud nodes seem to be failing to come up more frequently.  I
traced this down to the fact that `/etc/resolv.conf` in the `kube-proxy`
container contains both the AWS-provided DNS server and the on-premises
server set by Wireguard.  This evidently "works" correctly sometimes,
but not always.  When it doesn't, the `kube-proxy` cannot resolve the
Kubernetes API server address, and thus cannot create the necessary
netfilter rules to forward traffic correctly.  This causes pods to be
unable to communicate.

I am not entirely sure what the "correct" solution to this problem would
be, since there are various issues in play here.  Fortunately, cloud
nodes are only ever around for a short time, and never need to be
rebooted.  As such, we can use a "quick fix" and simply remove the
AWS-provided DNS configuration.
master
Dustin 2023-11-13 19:52:57 -06:00
parent 4a2a376409
commit 473e279a18
2 changed files with 13 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
terraform/terraform.tfstate
View File

@ -1,7 +1,7 @@
{ {
"version": 4, "version": 4,
"terraform_version": "1.2.9", "terraform_version": "1.6.2",
"serial": 88, "serial": 94,
"lineage": "a100be74-c98e-0769-2d6a-bf6a2c5f3ebf", "lineage": "a100be74-c98e-0769-2d6a-bf6a2c5f3ebf",
"outputs": {}, "outputs": {},
"resources": [ "resources": [
@ -15,9 +15,9 @@
"schema_version": 0, "schema_version": 0,
"attributes": { "attributes": {
"account_id": "566967686773", "account_id": "566967686773",
"arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1673405716573043213", "arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1699926524617521431",
"id": "566967686773", "id": "566967686773",
"user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1673405716573043213" "user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1699926524617521431"
}, },
"sensitive_attributes": [] "sensitive_attributes": []
} }
@ -126,7 +126,7 @@
"context": "", "context": "",
"default_cooldown": 300, "default_cooldown": 300,
"default_instance_warmup": 0, "default_instance_warmup": 0,
"desired_capacity": 1, "desired_capacity": 0,
"enabled_metrics": [], "enabled_metrics": [],
"force_delete": false, "force_delete": false,
"force_delete_warm_pool": false, "force_delete_warm_pool": false,
@ -264,7 +264,7 @@
"capacity_reservation_specification": [], "capacity_reservation_specification": [],
"cpu_options": [], "cpu_options": [],
"credit_specification": [], "credit_specification": [],
"default_version": 12, "default_version": 19,
"description": "", "description": "",
"disable_api_stop": false, "disable_api_stop": false,
"disable_api_termination": false, "disable_api_termination": false,
@ -275,7 +275,7 @@
"hibernation_options": [], "hibernation_options": [],
"iam_instance_profile": [], "iam_instance_profile": [],
"id": "lt-0789a3800bdaec215", "id": "lt-0789a3800bdaec215",
"image_id": "ami-0995531df014459c2", "image_id": "ami-0dcd72048e69236de",
"instance_initiated_shutdown_behavior": "", "instance_initiated_shutdown_behavior": "",
"instance_market_options": [ "instance_market_options": [
{ {
@ -287,7 +287,7 @@
"instance_type": "t4g.medium", "instance_type": "t4g.medium",
"kernel_id": "", "kernel_id": "",
"key_name": "dustin@rosalina", "key_name": "dustin@rosalina",
"latest_version": 12, "latest_version": 19,
"license_specification": [], "license_specification": [],
"maintenance_options": [], "maintenance_options": [],
"metadata_options": [], "metadata_options": [],
@ -311,7 +311,7 @@
"tags": {}, "tags": {},
"tags_all": {}, "tags_all": {},
"update_default_version": true, "update_default_version": true,
"user_data": "I2Nsb3VkLWNvbmZpZwpib290Y21kOgotIFsgZG5mLCBtb2R1bGUsIGVuYWJsZSwgJ2NyaS1vOjEuMjQnLCAteSBdCi0gWyBsbiwgLXNmLCAvcnVuL3N5c3RlbWQvcmVzb2x2ZS9zdHViLXJlc29sdi5jb25mLCAvZXRjL3Jlc29sdi5jb25mIF0KCnBhY2thZ2VzOgotIGNyaS1vCi0gY3JpLXRvb2xzCi0gZXRodG9vbAotIGlwdGFibGVzLW5mdAotIGlzY3NpLWluaXRpYXRvci11dGlscwotIGt1YmVybmV0ZXMta3ViZWFkbQotIGt1YmVybmV0ZXMtbm9kZQotIHJ1bmMKLSB3aXJlZ3VhcmQtdG9vbHMKCndyaXRlX2ZpbGVzOgotIHBhdGg6IC9ldGMvZG5mL2RuZi5jb25mCiAgY29udGVudDogfCsKICAgIGluc3RhbGxfd2Vha19kZXBzPUZhbHNlCiAgYXBwZW5kOiB0cnVlCi0gcGF0aDogL2V0Yy9tb2R1bGVzLWxvYWQuZC9rOHMuY29uZgogIGNvbnRlbnQ6IHwrCiAgICBicl9uZXRmaWx0ZXIKLSBwYXRoOiAvZXRjL3N5c2N0bC5kL2s4cy5jb25mCiAgY29udGVudDogfCsKICAgIG5ldC5icmlkZ2UuYnJpZGdlLW5mLWNhbGwtaXB0YWJsZXMgPSAxCiAgICBuZXQuYnJpZGdlLmJyaWRnZS1uZi1jYWxsLWlwNnRhYmxlcyA9IDEKICAgIG5ldC5pcHY0LmlwX2ZvcndhcmQgPSAxCi0gcGF0aDogL3Zhci9saWIvY2xvdWQvc2NyaXB0cy9wZXItaW5zdGFuY2Uva3ViZWFkbS1qb2luCiAgcGVybWlzc2lvbnM6ICcwNzU1JwogIGNvbnRlbnQ6IHwrCiAgICAjIS9iaW4vc2gKCiAgICBCQVNFX1VSTD1odHRwczovL2R5bms4cy1wcm92aXNpb25lci5weXJvY3VmZmxpbmsubmV0CgogICAgaW5zdGFuY2VfaWQ9JChjdXJsIC1zIDE2OS4yNTQuMTY5LjI1NC9sYXRlc3QvbWV0YS1kYXRhL2luc3RhbmNlLWlkKQogICAgYXo9JChjdXJsIC1zIDE2OS4yNTQuMTY5LjI1NC9sYXRlc3QvbWV0YS1kYXRhL3BsYWNlbWVudC9hdmFpbGFiaWxpdHktem9uZSkKCiAgICBjdXJsIC1mcyAiJHtCQVNFX1VSTH0iL3dpcmVndWFyZC9jb25maWcvJHtpbnN0YW5jZV9pZH0gXAogICAgICAgIC1vIC9ldGMvd2lyZWd1YXJkL3dnMC5jb25mIHx8IGV4aXQKICAgIHN5c3RlbWN0bCBlbmFibGUgLS1ub3cgd2ctcXVpY2tAd2cwIHx8IGV4aXQKCiAgICBtb2Rwcm9iZSBicl9uZXRmaWx0ZXIgfHwgZXhpdAogICAgc3lzY3RsIC13IC1mIC9ldGMvc3lzY3RsLmQvazhzLmNvbmYgfHwgZXhpdAoKICAgIHN3YXBvZmYgLWEgfHwgZXhpdAogICAgdG91Y2ggL2V0Yy9zeXN0ZW1kL3pyYW0tZ2VuZXJhdG9yLmNvbmYgfHwgZXhpdAogICAgc3lzdGVtY3RsIGRhZW1vbi1yZWxvYWQgfHwgZXhpdAogICAgc3lzdGVtY3RsIHN0b3AgJ3N5c3RlbWQtenJhbS1zZXR1cEAqJyB8fCBleGl0CgogICAgc3lzdGVtY3RsIGVuYWJsZSBjcmlvIGlzY3NpZCBrdWJlbGV0IHx8IGV4aXQKICAgIHN5c3RlbWN0bCBzdGFydCBjcmlvIGlzY3NpZCB8fCBleGl0CgogICAgaW50ZXJuYWxfaXA9JCgKICAgICAgaXAgYWRkcmVzcyBzaG93IGRldiB3ZzAgcHJpbWFyeSB8IFwKICAgICAgc2VkIC1ybiAncy8uKmluZXQgKFswLTkuXSspLiovXDEvcCcKICAgICkKCiAgICBjYXQgPiBsb25naG9ybi1pc3N1ZTQ5ODguY2lsIDw8RU9GCiAgICAoYWxsb3cgaXNjc2lkX3Qgc2VsZiAoY2FwYWJpbGl0eSAoZGFjX292ZXJyaWRlKSkpCiAgICBFT0YKICAgIHNlbW9kdWxlIC1pIGxvbmdob3JuLWlzc3VlNDk4OC5jaWwKCiAgICBjYXQgPiAvcnVuL2pvaW5jb25maWd1cmF0aW9uIDw8RU9GCiAgICBhcGlWZXJzaW9uOiBrdWJlYWRtLms4cy5pby92MWJldGEzCiAgICBraW5kOiBKb2luQ29uZmlndXJhdGlvbgogICAgbm9kZVJlZ2lzdHJhdGlvbjoKICAgICAga3ViZWxldEV4dHJhQXJnczoKICAgICAgICBwcm92aWRlci1pZDogYXdzOi8vLyR7YXp9LyR7aW5zdGFuY2VfaWR9CiAgICAgICAgbm9kZS1pcDogJHtpbnRlcm5hbF9pcH0KICAgIGRpc2NvdmVyeToKICAgICAgZmlsZToKICAgICAgICBrdWJlQ29uZmlnUGF0aDogJHtCQVNFX1VSTH0va3ViZWFkbS9rdWJlY29uZmlnLyR7aW5zdGFuY2VfaWR9CiAgICBFT0YKICAgIGt1YmVhZG0gam9pbiAtLWNvbmZpZz0vcnVuL2pvaW5jb25maWd1cmF0aW9uCgpydW5jbWQ6Ci0gWyBkbmYsIHJlbW92ZSwgLXksIHpyYW0tZ2VuZXJhdG9yIF0K", "user_data": "I2Nsb3VkLWNvbmZpZwpib290Y21kOgotIFsgZG5mLCBtb2R1bGUsIGVuYWJsZSwgJ2NyaS1vOjEuMjUnLCAteSBdCi0gWyBsbiwgLXNmLCAvcnVuL3N5c3RlbWQvcmVzb2x2ZS9zdHViLXJlc29sdi5jb25mLCAvZXRjL3Jlc29sdi5jb25mIF0KCnBhY2thZ2VzOgotIGNyaS1vCi0gY3JpLXRvb2xzCi0gZXRodG9vbAotIGlwdGFibGVzLW5mdAotIGlzY3NpLWluaXRpYXRvci11dGlscwotIGt1YmVybmV0ZXMta3ViZWFkbQotIGt1YmVybmV0ZXMtbm9kZQotIHJ1bmMKLSB3aXJlZ3VhcmQtdG9vbHMKCndyaXRlX2ZpbGVzOgotIHBhdGg6IC9ldGMvZG5mL2RuZi5jb25mCiAgY29udGVudDogfCsKICAgIGluc3RhbGxfd2Vha19kZXBzPUZhbHNlCiAgYXBwZW5kOiB0cnVlCi0gcGF0aDogL2V0Yy9tb2R1bGVzLWxvYWQuZC9rOHMuY29uZgogIGNvbnRlbnQ6IHwrCiAgICBicl9uZXRmaWx0ZXIKLSBwYXRoOiAvZXRjL3N5c2N0bC5kL2s4cy5jb25mCiAgY29udGVudDogfCsKICAgIG5ldC5icmlkZ2UuYnJpZGdlLW5mLWNhbGwtaXB0YWJsZXMgPSAxCiAgICBuZXQuYnJpZGdlLmJyaWRnZS1uZi1jYWxsLWlwNnRhYmxlcyA9IDEKICAgIG5ldC5pcHY0LmlwX2ZvcndhcmQgPSAxCi0gcGF0aDogL3Zhci9saWIvY2xvdWQvc2NyaXB0cy9wZXItaW5zdGFuY2Uva3ViZWFkbS1qb2luCiAgcGVybWlzc2lvbnM6ICcwNzU1JwogIGNvbnRlbnQ6IHwrCiAgICAjIS9iaW4vc2gKCiAgICBCQVNFX1VSTD1odHRwczovL2R5bms4cy1wcm92aXNpb25lci5weXJvY3VmZmxpbmsubmV0CgogICAgaW5zdGFuY2VfaWQ9JChjdXJsIC1zIDE2OS4yNTQuMTY5LjI1NC9sYXRlc3QvbWV0YS1kYXRhL2luc3RhbmNlLWlkKQogICAgYXo9JChjdXJsIC1zIDE2OS4yNTQuMTY5LjI1NC9sYXRlc3QvbWV0YS1kYXRhL3BsYWNlbWVudC9hdmFpbGFiaWxpdHktem9uZSkKCiAgICBjdXJsIC1mcyAiJHtCQVNFX1VSTH0iL3dpcmVndWFyZC9jb25maWcvJHtpbnN0YW5jZV9pZH0gXAogICAgICAgIC1vIC9ldGMvd2lyZWd1YXJkL3dnMC5jb25mIHx8IGV4aXQKICAgIHN5c3RlbWN0bCBlbmFibGUgLS1ub3cgd2ctcXVpY2tAd2cwIHx8IGV4aXQKCiAgICByZXNvbHZlY3RsIHJldmVydCBldGgwCgogICAgbW9kcHJvYmUgYnJfbmV0ZmlsdGVyIHx8IGV4aXQKICAgIHN5c2N0bCAtdyAtZiAvZXRjL3N5c2N0bC5kL2s4cy5jb25mIHx8IGV4aXQKCiAgICBzd2Fwb2ZmIC1hIHx8IGV4aXQKICAgIHRvdWNoIC9ldGMvc3lzdGVtZC96cmFtLWdlbmVyYXRvci5jb25mIHx8IGV4aXQKICAgIHN5c3RlbWN0bCBkYWVtb24tcmVsb2FkIHx8IGV4aXQKICAgIHN5c3RlbWN0bCBzdG9wICdzeXN0ZW1kLXpyYW0tc2V0dXBAKicgfHwgZXhpdAoKICAgIHN5c3RlbWN0bCBlbmFibGUgY3JpbyBpc2NzaWQga3ViZWxldCB8fCBleGl0CiAgICBzeXN0ZW1jdGwgc3RhcnQgY3JpbyBpc2NzaWQgfHwgZXhpdAoKICAgIGludGVybmFsX2lwPSQoCiAgICAgIGlwIGFkZHJlc3Mgc2hvdyBkZXYgd2cwIHByaW1hcnkgfCBcCiAgICAgIHNlZCAtcm4gJ3MvLippbmV0IChbMC05Ll0rKS4qL1wxL3AnCiAgICApCgogICAgY2F0ID4gbG9uZ2hvcm4taXNzdWU0OTg4LmNpbCA8PEVPRgogICAgKGFsbG93IGlzY3NpZF90IHNlbGYgKGNhcGFiaWxpdHkgKGRhY19vdmVycmlkZSkpKQogICAgRU9GCiAgICBzZW1vZHVsZSAtaSBsb25naG9ybi1pc3N1ZTQ5ODguY2lsCgogICAgY2F0ID4gL3J1bi9qb2luY29uZmlndXJhdGlvbiA8PEVPRgogICAgYXBpVmVyc2lvbjoga3ViZWFkbS5rOHMuaW8vdjFiZXRhMwogICAga2luZDogSm9pbkNvbmZpZ3VyYXRpb24KICAgIG5vZGVSZWdpc3RyYXRpb246CiAgICAgIGt1YmVsZXRFeHRyYUFyZ3M6CiAgICAgICAgcHJvdmlkZXItaWQ6IGF3czovLy8ke2F6fS8ke2luc3RhbmNlX2lkfQogICAgICAgIG5vZGUtaXA6ICR7aW50ZXJuYWxfaXB9CiAgICAgICAgY29uZmlnOiAvdmFyL2xpYi9rdWJlbGV0L2NvbmZpZy55YW1sCiAgICBkaXNjb3Zlcnk6CiAgICAgIGZpbGU6CiAgICAgICAga3ViZUNvbmZpZ1BhdGg6ICR7QkFTRV9VUkx9L2t1YmVhZG0va3ViZWNvbmZpZy8ke2luc3RhbmNlX2lkfQogICAgRU9GCiAgICBrdWJlYWRtIGpvaW4gLS1jb25maWc9L3J1bi9qb2luY29uZmlndXJhdGlvbgoKcnVuY21kOgotIFsgZG5mLCByZW1vdmUsIC15LCB6cmFtLWdlbmVyYXRvciBdCg==",
"vpc_security_group_ids": [] "vpc_security_group_ids": []
}, },
"sensitive_attributes": [], "sensitive_attributes": [],
@ -510,5 +510,6 @@
} }
] ]
} }
] ],
"check_results": null
} }

2
terraform/userdata.yml
View File

@ -41,6 +41,8 @@ write_files:
-o /etc/wireguard/wg0.conf || exit -o /etc/wireguard/wg0.conf || exit
systemctl enable --now wg-quick@wg0 || exit systemctl enable --now wg-quick@wg0 || exit
resolvectl revert eth0
modprobe br_netfilter || exit modprobe br_netfilter || exit
sysctl -w -f /etc/sysctl.d/k8s.conf || exit sysctl -w -f /etc/sysctl.d/k8s.conf || exit