From 84cd6022c06407342ed3390a1af6cedb479e9cc8 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sun, 29 Jun 2025 17:19:58 -0500 Subject: [PATCH] r/k8s-worker: Use K8s API to create join token Using the Kubernetes API to create bootstrap tokens makes it possible for the host-provisioner to automatically add new machines to the Kubernetes cluster. The host provisioner cannot connect to existing machines, and thus cannot run the `kubeadm token create` command on a control plane node. With the appropriate permissions assigned to the service account associated with the pod it runs in, though, it can directly create the secret via the API. There are actually two pieces of information required for a node to join a cluster, though: a bootstrap token and the CA certificate. When using the `kubeadm token create` command to issue a bootstrap token, it also provides (a hash of) the CA certificate with the command it prints. When creating the token manually, we need an alternative method for obtaining and distributing the CA certificate, so we use the `cluster-info` ConfigMap. This contains a stub `kubeconfig` file, which includes the CA certificate, which can be used by the `kubeadm join` command with a join configuration file. Generating both of these files may be a bit more involved than computing the CA certificate hash and passing that on the command line, but there are a couple of advantages. First, it's more extensible, as the join configuration file can specify additional configuration for the node (which we may want to use later). It's also somewhat more secure, since the token is not passed as a command-line argument. Interestingly, the most difficult part of this implementation was getting the expiration timestamp. Ansible exposes very little date math capability; notably lacking is the ability to construct a `timedelta` object, so the only way to get a timestamp in the future is to convert the `datetime` object returned by `now` to a Unix timestamp and add some number of seconds to it. Further, there is no direct way to get a `datetime` object from the computed Unix timestamp value, but we can rely on the fact that Python class methods can be called on instances, too, so `now().fromtimestamp()` works the same as `datetime.fromtimestamp()`. --- roles/k8s-worker/tasks/main.yml | 138 +++++++++++++++++++++++++++----- 1 file changed, 119 insertions(+), 19 deletions(-) diff --git a/roles/k8s-worker/tasks/main.yml b/roles/k8s-worker/tasks/main.yml index 9a1a075..61a4672 100644 --- a/roles/k8s-worker/tasks/main.yml +++ b/roles/k8s-worker/tasks/main.yml @@ -1,3 +1,6 @@ +- name: flush handlers + meta: flush_handlers + - name: stat /var/lib/kubelet/config.yaml stat: path: /var/lib/kubelet/config.yaml @@ -6,25 +9,122 @@ tags: - kubeadm-join -- name: generate bootstrap token - delegate_to: '{{ groups["k8s-controller"][0] }}' - command: - kubeadm token create - --kubeconfig /etc/kubernetes/admin.conf - --ttl 1h - --print-join-command +- name: add node to cluster when: - not stat_kublet_config.stat.exists - changed_when: true - register: kubeadm_token_create - tags: - - bootstrap-token - - kubeadm-join -- name: join the kubernetes cluster - command: >- - {{ kubeadm_token_create.stdout }} - when: - not stat_kublet_config.stat.exists - changed_when: true + stat_kubelet_config is not defined or not stat_kublet_config.stat.exists tags: - kubeadm-join + block: + - name: get kubernetes cluster info + set_fact: + cluster_info: >- + {{ query( + "kubernetes.core.k8s", + kind="ConfigMap", + namespace="kube-public", + resource_name="cluster-info", + )[0] }} + tags: + - cluster-info + + - name: generate bootstrap token + set_fact: + bootstrap_token_id: >- + {{ lookup("password", "/dev/null length=6 chars=ascii_lowercase,digits") }} + bootstrap_token_secret: >- + {{ lookup("password", "/dev/null length=16 chars=ascii_lowercase,digits") }} + cacheable: false + no_log: true + tags: + - bootstrap-token + + - name: create bootstrap token secret + delegate_to: localhost + become: false + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Secret + type: bootstrap.kubernetes.io/token + metadata: + name: bootstrap-token-{{ bootstrap_token_id }} + namespace: kube-system + stringData: + description: Bootstrap token for {{ inventory_hostname }} + token-id: '{{ bootstrap_token_id }}' + token-secret: '{{ bootstrap_token_secret }}' + expiration: >- + {{ now().utcfromtimestamp( + now().timestamp() + 300 + ).strftime("%Y-%m-%dT%H:%M:%SZ") + }} + usage-bootstrap-authentication: 'true' + usage-bootstrap-signing: 'true' + auth-extra-groups: 'system:bootstrappers:kubeadm:default-node-token' + no_log: true + tags: + - bootstrap-token + + - name: generate kubeconfig for kubeadm join + vars: + kubeconfig: '{{ cluster_info.data.kubeconfig | from_yaml }}' + config: + apiVersion: v1 + kind: Config + clusters: + - name: kubernetes + cluster: '{{ kubeconfig.clusters[0].cluster }}' + contexts: + - name: kubeadm + context: + cluster: kubernetes + user: kubeadm + current-context: kubeadm + users: + - name: kubeadm + user: + token: '{{ bootstrap_token_id }}.{{ bootstrap_token_secret }}' + copy: + dest: /tmp/kubeconfig + owner: root + group: root + mode: u=rw,go= + content: '{{ config | to_nice_yaml(indent=2) }}' + tags: + - kubeconfig + + - name: generate join configuration file + vars: + config: + apiVersion: kubeadm.k8s.io/v1beta3 + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + config: /var/lib/kubelet/config.yaml + discovery: + file: + kubeConfigPath: /tmp/kubeconfig + copy: + dest: /tmp/joinconfiguration + owner: root + group: root + mode: u=rw,go= + content: '{{ config | to_nice_yaml(indent=2) }}' + + - name: join the kubernetes cluster + command: >- + kubeadm join --config=/tmp/joinconfiguration + changed_when: true + tags: + - run-kubeadm-join + +- name: ensure temporary join configuration files are removed + file: + path: '{{ item }}' + state: absent + loop: + - /tmp/kubeconfig + - /tmp/joinconfiguration + tags: + - kubeadm-join-cleanup + - cleanup