From f7f408ca8cc9e3773b84bd00828033de403ca42b Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Tue, 25 Jun 2024 19:49:30 -0500 Subject: [PATCH] v-m: Redo vmstorage persistent volumes Longhorn does not work well for very large volumes. It takes ages to synchronize/rebuild them when migrating between nodes, which happens all too frequently. This consumes a lot of resources, which impacts the operation of the rest of the cluster, and can cause a cascading failure in some circumstances. Now that the cluster is set up to be able to mount storage directly from the Synology, it makes sense to move the Victoria Metrics data there as well. Similar to how I did this with Jenkins, I created PersistentVolume resources that map to iSCSI volumes, and patched the PersistentVolumeClaims (or rather the template for them defined by the StatefulSet) to use these. Each `vmstorage` pod then gets an iSCSI LUN, bypassing both Longhorn and QEMU to write directly to the NAS. The migration process was relatively straightforwrad. I started by scaling down the `vminsert` Deployment so the `vmagent` pods would queue the metrics they had collected while the storage layer was down. Next, I created a [native][0] export of all the time series in the database. Then, I deleted the `vmstorage` StatefulSet and its associated PVCs. Finally, I applied the updated configuration, including the new PVs and patched PVCs, and brought the `vminsert` pods back online. Once everything was up and running, I re-imported the exported data. [0]: https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-export-data-in-native-format --- victoria-metrics/kustomization.yaml | 20 +++- victoria-metrics/vmstorage-iscsi.yaml | 153 ++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 victoria-metrics/vmstorage-iscsi.yaml diff --git a/victoria-metrics/kustomization.yaml b/victoria-metrics/kustomization.yaml index f25beda..7ff0f78 100644 --- a/victoria-metrics/kustomization.yaml +++ b/victoria-metrics/kustomization.yaml @@ -14,6 +14,7 @@ labels: resources: - namespace.yaml - secrets.yaml +- vmstorage-iscsi.yaml - vmstorage.yaml - vmselect.yaml - vminsert.yaml @@ -59,7 +60,7 @@ replicas: # the storageNode value for vmselect and vminsert. Also, the # replicationFactor setting may need adjusted. - name: vmstorage - count: 3 + count: 4 - name: vmselect count: 2 - name: vminsert @@ -103,7 +104,7 @@ patches: - name: vmselect env: - name: vmselect_storageNode - value: vmstorage-0.vmstorage,vmstorage-1.vmstorage,vmstorage-2.vmstorage + value: vmstorage-0.vmstorage,vmstorage-1.vmstorage,vmstorage-2.vmstorage,vmstorage-3.vmstorage - name: vmselect_replicationFactor value: '2' @@ -119,7 +120,7 @@ patches: - name: vminsert env: - name: vminsert_storageNode - value: vmstorage-0.vmstorage,vmstorage-1.vmstorage,vmstorage-2.vmstorage + value: vmstorage-0.vmstorage,vmstorage-1.vmstorage,vmstorage-2.vmstorage,vmstorage-3.vmstorage - name: vminsert_dedup_minScrapeInterval value: 1m - name: vminsert_replicationFactor @@ -202,3 +203,16 @@ patches: - --storage.path=/alertmanager - --cluster.peer=alertmanager-0.alertmanager:9094 - --cluster.peer=alertmanager-1.alertmanager:9094 + +- patch: | + - op: add + path: /spec/volumeClaimTemplates/0/spec/storageClassName + value: '' + - op: replace + path: /spec/volumeClaimTemplates/0/spec/resources/requests/storage + value: 30G + target: + group: apps + version: v1 + kind: StatefulSet + name: vmstorage diff --git a/victoria-metrics/vmstorage-iscsi.yaml b/victoria-metrics/vmstorage-iscsi.yaml new file mode 100644 index 0000000..4946aaa --- /dev/null +++ b/victoria-metrics/vmstorage-iscsi.yaml @@ -0,0 +1,153 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: data-vmstorage-0 + namespace: victoria-metrics + labels: + app.kubernetes.io/name: vmstorage-iscsi-chap + app.kubernetes.io/component: vmstorage +spec: + accessModes: + - ReadWriteOnce + storageClassName: '' + capacity: + storage: 30G + iscsi: + targetPortal: '[fd68:c2d2:500e:3ea3:8d42:e33e:264b:7c30]:3260' + iqn: iqn.2000-01.com.synology:storage0.vmstorage-0.8181625090 + lun: 1 + chapAuthDiscovery: false + chapAuthSession: true + fsType: ext4 + secretRef: + name: vmstorage-iscsi-chap + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: network.du5t1n.me/storage + operator: In + values: + - 'true' + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: data-vmstorage-1 + namespace: victoria-metrics + labels: + app.kubernetes.io/name: vmstorage-iscsi-chap + app.kubernetes.io/component: vmstorage +spec: + accessModes: + - ReadWriteOnce + storageClassName: '' + capacity: + storage: 30G + iscsi: + targetPortal: '[fd68:c2d2:500e:3ea3:8d42:e33e:264b:7c30]:3260' + iqn: iqn.2000-01.com.synology:storage0.vmstorage-1.8181625090 + lun: 1 + chapAuthDiscovery: false + chapAuthSession: true + fsType: ext4 + secretRef: + name: vmstorage-iscsi-chap + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: network.du5t1n.me/storage + operator: In + values: + - 'true' + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: data-vmstorage-2 + namespace: victoria-metrics + labels: + app.kubernetes.io/name: vmstorage-iscsi-chap + app.kubernetes.io/component: vmstorage +spec: + accessModes: + - ReadWriteOnce + storageClassName: '' + capacity: + storage: 30G + iscsi: + targetPortal: '[fd68:c2d2:500e:3ea3:8d42:e33e:264b:7c30]:3260' + iqn: iqn.2000-01.com.synology:storage0.vmstorage-2.8181625090 + lun: 1 + chapAuthDiscovery: false + chapAuthSession: true + fsType: ext4 + secretRef: + name: vmstorage-iscsi-chap + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: network.du5t1n.me/storage + operator: In + values: + - 'true' + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: data-vmstorage-3 + namespace: victoria-metrics + labels: + app.kubernetes.io/name: vmstorage-iscsi-chap + app.kubernetes.io/component: vmstorage +spec: + accessModes: + - ReadWriteOnce + storageClassName: '' + capacity: + storage: 30G + iscsi: + targetPortal: '[fd68:c2d2:500e:3ea3:8d42:e33e:264b:7c30]:3260' + iqn: iqn.2000-01.com.synology:storage0.vmstorage-3.8181625090 + lun: 1 + chapAuthDiscovery: false + chapAuthSession: true + fsType: ext4 + secretRef: + name: vmstorage-iscsi-chap + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: network.du5t1n.me/storage + operator: In + values: + - 'true' + +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: vmstorage-iscsi-chap + namespace: victoria-metrics + labels: + app.kubernetes.io/name: vmstorage-iscsi-chap + app.kubernetes.io/component: vmstorage +spec: + encryptedData: + node.session.auth.password: AgBn6RzgncHMzWMrNK9xgARLSWImS1OzI1KiISOJJDXRbs/HmxIJ1f924n87tNJ6c6Z/qA5QjWEOMT1PYmfjWMNGLfTBwZGRPHfUGnLIyv1MRaBHcN8OCmYYC6fXclKjZz7Ql4Gw+egr/7v4drFRonswZ6k9OF8vr89AZLlBBmpK+nRTiKdO12OZNA6U2BKsUgNz+E+sCLy+yQ7IgztnMTExJ3tVgLYNVU75hW8cr2MXXpK0WOeaipcklCPFRwbPufFVY7Ma8AtrwCb6Ob6NzIjOtCyt9WXHv9VXw8Z0UjCUkqVEzjC+bjgf6pqy5cS2g3JkAtd6+OLzgB9l98uolHyKaZCxIwD3Jy2jG/hSDZWrgS0cBygei/UiYj1iNNH5rXAm4eNEHJ1JDJ6Kn1mgATCE0FJUnQ3uhwhNOFSQDchLirRZ3jCZgMUkcKPFwGEdiDTVcvKa8Wb2sJts7k3O92cCNXxgCa1mpxyCBUmIGZ1ANkh7wBgBxOzninmHlLq/HM/zoxmmeR6wIlF0Mmp1DLTi0n8TicOXXn7OtbJerTUXTVmzqXkyovK6FM1s0SYUXQcTUVODYFB+iwQoK1PmSrXYVdSKw7bARIp4HZTWy4O+GShsJEeUMYMmEca7720BF2NeXCp6ObTTXxHX+kp2jK1CX5D+cyugrzBa++28IYFo3/k7mYgGFUBw8b+aFKnTO0tWK50w15v0yArMf/k= + node.session.auth.password_in: AgAtvQGyPBfd96cxhwd8z2QBxm1mw4Xi3Y0Kp2NNqZaJKHMFaiNcu39itflyPdJULP63KDVImq5hpWDfElhJ83Ve4fKBtYJfMnWyIxlGeBukdYSg4g1Iu8AAKg84BM1FqHLccsYnDufzZ/3MGOXgOWDDWZZcTdV7d/MqtYnaIaz1TAJC9nX8F/+AAXWP1qZbIoYn85Dd+E+gGQA/Sg2wE8vOWSsmfhOOKX+wvL0vchWJ8OvuvUJd9SZNm3669bGCBsKn3p7Bd62r3XsCuM1C6VNHHTsxOQoSbNrrGqikGPYmoi0BeXSJ2fgm6KCO8YkhydG/QZ3uw1HPgBuyjJRB9NeG2di3mi1AZsl6wmpAkqRXob8hsBEq7CNo6YwLLSkRL6eiFuDH1j13stn5y820VGfFFux6aDTD/Av3VLTZH3rynq125erXvKHSYr6BSJw9roDaLifqYxrT0NHX1+NtuCx31uN/z3r8gxpTrWb950sG7eueION7u1wMX6ke6NFp2DgslalnhIIaFe3U+OgS3g0ZjYML7Yiw3RRisxnMiH0KbBzRtqH9z/vA1TVXxjln/xNQE/i/N9J74m23Dt/o37jFAGy5dHQe5+lA0g36P0ySvBaeMgCFU8W2sxNf02v4NLghRlJ7rAf+ZpGmxuHjgwb/nfD5GldtCsvZF9lrWey46ehpXt8djIREHNfSHBr+OjB0d507snYGAI0n0bk= + node.session.auth.username: AgBrMRiQfG2iNUb+ZN9NkcKaRbiyyWnLGpYV9Ps5dwNGMxVQCSE2s1zAc7x1V1KzFzntNvXFf/W5OjHXPgOJ0tD1JE64XcfXfqWiHUnQlg2Uv5giZ4THxFzreqqx+dQdBiPx6YNRsHEwDvk13l0xxyv3Ib1EoY+nQbk/lkLlPzpqN3FpGx408vMsPfkIohQ+qWDcNuh87IN8QqRzTjY8HX6amZ+3m10kgg8uGPjtLokT7AyxNJqdvHT9kIUjOp8ivq7qGlwV3BtKUMpWeEc05zCc79CROZP1cwIq5mTo2y6+2sG+1yeghx6bdauJqbKk3MoMN41lX/HvLOBsLzLqlAeIYCopg+BtrlrsZX8OK9vy11dzzd1ea6tv25DaMl4ciLyTSR0gC2gWrtuTQP4PLJg2gFBgkj69iNi6pKTOZVYoazP0+CeixTgd4pKxp2+WlmKrM+IHF647wv3M5Aj3+tzimq4POfcIESI84LGkdrbCEFqZ6yj4NRj+kQ5nMxw9LX3sgbULNnP1fiA2kFQaxq/u+n/gb6aNnrPqqx0n02C7/s3KgG3T7hDOEw2uIEz8nRuVrvvcHGnmVFMp7HlFT6FcArs2DupN+4BeXM9UYZ9f9qMCHkaU4ZBNhgTKkV65IEo9K3YypjU0wEZs26/VA2eCsCaEC0QXfUS6mqIpg0OiGm/w0d55kdgy4beCULaCs6iHQabhvliWks0= + node.session.auth.username_in: AgAcnIy+9/QmsSdnBW/AXGmgMNRcB3r0/6W3c5BZi3OgGuf5TJza4cvniEcIQbu4Dn6zRu02TTKea4Jez89+2aYSKuvVACCtxjigw1FjtnwQJmk1G3WMyDq/T3tm8Bnbq3ms+lDxZqhl0n95LUr0qKyDlWgxKaRZks25Ve5JqI6YOizyMmi2rJIrmEomDE8ZLAfY1I3jjwXpIoJ44aYzFurGliDbsZCywUbigJq6osNrQ94e43xB7lRuonOeEd5EMnvjCbM6squ7W3qFr/v/55/aaLWXF60VLCtCzVEY881o7JKqUiOHl4jwJYsmrNL7eW7bogdCYVqya4rt6tVnXyEHs6R18BQDHkruXuWOV8iSNY3aOHtNmF3Bfltfl2f/eIY3AtHH2OUeuxz6BfB7PV46+OBSN/NADlSzXwQ0tJptAZF1SSIFM+tfnD+7wC2b5AzGVNG86mK07c32EopXWa19J1RIwF+t691PTAB6Ra6wBGeD2bFCOsMsJ17bEmpPvJjNvBZcTu0BOWmk7YPxO+9arSaPg2yCVmp2JHmbrUPR2RA/Cz+Yu+agIRSLO76YxnhRQaDXJ1GZVmZEfNUZ5InhMoehMv/4vD+vqCiesqObiB5H/e7Eylyi0quHn7lEZI+JTFzBRztYcmijo8A56+mRvVC5lSp6Kf2tm49MHKdUVLBUpcgEV5fnU8VtiP2agbaNI4pW7td58g== + template: + metadata: + name: vmstorage-iscsi-chap + namespace: victoria-metrics + labels: + app.kubernetes.io/name: vmstorage-iscsi-chap + app.kubernetes.io/component: vmstorage