From 9561c687aa3275539b6b089d31f4217ba0cfbdee Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 23 Dec 2023 07:08:41 -0600 Subject: [PATCH] xactfetch: Run xactfetch in a CronJob I finally got *xactfetch* cleaned up enough to run in a headless container. --- xactfetch/README.md | 47 +++++++++++++++++++++++ xactfetch/kustomization.yaml | 22 +++++++++++ xactfetch/pvc.yaml | 15 ++++++++ xactfetch/rbw-config.json | 1 + xactfetch/secrets.yaml | 30 +++++++++++++++ xactfetch/xactfetch.env | 8 ++++ xactfetch/xactfetch.yaml | 72 ++++++++++++++++++++++++++++++++++++ 7 files changed, 195 insertions(+) create mode 100644 xactfetch/README.md create mode 100644 xactfetch/kustomization.yaml create mode 100644 xactfetch/pvc.yaml create mode 100644 xactfetch/rbw-config.json create mode 100644 xactfetch/secrets.yaml create mode 100644 xactfetch/xactfetch.env create mode 100644 xactfetch/xactfetch.yaml diff --git a/xactfetch/README.md b/xactfetch/README.md new file mode 100644 index 0000000..2496df9 --- /dev/null +++ b/xactfetch/README.md @@ -0,0 +1,47 @@ +# xactfetch—Automatically Import Bank Transactions + +*xactfetch* is a tool that automates importing bank account transactions into +[Firefly-III] using CSV exports from banks' websites. It uses [Playwright] to +automate navigating the online banking portals and downloads transaction +exports, then imports the data using the [Firefly-III Data Importer]. + + +## Online Banking Passwords + +Credentials for online banking websites are stored in a Bitwarden Vault. +*xactfetch* uses a dedicated account for accessing Bitwarden, which is a member +of a special Organization that shares the bank credentials. My normal user is +also a member of this Organization, which allows me to use and update the +credentials normally, and any changes will automatically be made available to +*xactfetch*. + + +## Chase SMS Verification + +The Chase website requires "verification" on a per-device basis. The first +time accessing the Chase website, a verification code will be sent to the SMS +number associated with the Chase account. That code must be provided in order +to log in. *xactfetch* does NOT automate this process. Instead, it tries to +appear like a device that has used the Chase website before by keeping a +persistent cookie store across executions. + +Sometimes, the cookie that indicates the device has been verified expires or +otherwise becomes invalid. To fix this, *xactfetch* must be run manually with +a non-headless browser: + +```sh +DEBUG_HEADLESS_BROWSER=0 python xactfetch.py +``` + +When the verification form is presented, follow the process to enter the code. +After *xactfetch* has completed successfully, copy the `cookies.json` file it +created to the Kubernetes PersistentVolume. One way to do this is to create +a Pod with the volume mounted, then use `kubectl` to copy the file: + +```sh +kubectl exec -i -n firefly-iii xactfetch-28388926-5r778 -- sh -c 'cat > /var/lib/xactfetch/cookies.json' < cookies.json +``` + +[Firefly-III]: https://www.firefly-iii.org/ +[Playwright]: https://playwright.dev/python/docs/intro +[Firefly-III Data Import Tool]: https://github.com/firefly-iii/data-importer/ diff --git a/xactfetch/kustomization.yaml b/xactfetch/kustomization.yaml new file mode 100644 index 0000000..cae0001 --- /dev/null +++ b/xactfetch/kustomization.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: firefly-iii + +labels: +- pairs: + app.kubernetes.io/instance: xactfetch + includeSelectors: true + +resources: +- pvc.yaml +- xactfetch.yaml +- secrets.yaml + +configMapGenerator: +- name: xactfetch + envs: + - xactfetch.env +- name: xactfetch-rbw + files: + - config.json=rbw-config.json diff --git a/xactfetch/pvc.yaml b/xactfetch/pvc.yaml new file mode 100644 index 0000000..6630d28 --- /dev/null +++ b/xactfetch/pvc.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: xactfetch + app.kubernetes.io/component: xactfetch + app.kubernetes.io/part-of: xactfetch + name: xactfetch +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Mi diff --git a/xactfetch/rbw-config.json b/xactfetch/rbw-config.json new file mode 100644 index 0000000..7cb7ff3 --- /dev/null +++ b/xactfetch/rbw-config.json @@ -0,0 +1 @@ +{"email":"xactfetch@pyrocufflink.net","base_url":"https://bitwarden.pyrocufflink.net/","identity_url":null,"lock_timeout":3600,"pinentry":"pinentry-stub"} diff --git a/xactfetch/secrets.yaml b/xactfetch/secrets.yaml new file mode 100644 index 0000000..6d0f67f --- /dev/null +++ b/xactfetch/secrets.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: xactfetch + namespace: firefly-iii +spec: + encryptedData: + rbw-vault.password: AgBEb1mbqd6z07wp8fMW73rxQOeHdDVz23BXEmmT3zyizaH/owEIOXJiMkjmFi3yYzjfMl9ILYNvcxuolM0vsGUlynaayawLzvZ19D4+lO81HPsMmxhCN0T3MTS6/Si/O6teVvs4bQi2DiGdXaQF3CVQuR8Fr6pTzZh3zu1Cpi3hpHqF5gwDchhFad84iHr/lbC9U4PTNdu1kxAv2EoZ5A2ZmHgk8cAaUB4MEU7idFpLLZ8BnsrD1WT1hhYzFXYrPFrgoQXe5Adx4T5LvHIhjGug8t2etPy6K/orz9gD14Ou8Mh+kh7FYFfGS5CnmyQM9W4HuqagrSc8cMz2o+2eAB4HNi3vzlQ12LEpGxAh1XfroNZe4hoBS2XlU1OM7dLJW7JMzq26do4Zi3PxdXK3zkVU3Huc/+cp306zsSxGUIXgHmWb/rMEMy3/w+J9wOY+9NSZp299rAXrgBc2VJYrGM3dj8QiiL+Ac2xQ65TKXRqmv5bTgbweD/mCptIU+f5o7SLGXpxGXEVQTss9LB09MVU8X14r02Zj3w+bbuvI5iDrsa+Y2MvhsCnuXUp9TkPPYpILJpMAcRE5ysrjzkG5ScHSils6u55t6MR33eNq6mwB+WN/poTb4EvEnUotxetq/F7TcBh/bpiVzYZB4FJ+Y4b7CoIAu/Nh+OxPN4sQ8lr/mns9FicJ67m8re3M5xEFw6gbu5Dxe1oBjmazypvvJQekDHP9Bihc89FVDfeNiqFz/bsXuidHf59S5tt0qIs= + firefly-import.secret: AgCWsgXqeJKGjdIjBoQdDKQJN0H62o8sVRRqrCGwf/O+oErwiI5FxihbZcLeJHrVYyUA+cxszu7QqXxlSGFyzxGDpRucFov3edLczkwjrrNSeXzSI5EAuoZLo6RqpQfkvEsoKIXxA+724h86xnUhg/Q3i01mG/ZLaPnbEydJtDmrSHYw49nt08+OMeMsmJxO+V8o6NRyRDUitU+ARKz5ha48HjTEaFekdEqoVJfjQUOLi226dP9Q++OeK8Re/78Kj7oi+lZ9fHETsJanyemPdFo0VpkANYALziaqWKMH6fDdji8T6cAYwU1teHWwNthLzZsfRqmA8CmDQDApgBOFiuN2FRPInex3FbWP6VH6mIoFqxTTW3BQcvpHbKdGMNwHkCyoFuQZ8yfjY9Ix4x1aeNTxkvHTIuI4rg1Jfj9YSp4mN9r7BSgXPqdsiYpHKAyRTBxeeWIIoWOP8N38b+/ZpphuXNjBU8wVhKOypo2Lzzf5PK1pVdmdoirntzqH2QPNu2WXQ5l15AjceYyJFRESgstEfxAMj9xNcr1vxBfK3B8yamueNyvoZLeMj0ZsI30WDJEMBZW/9fqF7XVFoRHdP6dfBM3TrltjacPPWNo/q3XMTLc0jhXifr2g/iej1VqTd+H530rTa4JYkIgd0kL7LAvFl9zN0e04GOa1ghRM9h3WsTw/2m7u5JYzoU07cBN4v4qUhP4Bdku21VAmwVp6G1feBKluHt2IcMo50YsG0RR+zQ== + firefly-import.password: AgAsUJE1zhOKFwG8rOGPHj5VtPFzfFN/JDwJlBIbZ7s0HFdtxiPajMOInhSE/efbSpN8BLk+mluoWe9mZLvyDYnaL8L5FSjP8s8q5oBWAfuzNka5LtFiMa3tdrY/4gTKrJ57FSM2TbM6dBjERCV1Xmt4Sk45VnafOmhWl68oycYe3zaDHWqYYtgTKDhruP+Yiw1YM24FwnHpmgdL1mnHjS/fiz/mrCz9XZBUajvwiLu3VdFc3iNWr2HRAV+snlI09pLFL8tF5wZFKw0KnwAPzj/q6jAP5i9B0BtedzhsMfSofH0tn3MdsluNTS8x3kTpdJZPrWf3lxOtotWh5mUuuMZiTGkct61DYg6NUXCt5cbIsLsU3f1L0BRte9mrJcAeqDGy3mCzHkjBgM7bxAnpViF3jRBVqAmNG05/ZXTetLdWopu6B1duw/jnWLDnfjcaEwPRZn4zrZD4Y+Q0vOFL2P+XqEXtsFic1StOxx0ViYDa/k9KdxF/L7xSWmw7Egkdw66B7nrhaxJCSYbvDt7Zm5USMkAgyPTEMNhcGoE9D5i30kiGOf5rI0SdtdT6calVWoDNyW3hnbV4Z4Tvv20fdJhLxFRqSjEGuqRfvUdtwDG84Nwx8YFjQrRVAvBw1pE4UiJW8jRNEqSIBN6L20BaZ2pCW6f7apFb6jYN3gYLGUTkACBs1DHEh9KtCuLd1/KeAaHZQsDc6E/qAr4biiwXMYZ2uh/WHDjYwuCsqR0ATkl+hw== + template: + metadata: + name: xactfetch + namespace: firefly-iii + +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: imagepull-gitea + namespace: firefly-iii +spec: + encryptedData: + .dockerconfigjson: AgCFtWPx0O9I2ZQhR9ycH/k/SzXw+L0eDx1jABvaSEb5YKNVWJKFLnHDcDgDpW94UqQtNAuVng2RiVA+FFhY07N3VnwwxSCDN1TfC9/Zzq2zbIo45zrmTuLZ1EgTyUPJlPTfFHBGcjJAQ5JVcwNZ41q09l8yFJ6t3dxdRPGVWInKMnv8i1B9ySl+2un/sm/XrANvAnZasUH+jj6gR4BL4KNzVJ7GwGQgUkdXfCTpYu7kA4A6v+3+pH4V03Z16S+Ykmtjfk8pFnm7Yzlz73RuW0Jv0X/yL03378ji9r54MKXXupCdcaWPqinX0S5Ih3vRog1gUUYOVXeG5rzh5/XSvsNy7bFHWFx3lMDlkgbl0Sfwt8QlMEx2guUXMQKpA79SFI6F7tMa767Er3xqHg6trR3wr0f0uZzz8D5TCFM/XDX3oW1a0LJe3WWT9yJnnCz2NLkMRSVMJyfa05G0hnexKcyMmPYMvPuBbhox1vITznezcTksfhxb1aOeGB+bKPJTkkyF8L50PDjwR80Y1fw9N1hlmWFJum6KujSVNtorN0r08gn5XLN+7vDN/EEnrNyM8Jjkmbo3RtJi7L6t9C8ZzudxAF/bty/dGgjLuXrSKKgCNvwDNyvHRUtX61tkRfcWsdouaZl7UkN+t5aupV6PepY2M6kn82LzxGck2Z8HI2PU7F2/xsRtqeW5NWegXJGJRzfXDU3PcjDmthcL9toery1QCkuH3nNtBWwoPCy/wn3qF5cmoI90YxnIZdLHbecMGVfBp3JutUSCT0zYTJ4DFfOpVDUw0SX9sx3TXjh++Lwzi8dbQnwlXWgHmpDur1BHKADmpEMQ8C7kXwu7HpiAyVWxB4UDv5DobILBNB9IDeIQaHK26T+Q3xHh4Osx61if24u42OsxPFmn2M5femy+p2NJ9WccTgu3n0qZW9rNAIcFbZi6Y3FhPcq+UaIfyAAuxYbdhytDr8kVFLEs7hUVSx0PmTycJBkCn0Lq/GtOz5esecVMcAu9b6hE9rhfLZk= + template: + metadata: + name: imagepull-gitea + namespace: firefly-iii + type: kubernetes.io/dockerconfigjson diff --git a/xactfetch/xactfetch.env b/xactfetch/xactfetch.env new file mode 100644 index 0000000..f6ced24 --- /dev/null +++ b/xactfetch/xactfetch.env @@ -0,0 +1,8 @@ +PINENTRY_PASSWORD_FILE=/run/secrets/xactfetch/rbw-vault.password +FIREFLY_IMPORT_SECRET_FILE=/run/secrets/xactfetch/firefly-import.secret +FIREFLY_IMPORT_PASSWORD_FILE=/run/secrets/xactfetch/firefly-import.password +FIREFLY_IMPORT_USER=svc.xactfetch +FIREFLY_III_URL=https://firefly.pyrocufflink.blue +FIREFLY_IMPORT_URL=https://firefly-importer.pyrocufflink.blue +NTFY_URL=https://ntfy.pyrocufflink.net +NTFY_TOPIC=dustin diff --git a/xactfetch/xactfetch.yaml b/xactfetch/xactfetch.yaml new file mode 100644 index 0000000..8511dbb --- /dev/null +++ b/xactfetch/xactfetch.yaml @@ -0,0 +1,72 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: xactfetch + labels: + app.kubernetes.io/name: xactfetch + app.kubernetes.io/component: xactfetch + app.kubernetes.io/part-of: xactfetch +spec: + schedule: 4 9 * * * + timeZone: America/Chicago + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + metadata: + labels: + app.kubernetes.io/name: xactfetch + app.kubernetes.io/component: xactfetch + app.kubernetes.io/part-of: xactfetch + spec: + restartPolicy: Never + imagePullSecrets: + - name: imagepull-gitea + initContainers: + - name: wait + image: registry.fedoraproject.org/fedora-minimal + command: + - sh + - -c + - sleep $((RANDOM % 3600)) + securityContext: + readOnlyRootFilesystem: true + runAsGroup: 999 + runAsUser: 999 + containers: + - name: xactfetch + image: git.pyrocufflink.net/packages/xactfetch + envFrom: + - configMapRef: + name: xactfetch + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/rbw + name: xactfetch-rbw + readOnly: true + - mountPath: /run/secrets/xactfetch + name: xactfetch-secrets + readOnly: true + - mountPath: /tmp + name: tmp + subPath: tmp + - mountPath: /var/lib/xactfetch + name: xactfetch-data + subPath: data + securityContext: + fsGroup: 2468 + runAsNonRoot: true + volumes: + - name: tmp + emptyDir: + medium: Memory + - name: xactfetch-data + persistentVolumeClaim: + claimName: xactfetch + - name: xactfetch-rbw + configMap: + name: xactfetch-rbw + - name: xactfetch-secrets + secret: + secretName: xactfetch