From c6f0ea97207d7cf66321f58b53bc9fb8e1ff3101 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Tue, 7 Nov 2023 20:06:34 -0600 Subject: [PATCH] r/repohost: Configure Yum package repo host So it turns out Gitea's RPM package repository feature is less than stellar. Since each organization/user can only have a single repository, separating packages by OS would be extremely cumbersome. Presumably, the feature was designed for projects that only build a single PRM for each version, but most of my packages need multiple builds, as they tend to link to system libraries. Further, only the repository owner can publish to user-scoped repositories, so e.g. Jenkins cannot publish anything to a repository under my *dustin* account. This means I would ultimately have to create an Organization for every OS/version I need to support, and make Jenkins a member of it. That sounds tedious and annoying, so I decided against using that feature for internal packages. Instead, I decided to return to the old ways, publishing packages with `rsync` and serving them with Apache. It's fairly straightforward to set this up: just need a directory with the appropriate permissions for users to upload packages, and configure Apache to serve from it. One advantage Gitea's feature had over a plain directory is its automatic management of repository metadata. Publishers only have to upload the RPMs they want to serve, and Gitea handles generating the index, database, etc. files necessary to make the packages available to Yum/dnf. With a plain file host, the publisher would need to use `createrepo` to generate the repository metadata and upload that as well. For repositories with multiple packages, the publisher would need a copy of every RPM file locally in order for them to be included in the repository metadata. This, too, seems like it would be too much trouble to be tenable, so I created a simple automatic metadata manager for the file-based repo host. Using `inotifywatch`, the `repohost-createrepo` script watches for file modifications in the repository base directory. Whenever a file is added or changed, the directory containing it is added to a queue. Every thirty seconds, the queue is processed; for each unique directory in the queue, repository metadata are generated. This implementation combines the flexibility of a plain file host, supporting an effectively unlimited number of repositories with fully-configurable permissions, and the ease of publishing of a simple file upload. --- group_vars/repohost.yml | 3 + hosts | 3 + repohost.yml | 4 + roles/repohost/defaults/main.yml | 1 + .../files/repohost-createrepo.service | 33 ++++++ roles/repohost/files/repohost-createrepo.sh | 35 ++++++ roles/repohost/files/yumrepos.httpd.conf | 6 ++ roles/repohost/handlers/main.yml | 9 ++ roles/repohost/tasks/main.yml | 102 ++++++++++++++++++ 9 files changed, 196 insertions(+) create mode 100644 group_vars/repohost.yml create mode 100644 repohost.yml create mode 100644 roles/repohost/defaults/main.yml create mode 100644 roles/repohost/files/repohost-createrepo.service create mode 100755 roles/repohost/files/repohost-createrepo.sh create mode 100644 roles/repohost/files/yumrepos.httpd.conf create mode 100644 roles/repohost/handlers/main.yml create mode 100644 roles/repohost/tasks/main.yml diff --git a/group_vars/repohost.yml b/group_vars/repohost.yml new file mode 100644 index 0000000..a0e0962 --- /dev/null +++ b/group_vars/repohost.yml @@ -0,0 +1,3 @@ +repohost_admins: +- dustin +- jenkins diff --git a/hosts b/hosts index 87407a9..c2de918 100644 --- a/hosts +++ b/hosts @@ -151,6 +151,9 @@ web0.pyrocufflink.blue [radius:children] samba-dc +[repohost] +file0.pyrocufflink.blue + [rw-root] serial0.pyrocufflink.blue diff --git a/repohost.yml b/repohost.yml new file mode 100644 index 0000000..0d36727 --- /dev/null +++ b/repohost.yml @@ -0,0 +1,4 @@ +- hosts: repohost + roles: + - role: repohost + tags: repohost diff --git a/roles/repohost/defaults/main.yml b/roles/repohost/defaults/main.yml new file mode 100644 index 0000000..fc2b986 --- /dev/null +++ b/roles/repohost/defaults/main.yml @@ -0,0 +1 @@ +repohost_admins: [] diff --git a/roles/repohost/files/repohost-createrepo.service b/roles/repohost/files/repohost-createrepo.service new file mode 100644 index 0000000..264aaea --- /dev/null +++ b/roles/repohost/files/repohost-createrepo.service @@ -0,0 +1,33 @@ +[Unit] +Description=Watch and regenerate Yum repositories +RequiresMountsFor=/srv/www/repohost + +[Service] +ExecStart=/usr/local/libexec/repohost-createrepo +User=repohost +DeviceAllow= +DevicePolicy=closed +LockPersonality=yes +MemoryDenyWriteExecute=yes +NoNewPrivileges=yes +PrivateDevices=yes +PrivateUsers=yes +PrivateTmp=yes +ProcSubset=pid +ProtectClock=yes +ProtectControlGroups=yes +ProtectHome=yes +ProtectHostname=yes +ProtectKernelLogs=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +ProtectProc=invisible +ProtectSystem=strict +ReadWritePaths=/srv/www/repohost +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes + +[Install] +WantedBy=multi-user.target diff --git a/roles/repohost/files/repohost-createrepo.sh b/roles/repohost/files/repohost-createrepo.sh new file mode 100755 index 0000000..9883404 --- /dev/null +++ b/roles/repohost/files/repohost-createrepo.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +QFILE="${HOME}"/createrepo.queue +REPOS_ROOT="${HOME}"/repos + +createrepo_loop() { + while sleep 30; do + [ -f "${QFILE}" ] || continue + mv "${QFILE}" "${QFILE}.work" + sort -u "${QFILE}.work" > "${QFILE}.sorted" + while read dir; do + printf 'Generating repository metadata for %s\n' "${dir}" + createrepo_c "${dir}" + done < "${QFILE}.sorted" + rm -f "${QFILE}.work" "${QFILE}.sorted" + done +} + +inotify_loop() { + stdbuf -o 0 inotifywait \ + --monitor \ + --event close_write,move,delete \ + --recursive \ + "${REPOS_ROOT}" \ + | stdbuf -o 0 grep -E '\.rpm$' \ + | while read dir _ _; do + flock "${QFILE}" sh -c 'echo "$1" >> "$2"' -- "${dir}" "${QFILE}" + done +} + +mkdir -p "${REPOS_ROOT}" + +createrepo_loop & +inotify_loop & +wait diff --git a/roles/repohost/files/yumrepos.httpd.conf b/roles/repohost/files/yumrepos.httpd.conf new file mode 100644 index 0000000..9bbba81 --- /dev/null +++ b/roles/repohost/files/yumrepos.httpd.conf @@ -0,0 +1,6 @@ +Alias /yum /srv/www/repohost/repos + + + Require all granted + Options +Indexes + diff --git a/roles/repohost/handlers/main.yml b/roles/repohost/handlers/main.yml new file mode 100644 index 0000000..4a3632d --- /dev/null +++ b/roles/repohost/handlers/main.yml @@ -0,0 +1,9 @@ +- name: restart repohost-createrepo + service: + name: repohost-createrepo + state: restarted + +- name: reload httpd + service: + name: httpd + state: reloaded diff --git a/roles/repohost/tasks/main.yml b/roles/repohost/tasks/main.yml new file mode 100644 index 0000000..a8b41f2 --- /dev/null +++ b/roles/repohost/tasks/main.yml @@ -0,0 +1,102 @@ +- name: ensure required packages are installed + package: + name: + - createrepo_c + - inotify-tools + state: present + tags: + - install + +- name: ensure repohost user exists + user: + name: repohost + system: true + createhome: false + home: /srv/www/repohost + +- name: ensure repohost base directory exists + file: + path: /srv/www/repohost + owner: repohost + group: repohost + mode: u=rwx,go=rx + state: directory + +- name: ensure repohost repos directory exists + file: + path: /srv/www/repohost/repos + owner: repohost + group: repohost + mode: ug=rwx,o=rx + state: directory + +- name: ensure repohost repos dir acl is set + acl: + entity: '{{ item }}' + etype: user + default: false + permissions: rwx + path: /srv/www/repohost/repos + loop: '{{ repohost_admins }}' +- name: ensure repohost repos dir default acl is set + acl: + entity: '{{ item }}' + etype: user + default: true + permissions: rwx + path: /srv/www/repohost/repos + loop: '{{ repohost_admins }}' + +- name: ensure repohost-createrepo script is installed + copy: + src: repohost-createrepo.sh + dest: /usr/local/libexec/repohost-createrepo + owner: root + group: root + mode: u=rwx,go=rx + setype: bin_t + notify: + - restart repohost-createrepo + tags: + - repohost-createrepo + - install + +- name: ensure repohost-createrepo systemd service unit is installed + copy: + src: repohost-createrepo.service + dest: /etc/systemd/system/repohost-createrepo.service + owner: root + group: root + mode: u=rw,go=r + tags: + - systemd + +- name: ensure repohost-createrepo service starts at boot + service: + name: repohost-createrepo + enabled: true + tags: + - service + +- name: flush handlers + meta: flush_handlers + +- name: ensure repohost-createrepo service is running + service: + name: repohost-createrepo + state: started + tags: + - service + +- name: ensure apache is configured to serve yum repos + copy: + src: yumrepos.httpd.conf + dest: /etc/httpd/conf.d/yumrepos.conf + owner: root + group: root + mode: u=rw,go=r + notify: + - reload httpd + tags: + - config + - httpd-config