From 34fbdc6e02c1bcb138e4e09b25c9b601c6803dcc Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Wed, 4 Sep 2024 21:20:18 -0500 Subject: [PATCH] Rework data model to group images into projects A "project" now refers to an application deployed into Kubernetes, which includes one or more "images." This is really the grouping I wanted in the beginning, which I tried to achieve using separate configuration files. Unfortunately, this made the original "projects" too independent, making it difficult to produce the manifest diff I wanted to add to the PR descriptions. It was also cumbersome managing multiple config files and therefore multiple CronJobs in Kubernetes. The new data model is a lot deeper than the original one, making TOML a lot less nice. YAML definitely handles nested data structures better, despite its shortcomings. Having to repeat nested table names in TOML is quite cumbersome. --- updatebot.py | 169 ++++++++++++++++++++++++++------------------------- 1 file changed, 85 insertions(+), 84 deletions(-) diff --git a/updatebot.py b/updatebot.py index c9f477e..eacac8c 100644 --- a/updatebot.py +++ b/updatebot.py @@ -7,10 +7,9 @@ import os import re import tempfile import threading -import tomllib import urllib.parse from pathlib import Path -from typing import ClassVar, Literal, Union, Optional +from typing import ClassVar, Iterable, Literal, Union, Optional import colorlog import pydantic @@ -112,19 +111,29 @@ Source = Union[ ] -class BaseProject(abc.ABC, pydantic.BaseModel): - path: Optional[Path] = None - source: Source +class ImageDef(pydantic.BaseModel): + name: str image: str + source: Source tag_format: str = '{version}' + +class BaseProject(abc.ABC, pydantic.BaseModel): + name: str + path: Path + + def __init__(self, **data) -> None: + data.setdefault('path', data.get('name')) + super().__init__(**data) + @abc.abstractmethod - def apply_update(self, path: Path, version: str) -> None: + def apply_updates(self, basedir: Path) -> Iterable[tuple[ImageDef, str]]: raise NotImplementedError class KustomizeProject(BaseProject): kind: Literal['kustomize'] + images: list[ImageDef] kustomize_files: ClassVar[list[str]] = [ 'kustomization.yaml', @@ -132,36 +141,36 @@ class KustomizeProject(BaseProject): 'Kustomization', ] - def apply_update(self, path: Path, version: str) -> None: + def apply_updates(self, basedir: Path) -> Iterable[tuple[ImageDef, str]]: + path = basedir / self.path for filename in self.kustomize_files: filepath = path / filename if filepath.is_file(): break else: - filepath = path / self.kustomize_files[0] - yaml = ruamel.yaml.YAML() - with filepath.open('rb') as f: - kustomization = yaml.load(f) - images = kustomization.setdefault('images', []) - new_tag = self.tag_format.format(version=version) - for image in images: - if image['name'] == self.image: - image['newTag'] = new_tag - break - else: - images.append({'name': self.image, 'newTag': new_tag}) - with filepath.open('wb') as f: - yaml.dump(kustomization, f) + raise ValueError( + f'Could not find Kustomize config for {self.name}' + ) + + for image in self.images: + yaml = ruamel.yaml.YAML() + with filepath.open('rb') as f: + kustomization = yaml.load(f) + images = kustomization.setdefault('images', []) + version = image.source.get_latest_version() + new_tag = image.tag_format.format(version=version) + for i in images: + if i['name'] == image.image: + i['newTag'] = new_tag + break + else: + images.append({'name': image.image, 'newTag': new_tag}) + with filepath.open('wb') as f: + yaml.dump(kustomization, f) + yield (image, version) -class DirectoryProject(BaseProject): - kind: Literal['dir'] | Literal['directory'] - - -Project = Union[ - KustomizeProject, - DirectoryProject, -] +Project = KustomizeProject class RepoConfig(pydantic.BaseModel): @@ -225,7 +234,7 @@ class RepoConfig(pydantic.BaseModel): class Config(pydantic.BaseModel): repo: RepoConfig - projects: dict[str, Project] + projects: list[Project] class Arguments: @@ -235,25 +244,24 @@ class Arguments: projects: list[str] -def update_project( - repo: git.Repo, name: str, project: Project -) -> Optional[git.Commit]: +def update_project(repo: git.Repo, project: Project) -> Optional[str]: basedir = Path(repo.working_dir) - log.debug('Checking for latest version of %s', name) - latest = project.source.get_latest_version() - log.info('Found version %s for %s', latest, name) - log.debug('Applying update for %s version %s', name, latest) - path = basedir / (project.path or name) - project.apply_update(path, latest) - if repo.index.diff(None): - log.debug('Committing changes to %s', path) - repo.index.add(str(path)) - c = repo.index.commit(f'{name}: Update to {latest}') - log.info('Commited %s %s', str(c)[:7], c.summary) - return c - else: - log.info('No changes to commit') - return None + title = None + for (image, version) in project.apply_updates(basedir): + log.info('Updating %s to %s', image.name, version) + if repo.index.diff(None): + log.debug('Committing changes to %s', project.path) + repo.index.add(str(project.path)) + c = repo.index.commit(f'{image.name}: Update to {version}') + log.info('Commited %s %s', str(c)[:7], c.summary) + if not title: + if not isinstance(c.summary, str): + title = bytes(c.summary).decode('utf-8') + else: + title = c.summary + else: + log.info('No changes to commit') + return title def parse_args() -> Arguments: @@ -296,23 +304,12 @@ def setup_logging() -> None: def main() -> None: setup_logging() args = parse_args() + yaml = ruamel.yaml.YAML() with args.config.open('rb') as f: - data = tomllib.load(f) + data = yaml.load(f) config = Config.model_validate(data) log.debug('Using configuration: %s', config) - all_projects = list(config.projects.keys()) - projects = [] - if args.projects: - for project in args.projects: - if project not in all_projects: - log.warning('Unknown project: %s', project) - continue - projects.append(project) - else: - projects = all_projects - if not projects: - log.error('No projects') - raise SystemExit(1) + projects = args.projects or [p.name for p in config.projects] if log.isEnabledFor(logging.INFO): log.info('Updating projects: %s', ', '.join(projects)) with tempfile.TemporaryDirectory(prefix='updatebot.') as d: @@ -321,29 +318,33 @@ def main() -> None: log.debug('Retreiving repository Git URL') repo_url = config.repo.get_git_url() repo = git.Repo.clone_from(repo_url, d, depth=1, b=config.repo.branch) - log.debug('Checking out new branch: %s', args.branch_name) - repo.heads[0].checkout(force=True, B=args.branch_name) - title = None - for project in projects: - commit = update_project(repo, project, config.projects[project]) - if commit and not title: - if not isinstance(commit.summary, str): - title = bytes(commit.summary).decode( - 'utf-8', errors='replace' - ) - else: - title = commit.summary - if not title: - log.info('No changes made') - return - repo.head.reference.set_tracking_branch( - git.RemoteReference( - repo, f'refs/remotes/origin/{args.branch_name}' + for project in config.projects: + log.debug('Checking out new branch: %s', args.branch_name) + repo.heads[0].checkout(force=True, B=args.branch_name) + title = None + if project.name not in projects: + continue + title = update_project(repo, project) + if not title: + log.info('No changes made') + continue + repo.head.reference.set_tracking_branch( + git.RemoteReference( + repo, f'refs/remotes/origin/{args.branch_name}' + ) ) - ) - if not args.dry_run: - repo.remote().push(force=True) - config.repo.create_pr(title, args.branch_name, config.repo.branch) + if not args.dry_run: + repo.remote().push(force=True) + config.repo.create_pr( + title, args.branch_name, config.repo.branch + ) + else: + log.info( + 'Would create PR %s → %s: %s', + config.repo.branch, + args.branch_name, + title, + ) if __name__ == '__main__':