infra
/
updatebot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework data model to group images into projects 

A "project" now refers to an application deployed into Kubernetes, which
includes one or more "images."  This is really the grouping I wanted in
the beginning, which I tried to achieve using separate configuration
files.  Unfortunately, this made the original "projects" too
independent, making it difficult to produce the manifest diff I wanted
to add to the PR descriptions.  It was also cumbersome managing multiple
config files and therefore multiple CronJobs in Kubernetes.

The new data model is a lot deeper than the original one, making TOML a
lot less nice.  YAML definitely handles nested data structures better,
despite its shortcomings.  Having to repeat nested table names in TOML
is quite cumbersome.
master
Dustin 2024-09-04 21:20:18 -05:00
parent 8126e5de21
commit 34fbdc6e02
1 changed files with 85 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
updatebot.py
View File

@ -7,10 +7,9 @@ import os
import re import re
import tempfile import tempfile
import threading import threading
import tomllib
import urllib.parse import urllib.parse
from pathlib import Path from pathlib import Path
from typing import ClassVar, Literal, Union, Optional from typing import ClassVar, Iterable, Literal, Union, Optional
import colorlog import colorlog
import pydantic import pydantic
@ -112,19 +111,29 @@ Source = Union[
] ]
class BaseProject(abc.ABC, pydantic.BaseModel): class ImageDef(pydantic.BaseModel):
path: Optional[Path] = None name: str
source: Source
image: str image: str
source: Source
tag_format: str = '{version}' tag_format: str = '{version}'
class BaseProject(abc.ABC, pydantic.BaseModel):
name: str
path: Path
def __init__(self, **data) -> None:
data.setdefault('path', data.get('name'))
super().__init__(**data)
@abc.abstractmethod @abc.abstractmethod
def apply_update(self, path: Path, version: str) -> None: def apply_updates(self, basedir: Path) -> Iterable[tuple[ImageDef, str]]:
raise NotImplementedError raise NotImplementedError
class KustomizeProject(BaseProject): class KustomizeProject(BaseProject):
kind: Literal['kustomize'] kind: Literal['kustomize']
images: list[ImageDef]
kustomize_files: ClassVar[list[str]] = [ kustomize_files: ClassVar[list[str]] = [
'kustomization.yaml', 'kustomization.yaml',
@ -132,36 +141,36 @@ class KustomizeProject(BaseProject):
'Kustomization', 'Kustomization',
] ]
def apply_update(self, path: Path, version: str) -> None: def apply_updates(self, basedir: Path) -> Iterable[tuple[ImageDef, str]]:
path = basedir / self.path
for filename in self.kustomize_files: for filename in self.kustomize_files:
filepath = path / filename filepath = path / filename
if filepath.is_file(): if filepath.is_file():
break break
else: else:
filepath = path / self.kustomize_files[0] raise ValueError(
f'Could not find Kustomize config for {self.name}'
)
for image in self.images:
yaml = ruamel.yaml.YAML() yaml = ruamel.yaml.YAML()
with filepath.open('rb') as f: with filepath.open('rb') as f:
kustomization = yaml.load(f) kustomization = yaml.load(f)
images = kustomization.setdefault('images', []) images = kustomization.setdefault('images', [])
new_tag = self.tag_format.format(version=version) version = image.source.get_latest_version()
for image in images: new_tag = image.tag_format.format(version=version)
if image['name'] == self.image: for i in images:
image['newTag'] = new_tag if i['name'] == image.image:
i['newTag'] = new_tag
break break
else: else:
images.append({'name': self.image, 'newTag': new_tag}) images.append({'name': image.image, 'newTag': new_tag})
with filepath.open('wb') as f: with filepath.open('wb') as f:
yaml.dump(kustomization, f) yaml.dump(kustomization, f)
yield (image, version)
class DirectoryProject(BaseProject): Project = KustomizeProject
kind: Literal['dir'] | Literal['directory']
Project = Union[
KustomizeProject,
DirectoryProject,
]
class RepoConfig(pydantic.BaseModel): class RepoConfig(pydantic.BaseModel):
@ -225,7 +234,7 @@ class RepoConfig(pydantic.BaseModel):
class Config(pydantic.BaseModel): class Config(pydantic.BaseModel):
repo: RepoConfig repo: RepoConfig
projects: dict[str, Project] projects: list[Project]
class Arguments: class Arguments:
@ -235,25 +244,24 @@ class Arguments:
projects: list[str] projects: list[str]
def update_project( def update_project(repo: git.Repo, project: Project) -> Optional[str]:
repo: git.Repo, name: str, project: Project
) -> Optional[git.Commit]:
basedir = Path(repo.working_dir) basedir = Path(repo.working_dir)
log.debug('Checking for latest version of %s', name) title = None
latest = project.source.get_latest_version() for (image, version) in project.apply_updates(basedir):
log.info('Found version %s for %s', latest, name) log.info('Updating %s to %s', image.name, version)
log.debug('Applying update for %s version %s', name, latest)
path = basedir / (project.path or name)
project.apply_update(path, latest)
if repo.index.diff(None): if repo.index.diff(None):
log.debug('Committing changes to %s', path) log.debug('Committing changes to %s', project.path)
repo.index.add(str(path)) repo.index.add(str(project.path))
c = repo.index.commit(f'{name}: Update to {latest}') c = repo.index.commit(f'{image.name}: Update to {version}')
log.info('Commited %s %s', str(c)[:7], c.summary) log.info('Commited %s %s', str(c)[:7], c.summary)
return c if not title:
if not isinstance(c.summary, str):
title = bytes(c.summary).decode('utf-8')
else:
title = c.summary
else: else:
log.info('No changes to commit') log.info('No changes to commit')
return None return title
def parse_args() -> Arguments: def parse_args() -> Arguments:
@ -296,23 +304,12 @@ def setup_logging() -> None:
def main() -> None: def main() -> None:
setup_logging() setup_logging()
args = parse_args() args = parse_args()
yaml = ruamel.yaml.YAML()
with args.config.open('rb') as f: with args.config.open('rb') as f:
data = tomllib.load(f) data = yaml.load(f)
config = Config.model_validate(data) config = Config.model_validate(data)
log.debug('Using configuration: %s', config) log.debug('Using configuration: %s', config)
all_projects = list(config.projects.keys()) projects = args.projects or [p.name for p in config.projects]
projects = []
if args.projects:
for project in args.projects:
if project not in all_projects:
log.warning('Unknown project: %s', project)
continue
projects.append(project)
else:
projects = all_projects
if not projects:
log.error('No projects')
raise SystemExit(1)
if log.isEnabledFor(logging.INFO): if log.isEnabledFor(logging.INFO):
log.info('Updating projects: %s', ', '.join(projects)) log.info('Updating projects: %s', ', '.join(projects))
with tempfile.TemporaryDirectory(prefix='updatebot.') as d: with tempfile.TemporaryDirectory(prefix='updatebot.') as d:
@ -321,21 +318,16 @@ def main() -> None:
log.debug('Retreiving repository Git URL') log.debug('Retreiving repository Git URL')
repo_url = config.repo.get_git_url() repo_url = config.repo.get_git_url()
repo = git.Repo.clone_from(repo_url, d, depth=1, b=config.repo.branch) repo = git.Repo.clone_from(repo_url, d, depth=1, b=config.repo.branch)
for project in config.projects:
log.debug('Checking out new branch: %s', args.branch_name) log.debug('Checking out new branch: %s', args.branch_name)
repo.heads[0].checkout(force=True, B=args.branch_name) repo.heads[0].checkout(force=True, B=args.branch_name)
title = None title = None
for project in projects: if project.name not in projects:
commit = update_project(repo, project, config.projects[project]) continue
if commit and not title: title = update_project(repo, project)
if not isinstance(commit.summary, str):
title = bytes(commit.summary).decode(
'utf-8', errors='replace'
)
else:
title = commit.summary
if not title: if not title:
log.info('No changes made') log.info('No changes made')
return continue
repo.head.reference.set_tracking_branch( repo.head.reference.set_tracking_branch(
git.RemoteReference( git.RemoteReference(
repo, f'refs/remotes/origin/{args.branch_name}' repo, f'refs/remotes/origin/{args.branch_name}'
@ -343,7 +335,16 @@ def main() -> None:
) )
if not args.dry_run: if not args.dry_run:
repo.remote().push(force=True) repo.remote().push(force=True)
config.repo.create_pr(title, args.branch_name, config.repo.branch) config.repo.create_pr(
title, args.branch_name, config.repo.branch
)
else:
log.info(
'Would create PR %s → %s: %s',
config.repo.branch,
args.branch_name,
title,
)
if __name__ == '__main__': if __name__ == '__main__':