infra/updatebot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework data model to group images into projects

Browse Source 
A "project" now refers to an application deployed into Kubernetes, which
includes one or more "images."  This is really the grouping I wanted in
the beginning, which I tried to achieve using separate configuration
files.  Unfortunately, this made the original "projects" too
independent, making it difficult to produce the manifest diff I wanted
to add to the PR descriptions.  It was also cumbersome managing multiple
config files and therefore multiple CronJobs in Kubernetes.

The new data model is a lot deeper than the original one, making TOML a
lot less nice.  YAML definitely handles nested data structures better,
despite its shortcomings.  Having to repeat nested table names in TOML
is quite cumbersome.
This commit is contained in:
Dustin
2024-09-04 21:20:18 -05:00
parent 8126e5de21
commit 34fbdc6e02
1 changed files with 85 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
updatebot.py
View File

@@ -7,10 +7,9 @@ import os
import re
import tempfile
import threading
import tomllib
import urllib.parse
from pathlib import Path
from typing import ClassVar, Literal, Union, Optional
from typing import ClassVar, Iterable, Literal, Union, Optional
import colorlog
import pydantic
@@ -112,19 +111,29 @@ Source = Union[
]
class BaseProject(abc.ABC, pydantic.BaseModel):
path: Optional[Path] = None
source: Source
class ImageDef(pydantic.BaseModel):
name: str
image: str
source: Source
tag_format: str = '{version}'
class BaseProject(abc.ABC, pydantic.BaseModel):
name: str
path: Path
def __init__(self, **data) -> None:
data.setdefault('path', data.get('name'))
super().__init__(**data)
@abc.abstractmethod
def apply_update(self, path: Path, version: str) -> None:
def apply_updates(self, basedir: Path) -> Iterable[tuple[ImageDef, str]]:
raise NotImplementedError
class KustomizeProject(BaseProject):
kind: Literal['kustomize']
images: list[ImageDef]
kustomize_files: ClassVar[list[str]] = [
'kustomization.yaml',
@@ -132,36 +141,36 @@ class KustomizeProject(BaseProject):
'Kustomization',
]
def apply_update(self, path: Path, version: str) -> None:
def apply_updates(self, basedir: Path) -> Iterable[tuple[ImageDef, str]]:
path = basedir / self.path
for filename in self.kustomize_files:
filepath = path / filename
if filepath.is_file():
break
else:
filepath = path / self.kustomize_files[0]
yaml = ruamel.yaml.YAML()
with filepath.open('rb') as f:
kustomization = yaml.load(f)
images = kustomization.setdefault('images', [])
new_tag = self.tag_format.format(version=version)
for image in images:
if image['name'] == self.image:
image['newTag'] = new_tag
break
else:
images.append({'name': self.image, 'newTag': new_tag})
with filepath.open('wb') as f:
yaml.dump(kustomization, f)
raise ValueError(
f'Could not find Kustomize config for {self.name}'
)
for image in self.images:
yaml = ruamel.yaml.YAML()
with filepath.open('rb') as f:
kustomization = yaml.load(f)
images = kustomization.setdefault('images', [])
version = image.source.get_latest_version()
new_tag = image.tag_format.format(version=version)
for i in images:
if i['name'] == image.image:
i['newTag'] = new_tag
break
else:
images.append({'name': image.image, 'newTag': new_tag})
with filepath.open('wb') as f:
yaml.dump(kustomization, f)
yield (image, version)
class DirectoryProject(BaseProject):
kind: Literal['dir'] | Literal['directory']
Project = Union[
KustomizeProject,
DirectoryProject,
]
Project = KustomizeProject
class RepoConfig(pydantic.BaseModel):
@@ -225,7 +234,7 @@ class RepoConfig(pydantic.BaseModel):
class Config(pydantic.BaseModel):
repo: RepoConfig
projects: dict[str, Project]
projects: list[Project]
class Arguments:
@@ -235,25 +244,24 @@ class Arguments:
projects: list[str]
def update_project(
repo: git.Repo, name: str, project: Project
) -> Optional[git.Commit]:
def update_project(repo: git.Repo, project: Project) -> Optional[str]:
basedir = Path(repo.working_dir)
log.debug('Checking for latest version of %s', name)
latest = project.source.get_latest_version()
log.info('Found version %s for %s', latest, name)
log.debug('Applying update for %s version %s', name, latest)
path = basedir / (project.path or name)
project.apply_update(path, latest)
if repo.index.diff(None):
log.debug('Committing changes to %s', path)
repo.index.add(str(path))
c = repo.index.commit(f'{name}: Update to {latest}')
log.info('Commited %s %s', str(c)[:7], c.summary)
return c
else:
log.info('No changes to commit')
return None
title = None
for (image, version) in project.apply_updates(basedir):
log.info('Updating %s to %s', image.name, version)
if repo.index.diff(None):
log.debug('Committing changes to %s', project.path)
repo.index.add(str(project.path))
c = repo.index.commit(f'{image.name}: Update to {version}')
log.info('Commited %s %s', str(c)[:7], c.summary)
if not title:
if not isinstance(c.summary, str):
title = bytes(c.summary).decode('utf-8')
else:
title = c.summary
else:
log.info('No changes to commit')
return title
def parse_args() -> Arguments:
@@ -296,23 +304,12 @@ def setup_logging() -> None:
def main() -> None:
setup_logging()
args = parse_args()
yaml = ruamel.yaml.YAML()
with args.config.open('rb') as f:
data = tomllib.load(f)
data = yaml.load(f)
config = Config.model_validate(data)
log.debug('Using configuration: %s', config)
all_projects = list(config.projects.keys())
projects = []
if args.projects:
for project in args.projects:
if project not in all_projects:
log.warning('Unknown project: %s', project)
continue
projects.append(project)
else:
projects = all_projects
if not projects:
log.error('No projects')
raise SystemExit(1)
projects = args.projects or [p.name for p in config.projects]
if log.isEnabledFor(logging.INFO):
log.info('Updating projects: %s', ', '.join(projects))
with tempfile.TemporaryDirectory(prefix='updatebot.') as d:
@@ -321,29 +318,33 @@ def main() -> None:
log.debug('Retreiving repository Git URL')
repo_url = config.repo.get_git_url()
repo = git.Repo.clone_from(repo_url, d, depth=1, b=config.repo.branch)
log.debug('Checking out new branch: %s', args.branch_name)
repo.heads[0].checkout(force=True, B=args.branch_name)
title = None
for project in projects:
commit = update_project(repo, project, config.projects[project])
if commit and not title:
if not isinstance(commit.summary, str):
title = bytes(commit.summary).decode(
'utf-8', errors='replace'
)
else:
title = commit.summary
if not title:
log.info('No changes made')
return
repo.head.reference.set_tracking_branch(
git.RemoteReference(
repo, f'refs/remotes/origin/{args.branch_name}'
for project in config.projects:
log.debug('Checking out new branch: %s', args.branch_name)
repo.heads[0].checkout(force=True, B=args.branch_name)
title = None
if project.name not in projects:
continue
title = update_project(repo, project)
if not title:
log.info('No changes made')
continue
repo.head.reference.set_tracking_branch(
git.RemoteReference(
repo, f'refs/remotes/origin/{args.branch_name}'
)
)
)
if not args.dry_run:
repo.remote().push(force=True)
config.repo.create_pr(title, args.branch_name, config.repo.branch)
if not args.dry_run:
repo.remote().push(force=True)
config.repo.create_pr(
title, args.branch_name, config.repo.branch
)
else:
log.info(
'Would create PR %s → %s: %s',
config.repo.branch,
args.branch_name,
title,
)
if __name__ == '__main__':