From 02b97364fcb315759c69d9b03fb03fdd4c7aad16 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sun, 26 Feb 2023 12:25:49 -0600 Subject: [PATCH] Implement run command The `ocivm run` command launches a QEMU microvm with an imported OCI container image as its root filesystem. The custom init program (`/linuxrc`) sets up the environment for the container's main process. Configuration information are passed to the VM via a VirtIO block device, which is read by the `/linuxrc` script. The main process is attached to a VirtIO serial console, which is in turn attached to the standart input/output streams of the `ocivm` process, thus allowing the process to be controlled interactively or redirecting its output. The `ocivm run` command supports mounting directories from the host (using 9pfs), setting environment variables, setting the working directory, and specifying the command to run in the VM, using a command-line syntax similar to `podman run`. After the specified command completes, the VM shuts down. For now, `/linuxrc` is implemented in POSIX shell, and thus requires that the container image must contain a complete userspace (including at least *coreutils*, *util-linux*, and *iproute2*. --- kconfig | 78 ++++++++++----------- src/ocivm/cli.py | 72 +++++++++++++++++++- src/ocivm/image.py | 2 +- src/ocivm/linuxrc | 30 +++++++-- src/ocivm/util.py | 2 +- src/ocivm/vm.py | 165 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 299 insertions(+), 50 deletions(-) create mode 100644 src/ocivm/vm.py diff --git a/kconfig b/kconfig index b6b81e0..5ac0ad6 100644 --- a/kconfig +++ b/kconfig @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86 6.1.12 Kernel Configuration +# Linux/x86 6.1.13 Kernel Configuration # CONFIG_CC_VERSION_TEXT="gcc (GCC) 12.2.1 20221121 (Red Hat 12.2.1-4)" CONFIG_CC_IS_GCC=y @@ -179,12 +179,12 @@ CONFIG_CC_NO_ARRAY_BOUNDS=y CONFIG_ARCH_SUPPORTS_INT128=y # CONFIG_CGROUPS is not set CONFIG_NAMESPACES=y -# CONFIG_UTS_NS is not set -# CONFIG_TIME_NS is not set -# CONFIG_IPC_NS is not set -# CONFIG_USER_NS is not set -# CONFIG_PID_NS is not set -# CONFIG_NET_NS is not set +CONFIG_UTS_NS=y +CONFIG_TIME_NS=y +CONFIG_IPC_NS=y +CONFIG_USER_NS=y +CONFIG_PID_NS=y +CONFIG_NET_NS=y # CONFIG_CHECKPOINT_RESTORE is not set # CONFIG_SCHED_AUTOGROUP is not set # CONFIG_SYSFS_DEPRECATED is not set @@ -198,7 +198,7 @@ CONFIG_LD_ORPHAN_WARN=y CONFIG_SYSCTL=y CONFIG_SYSCTL_EXCEPTION_TRACE=y CONFIG_HAVE_PCSPKR_PLATFORM=y -# CONFIG_EXPERT is not set +CONFIG_EXPERT=y CONFIG_MULTIUSER=y CONFIG_SGETMASK_SYSCALL=y CONFIG_SYSFS_SYSCALL=y @@ -225,9 +225,12 @@ CONFIG_KALLSYMS=y CONFIG_KALLSYMS_ABSOLUTE_PERCPU=y CONFIG_KALLSYMS_BASE_RELATIVE=y CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y +# CONFIG_KCMP is not set CONFIG_RSEQ=y +# CONFIG_DEBUG_RSEQ is not set # CONFIG_EMBEDDED is not set CONFIG_HAVE_PERF_EVENTS=y +# CONFIG_PC104 is not set # # Kernel Performance Events And Counters @@ -305,13 +308,14 @@ CONFIG_X86_MINIMUM_CPU_FAMILY=64 CONFIG_X86_DEBUGCTLMSR=y CONFIG_IA32_FEAT_CTL=y CONFIG_X86_VMX_FEATURE_NAMES=y +# CONFIG_PROCESSOR_SELECT is not set CONFIG_CPU_SUP_INTEL=y CONFIG_CPU_SUP_AMD=y CONFIG_CPU_SUP_HYGON=y CONFIG_CPU_SUP_CENTAUR=y CONFIG_CPU_SUP_ZHAOXIN=y CONFIG_HPET_TIMER=y -CONFIG_DMI=y +# CONFIG_DMI is not set # CONFIG_MAXSMP is not set CONFIG_NR_CPUS_RANGE_BEGIN=2 CONFIG_NR_CPUS_RANGE_END=512 @@ -450,6 +454,7 @@ CONFIG_HALTPOLL_CPUIDLE=y # # Bus options (PCI etc.) # +# CONFIG_ISA_BUS is not set CONFIG_ISA_DMA_API=y # end of Bus options (PCI etc.) @@ -673,6 +678,7 @@ CONFIG_SWAP=y # # CONFIG_SLAB is not set CONFIG_SLUB=y +# CONFIG_SLOB is not set CONFIG_SLAB_MERGE_DEFAULT=y # CONFIG_SLAB_FREELIST_RANDOM is not set CONFIG_SLAB_FREELIST_HARDENED=y @@ -712,6 +718,7 @@ CONFIG_GENERIC_EARLY_IOREMAP=y CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y CONFIG_ARCH_HAS_CURRENT_STACK_POINTER=y CONFIG_ARCH_HAS_PTE_DEVMAP=y +CONFIG_ARCH_HAS_ZONE_DMA_SET=y CONFIG_ZONE_DMA=y CONFIG_ZONE_DMA32=y CONFIG_ARCH_USES_HIGH_VMA_FLAGS=y @@ -923,9 +930,6 @@ CONFIG_PROC_EVENTS=y # CONFIG_EDD is not set CONFIG_FIRMWARE_MEMMAP=y -CONFIG_DMIID=y -# CONFIG_DMI_SYSFS is not set -CONFIG_DMI_SCAN_MACHINE_NON_EFI_FALLBACK=y # CONFIG_FW_CFG_SYSFS is not set # CONFIG_SYSFB_SIMPLEFB is not set # CONFIG_GOOGLE_FIRMWARE is not set @@ -945,8 +949,7 @@ CONFIG_BLK_DEV=y # CONFIG_BLK_DEV_NULL_BLK is not set # CONFIG_BLK_DEV_FD is not set # CONFIG_ZRAM is not set -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_LOOP_MIN_COUNT=8 +# CONFIG_BLK_DEV_LOOP is not set # CONFIG_BLK_DEV_DRBD is not set # CONFIG_BLK_DEV_NBD is not set # CONFIG_BLK_DEV_RAM is not set @@ -999,7 +1002,7 @@ CONFIG_SCSI_MOD=y CONFIG_SCSI_COMMON=y CONFIG_SCSI=y CONFIG_SCSI_DMA=y -CONFIG_SCSI_PROC_FS=y +# CONFIG_SCSI_PROC_FS is not set # # SCSI support type (disk, tape, CD-ROM) @@ -1008,10 +1011,10 @@ CONFIG_SCSI_PROC_FS=y # CONFIG_CHR_DEV_ST is not set # CONFIG_BLK_DEV_SR is not set # CONFIG_CHR_DEV_SG is not set -CONFIG_BLK_DEV_BSG=y +# CONFIG_BLK_DEV_BSG is not set # CONFIG_CHR_DEV_SCH is not set -CONFIG_SCSI_CONSTANTS=y -CONFIG_SCSI_LOGGING=y +# CONFIG_SCSI_CONSTANTS is not set +# CONFIG_SCSI_LOGGING is not set CONFIG_SCSI_SCAN_ASYNC=y # @@ -1019,17 +1022,17 @@ CONFIG_SCSI_SCAN_ASYNC=y # # CONFIG_SCSI_SPI_ATTRS is not set # CONFIG_SCSI_FC_ATTRS is not set -CONFIG_SCSI_ISCSI_ATTRS=y +# CONFIG_SCSI_ISCSI_ATTRS is not set # CONFIG_SCSI_SAS_ATTRS is not set # CONFIG_SCSI_SAS_LIBSAS is not set # CONFIG_SCSI_SRP_ATTRS is not set # end of SCSI Transports CONFIG_SCSI_LOWLEVEL=y -CONFIG_ISCSI_TCP=y +# CONFIG_ISCSI_TCP is not set # CONFIG_ISCSI_BOOT_SYSFS is not set # CONFIG_SCSI_DEBUG is not set -# CONFIG_SCSI_VIRTIO is not set +CONFIG_SCSI_VIRTIO=y # CONFIG_SCSI_DH is not set # end of SCSI device support @@ -1093,7 +1096,6 @@ CONFIG_INPUT=y CONFIG_INPUT_FF_MEMLESS=y # CONFIG_INPUT_SPARSEKMAP is not set # CONFIG_INPUT_MATRIXKMAP is not set -CONFIG_INPUT_VIVALDIFMAP=y # # Userland interfaces @@ -1107,7 +1109,7 @@ CONFIG_INPUT_VIVALDIFMAP=y # Input Device Drivers # CONFIG_INPUT_KEYBOARD=y -CONFIG_KEYBOARD_ATKBD=y +# CONFIG_KEYBOARD_ATKBD is not set # CONFIG_KEYBOARD_LKKBD is not set # CONFIG_KEYBOARD_NEWTON is not set # CONFIG_KEYBOARD_OPENCORES is not set @@ -1125,7 +1127,6 @@ CONFIG_INPUT_MISC=y # CONFIG_INPUT_UINPUT is not set # CONFIG_INPUT_ADXL34X is not set # CONFIG_INPUT_CMA3000 is not set -# CONFIG_INPUT_IDEAPAD_SLIDEBAR is not set # CONFIG_RMI4_CORE is not set # @@ -1133,10 +1134,10 @@ CONFIG_INPUT_MISC=y # CONFIG_SERIO=y CONFIG_ARCH_MIGHT_HAVE_PC_SERIO=y -CONFIG_SERIO_I8042=y +# CONFIG_SERIO_I8042 is not set CONFIG_SERIO_SERPORT=y # CONFIG_SERIO_CT82C710 is not set -CONFIG_SERIO_LIBPS2=y +# CONFIG_SERIO_LIBPS2 is not set # CONFIG_SERIO_RAW is not set # CONFIG_SERIO_ALTERA_PS2 is not set # CONFIG_SERIO_PS2MULT is not set @@ -1162,25 +1163,12 @@ CONFIG_LDISC_AUTOLOAD=y # # Serial drivers # -CONFIG_SERIAL_EARLYCON=y -CONFIG_SERIAL_8250=y -# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set -# CONFIG_SERIAL_8250_16550A_VARIANTS is not set -# CONFIG_SERIAL_8250_FINTEK is not set -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_8250_DMA=y -CONFIG_SERIAL_8250_NR_UARTS=1 -CONFIG_SERIAL_8250_RUNTIME_UARTS=1 -# CONFIG_SERIAL_8250_EXTENDED is not set -# CONFIG_SERIAL_8250_DW is not set -# CONFIG_SERIAL_8250_RT288X is not set +# CONFIG_SERIAL_8250 is not set # # Non-8250 serial port support # # CONFIG_SERIAL_UARTLITE is not set -CONFIG_SERIAL_CORE=y -CONFIG_SERIAL_CORE_CONSOLE=y # CONFIG_SERIAL_LANTIQ is not set # CONFIG_SERIAL_SCCNXP is not set # CONFIG_SERIAL_ALTERA_JTAGUART is not set @@ -1196,6 +1184,7 @@ CONFIG_SERIAL_CORE_CONSOLE=y CONFIG_HVC_DRIVER=y CONFIG_SERIAL_DEV_BUS=y CONFIG_SERIAL_DEV_CTRL_TTYPORT=y +# CONFIG_TTY_PRINTK is not set CONFIG_VIRTIO_CONSOLE=y # CONFIG_IPMI_HANDLER is not set CONFIG_HW_RANDOM=y @@ -1331,6 +1320,7 @@ CONFIG_BCMA_POSSIBLE=y # Graphics support # # CONFIG_DRM is not set +# CONFIG_DRM_DEBUG_MODESET_LOCK is not set # # ARM devices @@ -1855,6 +1845,7 @@ CONFIG_CRYPTO_NULL2=y # CONFIG_CRYPTO_PCRYPT is not set # CONFIG_CRYPTO_CRYPTD is not set # CONFIG_CRYPTO_AUTHENC is not set +# CONFIG_CRYPTO_TEST is not set # end of Crypto core or helper # @@ -2136,7 +2127,7 @@ CONFIG_CONSOLE_LOGLEVEL_DEFAULT=7 CONFIG_CONSOLE_LOGLEVEL_QUIET=4 CONFIG_MESSAGE_LOGLEVEL_DEFAULT=4 # CONFIG_BOOT_PRINTK_DELAY is not set -CONFIG_DYNAMIC_DEBUG=y +# CONFIG_DYNAMIC_DEBUG is not set CONFIG_DYNAMIC_DEBUG_CORE=y CONFIG_SYMBOLIC_ERRNAME=y CONFIG_DEBUG_BUGVERBOSE=y @@ -2159,9 +2150,11 @@ CONFIG_STRIP_ASM_SYMS=y # CONFIG_HEADERS_INSTALL is not set CONFIG_DEBUG_SECTION_MISMATCH=y CONFIG_SECTION_MISMATCH_WARN_ONLY=y +# CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B is not set CONFIG_FRAME_POINTER=y CONFIG_OBJTOOL=y # CONFIG_STACK_VALIDATION is not set +# CONFIG_VMLINUX_MAP is not set # CONFIG_DEBUG_FORCE_WEAK_PER_CPU is not set # end of Compile-time checks and compiler options @@ -2219,7 +2212,7 @@ CONFIG_ARCH_HAS_DEBUG_VM_PGTABLE=y # CONFIG_DEBUG_VM_PGTABLE is not set CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y # CONFIG_DEBUG_VIRTUAL is not set -CONFIG_DEBUG_MEMORY_INIT=y +# CONFIG_DEBUG_MEMORY_INIT is not set # CONFIG_DEBUG_PER_CPU_MAPS is not set CONFIG_ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP=y # CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP is not set @@ -2393,7 +2386,6 @@ CONFIG_RUNTIME_TESTING_MENU=y # CONFIG_TEST_FIRMWARE is not set # CONFIG_TEST_SYSCTL is not set # CONFIG_TEST_UDELAY is not set -# CONFIG_TEST_DYNAMIC_DEBUG is not set # CONFIG_TEST_MEMCAT_P is not set # CONFIG_TEST_MEMINIT is not set # CONFIG_TEST_FREE_PAGES is not set diff --git a/src/ocivm/cli.py b/src/ocivm/cli.py index 5c3a67e..20319dd 100644 --- a/src/ocivm/cli.py +++ b/src/ocivm/cli.py @@ -7,7 +7,7 @@ from typing import Optional import rich.console import rich.logging -from ocivm import image +from ocivm import image, vm log = logging.getLogger(__name__) @@ -31,6 +31,25 @@ class CLI: for name in image.list_images(): print(name) + def run( + self, + image: str, + name: Optional[str], + volumes: list[str], + cmd: list[str], + workdir: str, + env: Optional[dict[str, str]] = None, + memory: Optional[str] = None, + cpus: Optional[int] = None, + ) -> None: + machine = vm.VirtualMachine.create_from(image, name) + if memory is not None: + machine.memory = memory + if cpus is not None: + machine.cpus = cpus + machine.volumes = [tuple(v.split(':', 1)) for v in volumes] + machine.run(cmd, workdir, env) + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() @@ -55,6 +74,42 @@ def parse_args() -> argparse.Namespace: sp.add_parser('list', help='List VM images') + p_run = sp.add_parser('run', help='Run a virtual machine') + p_run.add_argument('--name', '-n', help='VM name') + p_run.add_argument( + '--volume', + '-v', + dest='volumes', + action='append', + default=[], + help='Mount a volume inside the VM', + ) + p_run.add_argument( + '--workdir', + '-w', + help='Working directory inside the VM', + ) + p_run.add_argument( + '--env', + '-e', + action='append', + default=[], + help='Set environment variables', + ) + p_run.add_argument( + '--memory', + '-m', + help='Memory allocation for the VM', + ) + p_run.add_argument( + '--cpus', + help='Number of CPUs', + ) + p_run.add_argument('image', help='VM image') + p_run.add_argument( + 'cmd', nargs='*', default=['/bin/bash'], help='Command to run' + ) + return parser.parse_args() @@ -73,3 +128,18 @@ def main() -> None: cli.import_image(args.name, args.size) case 'list': cli.list_images() + case 'run': + env = {} + for item in args.env: + key, __, value = item.partition('=') + env[key] = value + cli.run( + args.image, + args.name, + args.volumes, + args.cmd, + args.workdir, + env, + args.memory, + args.cpus, + ) diff --git a/src/ocivm/image.py b/src/ocivm/image.py index 5044ff2..26876a2 100644 --- a/src/ocivm/image.py +++ b/src/ocivm/image.py @@ -18,7 +18,7 @@ log = logging.getLogger(__name__) CONTAINERFILE_TMPL = string.Template( '''\ FROM ${name} -ADD linuxrc / +ADD --chmod=755 linuxrc / ''' ) diff --git a/src/ocivm/linuxrc b/src/ocivm/linuxrc index 805293a..e1e3c51 100755 --- a/src/ocivm/linuxrc +++ b/src/ocivm/linuxrc @@ -9,6 +9,7 @@ mkdir -p \ /tmp mountpoint -q /dev || mount -t devtmpfs devtmpfs /dev + mkdir -p /dev/pts /dev/shm mount -t devpts devpts /dev/pts mount -t tmpfs tmpfs /dev/shm @@ -18,16 +19,37 @@ mount -t sysfs sysfs /sys mount -t tmpfs tmpfs /tmp mount -t tmpfs tmpfs /run -mkdir -p /tmp/build -mount -t 9p -o trans=virtio,version=9p2000.L,msize=52428800 hostfiles /tmp/build +export PATH ip link set eth0 up ip address add 192.168.76.8/24 dev eth0 ip route add default via 192.168.76.2 echo nameserver 192.168.76.3 > /etc/resolv.conf +chmod u=rw,go=r /etc/resolv.conf -cd /tmp/build -exec /bin/bash +while read command args; do + case "${command}" in + CMD) + eval setsid -c ${args} < /dev/hvc1 > /dev/hvc1 2>&1 + ;; + ENV) + export "${args}" + ;; + WORKDIR) + cd "${args}" + ;; + MOUNT) + what="${args% *}" + where="${args#* }" + mkdir -p "${where}" + mount \ + -t 9p \ + -o trans=virtio,version=9p2000.L \ + "${what}" \ + "${where}" + ;; + esac +done < /dev/vdb echo 1 > /proc/sys/kernel/sysrq echo b > /proc/sysrq-trigger diff --git a/src/ocivm/util.py b/src/ocivm/util.py index 1b6d7f2..fd2e1a6 100644 --- a/src/ocivm/util.py +++ b/src/ocivm/util.py @@ -3,4 +3,4 @@ from typing import Any def list2cmdline(args: list[Any]) -> str: - return ' '.join(shlex.quote(a) for a in args) + return ' '.join(shlex.quote(str(a)) for a in args) diff --git a/src/ocivm/vm.py b/src/ocivm/vm.py new file mode 100644 index 0000000..20038ac --- /dev/null +++ b/src/ocivm/vm.py @@ -0,0 +1,165 @@ +import importlib.resources +import logging +import os +import shlex +import subprocess +import uuid +from pathlib import Path +from typing import Optional + +import xdg + +from .image import get_image_dir +from .util import list2cmdline + + +log = logging.getLogger(__name__) + + +def get_storage_dir() -> Path: + storage_dir = xdg.xdg_cache_home() / 'ocivm' / 'machines' + log.debug('Using storage directory: %s', storage_dir) + return storage_dir + + +def get_runtime_dir() -> Path: + runtime_dir = xdg.xdg_runtime_dir() or Path('~').expanduser() + runtime_dir /= 'ocivm' + log.debug('Using runtime directory: %s', runtime_dir) + return runtime_dir + + +def create_vm_disk(image: str, name: str) -> Path: + src = get_image_dir() / f'{image}.qcow2' + dest = get_storage_dir() / f'{name}.qcow2' + if not dest.parent.is_dir(): + dest.parent.mkdir(parents=True) + if dest.exists(): + dest.unlink() + cmd = [ + 'qemu-img', + 'create', + '-b', + src, + '-F', + 'qcow2', + '-f', + 'qcow2', + dest, + ] + if log.isEnabledFor(logging.DEBUG): + log.debug('Running command: %s', list2cmdline(cmd)) + subprocess.run( + cmd, + stdin=subprocess.DEVNULL, + capture_output=True, + ) + return dest + + +class VirtualMachine: + def __init__(self, name: str) -> None: + self.name = name + self.cpus = os.cpu_count() or 1 + self.memory = '8g' + self.volumes: list[tuple[str, str]] = [] + + @classmethod + def create_from(cls, image: str, name: Optional[str]) -> 'VirtualMachine': + if name is None: + name = str(uuid.uuid4()) + log.info('Creating VM %s from image %s', name, image) + create_vm_disk(image, name) + return cls(name) + + def run( + self, + command: list[str], + workdir: str = '/', + env: Optional[dict[str, str]] = None, + ) -> None: + storage_dir = get_storage_dir() + disk = storage_dir / f'{self.name}.qcow2' + console = storage_dir / f'{self.name}.log' + config = storage_dir / f'{self.name}.config' + kcmdline = ' '.join( + ( + 'console=hvc0', + 'panic=-1', + 'reboot=t', + 'root=/dev/vda', + 'rw', + 'init=/linuxrc', + ) + ) + res = importlib.resources.as_file( + importlib.resources.files(__package__).joinpath('kernel.img') + ) + with res as kernel: + cmd = [ + 'qemu-system-x86_64', + '-nodefaults', + '-no-user-config', + '-nographic', + '-no-reboot', + '-M', + 'microvm,x-option-roms=off,pit=off,pic=off,isa-serial=off,rtc=off', + '-no-acpi', + '-enable-kvm', + '-cpu', + 'host', + '-m', + f'{self.memory}', + '-smp', + f'{self.cpus}', + '-chardev', + 'stdio,id=out,signal=off', + '-device', + 'virtio-serial-device', + '-device', + 'virtconsole,chardev=console', + '-chardev', + f'file,id=console,path={console}', + '-device', + 'virtconsole,chardev=out', + '-drive', + f'id=disk0,file={disk},format=qcow2,if=none', + '-device', + 'virtio-blk-device,drive=disk0', + '-drive', + f'id=disk1,file={config},format=raw,if=none', + '-device', + 'virtio-blk-device,drive=disk1', + '-netdev', + 'user,id=net0,net=192.168.76.0/24,dhcpstart=192.168.76.9', + '-device', + 'virtio-net-device,netdev=net0', + ] + with config.open('w', encoding='utf-8') as f: + for idx, (source, target) in enumerate(self.volumes): + cmd += ( + '-fsdev', + f'local,path={source},security_model=none,id=files{idx}', + '-device', + f'virtio-9p-device,fsdev=files{idx},mount_tag=files{idx}', + ) + f.write(f'MOUNT files{idx} {target}\n') + if env: + for key, value in env.items(): + f.write(f'ENV {key}={shlex.quote(value)}\n') + f.write(f'WORKDIR {workdir}\n') + f.write(f'CMD {list2cmdline(command)}\n') + pos = f.tell() + end = (pos - 1 | 511) + 1 + f.seek(end) + f.truncate(end) + cmd += ('-kernel', str(kernel)) + cmd += ('-append', kcmdline) + if log.isEnabledFor(logging.DEBUG): + log.debug('Running command: %s', list2cmdline(cmd)) + p = subprocess.Popen(cmd, close_fds=False) + try: + p.wait() + except KeyboardInterrupt: + p.terminate() + log.info('Command exited with status code %d', p.returncode)