From d1d24ad4fcb97e26c249d4e42136c1cb31d828ed Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Fri, 5 Apr 2024 16:37:53 +0100 Subject: [PATCH 1/7] Start of QEMU K3s based CI --- .gitignore | 7 + script/vms/k3s.sh | 282 +++++++++++++++++++++++++++++++++++++ script/vms/metadata.yaml | 3 + script/vms/priv-shell.yaml | 37 +++++ 4 files changed, 329 insertions(+) create mode 100755 script/vms/k3s.sh create mode 100644 script/vms/metadata.yaml create mode 100644 script/vms/priv-shell.yaml diff --git a/.gitignore b/.gitignore index e917e5c..6353230 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,10 @@ Dockerfile.cross *.swp *.swo *~ + +# K3s VM testing +script/vms/*.img +script/vms/ssh_key* +script/vms/serial.log +script/vms/kubeconfig +script/vms/user-data.yaml diff --git a/script/vms/k3s.sh b/script/vms/k3s.sh new file mode 100755 index 0000000..49d8c25 --- /dev/null +++ b/script/vms/k3s.sh @@ -0,0 +1,282 @@ +#!/bin/sh -eu + +set_vars() { + echo "QEMU K3s Ports(SSH=${K3S_SSH_PORT:=2222} API=${K3S_API_PORT:=16443} web=${K3S_WEB_PORT:=8080})" + echo "Kubeconfig: ${KUBECONFIG:=kubeconfig}" + echo "Image: ${IMG:=container.tar}" + + rund=${XDG_RUNTIME_DIR-/tmp/$USER}/prem-operator + pid_file="${rund}/qemu.pid" + here=$(dirname "$0") + + echo "Runtime dir: $rund" +} + +# Function for 'boot-qemu-daemon' subcommand +boot_qemu_daemon() { + echo "Booting QEMU as a daemon..." + + mkdir -p $rund + + if [ -f "$pid_file" ] && ps -p $(cat "$pid_file") > /dev/null 2>&1; then + echo "QEMU is already running. Exiting." + exit 1 + fi + + if [ $# -lt 1 ]; then + QCOW=temp.img + rm -f temp.img + qemu-img create -F qcow2 -f qcow2 -b k3s-base.img temp.img + else + QCOW=$1 + fi + + qemu-system-x86_64 \ + -machine accel=kvm,type=q35 \ + -cpu host \ + -m 8G \ + -device virtio-net-pci,netdev=net0 \ + -netdev user,id=net0,hostfwd=tcp::${K3S_SSH_PORT}-:22,hostfwd=tcp::${K3S_API_PORT}-:6443,hostfwd=tcp::${K3S_WEB_PORT}-:80 \ + -drive if=virtio,format=qcow2,file=$QCOW,cache=none \ + -drive if=virtio,format=qcow2,file=seed.img \ + -serial file:serial.log \ + -pidfile "$pid_file" \ + -display none \ + -daemonize + + echo "Started QEMU waiting for SSH" + local ssh_opts="-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=5" + local timeout=120 + local start_time=$(date +%s) + + while ! ssh $ssh_opts -p "${K3S_SSH_PORT}" -i ssh_key ubuntu@0.0.0.0 true; do + local current_time=$(date +%s) + local elapsed_time=$((current_time - start_time)) + + if [ "$elapsed_time" -ge "$timeout" ]; then + echo "Serial log output:" + tail -n 100 serial.log + echo "Killing QEMU..." + kill $(cat "$pid_file") + rm -f "$pid_file" + exit 1 + fi + + echo "Retrying SSH..." + done + + echo "SSH connection established." +} + +boot_qemu() { + echo "Booting QEMU..." + + if [ $# -lt 1 ]; then + QCOW=temp.img + rm -f temp.img + qemu-img create -F qcow2 -f qcow2 -b k3s-base.img temp.img + else + QCOW=$1 + fi + + qemu-system-x86_64 \ + -machine accel=kvm,type=q35 \ + -cpu host \ + -m 8G \ + -device virtio-net-pci,netdev=net0 \ + -netdev user,id=net0,hostfwd=tcp::${K3S_SSH_PORT}-:22,hostfwd=tcp::${K3S_API_PORT}-:6443,hostfwd=tcp::${K3S_WEB_PORT}-:80 \ + -drive if=virtio,format=qcow2,file=$QCOW,cache=none \ + -drive if=virtio,format=qcow2,file=seed.img \ + -nographic +} + +stop_qemu() { + if [ -f "$pid_file" ]; then + pid=$(cat "$pid_file") + + if ps -p $pid > /dev/null 2>&1; then + # Check if the process is QEMU + if ps -p $pid -o comm= | grep -q "qemu-system"; then + echo "QEMU pid $pid found. Killing it..." + + kill $pid + else + echo "PID file found, but the process is not QEMU. Skipping termination." + fi + else + echo "PID file found, but the process is not running. Removing the PID file..." + fi + + rm -f "$pid_file" + else + echo "No QEMU PID file found." + fi +} + +shutdown_qemu() { + local timeout=120 + local interval=2 + + echo "Shutdown QEMU..." + ssh -o BatchMode=yes -p $K3S_SSH_PORT -i ssh_key ubuntu@0.0.0.0 sudo systemctl poweroff + + if [ -f "$pid_file" ]; then + local pid=$(cat "$pid_file") + + echo "Waiting for QEMU to exit..." + while [ $timeout -gt 0 ]; do + if kill -0 "$pid" 2>/dev/null; then + sleep $interval + else + echo "QEMU Process $pid has exited." + return + fi + timeout=$((timeout - interval)) + done + else + echo "No QEMU PID file found." + fi + + echo "Timeout reached. QEMU process $pid is still running." + stop_qemu +} + +install_k3s() { + echo "Installing k3s..." + + k3sup install --ip ${K3S_IP-0.0.0.0} --ssh-key ssh_key --ssh-port ${K3S_SSH_PORT-22} --user ${K3S_USER-ubuntu} + sed -i -e s/:6443/:${K3S_API_PORT}/ ./kubeconfig + k3sup ready --kubeconfig=./kubeconfig +} + +cloud_init() { + echo "Could init" + + if [ ! -f ssh_key ]; then + ssh-keygen -t ed25519 -f ssh_key -C "pou3" -N "" + else + echo "ssh_key exists, skipping key creation" + fi + local pub_key=$(cat ssh_key.pub) + cat > user-data.yaml < [options]" + echo "" + echo "Commands:" + echo " boot-qemu-daemon Boot QEMU as a daemon" + echo " boot-qemu Boot QEMU" + echo " stop-qemu Stop QEMU" + echo " shutdown-qemu Shutdown QEMU" + echo " install-k3s Install k3s to a running system" + echo " cloud-init Create a new VM and install K3s on it" + echo " setup-image Add the operator container image to a running system" + echo " tests Run e2e tests" + echo " clean Clean up all created files" + echo " help Show this help message" +} + +# Check if at least one argument is provided +if [ $# -lt 1 ]; then + show_help + exit 1 +fi + +original_pwd=$(pwd) +cd $(dirname $0) + +cleanup() { + cd "$original_pwd" +} +trap cleanup EXIT + +set_vars + +case "$1" in + boot-qemu-daemon) + boot_qemu_daemon "${@:2}" + ;; + boot-qemu) + boot_qemu "${@:2}" + ;; + stop-qemu) + stop_qemu "${@:2}" + ;; + shutdown-qemu) + shutdown_qemu "${@:2}" + ;; + install-k3s) + install_k3s "${@:2}" + ;; + cloud-init) + cloud_init "${@:2}" + ;; + setup-image) + setup_image "${@:2}" + ;; + tests) + tests "${@:2}" + ;; + clean) + clean + ;; + help) + show_help + ;; + *) + echo "Error: Unknown command '$1'" + show_help + exit 1 + ;; +esac diff --git a/script/vms/metadata.yaml b/script/vms/metadata.yaml new file mode 100644 index 0000000..c071628 --- /dev/null +++ b/script/vms/metadata.yaml @@ -0,0 +1,3 @@ +instance-id: pou3 +local-hostname: pou3 + diff --git a/script/vms/priv-shell.yaml b/script/vms/priv-shell.yaml new file mode 100644 index 0000000..a5f7d68 --- /dev/null +++ b/script/vms/priv-shell.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: shell + labels: + premai.io/util: shell +--- +apiVersion: v1 +kind: Pod +metadata: + name: shell + namespace: shell + labels: + premai.io/util: shell +spec: + hostNetwork: true + hostPID: true + hostIPC: true + volumes: + - name: root + hostPath: + path: / + containers: + - image: alpine + name: shell + securityContext: + privileged: true + procMount: Unmasked + seccompProfile: + type: Unconfined + command: + - /bin/sh + stdin: true + tty: true + volumeMounts: + - mountPath: / + name: root From 34a460b575c53defec4d0a17cae9e9e4eee5fbbd Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Tue, 9 Apr 2024 10:40:15 +0100 Subject: [PATCH 2/7] switch to bash --- script/vms/k3s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/vms/k3s.sh b/script/vms/k3s.sh index 49d8c25..1a4c21b 100755 --- a/script/vms/k3s.sh +++ b/script/vms/k3s.sh @@ -1,4 +1,4 @@ -#!/bin/sh -eu +#!/bin/bash -eu set_vars() { echo "QEMU K3s Ports(SSH=${K3S_SSH_PORT:=2222} API=${K3S_API_PORT:=16443} web=${K3S_WEB_PORT:=8080})" From bbbdf38edabefc4dff55a7ee302947faf791b512 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Tue, 9 Apr 2024 11:11:21 +0100 Subject: [PATCH 3/7] Docker ignore script/vms --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 0f04682..6ecfb85 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,5 @@ # Ignore build and test binaries. bin/ testbin/ +script/vms/ + From b3afbc0fe430e663a078d5a6e9f243e85e813b05 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Tue, 9 Apr 2024 11:18:25 +0100 Subject: [PATCH 4/7] Add docker save --- script/vms/k3s.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/vms/k3s.sh b/script/vms/k3s.sh index 1a4c21b..4ff17a7 100755 --- a/script/vms/k3s.sh +++ b/script/vms/k3s.sh @@ -186,6 +186,8 @@ EOF setup_image() { echo "Setup Image" + docker save -o $IMG controller:latest + export KUBECONFIG kubectl apply -f ./priv-shell.yaml --prune -l premai.io/util=shell From 21525a630b30e1860bab732e6116c63abf0e0b0e Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Wed, 10 Apr 2024 08:54:40 +0100 Subject: [PATCH 5/7] Add vfio-pci --- script/vms/k3s.sh | 70 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/script/vms/k3s.sh b/script/vms/k3s.sh index 4ff17a7..ff6907f 100755 --- a/script/vms/k3s.sh +++ b/script/vms/k3s.sh @@ -4,6 +4,7 @@ set_vars() { echo "QEMU K3s Ports(SSH=${K3S_SSH_PORT:=2222} API=${K3S_API_PORT:=16443} web=${K3S_WEB_PORT:=8080})" echo "Kubeconfig: ${KUBECONFIG:=kubeconfig}" echo "Image: ${IMG:=container.tar}" + echo "GPU PCI Address: ${VFIO_PCI:=none}" rund=${XDG_RUNTIME_DIR-/tmp/$USER}/prem-operator pid_file="${rund}/qemu.pid" @@ -12,7 +13,6 @@ set_vars() { echo "Runtime dir: $rund" } -# Function for 'boot-qemu-daemon' subcommand boot_qemu_daemon() { echo "Booting QEMU as a daemon..." @@ -25,24 +25,35 @@ boot_qemu_daemon() { if [ $# -lt 1 ]; then QCOW=temp.img + UEFI_VARS=uefi-vars-temp.img rm -f temp.img - qemu-img create -F qcow2 -f qcow2 -b k3s-base.img temp.img + qemu-img create -F qcow2 -f qcow2 -b k3s-base.img $QCOW + qemu-img create -F qcow2 -f qcow2 -b uefi-vars-base.img $UEFI_VARS else QCOW=$1 + UEFI_VARS=uefi-vars-base.img + fi + + local opt_vars="" + if [ ! "$VFIO_PCI" = "none" ]; then + opt_vars="-device vfio-pci,host=$VFIO_PCI" fi qemu-system-x86_64 \ -machine accel=kvm,type=q35 \ -cpu host \ - -m 8G \ + -m 16G \ -device virtio-net-pci,netdev=net0 \ -netdev user,id=net0,hostfwd=tcp::${K3S_SSH_PORT}-:22,hostfwd=tcp::${K3S_API_PORT}-:6443,hostfwd=tcp::${K3S_WEB_PORT}-:80 \ -drive if=virtio,format=qcow2,file=$QCOW,cache=none \ -drive if=virtio,format=qcow2,file=seed.img \ + -drive if=pflash,format=raw,readonly=on,file=OVMF_CODE_4M.fd \ + -drive if=pflash,format=qcow2,file=$UEFI_VARS \ -serial file:serial.log \ -pidfile "$pid_file" \ -display none \ - -daemonize + -daemonize \ + $opt_vars echo "Started QEMU waiting for SSH" local ssh_opts="-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=5" @@ -62,6 +73,8 @@ boot_qemu_daemon() { exit 1 fi + # At points during bootup the connection is reset and SSH itself will not wait or retry + sleep 1 echo "Retrying SSH..." done @@ -71,15 +84,23 @@ boot_qemu_daemon() { boot_qemu() { echo "Booting QEMU..." - if [ $# -lt 1 ]; then + if [ $# -lt 1 ]; then QCOW=temp.img + UEFI_VARS=uefi-vars-temp.img rm -f temp.img - qemu-img create -F qcow2 -f qcow2 -b k3s-base.img temp.img + qemu-img create -F qcow2 -f qcow2 -b k3s-base.img $QCOW + qemu-img create -F qcow2 -f qcow2 -b uefi-vars-base.img $UEFI_VARS else QCOW=$1 + UEFI_VARS=uefi-vars-base.img fi - qemu-system-x86_64 \ + local opt_vars="" + if [ ! "$VFIO_PCI" = "none" ]; then + opt_vars="-device vifio-pci,host=$VFIO_PCI" + fi + + qemu-system-x86_64 \ -machine accel=kvm,type=q35 \ -cpu host \ -m 8G \ @@ -87,7 +108,10 @@ boot_qemu() { -netdev user,id=net0,hostfwd=tcp::${K3S_SSH_PORT}-:22,hostfwd=tcp::${K3S_API_PORT}-:6443,hostfwd=tcp::${K3S_WEB_PORT}-:80 \ -drive if=virtio,format=qcow2,file=$QCOW,cache=none \ -drive if=virtio,format=qcow2,file=seed.img \ - -nographic + -drive if=pflash,format=raw,readonly=on,file=OVMF_CODE_4M.fd \ + -drive if=pflash,format=qcow2,file=$UEFI_VARS \ + -nographic \ + $opt_vars } stop_qemu() { @@ -152,6 +176,14 @@ install_k3s() { cloud_init() { echo "Could init" + if [ ! -f OVMF_CODE_4M.fd ]; then + cp /usr/share/OVMF/OVMF_CODE_4M.fd ./ + fi + + if [ ! -f OVMF_VARS_4M.fd ]; then + cp /usr/share/OVMF/OVMF_VARS_4M.fd ./ + fi + if [ ! -f ssh_key ]; then ssh-keygen -t ed25519 -f ssh_key -C "pou3" -N "" else @@ -176,6 +208,7 @@ EOF echo "ubuntu-base.img exists, skipping download" fi qemu-img create -F qcow2 -f qcow2 -b ubuntu-base.img k3s-base.img 200G + qemu-img create -F raw -f qcow2 -b OVMF_VARS_4M.fd uefi-vars-base.img 4M ssh-keygen -R [0.0.0.0]:2222 boot_qemu_daemon k3s-base.img @@ -213,6 +246,23 @@ clean() { rm -r ./user-data.yaml } +find_vfio_gpus() { + local lspci_output=$(lspci -vmm -k -d 10de:) + + echo -e "\nFound GPUs with vfio-pci driver:" + echo "$lspci_output" | awk ' + BEGIN { RS=""; FS="\n" } + { + for(i=1; i<=NF; i++) { + if($i ~ /Driver:.*vfio-pci/) { + print $0 + print "" + next + } + } + }' | awk '/Slot/ { print $2 }' +} + show_help() { echo "Usage: $0 [options]" echo "" @@ -226,6 +276,7 @@ show_help() { echo " setup-image Add the operator container image to a running system" echo " tests Run e2e tests" echo " clean Clean up all created files" + echo " find-vfio-gpus Find VFIO GPUs" echo " help Show this help message" } @@ -273,6 +324,9 @@ case "$1" in clean) clean ;; + find-vfio-gpus) + find_vfio_gpus "${@:2}" + ;; help) show_help ;; From b30cf674335c9b06f6df744ca65b85c67d86b1d4 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Wed, 10 Apr 2024 08:55:11 +0100 Subject: [PATCH 6/7] ignore OVMF --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6353230..4f8fad7 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ script/vms/ssh_key* script/vms/serial.log script/vms/kubeconfig script/vms/user-data.yaml +script/vms/*.fd + From 0d27e25fe7be682c075d021bb5827954e5168a71 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Wed, 10 Apr 2024 09:34:42 +0100 Subject: [PATCH 7/7] s/vifio/vfio/ --- script/vms/k3s.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/vms/k3s.sh b/script/vms/k3s.sh index ff6907f..0729e5b 100755 --- a/script/vms/k3s.sh +++ b/script/vms/k3s.sh @@ -97,7 +97,7 @@ boot_qemu() { local opt_vars="" if [ ! "$VFIO_PCI" = "none" ]; then - opt_vars="-device vifio-pci,host=$VFIO_PCI" + opt_vars="-device vfio-pci,host=$VFIO_PCI" fi qemu-system-x86_64 \