From c5c1b1953b45f81dca60448acdb3ce981fbc5c58 Mon Sep 17 00:00:00 2001 From: Yaron Bar Date: Tue, 23 Dec 2025 11:13:38 +0200 Subject: [PATCH] fix: make /lib/modules directory overlayfs over the hostpath --- .../weka-operator/resources/weka_runtime.py | 75 +++++++++++++ internal/controllers/resources/drivers.go | 105 ++++++++++++++++++ internal/controllers/resources/pod.go | 99 +---------------- 3 files changed, 181 insertions(+), 98 deletions(-) create mode 100644 internal/controllers/resources/drivers.go diff --git a/charts/weka-operator/resources/weka_runtime.py b/charts/weka-operator/resources/weka_runtime.py index f3aa3bcce..d58d8fcb5 100644 --- a/charts/weka-operator/resources/weka_runtime.py +++ b/charts/weka-operator/resources/weka_runtime.py @@ -1137,6 +1137,69 @@ async def copy_drivers(): logging.info("done copying drivers") +async def setup_overlayfs_for_lib_modules(): + """ + Set up overlayfs for /lib/modules to allow writes on top of readonly host mount. + This is needed for drivers-loader mode where we need to load kernel modules + but want to keep the host /lib/modules readonly. + + Uses tmpfs for overlay storage and handles /lib -> usr/lib symlinks properly. + """ + logging.info("Setting up overlayfs for /lib/modules") + + # Get the real path of /lib/modules (handles symlinks like /lib -> usr/lib) + stdout, stderr, ec = await run_command("readlink -f /lib/modules") + if ec != 0: + raise Exception(f"Failed to get real path of /lib/modules: {stderr.decode('utf-8')}") + real_path = stdout.decode('utf-8').strip() + logging.info(f"Real path of /lib/modules: {real_path}") + + # Setup paths + ovl_root = "/tmp/ovl-libmodules" + upper_dir = f"{ovl_root}/upper" + work_dir = f"{ovl_root}/work" + ovl_mnt = f"{ovl_root}/mnt" + + # Create overlay root directory + os.makedirs(ovl_root, exist_ok=True) + + # Check if tmpfs is already mounted, if not mount it + stdout, stderr, ec = await run_command(f"mountpoint -q {ovl_root}") + if ec != 0: + # Not mounted yet, create tmpfs + stdout, stderr, ec = await run_command(f"mount -t tmpfs -o size=512m tmpfs {ovl_root}") + if ec != 0: + raise Exception(f"Failed to mount tmpfs at {ovl_root}: {stderr.decode('utf-8')}") + logging.info(f"Mounted tmpfs at {ovl_root}") + + # Create overlay directories + os.makedirs(upper_dir, exist_ok=True) + os.makedirs(work_dir, exist_ok=True) + os.makedirs(ovl_mnt, exist_ok=True) + + # Create overlay view + overlay_opts = f"lowerdir={real_path},upperdir={upper_dir},workdir={work_dir}" + stdout, stderr, ec = await run_command(f"mount -t overlay overlay -o {overlay_opts} {ovl_mnt}") + if ec != 0: + raise Exception(f"Failed to mount overlayfs: {stderr.decode('utf-8')}") + logging.info(f"Created overlay at {ovl_mnt}") + + # Overmount the real path in this mount namespace + stdout, stderr, ec = await run_command(f"mount --bind {ovl_mnt} {real_path}") + if ec != 0: + raise Exception(f"Failed to bind mount overlay over {real_path}: {stderr.decode('utf-8')}") + logging.info(f"Overmounted {real_path} with overlay") + + # If /lib/modules is not the same as real_path, keep /lib/modules consistent too + if real_path != "/lib/modules": + stdout, stderr, ec = await run_command(f"mount --bind {real_path} /lib/modules") + if ec != 0: + raise Exception(f"Failed to bind mount {real_path} to /lib/modules: {stderr.decode('utf-8')}") + logging.info("Ensured /lib/modules points to overlay") + + logging.info(f"Overlayfs active: lower={real_path} upper={upper_dir} work={work_dir}") + + async def cos_build_drivers(): weka_driver_version = version_params["wekafs"] weka_driver_file_version = weka_driver_version.rsplit("-", 1)[0] @@ -3280,6 +3343,18 @@ async def main(): # 2 minutes timeout for driver loading end_time = time.time() + 120 await disable_driver_signing() + + # Set up overlayfs for /lib/modules to allow writes on readonly host mount + try: + await setup_overlayfs_for_lib_modules() + except Exception as e: + logging.error(f"Failed to set up overlayfs: {e}") + write_results(dict( + err=f"Failed to set up overlayfs: {str(e)}", + drivers_loaded=False, + )) + return + loaded = False while time.time() < end_time: try: diff --git a/internal/controllers/resources/drivers.go b/internal/controllers/resources/drivers.go new file mode 100644 index 000000000..ef36e9dd4 --- /dev/null +++ b/internal/controllers/resources/drivers.go @@ -0,0 +1,105 @@ +package resources + +import ( + weka "github.com/weka/weka-k8s-api/api/v1alpha1" + "github.com/weka/weka-operator/internal/config" + corev1 "k8s.io/api/core/v1" +) + +func (f *PodFactory) setDriverDependencies(pod *corev1.Pod) { + if f.container.Spec.Instructions != nil && f.container.Spec.Instructions.Type == + weka.InstructionCopyWekaFilesToDriverLoader { + f.copyWekaVersionToDriverLoader(pod) + } + if f.nodeInfo.IsCos() { + // in COS we can't load it in the drivers-loader pod because of /lib/modules override + addUIOLoaderInitContainer(pod) + allowCosDisableDriverSigning := config.Config.GkeCompatibility.DisableDriverSigning + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "weka-boot-scripts", + MountPath: "/devenv.sh", + SubPath: "devenv.sh", + }) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "proc-sysrq-trigger", + MountPath: "/hostside/proc/sysrq-trigger", + }) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "proc-cmdline", + MountPath: "/hostside/proc/cmdline", + }) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "proc-sysrq-trigger", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/proc/sysrq-trigger", + Type: &[]corev1.HostPathType{corev1.HostPathFile}[0], + }, + }, + }) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "proc-cmdline", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/proc/cmdline", + Type: &[]corev1.HostPathType{corev1.HostPathFile}[0], + }, + }, + }) + if allowCosDisableDriverSigning { + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ + Name: "WEKA_COS_ALLOW_DISABLE_DRIVER_SIGNING", + Value: "true", + }) + } + + if f.container.IsDriversBuilder() { + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "gcloud-credentials", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: config.Config.GkeCompatibility.ServiceAccountSecret, + }, + }, + }) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "gcloud-credentials", + MountPath: "/var/secrets/google", + }) + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ + Name: "GOOGLE_APPLICATION_CREDENTIALS", + Value: "/var/secrets/google/service-account.json", + }) + } + } else { + libModulesPath := "/lib/modules" + usrSrcPath := "/usr/src" + + // adding mount of headers only for case of drivers-related container + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "libmodules", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/lib/modules", + }, + }, + }) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "usrsrc", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/usr/src", + }, + }, + }) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "libmodules", + MountPath: libModulesPath, + ReadOnly: true, + }) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "usrsrc", + MountPath: usrSrcPath, + }) + } +} diff --git a/internal/controllers/resources/pod.go b/internal/controllers/resources/pod.go index 6da4d3bc4..b46d50a45 100644 --- a/internal/controllers/resources/pod.go +++ b/internal/controllers/resources/pod.go @@ -735,104 +735,7 @@ func (f *PodFactory) Create(ctx context.Context, podImage *string) (*corev1.Pod, } if f.container.IsDriversContainer() { // Dependencies for driver-loader probably can be reduced - if f.container.Spec.Instructions != nil && f.container.Spec.Instructions.Type == - weka.InstructionCopyWekaFilesToDriverLoader { - f.copyWekaVersionToDriverLoader(pod) - } - if f.nodeInfo.IsCos() { - // in COS we can't load it in the drivers-loader pod because of /lib/modules override - addUIOLoaderInitContainer(pod) - allowCosDisableDriverSigning := config.Config.GkeCompatibility.DisableDriverSigning - pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "weka-boot-scripts", - MountPath: "/devenv.sh", - SubPath: "devenv.sh", - }) - pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "proc-sysrq-trigger", - MountPath: "/hostside/proc/sysrq-trigger", - }) - pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "proc-cmdline", - MountPath: "/hostside/proc/cmdline", - }) - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ - Name: "proc-sysrq-trigger", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/proc/sysrq-trigger", - Type: &[]corev1.HostPathType{corev1.HostPathFile}[0], - }, - }, - }) - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ - Name: "proc-cmdline", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/proc/cmdline", - Type: &[]corev1.HostPathType{corev1.HostPathFile}[0], - }, - }, - }) - if allowCosDisableDriverSigning { - pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ - Name: "WEKA_COS_ALLOW_DISABLE_DRIVER_SIGNING", - Value: "true", - }) - } - - if f.container.IsDriversBuilder() { - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ - Name: "gcloud-credentials", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: config.Config.GkeCompatibility.ServiceAccountSecret, - }, - }, - }) - pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "gcloud-credentials", - MountPath: "/var/secrets/google", - }) - pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ - Name: "GOOGLE_APPLICATION_CREDENTIALS", - Value: "/var/secrets/google/service-account.json", - }) - } - } else { - libModulesPath := "/lib/modules" - usrSrcPath := "/usr/src" - if f.nodeInfo.IsRhCos() { - libModulesPath = "/hostpath/lib/modules" - usrSrcPath = "/hostpath/usr/src" - } - - // adding mount of headers only for case of drivers-related container - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ - Name: "libmodules", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/lib/modules", - }, - }, - }) - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ - Name: "usrsrc", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/usr/src", - }, - }, - }) - pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "libmodules", - MountPath: libModulesPath, - }) - pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "usrsrc", - MountPath: usrSrcPath, - }) - } + f.setDriverDependencies(pod) } // for Dist container, if running on OCP / COS / other containerized OS