From 7999d4056e294d50db213563d8446af5ae8dda6c Mon Sep 17 00:00:00 2001 From: Yaron Bar Date: Tue, 23 Dec 2025 11:19:13 +0200 Subject: [PATCH] fix: refactor hugepages logic to its own file --- internal/controllers/resources/hugepages.go | 88 +++++++++++++++++ internal/controllers/resources/pod.go | 100 +++++--------------- 2 files changed, 111 insertions(+), 77 deletions(-) create mode 100644 internal/controllers/resources/hugepages.go diff --git a/internal/controllers/resources/hugepages.go b/internal/controllers/resources/hugepages.go new file mode 100644 index 000000000..e1593dfeb --- /dev/null +++ b/internal/controllers/resources/hugepages.go @@ -0,0 +1,88 @@ +package resources + +import ( + "fmt" + "strings" + + weka "github.com/weka/weka-k8s-api/api/v1alpha1" + corev1 "k8s.io/api/core/v1" +) + +type HugePagesDetails struct { + HugePagesStr string + HugePagesK8sSuffix string + HugePagesMb int + WekaMemoryString string + HugePagesResourceName corev1.ResourceName +} + +func (f *PodFactory) getHugePagesDetails() HugePagesDetails { + hugePagesStr := "" + hugePagesK8sSuffix := "2Mi" + wekaMemoryString := "" + if f.container.Spec.HugepagesSize == "1Gi" { + hugePagesK8sSuffix = f.container.Spec.HugepagesSize + hugePagesStr = fmt.Sprintf("%dGi", f.container.Spec.Hugepages/1000) + wekaMemoryString = fmt.Sprintf("%dGiB", f.container.Spec.Hugepages/1000) + } else { + hugePagesStr = fmt.Sprintf("%dMi", f.container.Spec.Hugepages) + hugePagesK8sSuffix = "2Mi" + offset := f.getHugePagesOffset() + wekaMemoryString = fmt.Sprintf("%dMiB", f.container.Spec.Hugepages-offset) + } + + if f.container.Spec.HugepagesOverride != "" { + wekaMemoryString = f.container.Spec.HugepagesOverride + } + + hugePagesName := corev1.ResourceName( + strings.Join( + []string{corev1.ResourceHugePagesPrefix, hugePagesK8sSuffix}, + "")) + + return HugePagesDetails{ + HugePagesStr: hugePagesStr, + HugePagesK8sSuffix: hugePagesK8sSuffix, + WekaMemoryString: wekaMemoryString, + HugePagesResourceName: hugePagesName, + HugePagesMb: f.container.Spec.Hugepages, + } +} + +func (f *PodFactory) getHugePagesOffset() int { + offset := f.container.Spec.HugepagesOffset + // get default if not set + if offset == 0 { + if f.container.Spec.Mode == weka.WekaContainerModeDrive { + offset = 200 * f.container.Spec.NumDrives + } else { + offset = 200 + } + } + return offset +} + +func (f *PodFactory) setHugePages(pod *corev1.Pod) { + // Skip hugepages for drivers containers (they only load kernel modules, don't run Weka processes) + if f.container.IsDriversContainer() { + return + } + + hgDetails := f.getHugePagesDetails() + + // Add hugepages volume + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "hugepages", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMedium(fmt.Sprintf("HugePages-%s", hgDetails.HugePagesK8sSuffix)), + }, + }, + }) + + // Add hugepages volume mount + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "hugepages", + MountPath: "/dev/hugepages", + }) +} diff --git a/internal/controllers/resources/pod.go b/internal/controllers/resources/pod.go index b46d50a45..b1a273582 100644 --- a/internal/controllers/resources/pod.go +++ b/internal/controllers/resources/pod.go @@ -233,10 +233,6 @@ func (f *PodFactory) Create(ctx context.Context, podImage *string) (*corev1.Pod, Name: "run", MountPath: "/host/run", }, - { - Name: "hugepages", - MountPath: "/dev/hugepages", - }, { Name: "sys", MountPath: "/sys", @@ -433,14 +429,6 @@ func (f *PodFactory) Create(ctx context.Context, podImage *string) (*corev1.Pod, }, }, Volumes: []corev1.Volume{ - { - Name: "hugepages", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMedium(fmt.Sprintf("HugePages-%s", f.getHugePagesDetails().HugePagesK8sSuffix)), - }, - }, - }, { Name: "osrelease", VolumeSource: corev1.VolumeSource{ @@ -489,6 +477,9 @@ func (f *PodFactory) Create(ctx context.Context, podImage *string) (*corev1.Pod, }, } + // Set up hugepages volume and mount (skipped for drivers containers) + f.setHugePages(pod) + if f.container.Spec.GetOverrides().PreRunScript != "" { // encode in base64 and write into env var base64str := base64.StdEncoding.EncodeToString([]byte(f.container.Spec.GetOverrides().PreRunScript)) @@ -953,60 +944,6 @@ func GetWekaPodTolerations(container *weka.WekaContainer) []corev1.Toleration { return tolerations } -type HugePagesDetails struct { - HugePagesStr string - HugePagesK8sSuffix string - HugePagesMb int - WekaMemoryString string - HugePagesResourceName corev1.ResourceName -} - -func (f *PodFactory) getHugePagesDetails() HugePagesDetails { - hugePagesStr := "" - hugePagesK8sSuffix := "2Mi" - wekaMemoryString := "" - if f.container.Spec.HugepagesSize == "1Gi" { - hugePagesK8sSuffix = f.container.Spec.HugepagesSize - hugePagesStr = fmt.Sprintf("%dGi", f.container.Spec.Hugepages/1000) - wekaMemoryString = fmt.Sprintf("%dGiB", f.container.Spec.Hugepages/1000) - } else { - hugePagesStr = fmt.Sprintf("%dMi", f.container.Spec.Hugepages) - hugePagesK8sSuffix = "2Mi" - offset := f.getHugePagesOffset() - wekaMemoryString = fmt.Sprintf("%dMiB", f.container.Spec.Hugepages-offset) - } - - if f.container.Spec.HugepagesOverride != "" { - wekaMemoryString = f.container.Spec.HugepagesOverride - } - - hugePagesName := corev1.ResourceName( - strings.Join( - []string{corev1.ResourceHugePagesPrefix, hugePagesK8sSuffix}, - "")) - - return HugePagesDetails{ - HugePagesStr: hugePagesStr, - HugePagesK8sSuffix: hugePagesK8sSuffix, - WekaMemoryString: wekaMemoryString, - HugePagesResourceName: hugePagesName, - HugePagesMb: f.container.Spec.Hugepages, - } -} - -func (f *PodFactory) getHugePagesOffset() int { - offset := f.container.Spec.HugepagesOffset - // get default if not set - if offset == 0 { - if f.container.Spec.Mode == weka.WekaContainerModeDrive { - offset = 200 * f.container.Spec.NumDrives - } else { - offset = 200 - } - } - return offset -} - func (f *PodFactory) setResources(ctx context.Context, pod *corev1.Pod) error { totalNumCores := f.container.Spec.NumCores if f.container.Spec.Mode == weka.WekaContainerModeCompute { @@ -1214,18 +1151,27 @@ func (f *PodFactory) setResources(ctx context.Context, pod *corev1.Pod) error { // since this is HT, we are doubling num of cores on allocation logger.SetValues("cpuRequestStr", cpuRequestStr, "cpuLimitStr", cpuLimitStr, "memRequest", memRequest, "hugePages", hgDetails.HugePagesStr) + + // Build resource lists + limitsResourceList := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(cpuLimitStr), + corev1.ResourceMemory: resource.MustParse(memLimit), + } + requestsResourceList := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(cpuRequestStr), + corev1.ResourceMemory: resource.MustParse(memRequest), + corev1.ResourceEphemeralStorage: resource.MustParse(requestedEphemeralStorage), + } + + // Add hugepages only for non-drivers containers + if !f.container.IsDriversContainer() { + limitsResourceList[hgDetails.HugePagesResourceName] = resource.MustParse(hgDetails.HugePagesStr) + requestsResourceList[hgDetails.HugePagesResourceName] = resource.MustParse(hgDetails.HugePagesStr) + } + pod.Spec.Containers[0].Resources = corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(cpuLimitStr), - hgDetails.HugePagesResourceName: resource.MustParse(hgDetails.HugePagesStr), - corev1.ResourceMemory: resource.MustParse(memLimit), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(cpuRequestStr), - hgDetails.HugePagesResourceName: resource.MustParse(hgDetails.HugePagesStr), - corev1.ResourceMemory: resource.MustParse(memRequest), - corev1.ResourceEphemeralStorage: resource.MustParse(requestedEphemeralStorage), - }, + Limits: limitsResourceList, + Requests: requestsResourceList, } if f.container.Spec.Mode == weka.WekaContainerModeDrive {