Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 88 additions & 0 deletions internal/controllers/resources/hugepages.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package resources

import (
"fmt"
"strings"

weka "github.com/weka/weka-k8s-api/api/v1alpha1"
corev1 "k8s.io/api/core/v1"
)

type HugePagesDetails struct {
HugePagesStr string
HugePagesK8sSuffix string
HugePagesMb int
WekaMemoryString string
HugePagesResourceName corev1.ResourceName
}

func (f *PodFactory) getHugePagesDetails() HugePagesDetails {
hugePagesStr := ""
hugePagesK8sSuffix := "2Mi"
wekaMemoryString := ""
if f.container.Spec.HugepagesSize == "1Gi" {
hugePagesK8sSuffix = f.container.Spec.HugepagesSize
hugePagesStr = fmt.Sprintf("%dGi", f.container.Spec.Hugepages/1000)
wekaMemoryString = fmt.Sprintf("%dGiB", f.container.Spec.Hugepages/1000)
} else {
hugePagesStr = fmt.Sprintf("%dMi", f.container.Spec.Hugepages)
hugePagesK8sSuffix = "2Mi"
offset := f.getHugePagesOffset()
wekaMemoryString = fmt.Sprintf("%dMiB", f.container.Spec.Hugepages-offset)
}

if f.container.Spec.HugepagesOverride != "" {
wekaMemoryString = f.container.Spec.HugepagesOverride
}

hugePagesName := corev1.ResourceName(
strings.Join(
[]string{corev1.ResourceHugePagesPrefix, hugePagesK8sSuffix},
""))

return HugePagesDetails{
HugePagesStr: hugePagesStr,
HugePagesK8sSuffix: hugePagesK8sSuffix,
WekaMemoryString: wekaMemoryString,
HugePagesResourceName: hugePagesName,
HugePagesMb: f.container.Spec.Hugepages,
}
}

func (f *PodFactory) getHugePagesOffset() int {
offset := f.container.Spec.HugepagesOffset
// get default if not set
if offset == 0 {
if f.container.Spec.Mode == weka.WekaContainerModeDrive {
offset = 200 * f.container.Spec.NumDrives
} else {
offset = 200
}
}
return offset
}

func (f *PodFactory) setHugePages(pod *corev1.Pod) {
// Skip hugepages for drivers containers (they only load kernel modules, don't run Weka processes)
if f.container.IsDriversContainer() {
return
}

hgDetails := f.getHugePagesDetails()

// Add hugepages volume
pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
Name: "hugepages",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{
Medium: corev1.StorageMedium(fmt.Sprintf("HugePages-%s", hgDetails.HugePagesK8sSuffix)),
},
},
})

// Add hugepages volume mount
pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "hugepages",
MountPath: "/dev/hugepages",
})
}
100 changes: 23 additions & 77 deletions internal/controllers/resources/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -233,10 +233,6 @@ func (f *PodFactory) Create(ctx context.Context, podImage *string) (*corev1.Pod,
Name: "run",
MountPath: "/host/run",
},
{
Name: "hugepages",
MountPath: "/dev/hugepages",
},
{
Name: "sys",
MountPath: "/sys",
Expand Down Expand Up @@ -433,14 +429,6 @@ func (f *PodFactory) Create(ctx context.Context, podImage *string) (*corev1.Pod,
},
},
Volumes: []corev1.Volume{
{
Name: "hugepages",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{
Medium: corev1.StorageMedium(fmt.Sprintf("HugePages-%s", f.getHugePagesDetails().HugePagesK8sSuffix)),
},
},
},
{
Name: "osrelease",
VolumeSource: corev1.VolumeSource{
Expand Down Expand Up @@ -489,6 +477,9 @@ func (f *PodFactory) Create(ctx context.Context, podImage *string) (*corev1.Pod,
},
}

// Set up hugepages volume and mount (skipped for drivers containers)
f.setHugePages(pod)

if f.container.Spec.GetOverrides().PreRunScript != "" {
// encode in base64 and write into env var
base64str := base64.StdEncoding.EncodeToString([]byte(f.container.Spec.GetOverrides().PreRunScript))
Expand Down Expand Up @@ -953,60 +944,6 @@ func GetWekaPodTolerations(container *weka.WekaContainer) []corev1.Toleration {
return tolerations
}

type HugePagesDetails struct {
HugePagesStr string
HugePagesK8sSuffix string
HugePagesMb int
WekaMemoryString string
HugePagesResourceName corev1.ResourceName
}

func (f *PodFactory) getHugePagesDetails() HugePagesDetails {
hugePagesStr := ""
hugePagesK8sSuffix := "2Mi"
wekaMemoryString := ""
if f.container.Spec.HugepagesSize == "1Gi" {
hugePagesK8sSuffix = f.container.Spec.HugepagesSize
hugePagesStr = fmt.Sprintf("%dGi", f.container.Spec.Hugepages/1000)
wekaMemoryString = fmt.Sprintf("%dGiB", f.container.Spec.Hugepages/1000)
} else {
hugePagesStr = fmt.Sprintf("%dMi", f.container.Spec.Hugepages)
hugePagesK8sSuffix = "2Mi"
offset := f.getHugePagesOffset()
wekaMemoryString = fmt.Sprintf("%dMiB", f.container.Spec.Hugepages-offset)
}

if f.container.Spec.HugepagesOverride != "" {
wekaMemoryString = f.container.Spec.HugepagesOverride
}

hugePagesName := corev1.ResourceName(
strings.Join(
[]string{corev1.ResourceHugePagesPrefix, hugePagesK8sSuffix},
""))

return HugePagesDetails{
HugePagesStr: hugePagesStr,
HugePagesK8sSuffix: hugePagesK8sSuffix,
WekaMemoryString: wekaMemoryString,
HugePagesResourceName: hugePagesName,
HugePagesMb: f.container.Spec.Hugepages,
}
}

func (f *PodFactory) getHugePagesOffset() int {
offset := f.container.Spec.HugepagesOffset
// get default if not set
if offset == 0 {
if f.container.Spec.Mode == weka.WekaContainerModeDrive {
offset = 200 * f.container.Spec.NumDrives
} else {
offset = 200
}
}
return offset
}

func (f *PodFactory) setResources(ctx context.Context, pod *corev1.Pod) error {
totalNumCores := f.container.Spec.NumCores
if f.container.Spec.Mode == weka.WekaContainerModeCompute {
Expand Down Expand Up @@ -1214,18 +1151,27 @@ func (f *PodFactory) setResources(ctx context.Context, pod *corev1.Pod) error {

// since this is HT, we are doubling num of cores on allocation
logger.SetValues("cpuRequestStr", cpuRequestStr, "cpuLimitStr", cpuLimitStr, "memRequest", memRequest, "hugePages", hgDetails.HugePagesStr)

// Build resource lists
limitsResourceList := corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(cpuLimitStr),
corev1.ResourceMemory: resource.MustParse(memLimit),
}
requestsResourceList := corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(cpuRequestStr),
corev1.ResourceMemory: resource.MustParse(memRequest),
corev1.ResourceEphemeralStorage: resource.MustParse(requestedEphemeralStorage),
}

// Add hugepages only for non-drivers containers
if !f.container.IsDriversContainer() {
limitsResourceList[hgDetails.HugePagesResourceName] = resource.MustParse(hgDetails.HugePagesStr)
requestsResourceList[hgDetails.HugePagesResourceName] = resource.MustParse(hgDetails.HugePagesStr)
}

pod.Spec.Containers[0].Resources = corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(cpuLimitStr),
hgDetails.HugePagesResourceName: resource.MustParse(hgDetails.HugePagesStr),
corev1.ResourceMemory: resource.MustParse(memLimit),
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(cpuRequestStr),
hgDetails.HugePagesResourceName: resource.MustParse(hgDetails.HugePagesStr),
corev1.ResourceMemory: resource.MustParse(memRequest),
corev1.ResourceEphemeralStorage: resource.MustParse(requestedEphemeralStorage),
},
Limits: limitsResourceList,
Requests: requestsResourceList,
}

if f.container.Spec.Mode == weka.WekaContainerModeDrive {
Expand Down
Loading