diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 7752a1ce0..57c3bc32c 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -49,6 +49,8 @@ const ( balloonKey = "balloon." + PolicyName + "." + kubernetes.ResmgrKeyNamespace // hideHyperthreadsKey is a pod annotation key for pod/container-specific hyperthread allowance. hideHyperthreadsKey = "hide-hyperthreads." + kubernetes.ResmgrKeyNamespace + // schedulingClassKey is a pod annotation key for pod/container-specific scheduling class. + schedulingClassKey = "scheduling-class." + kubernetes.ResmgrKeyNamespace // reservedBalloonDefName is the name in the reserved balloon definition. reservedBalloonDefName = "reserved" // defaultBalloonDefName is the name in the default balloon definition. @@ -1509,6 +1511,7 @@ func (p *balloons) applyBalloonDef(balloons *[]*Balloon, blnDef *BalloonDef, fre func (p *balloons) validateConfig(bpoptions *BalloonsOptions) error { seenNames := map[string]struct{}{} undefinedLoadClasses := map[string]struct{}{} + undefinedSchedulingClasses := map[string]struct{}{} compositeBlnDefs := map[string]*BalloonDef{} for _, blnDef := range bpoptions.BalloonDefs { if blnDef.Name == "" { @@ -1581,6 +1584,9 @@ func (p *balloons) validateConfig(bpoptions *BalloonsOptions) error { for _, load := range blnDef.Loads { undefinedLoadClasses[load] = struct{}{} } + if blnDef.SchedulingClass != "" { + undefinedSchedulingClasses[blnDef.SchedulingClass] = struct{}{} + } } for lcIndex, loadClass := range bpoptions.LoadClasses { delete(undefinedLoadClasses, loadClass.Name) @@ -1594,6 +1600,12 @@ func (p *balloons) validateConfig(bpoptions *BalloonsOptions) error { if len(undefinedLoadClasses) > 0 { return balloonsError("loads defined in balloonTypes but missing from loadClasses: %v", undefinedLoadClasses) } + for _, schedClass := range bpoptions.SchedulingClasses { + delete(undefinedSchedulingClasses, schedClass.Name) + } + if len(undefinedSchedulingClasses) > 0 { + return balloonsError("schedulingClass(es) defined in balloonTypes but missing from schedulingClasses: %v", undefinedSchedulingClasses) + } var circularCheck func(name string, seen map[string]int) error circularCheck = func(name string, seen map[string]int) error { if seen[name] > 0 { @@ -2124,6 +2136,81 @@ func (bln *Balloon) updateGroups(c cache.Container, delta int) { } } +// applyProcessScheduling configures container's scheduling and IO priorities +func applyProcessScheduling(c cache.Container, sc *SchedulingClass) { + if sc == nil { + return + } + log.Debug(" - applying scheduling class %q to %s", sc.Name, c.PrettyName()) + if sc.Policy != "" { + if pol, err := sc.Policy.ToNRI(); err == nil { + c.SetSchedulingPolicy(pol) + log.Debug(" - scheduling policy %q (%s)", sc.Policy, pol) + } else { + log.Debug(" - invalid scheduling policy %q in scheduling class %q: %v", sc.Policy, sc.Name, err) + } + } + if sc.Priority != nil { + c.SetSchedulingPriority(int32(*sc.Priority)) + log.Debug(" - scheduling priority %d", *sc.Priority) + } + if len(sc.Flags) > 0 { + if flags, err := sc.Flags.ToNRI(); err == nil { + c.SetSchedulingFlags(flags) + log.Debug(" - scheduling flags %q", sc.Flags) + } else { + log.Debug(" - invalid scheduling flags %q in scheduling class %q: %v", sc.Flags, sc.Name, err) + } + } + if sc.Nice != nil { + c.SetSchedulingNice(int32(*sc.Nice)) + log.Debug(" - nice value %d", *sc.Nice) + } + if sc.Runtime != nil { + c.SetSchedulingRuntime(*sc.Runtime) + log.Debug(" - scheduling runtime %d", *sc.Runtime) + } + if sc.Deadline != nil { + c.SetSchedulingDeadline(*sc.Deadline) + log.Debug(" - scheduling deadline %d", *sc.Deadline) + } + if sc.Period != nil { + c.SetSchedulingPeriod(*sc.Period) + log.Debug(" - scheduling period %d", *sc.Period) + } + if sc.IOClass != "" { + if ioClass, err := sc.IOClass.ToNRI(); err == nil { + c.SetSchedulingIOClass(ioClass) + log.Debug(" - IO class %q", sc.IOClass) + } else { + log.Debug(" - invalid IO class %q in scheduling class %q: %v", sc.IOClass, sc.Name, err) + } + } + if sc.IOPriority != nil { + c.SetSchedulingIOPriority(int32(*sc.IOPriority)) + log.Debug(" - IO priority %d", *sc.IOPriority) + } +} + +func (p *balloons) applyProcessProperties(c cache.Container, bln *Balloon) { + effSc := bln.Def.SchedulingClass + if annSc, annExists := c.GetEffectiveAnnotation(schedulingClassKey); annExists { + if annSc != effSc { + log.Debug(" - container %s overrides balloon scheduling class %q with annotation %q", + c.PrettyName(), effSc, annSc) + } + effSc = annSc + } + if effSc != "" { + for _, sc := range p.bpoptions.SchedulingClasses { + if sc.Name == effSc { + applyProcessScheduling(c, sc) + break + } + } + } +} + // assignContainer adds a container to a balloon func (p *balloons) assignContainer(c cache.Container, bln *Balloon) { log.Info("assigning container %s to balloon %s", c.PrettyName(), bln) @@ -2131,6 +2218,7 @@ func (p *balloons) assignContainer(c cache.Container, bln *Balloon) { bln.PodIDs[podID] = append(bln.PodIDs[podID], c.GetID()) bln.updateGroups(c, 1) p.updatePinning(bln) + p.applyProcessProperties(c, bln) } // dismissContainer removes a container from a balloon diff --git a/cmd/plugins/balloons/policy/flags.go b/cmd/plugins/balloons/policy/flags.go index dafbd551a..19889ea15 100644 --- a/cmd/plugins/balloons/policy/flags.go +++ b/cmd/plugins/balloons/policy/flags.go @@ -23,6 +23,7 @@ type ( BalloonsOptions = cfgapi.Config BalloonDef = cfgapi.BalloonDef LoadClass = cfgapi.LoadClass + SchedulingClass = cfgapi.SchedulingClass CPUTopologyLevel = cfgapi.CPUTopologyLevel ) diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index 09838f8f6..9477f94b0 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -479,6 +479,33 @@ func (m *mockContainer) SetMemoryLimit(int64) { func (m *mockContainer) SetMemorySwap(int64) { panic("unimplemented") } +func (m *mockContainer) SetSchedulingPolicy(nri.LinuxSchedulerPolicy) { + panic("unimplemented") +} +func (m *mockContainer) SetSchedulingNice(int32) { + panic("unimplemented") +} +func (m *mockContainer) SetSchedulingPriority(int32) { + panic("unimplemented") +} +func (m *mockContainer) SetSchedulingFlags([]nri.LinuxSchedulerFlag) { + panic("unimplemented") +} +func (m *mockContainer) SetSchedulingRuntime(uint64) { + panic("unimplemented") +} +func (m *mockContainer) SetSchedulingDeadline(uint64) { + panic("unimplemented") +} +func (m *mockContainer) SetSchedulingPeriod(uint64) { + panic("unimplemented") +} +func (m *mockContainer) SetSchedulingIOClass(nri.IOPrioClass) { + panic("unimplemented") +} +func (m *mockContainer) SetSchedulingIOPriority(int32) { + panic("unimplemented") +} func (m *mockContainer) GetPendingAdjustment() *nri.ContainerAdjustment { panic("unimplemented") } diff --git a/config/crd/bases/config.nri_balloonspolicies.yaml b/config/crd/bases/config.nri_balloonspolicies.yaml index daf5dda6a..875a88a08 100644 --- a/config/crd/bases/config.nri_balloonspolicies.yaml +++ b/config/crd/bases/config.nri_balloonspolicies.yaml @@ -314,6 +314,14 @@ spec: placed on separate balloons. The default is false: prefer placing containers of a pod to the same balloon(s). type: boolean + schedulingClass: + description: |- + SchedulingClass is the name of the scheduling class from + which default Linux scheduling and IO priority parameters + are applied on new containers created in these + balloons. Parameters for the class are specified in the + schedulingClasses list. + type: string shareIdleCPUsInSame: description: |- ShareIdleCpusInSame : if there are idle @@ -964,6 +972,92 @@ spec: type: string description: Reserved (CPU) resources for kube-system namespace. type: object + schedulingClasses: + description: |- + SchedulingClasses specify scheduling classes available in + balloon types. + items: + description: |- + SchedulingClass specifies the default Linux scheduling and IO + priority parameters for containers assigned into this class. + properties: + deadline: + description: Deadline is the Linux SCHED_DEADLINE deadline value + to use (in microseconds). + format: int64 + type: integer + flags: + description: |- + Flags is a list of Linux scheduling flags to set. + SCHED_FLAG_ translates to etc. + enum: + - reset-on-fork + - reclaim + - dl-overrun + - keep-policy + - keep-params + - util-clamp-min + - util-clamp-max + items: + type: string + type: array + ioClass: + description: |- + IOClass is the IO scheduling class to use. + IOPRIO_CLASS_ translates to . + Refer to ioprio_set(2) and ionice(1) for details. + enum: + - none + - rt + - be + - idle + type: string + ioPriority: + description: |- + IOPriority is the IO priority within the selected IO class to use. + Valid range depends on the selected class. + Refer to ionice(1) for details. + type: integer + name: + description: Name of the scheduling class. + type: string + nice: + description: Nice is the Linux nice value to use. + type: integer + period: + description: Period is the Linux SCHED_DEADLINE period value + to use (in microseconds). + format: int64 + type: integer + policy: + description: |- + Policy is the Linux scheduling policy to use. + SCHED_ translates to etc. + enum: + - none + - other + - fifo + - rr + - batch + - iso + - idle + - deadline + type: string + priority: + description: |- + Priority is the Linux scheduling priority to use. + Valid range depends on the selected policy. + Refer to sched_setscheduler(2) for details. + type: integer + runtime: + description: Runtime is the Linux SCHED_DEADLINE runtime value + to use (in microseconds). + format: int64 + type: integer + required: + - name + type: object + type: array showContainersInNrt: description: |- ShowContainersInNrt controls whether containers in balloons diff --git a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml index daf5dda6a..875a88a08 100644 --- a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml +++ b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml @@ -314,6 +314,14 @@ spec: placed on separate balloons. The default is false: prefer placing containers of a pod to the same balloon(s). type: boolean + schedulingClass: + description: |- + SchedulingClass is the name of the scheduling class from + which default Linux scheduling and IO priority parameters + are applied on new containers created in these + balloons. Parameters for the class are specified in the + schedulingClasses list. + type: string shareIdleCPUsInSame: description: |- ShareIdleCpusInSame : if there are idle @@ -964,6 +972,92 @@ spec: type: string description: Reserved (CPU) resources for kube-system namespace. type: object + schedulingClasses: + description: |- + SchedulingClasses specify scheduling classes available in + balloon types. + items: + description: |- + SchedulingClass specifies the default Linux scheduling and IO + priority parameters for containers assigned into this class. + properties: + deadline: + description: Deadline is the Linux SCHED_DEADLINE deadline value + to use (in microseconds). + format: int64 + type: integer + flags: + description: |- + Flags is a list of Linux scheduling flags to set. + SCHED_FLAG_ translates to etc. + enum: + - reset-on-fork + - reclaim + - dl-overrun + - keep-policy + - keep-params + - util-clamp-min + - util-clamp-max + items: + type: string + type: array + ioClass: + description: |- + IOClass is the IO scheduling class to use. + IOPRIO_CLASS_ translates to . + Refer to ioprio_set(2) and ionice(1) for details. + enum: + - none + - rt + - be + - idle + type: string + ioPriority: + description: |- + IOPriority is the IO priority within the selected IO class to use. + Valid range depends on the selected class. + Refer to ionice(1) for details. + type: integer + name: + description: Name of the scheduling class. + type: string + nice: + description: Nice is the Linux nice value to use. + type: integer + period: + description: Period is the Linux SCHED_DEADLINE period value + to use (in microseconds). + format: int64 + type: integer + policy: + description: |- + Policy is the Linux scheduling policy to use. + SCHED_ translates to etc. + enum: + - none + - other + - fifo + - rr + - batch + - iso + - idle + - deadline + type: string + priority: + description: |- + Priority is the Linux scheduling priority to use. + Valid range depends on the selected policy. + Refer to sched_setscheduler(2) for details. + type: integer + runtime: + description: Runtime is the Linux SCHED_DEADLINE runtime value + to use (in microseconds). + format: int64 + type: integer + required: + - name + type: object + type: array showContainersInNrt: description: |- ShowContainersInNrt controls whether containers in balloons diff --git a/docs/resource-policy/policy/balloons.md b/docs/resource-policy/policy/balloons.md index 2fdcfad49..2d277bb62 100644 --- a/docs/resource-policy/policy/balloons.md +++ b/docs/resource-policy/policy/balloons.md @@ -275,6 +275,10 @@ Balloons policy parameters: should get their CPUs from separate cache blocks for best performance. Every listed class must be specified in `loadClasses`. + - `schedulingClass` specifies the name of the scheduling class + according to which containers in balloons are scheduled. Class + properties are defined in separate `schedulingClasses` objects, + see below. - `components` list includes component balloon types. If non-empty, the balloon is a composite balloon whose CPUs are not allocated directly to itself but its CPUs are the union of CPUs of its @@ -318,6 +322,33 @@ Balloons policy parameters: physical `core` or different `l2cache` block. The default is `false`, that is, locality of balloon's CPUs is seen more important than avoiding balloon's own load. +- `schedulingClasses`: is a list of scheduling related parameters + organized in classes. These parameters tune containers when they are + created but do not affect already running containers. The class of a + container is defined with the `schedulingClass: ` option in + container's balloon type, and can be overridden by + `scheduling-class.resource-policy.nri.io` pod annotation. Each class + in the list has following properties. + - `name` is the name of the scheduling class. + - `policy` is the Linux scheduling policy. Supported policies are: + `none`, `other`, `fifo`, `rr`, `batch`, `idle`, and `deadline`. + - `priority` is the scheduling priority. Refer to + sched_setscheduler(2) documentation for valid values depending on + the policy. + - `flags` is a list of scheduling flags. Supported flags are: + `reset-on-fork`, `reclaim`, `dl-overrun`, `keep-policy`, + `keep-params`, `util-clamp-min`, `util-clamp-max`. + - `nice`: nice value for the container process. + - `runtime`: runtime value for `deadline` scheduling policy (in + microseconds). + - `deadline`: deadline value for `deadline` scheduling policy (in + microseconds). + - `period`: period value for `deadline` scheduling policy (in + microseconds). + - `ioClass`: IO class for the container process. Supported classes + are: `none`, `rt` for realtime, `be` for best-effort, and `idle`. + - `ioPriority`: IO priority for the container process. Refer + to ionice(1) documentation for valid values. - `control.cpu.classes`: defines CPU classes and their properties. Class names are keys followed by properties: - `disabledCstates` is a list of c-state names that are disabled @@ -389,12 +420,19 @@ spec: allocatorTopologyBalancing: true idleCPUClass: lowpower balloonTypes: - - name: "quad" - maxCPUs: 4 - cpuClass: dynamic - namespaces: - - "*" - showContainersInNrt: true + - name: "quad" + maxCPUs: 4 + cpuClass: dynamic + namespaces: + - "*" + showContainersInNrt: true + schedulingClasses: + - name: run-when-idle + policy: idle + ioClass: idle + - name: high-priority + nice: -10 + ioClass: rt control: cpu: classes: @@ -415,6 +453,14 @@ spec: prometheusExport: true ``` +Because all namespaces are assigned to the "quad" balloon type, all +containers (except for `kube-system`) will be assigned to balloons of +this type unless overridden by pod annotation. The configuration +specifies `run-when-idle` and `high-priority` classes in +`schedulingClasses` to allow higher priority containers to be +prioritized over lower priority containers should they share the same +CPUs, and even if they request equally many CPUs. + ## Assigning a Container to a Balloon The balloon type of a container can be defined in pod annotations. In diff --git a/go.mod b/go.mod index b3e709a1b..e5fe6321a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25.0 require ( github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f - github.com/containerd/nri v0.9.1-0.20250530003506-6120e633d4ad + github.com/containerd/nri v0.11.0 github.com/containerd/otelttrpc v0.0.0-20240305015340-ea5083fda723 github.com/containerd/ttrpc v1.2.7 github.com/containers/nri-plugins/pkg/topology v0.0.0 @@ -69,12 +69,12 @@ require ( github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/knqyf263/go-plugin v0.8.1-0.20240827022226-114c6257e441 // indirect + github.com/knqyf263/go-plugin v0.9.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/runtime-spec v1.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect @@ -82,7 +82,7 @@ require ( github.com/prometheus/otlptranslator v0.0.2 // indirect github.com/prometheus/procfs v0.17.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tetratelabs/wazero v1.9.0 // indirect + github.com/tetratelabs/wazero v1.10.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect diff --git a/go.sum b/go.sum index 0c5fdf835..8f3a66f9d 100644 --- a/go.sum +++ b/go.sum @@ -645,8 +645,8 @@ github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/nri v0.9.1-0.20250530003506-6120e633d4ad h1:FiRhXzn9B9ToI30hX/2i4dRUKUoVELTc7PlEqvFLwGE= -github.com/containerd/nri v0.9.1-0.20250530003506-6120e633d4ad/go.mod h1:zA1mhuTD3Frj9fyyIp+1+H2AXS/IueLvxRpkAzmP6RQ= +github.com/containerd/nri v0.11.0 h1:26mcQwNG58AZn0YkOrlJQ0yxQVmyZooflnVWJTqQrqQ= +github.com/containerd/nri v0.11.0/go.mod h1:bjGTLdUA58WgghKHg8azFMGXr05n1wDHrt3NSVBHiGI= github.com/containerd/otelttrpc v0.0.0-20240305015340-ea5083fda723 h1:swk9KxrmARZjSMrHc1Lzb39XhcDwAhYpqkBhinCFLCQ= github.com/containerd/otelttrpc v0.0.0-20240305015340-ea5083fda723/go.mod h1:ZKzztepTSz/LKtbUSzfBNVwgqBEPABVZV9PQF/l53+Q= github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= @@ -862,8 +862,8 @@ github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHU github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/knqyf263/go-plugin v0.8.1-0.20240827022226-114c6257e441 h1:Q/sZeuWkXprbKJSs7AwXryuZKSEL/a8ltC7e7xSspN0= -github.com/knqyf263/go-plugin v0.8.1-0.20240827022226-114c6257e441/go.mod h1:CvCrNDMiKFlAlLFLmcoEfsTROEfNKbEZAMMrwQnLXCM= +github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI= +github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -897,8 +897,8 @@ github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= -github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= -github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= +github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= @@ -962,8 +962,8 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= -github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= +github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8= +github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go index 95cf8e055..cab261dd7 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go @@ -31,6 +31,7 @@ type ( AmountKind = policy.AmountKind CPUTopologyLevel = policy.CPUTopologyLevel ComponentCreationStrategy = policy.ComponentCreationStrategy + SchedulingClass = policy.SchedulingClass ) const ( @@ -51,6 +52,22 @@ const ( ComponentCreationAll = policy.ComponentCreationAll ComponentCreationBalanceBalloons = policy.ComponentCreationBalanceBalloons + + SchedulingPolicyUndefined = policy.SchedulingPolicyUndefined + SchedulingPolicyNone = policy.SchedulingPolicyNone + SchedulingPolicyOther = policy.SchedulingPolicyOther + SchedulingPolicyFifo = policy.SchedulingPolicyFifo + SchedulingPolicyRr = policy.SchedulingPolicyRr + SchedulingPolicyBatch = policy.SchedulingPolicyBatch + SchedulingPolicyIdle = policy.SchedulingPolicyIdle + SchedulingPolicyDeadline = policy.SchedulingPolicyDeadline + SchedulingFlagResetOnFork = policy.SchedulingFlagResetOnFork + SchedulingFlagReclaimable = policy.SchedulingFlagReclaimable + SchedulingFlagDlOverrun = policy.SchedulingFlagDlOverrun + SchedulingFlagKeepPolicy = policy.SchedulingFlagKeepPolicy + SchedulingFlagKeepParams = policy.SchedulingFlagKeepParams + SchedulingFlagUtilClampMin = policy.SchedulingFlagUtilClampMin + SchedulingFlagUtilClampMax = policy.SchedulingFlagUtilClampMax ) var ( @@ -114,6 +131,9 @@ type Config struct { ShowContainersInNrt *bool `json:"showContainersInNrt,omitempty"` // LoadClasses specify available loads in balloon types. LoadClasses []LoadClass `json:"loadClasses,omitempty"` + // SchedulingClasses specify scheduling classes available in + // balloon types. + SchedulingClasses []*SchedulingClass `json:"schedulingClasses,omitempty"` } // BalloonDef contains a balloon definition. @@ -255,6 +275,12 @@ type BalloonDef struct { // +optional // +kubebuilder:validation:Enum=efficient;performance PreferCoreType string `json:"preferCoreType,omitempty"` + // SchedulingClass is the name of the scheduling class from + // which default Linux scheduling and IO priority parameters + // are applied on new containers created in these + // balloons. Parameters for the class are specified in the + // schedulingClasses list. + SchedulingClass string `json:"schedulingClass,omitempty"` // ShowContainersInNrt controls showing containers and their // resource affinities as part of // NodeResourceTopology. Overrides the policy level diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go index 5ab11f2f8..74276ce1d 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go @@ -174,6 +174,17 @@ func (in *Config) DeepCopyInto(out *Config) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.SchedulingClasses != nil { + in, out := &in.SchedulingClasses, &out.SchedulingClasses + *out = make([]*SchedulingClass, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(SchedulingClass) + (*in).DeepCopyInto(*out) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/config.go index c1869211b..4ece64b65 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/config.go @@ -20,6 +20,8 @@ import ( "github.com/containers/nri-plugins/pkg/utils/cpuset" "k8s.io/apimachinery/pkg/api/resource" + + nriapi "github.com/containerd/nri/pkg/api" ) type ( @@ -29,6 +31,10 @@ type ( AmountKind int CPUTopologyLevel string ComponentCreationStrategy string + SchedulingPolicy string + SchedulingFlag string + SchedulingFlags []SchedulingFlag + IOPriorityClass string ) const ( @@ -52,6 +58,32 @@ const ( ComponentCreationAll ComponentCreationStrategy = "all" ComponentCreationBalanceBalloons ComponentCreationStrategy = "balance-balloons" + + // Scheduling policy and flag user-facing strings are formed from + // SCHED_* and SCHED_FLAG_* by stripping prefix, lowercase and s/_/-/g. + SchedulingPolicyUndefined SchedulingPolicy = "" + SchedulingPolicyNone SchedulingPolicy = "none" + SchedulingPolicyOther SchedulingPolicy = "other" + SchedulingPolicyFifo SchedulingPolicy = "fifo" + SchedulingPolicyRr SchedulingPolicy = "rr" + SchedulingPolicyBatch SchedulingPolicy = "batch" + SchedulingPolicyIdle SchedulingPolicy = "idle" + SchedulingPolicyDeadline SchedulingPolicy = "deadline" + SchedulingFlagResetOnFork SchedulingFlag = "reset-on-fork" + SchedulingFlagReclaimable SchedulingFlag = "reclaimable" + SchedulingFlagDlOverrun SchedulingFlag = "dl-overrun" + SchedulingFlagKeepPolicy SchedulingFlag = "keep-policy" + SchedulingFlagKeepParams SchedulingFlag = "keep-params" + SchedulingFlagUtilClampMin SchedulingFlag = "util-clamp-min" + SchedulingFlagUtilClampMax SchedulingFlag = "util-clamp-max" + + // IO priority classes are constructed from the + // IOPRIO_CLASS_* constants by stripping prefix and lowercase. + IOPriorityClassUndefined IOPriorityClass = "" + IOPriorityClassNone IOPriorityClass = "none" + IOPriorityClassRt IOPriorityClass = "rt" + IOPriorityClassBe IOPriorityClass = "be" + IOPriorityClassIdle IOPriorityClass = "idle" ) var ( @@ -114,3 +146,54 @@ func (l CPUTopologyLevel) Value() int { } return cpuTopologyLevelValues[CPUTopologyLevelUndefined] } + +func (sp SchedulingPolicy) String() string { + return string(sp) +} + +func (sp SchedulingPolicy) ToNRI() (nriapi.LinuxSchedulerPolicy, error) { + cstyleSp := "SCHED_" + strings.ToUpper(strings.ReplaceAll(string(sp), "-", "_")) + n, ok := nriapi.LinuxSchedulerPolicy_value[cstyleSp] + if !ok { + return 0, fmt.Errorf("unknown scheduling policy '%s'", sp) + } + return nriapi.LinuxSchedulerPolicy(n), nil +} + +func (sf SchedulingFlag) String() string { + return string(sf) +} + +func (sf SchedulingFlag) ToNRI() (nriapi.LinuxSchedulerFlag, error) { + cstyleSf := "SCHED_FLAG_" + strings.ToUpper(strings.ReplaceAll(string(sf), "-", "_")) + n, ok := nriapi.LinuxSchedulerFlag_value[cstyleSf] + if !ok { + return 0, fmt.Errorf("unknown scheduling flag '%s'", sf) + } + return nriapi.LinuxSchedulerFlag(n), nil +} + +func (sfl SchedulingFlags) ToNRI() ([]nriapi.LinuxSchedulerFlag, error) { + var nriFlags []nriapi.LinuxSchedulerFlag + for _, sf := range sfl { + nriFlag, err := sf.ToNRI() + if err != nil { + return nil, err + } + nriFlags = append(nriFlags, nriFlag) + } + return nriFlags, nil +} + +func (ioc IOPriorityClass) String() string { + return string(ioc) +} + +func (ioc IOPriorityClass) ToNRI() (nriapi.IOPrioClass, error) { + cstyleIoc := "IOPRIO_CLASS_" + strings.ToUpper(strings.ReplaceAll(string(ioc), "-", "_")) + n, ok := nriapi.IOPrioClass_value[cstyleIoc] + if !ok { + return 0, fmt.Errorf("unknown IO priority class '%s'", ioc) + } + return nriapi.IOPrioClass(n), nil +} diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/schedulingclass.go b/pkg/apis/config/v1alpha1/resmgr/policy/schedulingclass.go new file mode 100644 index 000000000..186bcac72 --- /dev/null +++ b/pkg/apis/config/v1alpha1/resmgr/policy/schedulingclass.go @@ -0,0 +1,54 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +// SchedulingClass specifies the default Linux scheduling and IO +// priority parameters for containers assigned into this class. +// +k8s:deepcopy-gen=true +type SchedulingClass struct { + // Name of the scheduling class. + // +kube:validation:Required + Name string `json:"name"` + // Policy is the Linux scheduling policy to use. + // SCHED_ translates to etc. + // +kubebuilder:validation:Enum=none;other;fifo;rr;batch;iso;idle;deadline + // +kubebuilder:validation:Format:string + Policy SchedulingPolicy `json:"policy,omitempty"` + // Priority is the Linux scheduling priority to use. + // Valid range depends on the selected policy. + // Refer to sched_setscheduler(2) for details. + Priority *int `json:"priority,omitempty"` + // Flags is a list of Linux scheduling flags to set. + // SCHED_FLAG_ translates to etc. + // +kubebuilder:validation:Enum=reset-on-fork;reclaim;dl-overrun;keep-policy;keep-params;util-clamp-min;util-clamp-max + Flags SchedulingFlags `json:"flags,omitempty"` + // Nice is the Linux nice value to use. + Nice *int `json:"nice,omitempty"` + // Runtime is the Linux SCHED_DEADLINE runtime value to use (in microseconds). + Runtime *uint64 `json:"runtime,omitempty"` + // Deadline is the Linux SCHED_DEADLINE deadline value to use (in microseconds). + Deadline *uint64 `json:"deadline,omitempty"` + // Period is the Linux SCHED_DEADLINE period value to use (in microseconds). + Period *uint64 `json:"period,omitempty"` + // IOClass is the IO scheduling class to use. + // IOPRIO_CLASS_ translates to . + // Refer to ioprio_set(2) and ionice(1) for details. + // +kubebuilder:validation:Enum=none;rt;be;idle + IOClass IOPriorityClass `json:"ioClass,omitempty"` + // IOPriority is the IO priority within the selected IO class to use. + // Valid range depends on the selected class. + // Refer to ionice(1) for details. + IOPriority *int `json:"ioPriority,omitempty"` +} diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go new file mode 100644 index 000000000..3bef85a34 --- /dev/null +++ b/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go @@ -0,0 +1,71 @@ +//go:build !ignore_autogenerated + +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by controller-gen. DO NOT EDIT. + +package policy + +import () + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulingClass) DeepCopyInto(out *SchedulingClass) { + *out = *in + if in.Priority != nil { + in, out := &in.Priority, &out.Priority + *out = new(int) + **out = **in + } + if in.Flags != nil { + in, out := &in.Flags, &out.Flags + *out = make(SchedulingFlags, len(*in)) + copy(*out, *in) + } + if in.Nice != nil { + in, out := &in.Nice, &out.Nice + *out = new(int) + **out = **in + } + if in.Runtime != nil { + in, out := &in.Runtime, &out.Runtime + *out = new(uint64) + **out = **in + } + if in.Deadline != nil { + in, out := &in.Deadline, &out.Deadline + *out = new(uint64) + **out = **in + } + if in.Period != nil { + in, out := &in.Period, &out.Period + *out = new(uint64) + **out = **in + } + if in.IOPriority != nil { + in, out := &in.IOPriority, &out.IOPriority + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingClass. +func (in *SchedulingClass) DeepCopy() *SchedulingClass { + if in == nil { + return nil + } + out := new(SchedulingClass) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/resmgr/cache/cache.go b/pkg/resmgr/cache/cache.go index 77248a502..fef120d12 100644 --- a/pkg/resmgr/cache/cache.go +++ b/pkg/resmgr/cache/cache.go @@ -265,6 +265,25 @@ type Container interface { // SetMemorySwap sets the swap limit in bytes for the container. SetMemorySwap(int64) + // SetSchedulingPolicy sets the scheduling policy for the container. + SetSchedulingPolicy(nri.LinuxSchedulerPolicy) + // SetSchedulingNice sets the nice value for the container. + SetSchedulingNice(int32) + // SetSchedulingPriority sets the real-time priority for the container. + SetSchedulingPriority(int32) + // SetSchedulingFlags sets the scheduling flags for the container. + SetSchedulingFlags([]nri.LinuxSchedulerFlag) + // SetSchedulingRuntime sets the runtime for the container in microseconds. + SetSchedulingRuntime(uint64) + // SetSchedulingDeadline sets the deadline for the container in microseconds. + SetSchedulingDeadline(uint64) + // SetSchedulingPeriod sets the scheduling period for the container in microseconds. + SetSchedulingPeriod(uint64) + // SetSchedulingIOClass sets the I/O priority class for the container. + SetSchedulingIOClass(nri.IOPrioClass) + // SetSchedulingIOPriority sets the I/O priority for the container. + SetSchedulingIOPriority(int32) + // GetCPUShares gets the CFS CPU shares of the container. GetCPUShares() int64 // GetCPUQuota gets the CFS CPU quota of the container. @@ -340,7 +359,9 @@ type container struct { ResourceUpdates *v1.ResourceRequirements request interface{} - Resources *nri.LinuxResources + Resources *nri.LinuxResources + LinuxScheduler *nri.LinuxScheduler + LinuxIOPriority *nri.LinuxIOPriority TopologyHints topology.Hints // Set of topology hints for all containers within Pod Tags map[string]string // container tags (local dynamic labels) diff --git a/pkg/resmgr/cache/container.go b/pkg/resmgr/cache/container.go index 446c94178..f9ec72118 100644 --- a/pkg/resmgr/cache/container.go +++ b/pkg/resmgr/cache/container.go @@ -601,6 +601,14 @@ func (c *container) GetLinuxResources() *nri.LinuxResources { return c.Resources } +func (c *container) GetLinuxScheduler() *nri.LinuxScheduler { + return c.LinuxScheduler +} + +func (c *container) GetLinuxIOPriority() *nri.LinuxIOPriority { + return c.LinuxIOPriority +} + func (c *container) GetTopologyHints() topology.Hints { return c.TopologyHints } @@ -687,6 +695,18 @@ func (c *container) ensureLinuxResourcesMemory() { } } +func (c *container) ensureLinuxScheduler() { + if c.LinuxScheduler == nil { + c.LinuxScheduler = &nri.LinuxScheduler{} + } +} + +func (c *container) ensureLinuxIOPriority() { + if c.LinuxIOPriority == nil { + c.LinuxIOPriority = &nri.LinuxIOPriority{} + } +} + func (c *container) SetCPUShares(value int64) { switch req := c.getPendingRequest().(type) { case *nri.ContainerAdjustment: @@ -806,6 +826,93 @@ func (c *container) SetMemorySwap(value int64) { c.Ctr.Linux.Resources.Memory.Swap = nri.Int64(value) } +// Scheduling properties can be set only during container creation. +// setSchedulingOnAdjustment is a helper to set scheduling parameters +func (c *container) setSchedulingOnAdjustment(setter func(sch *nri.LinuxScheduler)) { + switch req := c.getPendingRequest().(type) { + case *nri.ContainerAdjustment: + c.ensureLinuxScheduler() + sch := c.GetLinuxScheduler() + setter(sch) + req.SetLinuxScheduler(sch) + default: + log.Error("%s: can't set scheduling parameter: incorrect pending request type %T", + c.PrettyName(), c.request) + return + } + c.markPending(NRI) +} + +func (c *container) SetSchedulingPolicy(value nri.LinuxSchedulerPolicy) { + c.setSchedulingOnAdjustment(func(sch *nri.LinuxScheduler) { + sch.Policy = value + }) +} + +func (c *container) SetSchedulingNice(value int32) { + c.setSchedulingOnAdjustment(func(sch *nri.LinuxScheduler) { + sch.Nice = value + }) +} + +func (c *container) SetSchedulingPriority(value int32) { + c.setSchedulingOnAdjustment(func(sch *nri.LinuxScheduler) { + sch.Priority = value + }) +} + +func (c *container) SetSchedulingFlags(value []nri.LinuxSchedulerFlag) { + c.setSchedulingOnAdjustment(func(sch *nri.LinuxScheduler) { + sch.Flags = make([]nri.LinuxSchedulerFlag, len(value)) + copy(sch.Flags, value) + }) +} + +func (c *container) SetSchedulingRuntime(value uint64) { + c.setSchedulingOnAdjustment(func(sch *nri.LinuxScheduler) { + sch.Runtime = value + }) +} + +func (c *container) SetSchedulingDeadline(value uint64) { + c.setSchedulingOnAdjustment(func(sch *nri.LinuxScheduler) { + sch.Deadline = value + }) +} + +func (c *container) SetSchedulingPeriod(value uint64) { + c.setSchedulingOnAdjustment(func(sch *nri.LinuxScheduler) { + sch.Period = value + }) +} + +func (c *container) setIOPriorityOnAdjustment(setter func(iop *nri.LinuxIOPriority)) { + switch req := c.getPendingRequest().(type) { + case *nri.ContainerAdjustment: + c.ensureLinuxIOPriority() + iop := c.GetLinuxIOPriority() + setter(iop) + req.SetLinuxIOPriority(iop) + default: + log.Error("%s: can't set IO priority parameter: incorrect pending request type %T", + c.PrettyName(), c.request) + return + } + c.markPending(NRI) +} + +func (c *container) SetSchedulingIOClass(value nri.IOPrioClass) { + c.setIOPriorityOnAdjustment(func(iop *nri.LinuxIOPriority) { + iop.Class = value + }) +} + +func (c *container) SetSchedulingIOPriority(value int32) { + c.setIOPriorityOnAdjustment(func(iop *nri.LinuxIOPriority) { + iop.Priority = value + }) +} + func (c *container) GetCPUShares() int64 { return int64(c.Ctr.GetLinux().GetResources().GetCpu().GetShares().GetValue()) } diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/balloons-cstates.cfg b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/balloons-cstates.cfg similarity index 74% rename from test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/balloons-cstates.cfg rename to test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/balloons-cstates.cfg index fe7f80b55..215432bf1 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/balloons-cstates.cfg +++ b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/balloons-cstates.cfg @@ -14,6 +14,7 @@ config: balloonTypes: - name: lowlatency-bln cpuClass: lowlatency-class + schedulingClass: realtime control: cpu: @@ -22,9 +23,22 @@ config: disabledCstates: [C4, C6, C8, C10] default-class: disabledCstates: [] + + schedulingClasses: + - name: realtime + policy: fifo # SCHED_FIFO + priority: 42 + + - name: run-when-free + policy: idle + nice: 17 + ioClass: be + ioPriority: 6 + log: debug: - policy + - nri-plugin - cpu extraEnv: OVERRIDE_SYS_CSTATES: '''[{"cpus": "0-15", "names": ["C1E", "C2", "C4", "C8"], "files": {"disable": "0"}}]''' diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh similarity index 75% rename from test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/code.var.sh rename to test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh index a5c250696..d89ef3763 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates-scheduling/code.var.sh @@ -35,6 +35,24 @@ verify-cstates() { done } +verify-sched() { + local podXcY=$1 + vm-command "cat /proc/\$(pgrep -f $podXcY)/sched" || command-error "cannot get /proc/PID/sched for $podXcY" + + if [ "$expected_policy" != "" ]; then + echo "verify scheduling policy of $podXcY is $expected_policy" + grep -q -E "policy .* $expected_policy" <<< $COMMAND_OUTPUT || + error "expected policy $expected_policy not found" + + fi + + if [ "$expected_prio" != "" ]; then + echo "verify scheduling priority of $podXcY is $expected_prio" + grep -q -E "prio .* $expected_prio" <<< $COMMAND_OUTPUT || + error "expected priority $expected_prio not found" + fi +} + # verify-cstates-no-writes checks that any c-states of given CPUs have not been written verify-cstates-no-writes() { local cpu_ids=$1 # e.g. "1 2 4" @@ -52,7 +70,7 @@ cleanup() { } echo "verify that all c-states of all available CPUs are enabled" -verify-cstates "2 3 4 5 6 7 11 12 13" "C1E C2 C4 C8" "" 36 +verify-cstates "2 3 4 5 6 7 11 12 13" "C1E C2 C4 C8" "" 40 echo "verify that c-states of CPUs outside AvailableResources have not been written" verify-cstates-no-writes "0 1 8 9 14 15" @@ -63,15 +81,27 @@ report allowed verify 'len(cpus["pod0c0"]) == 1' echo "verify that CPUs of low-latency pod0 cannot enter C4 or C8" verify-cstates "$(cpuids-of pod0c0)" "C1E C2" "C4 C8" 4 +expected_policy=1 expected_prio=$((99 - 42)) verify-sched pod0c0 # expect SCHED_FIFO, prio 56 CPUREQ="3" MEMREQ="100M" CPULIM="" MEMLIM="" -POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: lowlatency-bln" CONTCOUNT=1 create balloons-busybox +POD_ANNOTATION=( + "balloon.balloons.resource-policy.nri.io: lowlatency-bln" + "scheduling-class.resource-policy.nri.io: run-when-free" +) +CONTCOUNT=1 create balloons-busybox report allowed verify 'cpus["pod0c0"] == cpus["pod1c0"]' \ 'len(cpus["pod0c0"]) == 4' echo "verify that CPUs of low-latency pods pod0 and pod1 cannot enter C4 or C8" verify-cstates "$(cpuids-of pod1c0)" "C1E C2" "C4 C8" 16 +expected_policy=5 expected_prio=$((120 + 17)) verify-sched pod1c0 # expect SCHED_IDLE, prio 137 +vm-command "ionice -p \$(pgrep -f pod1c0)" || + command-error "cannot get ionice for pod1c0" +expected_ionice="best-effort: prio 6" +[[ "$COMMAND_OUTPUT" == "$expected_ionice" ]] || + command-error "expected ionice output '$expected_ionice'" + # store CPU ids of maximal cpuset before deleting pods max_lowlatency_cpus="$(echo $(cpuids-of pod1c0) )"