From 0123bb1d2214ecc0a1dc9541e158b986f8db1add Mon Sep 17 00:00:00 2001 From: james Date: Thu, 6 Nov 2025 17:10:59 +0800 Subject: [PATCH 01/33] feat: support hami ascend device Signed-off-by: james --- .../api/devices/ascend/device_info.go | 704 ++++++++++++++++++ .../api/devices/ascend/device_info_test.go | 273 +++++++ pkg/scheduler/api/devices/config/config.go | 19 + pkg/scheduler/api/devices/config/vnpu.go | 36 + pkg/scheduler/api/devices/device_info.go | 219 ++++++ pkg/scheduler/api/devices/util.go | 106 +++ pkg/scheduler/api/node_info.go | 21 + pkg/scheduler/api/shared_device_pool.go | 6 + .../plugins/deviceshare/deviceshare.go | 49 +- 9 files changed, 1419 insertions(+), 14 deletions(-) create mode 100644 pkg/scheduler/api/devices/ascend/device_info.go create mode 100644 pkg/scheduler/api/devices/ascend/device_info_test.go create mode 100644 pkg/scheduler/api/devices/config/vnpu.go create mode 100644 pkg/scheduler/api/devices/device_info.go diff --git a/pkg/scheduler/api/devices/ascend/device_info.go b/pkg/scheduler/api/devices/ascend/device_info.go new file mode 100644 index 0000000000..2d4b30410c --- /dev/null +++ b/pkg/scheduler/api/devices/ascend/device_info.go @@ -0,0 +1,704 @@ +/* +Copyright 2025 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ascend + +import ( + "encoding/json" + "flag" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "volcano.sh/volcano/pkg/scheduler/api/devices" + "volcano.sh/volcano/pkg/scheduler/api/devices/config" + "volcano.sh/volcano/pkg/scheduler/plugins/util/nodelock" +) + +const ( + NodeLockAscend = "hami.io/mutex.lock" + Ascend910Prefix = "Ascend910" + Ascend910NetworkWeight = 10 + // binpack means the lower device memory remained after this allocation, the better + binpackPolicy = "binpack" + // spread means better put this task into an idle GPU card than a shared GPU card + spreadPolicy = "spread" + binpackMultiplier = 100 + spreadMultiplier = 100 + CMName = "volcano-vgpu-device-config" + CMNamespace = "kube-system" +) + +type AscendDevice struct { + config config.VNPUConfig + nodeRegisterAnno string + useUUIDAnno string + noUseUUIDAnno string + handshakeAnno string + DeviceInfo *devices.DeviceInfo + DeviceUsage *devices.DeviceUsage + Score float64 +} + +type AscendDevices struct { + NodeName string + Type string + Devices map[string]*AscendDevice + Policy string +} + +type RuntimeInfo struct { + UUID string `json:"UUID,omitempty"` + Temp string `json:"temp,omitempty"` +} + +var ( + AscendVNPUEnable bool + configFile string + NodeLockEnable bool +) + +func NewAscendDevices(name string, node *v1.Node) map[string]*AscendDevices { + ascend_devices := make(map[string]*AscendDevices) + if node == nil { + klog.Warningf("Node is nil for node %s, returning empty AscendDevices", name) + return ascend_devices + } + cur_config := config.GetConfig() + if cur_config == nil { + klog.V(5).InfoS("cur config is null. call InitDevicesConfig") + config.InitDevicesConfig(CMName, CMNamespace) + cur_config = config.GetConfig() + } + devs := InitDevices(cur_config.VNPUs) + for _, dev := range devs { + node_devices, err := dev.GetNodeDevices(*node) + if err != nil { + klog.Warningf("Failed to get node devices. nodeName %s, deviceType %s, error %s" , node.Name, dev.CommonWord(), err) + continue + } + as_devices := &AscendDevices{ + NodeName: name, + Type: dev.CommonWord(), + Devices: make(map[string]*AscendDevice), + } + for _, nd := range node_devices { + cur_dev := &AscendDevice{ + config: dev.config, + nodeRegisterAnno: dev.nodeRegisterAnno, + useUUIDAnno: dev.useUUIDAnno, + noUseUUIDAnno: dev.noUseUUIDAnno, + handshakeAnno: dev.handshakeAnno, + DeviceInfo: nd, + DeviceUsage: &devices.DeviceUsage{ + Used: 0, + Usedmem: 0, + Usedcores: 0, + }, + } + as_devices.Devices[nd.ID] = cur_dev + klog.V(5).Infof("add device. ID %s dev_info %+v", cur_dev.DeviceInfo.ID, cur_dev.DeviceInfo) + } + ascend_devices[dev.CommonWord()] = as_devices + } + return ascend_devices +} + +func GetAscendDeviceNames() ([]string){ + cur_config := config.GetConfig() + if cur_config == nil { + config.InitDevicesConfig(CMName, CMNamespace) + cur_config = config.GetConfig() + } + deviceNames := make([]string, 0, len(cur_config.VNPUs)) + for _, vnpu := range cur_config.VNPUs { + deviceNames = append(deviceNames, vnpu.CommonWord) + } + return deviceNames +} + +func (ads *AscendDevices) AddResourceUsage(id string, cores int32, mem int32) error { + dev, ok := ads.Devices[id] + if !ok { + return fmt.Errorf("ascend device %s not found", id) + } + dev.DeviceUsage.Used++ + dev.DeviceUsage.Usedcores += cores + dev.DeviceUsage.Usedmem += mem + return nil +} + +func (ads *AscendDevices) SubResourceUsage(id string, cores int32, mem int32) error { + dev, ok := ads.Devices[id] + if !ok { + return fmt.Errorf("ascend device %s not found", id) + } + dev.DeviceUsage.Used-- + dev.DeviceUsage.Usedcores -= cores + dev.DeviceUsage.Usedmem -= mem + return nil +} + +func (ads *AscendDevices) AddResource(pod *v1.Pod) { + if ads == nil { + return + } + ads.addResource(pod.Annotations, pod) +} + +func (ads *AscendDevices) SubResource(pod *v1.Pod) { + if ads == nil { + return + } + ano_key := devices.InRequestDevices[ads.Type] + ano, ok := pod.Annotations[ano_key] + if !ok { + return + } + con_devs, err := devices.DecodeContainerDevices(ano) + if err != nil { + klog.ErrorS(err, "failed to decode container devices", "pod", pod.Name, "annotation", ano) + return + } + for _, cono_dev := range con_devs { + ads.SubResourceUsage(cono_dev.UUID, cono_dev.Usedcores, cono_dev.Usedmem) + } +} + +func (ads *AscendDevices) addResource(annotations map[string]string, pod *v1.Pod) { + ano_key := devices.InRequestDevices[ads.Type] + ano, ok := annotations[ano_key] + if !ok { + return + } + con_devs, err := devices.DecodeContainerDevices(ano) + if err != nil { + klog.ErrorS(err, "failed to decode container devices", "pod", pod.Name, "annotation", ano) + return + } + for _, cono_dev := range con_devs { + ads.AddResourceUsage(cono_dev.UUID, cono_dev.Usedcores, cono_dev.Usedmem) + } +} + +func (ads *AscendDevices) AddQueueResource(pod *v1.Pod) map[string]float64 { + return map[string]float64{} +} + +func (ads *AscendDevices) HasDeviceRequest(pod *v1.Pod) bool { + if !AscendVNPUEnable { + return false + } + rand_dev, err := ads.getRandomDevice() + if rand_dev == nil || err != nil { + return false + } + var vnpu_config = rand_dev.config + for _, container := range pod.Spec.Containers { + _, ok := container.Resources.Limits[v1.ResourceName(vnpu_config.ResourceName)] + if ok { + klog.V(5).Infof("%s check HasDeviceRequest ok. %s", ads.Type, vnpu_config.ResourceName) + return true + } + _, ok = container.Resources.Limits[v1.ResourceName(vnpu_config.ResourceMemoryName)] + if ok { + klog.V(5).Infof("%s check HasDeviceRequest ok. %s", ads.Type, vnpu_config.ResourceMemoryName) + return true + } + } + klog.V(5).Infof("%s check HasDeviceRequest false", ads.Type) + return false +} + +func (ads *AscendDevices) FilterNode(pod *v1.Pod, policy string) (int, string, error) { + _, err := ads.selectDevices(pod, policy) + if err != nil { + return devices.Error, "no ascend device available", err + } + klog.V(4).Infoln("ascend DeviceSharing successfully filters pods. device_type:", ads.Type) + return devices.Success, "", nil +} + +func (ads *AscendDevices) ScoreNode(pod *v1.Pod, policy string) float64 { + ads.Policy = policy + pod_devs, err := ads.selectDevices(pod, policy) + if err != nil { + return 0 + } + score := 0.0 + var used_devs []*AscendDevice + for _, dev := range pod_devs { + dev, ok := ads.Devices[dev[0].UUID] + if !ok { + return 0 + } + used_devs = append(used_devs, dev) + score += CalScore(policy, dev.DeviceUsage, dev.DeviceInfo) + } + + if strings.HasPrefix(ads.Type, Ascend910Prefix) && hasNetworkID(used_devs) { + klog.V(4).Infof("all devices have NetworkID. device CommonWord %s", ads.Type) + cntMap := make(map[int]int) + for _, dev := range used_devs { + if dev.DeviceInfo.CustomInfo == nil { + return 0 + } + if networkID, ok := dev.DeviceInfo.CustomInfo["NetworkID"]; ok { + if id, ok := networkID.(float64); ok { + cntMap[int(id)]++ + } + } else { + return 0 + } + } + maxCnt, totalCnt := 0, 0 + for _, cnt := range cntMap { + if cnt > maxCnt { + maxCnt = cnt + } + totalCnt += cnt + } + if totalCnt == 0 { + return 0 + } + score += (float64(maxCnt) / float64(totalCnt)) * Ascend910NetworkWeight + } + return score +} + +func (ads *AscendDevices) Allocate(kubeClient kubernetes.Interface, pod *v1.Pod) error { + klog.V(4).Infof("Allocate device %s to Pod %s", ads.Type, pod.Name) + if NodeLockEnable { + nodelock.UseClient(kubeClient) + err := nodelock.LockNode(ads.NodeName, ads.Type) + if err != nil { + return errors.Errorf("node %s locked for %s hamivgpu lockname %s", ads.NodeName, pod.Name, err.Error()) + } + } + pod_devs, err := ads.selectDevices(pod, ads.Policy) + if err != nil { + return errors.Errorf("failed to select ascend devices for pod %s: %v", pod.Name, err) + } + annotations := make(map[string]string) + ads.PatchAnnotations(pod, &annotations, pod_devs) + + ads.addResource(annotations, pod) + annotations[devices.AssignedNodeAnnotations] = ads.NodeName + annotations[devices.AssignedTimeAnnotations] = strconv.FormatInt(time.Now().Unix(), 10) + annotations[devices.DeviceBindPhase] = "allocating" + annotations[devices.BindTimeAnnotations] = strconv.FormatInt(time.Now().Unix(), 10) + + err = devices.PatchPodAnnotations(kubeClient, pod, annotations) + if err != nil { + return err + } + if NodeLockEnable { + nodelock.ReleaseNodeLock(ads.NodeName, ads.Type) + } + klog.V(4).Infof("Allocate Success. device %s Pod %s", ads.Type, pod.Name) + return nil +} + +func (ads *AscendDevices) Release(kubeClient kubernetes.Interface, pod *v1.Pod) error { + return nil +} + +func (ads *AscendDevices) GetIgnoredDevices() []string { + rand_dev, err := ads.getRandomDevice() + if rand_dev == nil || err != nil { + return []string{""} + } + vnpu_config := rand_dev.config + return []string{vnpu_config.ResourceMemoryName} +} + +func (ads *AscendDevices) GetStatus() string { + return "" +} + +func (ads *AscendDevices) selectDevices(pod *v1.Pod, schedulePolicy string) (devices.PodSingleDevice, error) { + dup_devs := getDeviceSnapshot(ads) + if len(dup_devs) == 0 { + return nil, errors.Errorf("no ascend device available") + } + for _, dev := range dup_devs { + dev.Score = CalScore(schedulePolicy, dev.DeviceUsage, dev.DeviceInfo) + } + sort.Slice(dup_devs, func(i, j int) bool { + return dup_devs[i].Score > dup_devs[j].Score + }) + needTopology := false + if strings.HasPrefix(ads.Type, Ascend910Prefix) && hasNetworkID(dup_devs) { + klog.V(4).Infof("all devices have NetworkID. device CommonWord %s", ads.Type) + needTopology = true + } + reqs := dup_devs[0].ResourceReqs(pod) + var pod_devs devices.PodSingleDevice + used_devs := make([]*AscendDevice, 0) + for _, req := range reqs { + klog.V(5).Infof("req %+v", req) + available_devs := make([]*AscendDevice, 0) + for _, dev := range dup_devs { + selected := false + for _, used_dev := range used_devs { + if used_dev.DeviceInfo.ID == dev.DeviceInfo.ID { + selected = true + break + } + } + if !selected { + available_devs = append(available_devs, dev) + } + } + req_nums := req.Nums + selected_devs := make([]*AscendDevice, 0) + for _, dev := range available_devs { + klog.V(5).Infof("check fit. req %+v dev_info %+v dev_usage %+v", req, dev.DeviceInfo, dev.DeviceUsage) + if fit(&req, dev) == false { + klog.V(5).Infof("fit false. dev ID %s", dev.DeviceInfo.ID) + continue + } + selected_devs = append(selected_devs, dev) + req_nums -= 1 + if req_nums <= 0 && !needTopology { + break + } + } + if req_nums > 0 { + klog.V(5).Infof("no enough ascend device available! raw req_nums %d cur req_nums %d", req.Nums, req_nums) + return nil, errors.Errorf("no enough ascend device available") + } + if needTopology { + selected_devs = selectDevicesWithTopology(int(req.Nums), selected_devs) + } + used_devs = append(used_devs, selected_devs...) + var con_devs devices.ContainerDevices + for _, dev := range selected_devs { + con_devs = append(con_devs, devices.ContainerDevice{ + UUID: dev.DeviceInfo.ID, + Type: ads.Type, + Usedmem: req.Memreq, + Usedcores: req.Coresreq, + CustomInfo: dev.DeviceInfo.CustomInfo, + }) + } + pod_devs = append(pod_devs, con_devs) + } + return pod_devs, nil +} + +func hasNetworkID(devices []*AscendDevice) bool { + for _, dev := range devices { + if dev.DeviceInfo.CustomInfo == nil { + return false + } + if _, ok := dev.DeviceInfo.CustomInfo["NetworkID"]; !ok { + return false + } + } + return true +} + +func fit(req *devices.ContainerDeviceRequest, dev *AscendDevice) bool { + if req.Type != dev.config.CommonWord { + return false + } + device_usage := dev.DeviceUsage + device_info := dev.DeviceInfo + if device_info.Count < device_usage.Used { + return false + } + if device_info.Devmem-device_usage.Usedmem < req.Memreq { + return false + } + if device_info.Devcore-device_usage.Usedcores < req.Coresreq { + return false + } + if device_info.Devcore == 100 && req.Coresreq == 100 && device_usage.Used > 0 { + return false + } + if device_info.Devcore != 0 && device_usage.Usedcores == device_info.Devcore && req.Coresreq == 0 { + return false + } + return true +} + +func getDeviceSnapshot(ads *AscendDevices) []*AscendDevice { + dup_devs := make([]*AscendDevice, 0, len(ads.Devices)) + for _, dev := range ads.Devices { + dup_dev := &AscendDevice{ + config: dev.config, + nodeRegisterAnno: dev.nodeRegisterAnno, + useUUIDAnno: dev.useUUIDAnno, + noUseUUIDAnno: dev.noUseUUIDAnno, + handshakeAnno: dev.handshakeAnno, + DeviceInfo: dev.DeviceInfo, + DeviceUsage: &devices.DeviceUsage{ + Used: dev.DeviceUsage.Used, + Usedmem: dev.DeviceUsage.Usedmem, + Usedcores: dev.DeviceUsage.Usedcores, + }, + } + dup_devs = append(dup_devs, dup_dev) + } + return dup_devs +} + +func selectDevicesWithTopology(req_nums int, selected_devs []*AscendDevice) []*AscendDevice { + network_map := make(map[int][]*AscendDevice) + + for _, dev := range selected_devs { + if dev.DeviceInfo.CustomInfo != nil { + if networkID, ok := dev.DeviceInfo.CustomInfo["NetworkID"]; ok { + if id, ok := networkID.(float64); ok { + network_map[int(id)] = append(network_map[int(id)], dev) + } + } + } + } + type NetworkDeviceCount struct { + NetworkID int + Count int + } + var sortedNetworks []NetworkDeviceCount + for networkID, devices := range network_map { + sortedNetworks = append(sortedNetworks, NetworkDeviceCount{ + NetworkID: networkID, + Count: len(devices), + }) + } + sort.Slice(sortedNetworks, func(i, j int) bool { + return sortedNetworks[i].Count > sortedNetworks[j].Count + }) + devs := make([]*AscendDevice, 0) + for _, item := range sortedNetworks { + for _, dev := range network_map[item.NetworkID] { + devs = append(devs, dev) + if len(devs) == req_nums { + return devs + } + } + } + return devs +} + +func (ads *AscendDevices) getRandomDevice() (*AscendDevice, error) { + if len(ads.Devices) == 0 { + return nil, errors.New("no ascend device available") + } + for _, dev := range ads.Devices { + return dev, nil + } + return nil, errors.New("no ascend device available") +} + +func (dev *AscendDevice) trimMemory(m int64) (int64, string) { + for i := range dev.config.Templates { + if m <= dev.config.Templates[i].Memory { + return dev.config.Templates[i].Memory, dev.config.Templates[i].Name + } + } + if m <= dev.config.MemoryCapacity { + return dev.config.MemoryAllocatable, "" + } + return 0, "" +} + +func InitDevices(config []config.VNPUConfig) []*AscendDevice { + devs := make([]*AscendDevice, 0) + for _, vnpu := range config { + commonWord := vnpu.CommonWord + dev := &AscendDevice{ + config: vnpu, + nodeRegisterAnno: fmt.Sprintf("hami.io/node-register-%s", commonWord), + useUUIDAnno: fmt.Sprintf("hami.io/use-%s-uuid", commonWord), + noUseUUIDAnno: fmt.Sprintf("hami.io/no-use-%s-uuid", commonWord), + handshakeAnno: fmt.Sprintf("hami.io/node-handshake-%s", commonWord), + } + sort.Slice(dev.config.Templates, func(i, j int) bool { + return dev.config.Templates[i].Memory < dev.config.Templates[j].Memory + }) + _, ok := devices.InRequestDevices[commonWord] + if !ok { + devices.InRequestDevices[commonWord] = fmt.Sprintf("hami.io/%s-devices-to-allocate", commonWord) + devices.SupportDevices[commonWord] = fmt.Sprintf("hami.io/%s-devices-allocated", commonWord) + // util.HandshakeAnnos[commonWord] = dev.handshakeAnno + } + devs = append(devs, dev) + klog.Infof("load ascend vnpu config %s: %v", commonWord, dev.config) + } + return devs +} + +func ParseConfig(fs *flag.FlagSet) { + fs.BoolVar(&AscendVNPUEnable, "AscendVNPUEnable", false, "enable ascend device") +} + +func (dev *AscendDevice) CommonWord() string { + return dev.config.CommonWord +} + +func (dev *AscendDevice) GetNodeDevices(n v1.Node) ([]*devices.DeviceInfo, error) { + anno, ok := n.Annotations[dev.nodeRegisterAnno] + if !ok { + return []*devices.DeviceInfo{}, fmt.Errorf("annos not found %s", dev.nodeRegisterAnno) + } + nodeDevices, err := devices.UnMarshalNodeDevices(anno) + if err != nil { + klog.ErrorS(err, "failed to unmarshal node devices", "node", n.Name, "device annotation", anno) + return []*devices.DeviceInfo{}, err + } + if len(nodeDevices) == 0 { + klog.InfoS("no gpu device found", "node", n.Name, "device annotation", anno) + return []*devices.DeviceInfo{}, errors.New("no device found on node") + } + return nodeDevices, nil +} + +func (dev *AscendDevice) GenerateResourceRequests(ctr *v1.Container) devices.ContainerDeviceRequest { + ascendResourceCount := v1.ResourceName(dev.config.ResourceName) + ascendResourceMem := v1.ResourceName(dev.config.ResourceMemoryName) + v, ok := ctr.Resources.Limits[ascendResourceCount] + if !ok { + v, ok = ctr.Resources.Requests[ascendResourceCount] + } + if ok { + if n, ok := v.AsInt64(); ok { + memnum := 0 + mem, ok := ctr.Resources.Limits[ascendResourceMem] + if !ok { + mem, ok = ctr.Resources.Requests[ascendResourceMem] + } + if ok { + memnums, ok := mem.AsInt64() + if ok { + m, _ := dev.trimMemory(memnums) + memnum = int(m) + } + klog.V(5).Infof("raw mem %d memnum %d", memnums, memnum) + } + corenum := int32(0) + + mempnum := 0 + if memnum == 0 { + mempnum = 100 + } + + if corenum > 100 { + klog.ErrorS(nil, "core limit can't exceed 100", "device", dev.config.CommonWord) + corenum = 100 + } + if mempnum != 0 && memnum == 0 { + memnum = int(dev.DeviceInfo.Devmem) * mempnum / 100 + klog.V(5).Infof("new memreq %d totalmem %d mempercentage %d", memnum, dev.DeviceInfo.Devmem, mempnum) + } + + return devices.ContainerDeviceRequest{ + Nums: int32(n), + Type: dev.CommonWord(), + Memreq: int32(memnum), + MemPercentagereq: int32(mempnum), + Coresreq: corenum, + } + } + } + return devices.ContainerDeviceRequest{} +} + +func (dev *AscendDevice) ResourceReqs(pod *v1.Pod) []devices.ContainerDeviceRequest { + var reqs []devices.ContainerDeviceRequest + for _, ctr := range pod.Spec.Containers { + req := dev.GenerateResourceRequests(&ctr) + if req.Nums > 0 { + reqs = append(reqs, req) + } + } + return reqs +} + +func (ads *AscendDevices) PatchAnnotations(pod *v1.Pod, annoInput *map[string]string, devList devices.PodSingleDevice) map[string]string { + dev, err := ads.getRandomDevice() + if err != nil { + return *annoInput + } + commonWord := dev.CommonWord() + + (*annoInput)[devices.InRequestDevices[commonWord]] = devices.EncodePodSingleDevice(devList) + (*annoInput)[devices.SupportDevices[commonWord]] = devices.EncodePodSingleDevice(devList) + (*annoInput)["predicate-time"] = strconv.FormatInt(time.Now().Unix(), 10) + allocateStr := fmt.Sprintf("huawei.com/%s", dev.CommonWord()) + var rtInfo []RuntimeInfo + for _, dp := range devList { + for _, val := range dp { + _, temp := dev.trimMemory(int64(val.Usedmem)) + rtInfo = append(rtInfo, RuntimeInfo{ + UUID: val.UUID, + Temp: temp, + }) + } + } + s, err := json.Marshal(rtInfo) + if err != nil { + klog.ErrorS(err, "failed to marshal runtime info", "runtime info", rtInfo) + } + (*annoInput)[allocateStr] = string(s) + + return *annoInput +} + +func (dev *AscendDevice) GetResourceNames() devices.ResourceNames { + return devices.ResourceNames{ + ResourceCountName: dev.config.ResourceName, + ResourceMemoryName: dev.config.ResourceMemoryName, + ResourceCoreName: "", + } +} + +func CalScore(schedulePolicy string, dev_usage *devices.DeviceUsage, dev_info *devices.DeviceInfo) float64 { + var score float64 + switch schedulePolicy { + case binpackPolicy: + score = binpackMultiplier * (float64(dev_usage.Usedmem) / float64(dev_info.Devmem)) + case spreadPolicy: + if dev_usage.Used == 1 { + score = spreadMultiplier + } + default: + score = float64(0) + } + return score +} diff --git a/pkg/scheduler/api/devices/ascend/device_info_test.go b/pkg/scheduler/api/devices/ascend/device_info_test.go new file mode 100644 index 0000000000..eebb04ee43 --- /dev/null +++ b/pkg/scheduler/api/devices/ascend/device_info_test.go @@ -0,0 +1,273 @@ +/* +Copyright 2025 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ascend + +import ( + "fmt" + "testing" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" + "volcano.sh/volcano/pkg/scheduler/api/devices" + "volcano.sh/volcano/pkg/scheduler/api/devices/config" +) + +var config_yaml = ` +vnpus: +- chipName: 310P3 + + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 +- chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 +nvidia: + resourceCountName: volcano.sh/vgpu-number + resourceMemoryName: volcano.sh/vgpu-memory + resourceMemoryPercentageName: volcano.sh/vgpu-memory-percentage + resourceCoreName: volcano.sh/vgpu-cores + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + deviceSplitCount: 10 + deviceMemoryScaling: 1 + deviceCoreScaling: 1 + gpuMemoryFactor: 1 + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - group: group1 + geometries: + - name: 1g.6gb + memory: 6144 + count: 4 + - group: group2 + geometries: + - name: 2g.12gb + memory: 12288 + count: 2 + - group: group3 + geometries: + - name: 4g.24gb + memory: 24576 + count: 1 +` + +func yamlStringToConfig(yamlStr string) (*config.Config, error) { + var config config.Config + err := yaml.Unmarshal([]byte(yamlStr), &config) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal YAML: %v", err) + } + return &config, nil +} + +func Test_trimMemory(t *testing.T) { + _, err := yamlStringToConfig(config_yaml) + conf, err := yamlStringToConfig(config_yaml) + assert.Nil(t, err) + dev := AscendDevice{ + config: conf.VNPUs[0], + } + tests := []struct { + name string + inputMem int64 + wantMem int64 + }{ + {"test1", 0, 3072}, + {"test2", 1, 3072}, + {"test3", 100, 3072}, + {"test4", 3071, 3072}, + {"test5", 3072, 3072}, + {"test6", 3073, 6144}, + {"test7", 6143, 6144}, + {"test8", 6144, 6144}, + {"test9", 6144, 6144}, + {"test10", 6145, 12288}, + {"test11", 12288, 12288}, + {"test12", 12289, 21527}, + {"test13", 21527, 21527}, + {"test14", 21528, 21527}, + {"test15", 24576, 21527}, + {"test16", 24577, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, _ := dev.trimMemory(tt.inputMem) + assert.Equal(t, tt.wantMem, got) + }) + } +} + +func Test_fit(t *testing.T) { + _, err := yamlStringToConfig(config_yaml) + conf, err := yamlStringToConfig(config_yaml) + assert.Nil(t, err) + device_info := &devices.DeviceInfo{ + ID: "68496E64-20E05477-92C31323-6E78030A-BD003019", + Index: 0, + Count: 7, + Devcore: 8, + Devmem: 21527, + } + tests := []struct { + name string + req *devices.ContainerDeviceRequest + dev *AscendDevice + result bool + }{ + { + "test1", + &devices.ContainerDeviceRequest { + Nums: 1, + Type: "Ascend310P", + Memreq: 1024, + }, + &AscendDevice { + config: conf.VNPUs[0], + DeviceInfo: device_info, + DeviceUsage: &devices.DeviceUsage{ + Used: 1, + Usedmem: 3072, + }, + }, + true, + }, + { + "test2", + &devices.ContainerDeviceRequest { + Nums: 1, + Type: "Ascend310P", + Memreq: 21527, + }, + &AscendDevice { + config: conf.VNPUs[0], + DeviceInfo: device_info, + DeviceUsage: &devices.DeviceUsage{ + Used: 1, + Usedmem: 3072, + }, + }, + false, + }, + { + "test3", + &devices.ContainerDeviceRequest { + Nums: 1, + Type: "Ascend310P", + Memreq: 6144, + }, + &AscendDevice { + config: conf.VNPUs[0], + DeviceInfo: device_info, + DeviceUsage: &devices.DeviceUsage{ + Used: 1, + Usedmem: 12288, + }, + }, + true, + }, + { + "test4", + &devices.ContainerDeviceRequest { + Nums: 1, + Type: "Ascend310P", + Memreq: 24576, + }, + &AscendDevice { + config: conf.VNPUs[0], + DeviceInfo: device_info, + DeviceUsage: &devices.DeviceUsage{ + Used: 0, + Usedmem: 0, + }, + }, + false, + }, + { + "test5_core", + &devices.ContainerDeviceRequest { + Nums: 1, + Type: "Ascend310P", + Memreq: 6144, + Coresreq: 4, + }, + &AscendDevice { + config: conf.VNPUs[0], + DeviceInfo: device_info, + DeviceUsage: &devices.DeviceUsage{ + Used: 1, + Usedmem: 12288, + Usedcores: 6, + }, + }, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ret := fit(tt.req, tt.dev) + assert.Equal(t, tt.result, ret) + }) + } +} diff --git a/pkg/scheduler/api/devices/config/config.go b/pkg/scheduler/api/devices/config/config.go index ad61bacbf2..d5dc56d5dc 100644 --- a/pkg/scheduler/api/devices/config/config.go +++ b/pkg/scheduler/api/devices/config/config.go @@ -44,11 +44,30 @@ const ( hygon: resourceCountName: "volcano.sh/vdcu" ... + vnpus: + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 */ type Config struct { //NvidiaConfig is used for vGPU feature for nvidia, gpushare is not using this config NvidiaConfig NvidiaConfig `yaml:"nvidia"` + VNPUs []VNPUConfig `yaml:"vnpus"` } var ( diff --git a/pkg/scheduler/api/devices/config/vnpu.go b/pkg/scheduler/api/devices/config/vnpu.go new file mode 100644 index 0000000000..dc02910352 --- /dev/null +++ b/pkg/scheduler/api/devices/config/vnpu.go @@ -0,0 +1,36 @@ +/* +Copyright 2025 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +type Template struct { + Name string `yaml:"name"` + Memory int64 `yaml:"memory"` + AICore int32 `yaml:"aiCore,omitempty"` + AICPU int32 `yaml:"aiCPU,omitempty"` +} + +type VNPUConfig struct { + CommonWord string `yaml:"commonWord"` + ChipName string `yaml:"chipName"` + ResourceName string `yaml:"resourceName"` + ResourceMemoryName string `yaml:"resourceMemoryName"` + MemoryAllocatable int64 `yaml:"memoryAllocatable"` + MemoryCapacity int64 `yaml:"memoryCapacity"` + AICore int32 `yaml:"aiCore"` + AICPU int32 `yaml:"aiCPU"` + Templates []Template `yaml:"templates"` +} \ No newline at end of file diff --git a/pkg/scheduler/api/devices/device_info.go b/pkg/scheduler/api/devices/device_info.go new file mode 100644 index 0000000000..be13a22ebc --- /dev/null +++ b/pkg/scheduler/api/devices/device_info.go @@ -0,0 +1,219 @@ +/* +Copyright 2023 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package devices + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" +) + +type DeviceInfo struct { + ID string `json:"id,omitempty"` + Index uint `json:"index,omitempty"` + Count int32 `json:"count,omitempty"` + Devmem int32 `json:"devmem,omitempty"` + Devcore int32 `json:"devcore,omitempty"` + Type string `json:"type,omitempty"` + Numa int `json:"numa,omitempty"` + Mode string `json:"mode,omitempty"` + // MIGTemplate []Geometry `json:"migtemplate,omitempty"` + Health bool `json:"health,omitempty"` + DeviceVendor string `json:"devicevendor,omitempty"` + CustomInfo map[string]any `json:"custominfo,omitempty"` + DevicePairScore DevicePairScore `json:"devicepairscore,omitempty"` +} + +type DevicePairScores []DevicePairScore +type DevicePairScore struct { + ID string `json:"uuid,omitempty"` + Scores map[string]int `json:"score,omitempty"` +} + +type DeviceUsage struct { + // ID string + // Index uint + Used int32 + // Count int32 + Usedmem int32 + // Totalmem int32 + // Totalcore int32 + Usedcores int32 + // Mode string + // MigTemplate []Geometry + // MigUsage MigInUse + // Numa int + // Type string + // Health bool + // CustomInfo map[string]any +} + +type ResourceNames struct { + ResourceCountName string + ResourceMemoryName string + ResourceCoreName string +} + +type ContainerDevice struct { + // TODO current Idx cannot use, because EncodeContainerDevices method not encode this filed. + Idx int + UUID string + Type string + Usedmem int32 + Usedcores int32 + CustomInfo map[string]any +} + +type ContainerDeviceRequest struct { + Nums int32 + Type string + Memreq int32 + MemPercentagereq int32 + Coresreq int32 +} + +const ( + // OneContainerMultiDeviceSplitSymbol this is when one container use multi device, use : symbol to join device info. + OneContainerMultiDeviceSplitSymbol = ":" + + // OnePodMultiContainerSplitSymbol this is when one pod having multi container and more than one container use device, use ; symbol to join device info. + OnePodMultiContainerSplitSymbol = ";" +) + +var ( + HandshakeAnnos = map[string]string{} + InRequestDevices = map[string]string{} + SupportDevices = map[string]string{} +) + +type ContainerDevices []ContainerDevice +type ContainerDeviceRequests map[string]ContainerDeviceRequest +type PodDeviceRequests []ContainerDeviceRequests +type PodSingleDevice []ContainerDevices +type PodDevices map[string]PodSingleDevice + +func CheckHealth(devType string, n *corev1.Node) (bool, bool) { + handshake := n.Annotations[HandshakeAnnos[devType]] + if strings.Contains(handshake, "Requesting") { + formertime, _ := time.Parse(time.DateTime, strings.Split(handshake, "_")[1]) + return time.Now().Before(formertime.Add(time.Second * 60)), false + } else if strings.Contains(handshake, "Deleted") { + return true, false + } else { + return true, true + } +} + +func UnMarshalNodeDevices(str string) ([]*DeviceInfo, error) { + var dlist []*DeviceInfo + err := json.Unmarshal([]byte(str), &dlist) + return dlist, err +} + +func EncodeContainerDevices(cd ContainerDevices) string { + tmp := "" + for _, val := range cd { + tmp += val.UUID + "," + val.Type + "," + strconv.Itoa(int(val.Usedmem)) + "," + strconv.Itoa(int(val.Usedcores)) + OneContainerMultiDeviceSplitSymbol + } + klog.Infof("Encoded container Devices: %s", tmp) + return tmp + //return strings.Join(cd, ",") +} + +func EncodePodSingleDevice(pd PodSingleDevice) string { + res := "" + for _, ctrdevs := range pd { + res = res + EncodeContainerDevices(ctrdevs) + res = res + OnePodMultiContainerSplitSymbol + } + klog.Infof("Encoded pod single devices %s", res) + return res +} + +func DecodeContainerDevices(str string) (ContainerDevices, error) { + if len(str) == 0 { + return ContainerDevices{}, nil + } + cd := strings.Split(str, OneContainerMultiDeviceSplitSymbol) + contdev := ContainerDevices{} + tmpdev := ContainerDevice{} + klog.V(5).Infof("Start to decode container device %s", str) + for _, val := range cd { + if strings.Contains(val, ",") { + //fmt.Println("cd is ", val) + tmpstr := strings.Split(val, ",") + if len(tmpstr) < 4 { + return ContainerDevices{}, fmt.Errorf("pod annotation format error; information missing, please do not use nodeName field in task") + } + tmpdev.UUID = tmpstr[0] + tmpdev.Type = tmpstr[1] + devmem, _ := strconv.ParseInt(tmpstr[2], 10, 32) + tmpdev.Usedmem = int32(devmem) + devcores, _ := strconv.ParseInt(tmpstr[3], 10, 32) + tmpdev.Usedcores = int32(devcores) + contdev = append(contdev, tmpdev) + } + } + klog.V(5).Infof("Finished decoding container devices. Total devices: %d", len(contdev)) + return contdev, nil +} + +func DecodePodDevices(checklist map[string]string, annos map[string]string) (PodDevices, error) { + klog.V(5).Infof("checklist is [%+v], annos is [%+v]", checklist, annos) + if len(annos) == 0 { + return PodDevices{}, nil + } + pd := make(PodDevices) + for devID, devs := range checklist { + str, ok := annos[devs] + if !ok { + continue + } + pd[devID] = make(PodSingleDevice, 0) + for s := range strings.SplitSeq(str, OnePodMultiContainerSplitSymbol) { + cd, err := DecodeContainerDevices(s) + if err != nil { + return PodDevices{}, nil + } + if len(cd) == 0 { + continue + } + pd[devID] = append(pd[devID], cd) + } + } + klog.V(5).InfoS("Decoded pod annos", "poddevices", pd) + return pd, nil +} diff --git a/pkg/scheduler/api/devices/util.go b/pkg/scheduler/api/devices/util.go index e23829f663..e04499e512 100644 --- a/pkg/scheduler/api/devices/util.go +++ b/pkg/scheduler/api/devices/util.go @@ -31,6 +31,15 @@ limitations under the License. package devices import ( + "context" + "encoding/json" + "fmt" + "time" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" @@ -59,6 +68,16 @@ const ( Skip ) +const ( + AssignedNodeAnnotations = "hami.io/vgpu-node" + AssignedTimeAnnotations = "hami.io/vgpu-time" + BindTimeAnnotations = "hami.io/bind-time" + DeviceBindPhase = "hami.io/bind-phase" + DeviceBindAllocating = "allocating" + DeviceBindFailed = "failed" + DeviceBindSuccess = "success" +) + var kubeClient *kubernetes.Clientset func GetClient() kubernetes.Interface { @@ -81,3 +100,90 @@ func NewClient() (*kubernetes.Clientset, error) { client, err := kubernetes.NewForConfig(config) return client, err } + +func GetNode(nodename string) (*v1.Node, error) { + if nodename == "" { + klog.ErrorS(nil, "Node name is empty") + return nil, fmt.Errorf("nodename is empty") + } + + klog.V(5).InfoS("Fetching node", "nodeName", nodename) + n, err := GetClient().CoreV1().Nodes().Get(context.Background(), nodename, metav1.GetOptions{}) + if err != nil { + switch { + case apierrors.IsNotFound(err): + klog.ErrorS(err, "Node not found", "nodeName", nodename) + return nil, fmt.Errorf("node %s not found", nodename) + case apierrors.IsUnauthorized(err): + klog.ErrorS(err, "Unauthorized to access node", "nodeName", nodename) + return nil, fmt.Errorf("unauthorized to access node %s", nodename) + default: + klog.ErrorS(err, "Failed to get node", "nodeName", nodename) + return nil, fmt.Errorf("failed to get node %s: %v", nodename, err) + } + } + + klog.V(5).InfoS("Successfully fetched node", "nodeName", nodename) + return n, nil +} + +func MarkAnnotationsToDelete(devType string, nn string) error { + tmppat := make(map[string]string) + tmppat[devType] = "Deleted_" + time.Now().Format(time.DateTime) + n, err := GetNode(nn) + if err != nil { + klog.Errorln("get node failed", err.Error()) + return err + } + return PatchNodeAnnotations(n, tmppat) +} + +func PatchPodAnnotations(kubeClient kubernetes.Interface, pod *v1.Pod, annotations map[string]string) error { + type patchMetadata struct { + Annotations map[string]string `json:"annotations,omitempty"` + } + type patchPod struct { + Metadata patchMetadata `json:"metadata"` + //Spec patchSpec `json:"spec,omitempty"` + } + + p := patchPod{} + p.Metadata.Annotations = annotations + + bytes, err := json.Marshal(p) + if err != nil { + return err + } + _, err = kubeClient.CoreV1().Pods(pod.Namespace). + Patch(context.Background(), pod.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{}) + if err != nil { + klog.Errorf("patch pod %v failed, %v", pod.Name, err) + } + + return err +} + +func PatchNodeAnnotations(node *v1.Node, annotations map[string]string) error { + type patchMetadata struct { + Annotations map[string]string `json:"annotations,omitempty"` + } + type patchPod struct { + Metadata patchMetadata `json:"metadata"` + //Spec patchSpec `json:"spec,omitempty"` + } + + p := patchPod{} + p.Metadata.Annotations = annotations + + bytes, err := json.Marshal(p) + if err != nil { + return err + } + _, err = GetClient().CoreV1().Nodes(). + Patch(context.Background(), node.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{}) + if err != nil { + klog.Infoln("annotations=", annotations) + klog.Infof("patch node %v failed, %v", node.Name, err) + } + return err +} diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index 12306aeae7..3f9e3ccaf2 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -24,6 +24,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" fwk "k8s.io/kube-scheduler/framework" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend" "volcano.sh/apis/pkg/apis/scheduling" "volcano.sh/apis/pkg/apis/scheduling/v1beta1" @@ -350,10 +351,16 @@ func (ni *NodeInfo) setNodeOthersResource(node *v1.Node) { ni.Others[gpushare.DeviceName] = gpushare.NewGPUDevices(ni.Name, node) ni.Others[vgpu.DeviceName] = vgpu.NewGPUDevices(ni.Name, node) ni.Others[vnpu.DeviceName] = vnpu.NewNPUDevices(ni.Name, node) + ascend_ignored_list := []string{} + for device_name, devices := range ascend.NewAscendDevices(ni.Name, node) { + ni.Others[device_name] = devices + ascend_ignored_list = append(ascend_ignored_list, devices.GetIgnoredDevices()...) + } IgnoredDevicesList.Set( ni.Others[gpushare.DeviceName].(Devices).GetIgnoredDevices(), ni.Others[vgpu.DeviceName].(Devices).GetIgnoredDevices(), ni.Others[vnpu.DeviceName].(Devices).GetIgnoredDevices(), + ascend_ignored_list, ) } @@ -507,6 +514,13 @@ func (ni *NodeInfo) addResource(pod *v1.Pod) { } ni.Others[vgpu.DeviceName].(Devices).AddResource(pod) ni.Others[vnpu.DeviceName].(Devices).AddResource(pod) + for _, name := range ascend.GetAscendDeviceNames() { + if other, exists := ni.Others[name]; exists { + if devices, ok := other.(Devices); ok { + devices.AddResource(pod) + } + } + } } // subResource is used to subtract sharable devices @@ -516,6 +530,13 @@ func (ni *NodeInfo) subResource(pod *v1.Pod) { } ni.Others[vgpu.DeviceName].(Devices).SubResource(pod) ni.Others[vnpu.DeviceName].(Devices).SubResource(pod) + for _, name := range ascend.GetAscendDeviceNames() { + if other, exists := ni.Others[name]; exists { + if devices, ok := other.(Devices); ok { + devices.SubResource(pod) + } + } + } } // UpdateTask is used to update a task in nodeInfo object. diff --git a/pkg/scheduler/api/shared_device_pool.go b/pkg/scheduler/api/shared_device_pool.go index c6e524140a..b96196098f 100644 --- a/pkg/scheduler/api/shared_device_pool.go +++ b/pkg/scheduler/api/shared_device_pool.go @@ -23,6 +23,7 @@ import ( "k8s.io/client-go/kubernetes" "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/gpushare" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/vgpu" ) @@ -78,6 +79,7 @@ type Devices interface { var _ Devices = new(gpushare.GPUDevices) var _ Devices = new(vgpu.GPUDevices) var _ Devices = new(vnpu.NPUDevices) +var _ Devices = new(ascend.AscendDevices) var RegisteredDevices = []string{ gpushare.DeviceName, @@ -85,6 +87,10 @@ var RegisteredDevices = []string{ vnpu.DeviceName, } +func RegisterDevice(deviceName string) { + RegisteredDevices = append(RegisteredDevices, deviceName) +} + var IgnoredDevicesList = ignoredDevicesList{} type ignoredDevicesList struct { diff --git a/pkg/scheduler/plugins/deviceshare/deviceshare.go b/pkg/scheduler/plugins/deviceshare/deviceshare.go index fc59e36d41..d574c4ab48 100644 --- a/pkg/scheduler/plugins/deviceshare/deviceshare.go +++ b/pkg/scheduler/plugins/deviceshare/deviceshare.go @@ -21,6 +21,7 @@ import ( "fmt" "math" "reflect" + "sync" v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" @@ -28,6 +29,7 @@ import ( "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/api/devices" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend" "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/api/devices/config" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/gpushare" @@ -46,7 +48,8 @@ const ( VGPUEnable = "deviceshare.VGPUEnable" - ASCEND310PvGPU = "deviceshare.ASCEND310PVNPUEnable" + ASCEND310PvGPU = "deviceshare.ASCEND310PVGPUEnable" + AscendVNPUEnable = "deviceshare.AscendVNPUEnable" SchedulePolicyArgument = "deviceshare.SchedulePolicy" ScheduleWeight = "deviceshare.ScheduleWeight" @@ -55,6 +58,10 @@ const ( KnownGeometriesCMNamespace = "deviceshare.KnownGeometriesCMNamespace" ) +var ( + once sync.Once +) + type deviceSharePlugin struct { // Arguments given for the plugin pluginArguments framework.Arguments @@ -82,6 +89,7 @@ func enablePredicate(dsp *deviceSharePlugin) { args.GetBool(&nodeLockEnable, NodeLockEnable) args.GetBool(&vgpu.VGPUEnable, VGPUEnable) args.GetBool(&vnpu.Ascend310pvNPUEnable, ASCEND310PvGPU) + args.GetBool(&ascend.AscendVNPUEnable, AscendVNPUEnable) gpushare.NodeLockEnable = nodeLockEnable vgpu.NodeLockEnable = nodeLockEnable @@ -96,14 +104,23 @@ func enablePredicate(dsp *deviceSharePlugin) { klog.Fatal("gpu-share and vgpu can't be used together") } - if !vgpu.VGPUEnable { - return - } knownGeometriesCMName := "volcano-vgpu-device-config" args.GetString(&knownGeometriesCMName, KnownGeometriesCMName) knownGeometriesCMNamespace := "kube-system" args.GetString(&knownGeometriesCMNamespace, KnownGeometriesCMNamespace) config.InitDevicesConfig(knownGeometriesCMName, knownGeometriesCMNamespace) + registerDevices() +} + +func registerDevices() { + once.Do(func() { + if ascend.AscendVNPUEnable { + for _, vnpu := range config.GetConfig().VNPUs { + klog.Infof("register device %s", vnpu.CommonWord) + api.RegisterDevice(vnpu.CommonWord) + } + } + }) } func createStatus(code int, reason string) *api.Status { @@ -118,16 +135,12 @@ func getDeviceScore(ctx context.Context, pod *v1.Pod, node *api.NodeInfo, schedu s := float64(0) for deviceType, device := range node.Others { if device.(api.Devices).HasDeviceRequest(pod) { - var ns float64 // Only process device types that use NodeOrderFn (vgpu and gpushare) // vnpu devices use BatchNodeOrderFn, skip them here - if deviceType == vgpu.DeviceName || deviceType == gpushare.DeviceName { - ns = device.(api.Devices).ScoreNode(pod, schedulePolicy) - } else { - // Other device types (like vnpu) use BatchNodeOrderFn, skip scoring here - continue + if deviceType != vnpu.NPUDevices { + ns := device.(api.Devices).ScoreNode(pod, schedulePolicy) + s += ns } - s += ns } } klog.V(4).Infof("deviceScore for task %s/%s is: %v", pod.Namespace, pod.Name, s) @@ -172,11 +185,14 @@ func initializeDevicesWithSession(ssn *framework.Session) { func initializeDevice(device api.Devices, ssn *framework.Session, nodeInfo *api.NodeInfo) error { switch d := device.(type) { case *vnpu.NPUDevices: - klog.V(3).Infof("initialize ascend310p device.") - return vnpu310p.InitVNPUDevice(d, ssn, nodeInfo) + if vnpu.Ascend310pvNPUEnable { + klog.V(3).Infof("initialize ascend310p device.") + return vnpu310p.InitVNPUDevice(d, ssn, nodeInfo) + } default: return nil } + return nil } func (dp *deviceSharePlugin) OnSessionOpen(ssn *framework.Session) { @@ -191,7 +207,7 @@ func (dp *deviceSharePlugin) OnSessionOpen(ssn *framework.Session) { if dev, ok := node.Others[val].(api.Devices); ok { if reflect.ValueOf(dev).IsNil() { // TODO When a pod requests a device of the current type, but the current node does not have such a device, an error is thrown - if dev == nil || dev.HasDeviceRequest(task.Pod) { + if dev == nil { predicateStatus = append(predicateStatus, &api.Status{ Code: devices.Unschedulable, Reason: "node not initialized with device" + val, @@ -202,8 +218,13 @@ func (dp *deviceSharePlugin) OnSessionOpen(ssn *framework.Session) { klog.V(4).Infof("pod %s/%s did not request device %s on %s, skipping it", task.Pod.Namespace, task.Pod.Name, val, node.Name) continue } + if !dev.HasDeviceRequest(task.Pod) { + klog.V(4).Infof("pod %s/%s did not request device %s on %s, skipping it", task.Pod.Namespace, task.Pod.Name, val, node.Name) + continue + } code, msg, err := dev.FilterNode(task.Pod, dp.schedulePolicy) if err != nil { + klog.V(4).Infof("pod %s/%s fit failed. device %s node %s err %v", task.Pod.Namespace, task.Pod.Name, val, node.Name, err) predicateStatus = append(predicateStatus, createStatus(code, msg)) return api.NewFitErrWithStatus(task, node, predicateStatus...) } From 11657ea44234e136bdbf2b0af1543c5cab994746 Mon Sep 17 00:00:00 2001 From: james Date: Thu, 6 Nov 2025 17:38:50 +0800 Subject: [PATCH 02/33] refactor: revert ASCEND310PvGPU Signed-off-by: james --- pkg/scheduler/plugins/deviceshare/deviceshare.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/scheduler/plugins/deviceshare/deviceshare.go b/pkg/scheduler/plugins/deviceshare/deviceshare.go index d574c4ab48..80bfa54e97 100644 --- a/pkg/scheduler/plugins/deviceshare/deviceshare.go +++ b/pkg/scheduler/plugins/deviceshare/deviceshare.go @@ -48,7 +48,7 @@ const ( VGPUEnable = "deviceshare.VGPUEnable" - ASCEND310PvGPU = "deviceshare.ASCEND310PVGPUEnable" + ASCEND310PvGPU = "deviceshare.ASCEND310PVNPUEnable" AscendVNPUEnable = "deviceshare.AscendVNPUEnable" SchedulePolicyArgument = "deviceshare.SchedulePolicy" @@ -59,7 +59,7 @@ const ( ) var ( - once sync.Once + once sync.Once ) type deviceSharePlugin struct { @@ -189,8 +189,6 @@ func initializeDevice(device api.Devices, ssn *framework.Session, nodeInfo *api. klog.V(3).Infof("initialize ascend310p device.") return vnpu310p.InitVNPUDevice(d, ssn, nodeInfo) } - default: - return nil } return nil } From 5740a812e0336c86b612ea6e7cb4e8ef4dcd405c Mon Sep 17 00:00:00 2001 From: james Date: Thu, 6 Nov 2025 18:26:12 +0800 Subject: [PATCH 03/33] fix: fix gemini comment Signed-off-by: james --- .../api/devices/ascend/device_info.go | 184 ++++++++---------- .../api/devices/ascend/device_info_test.go | 15 -- pkg/scheduler/api/devices/device_info.go | 16 +- pkg/scheduler/api/devices/util.go | 12 -- pkg/scheduler/api/node_info.go | 16 +- .../plugins/deviceshare/deviceshare.go | 2 +- 6 files changed, 95 insertions(+), 150 deletions(-) diff --git a/pkg/scheduler/api/devices/ascend/device_info.go b/pkg/scheduler/api/devices/ascend/device_info.go index 2d4b30410c..6af30b4324 100644 --- a/pkg/scheduler/api/devices/ascend/device_info.go +++ b/pkg/scheduler/api/devices/ascend/device_info.go @@ -14,20 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package ascend import ( @@ -59,8 +45,8 @@ const ( spreadPolicy = "spread" binpackMultiplier = 100 spreadMultiplier = 100 - CMName = "volcano-vgpu-device-config" - CMNamespace = "kube-system" + CMName = "volcano-vgpu-device-config" + CMNamespace = "kube-system" ) type AscendDevice struct { @@ -87,65 +73,65 @@ type RuntimeInfo struct { } var ( - AscendVNPUEnable bool - configFile string - NodeLockEnable bool + AscendVNPUEnable bool + configFile string + NodeLockEnable bool ) func NewAscendDevices(name string, node *v1.Node) map[string]*AscendDevices { - ascend_devices := make(map[string]*AscendDevices) + ascendDevices := make(map[string]*AscendDevices) if node == nil { klog.Warningf("Node is nil for node %s, returning empty AscendDevices", name) - return ascend_devices + return ascendDevices } - cur_config := config.GetConfig() - if cur_config == nil { + curConfig := config.GetConfig() + if curConfig == nil { klog.V(5).InfoS("cur config is null. call InitDevicesConfig") config.InitDevicesConfig(CMName, CMNamespace) - cur_config = config.GetConfig() + curConfig = config.GetConfig() } - devs := InitDevices(cur_config.VNPUs) + devs := InitDevices(curConfig.VNPUs) for _, dev := range devs { - node_devices, err := dev.GetNodeDevices(*node) + nodeDevices, err := dev.GetNodeDevices(*node) if err != nil { - klog.Warningf("Failed to get node devices. nodeName %s, deviceType %s, error %s" , node.Name, dev.CommonWord(), err) + klog.Warningf("Failed to get node devices. nodeName %s, deviceType %s, error %s", node.Name, dev.CommonWord(), err) continue } - as_devices := &AscendDevices{ + asDevices := &AscendDevices{ NodeName: name, Type: dev.CommonWord(), Devices: make(map[string]*AscendDevice), } - for _, nd := range node_devices { + for _, nd := range nodeDevices { cur_dev := &AscendDevice{ config: dev.config, nodeRegisterAnno: dev.nodeRegisterAnno, useUUIDAnno: dev.useUUIDAnno, noUseUUIDAnno: dev.noUseUUIDAnno, handshakeAnno: dev.handshakeAnno, - DeviceInfo: nd, - DeviceUsage: &devices.DeviceUsage{ - Used: 0, - Usedmem: 0, - Usedcores: 0, - }, + DeviceInfo: nd, + DeviceUsage: &devices.DeviceUsage{ + Used: 0, + Usedmem: 0, + Usedcores: 0, + }, } - as_devices.Devices[nd.ID] = cur_dev + asDevices.Devices[nd.ID] = cur_dev klog.V(5).Infof("add device. ID %s dev_info %+v", cur_dev.DeviceInfo.ID, cur_dev.DeviceInfo) } - ascend_devices[dev.CommonWord()] = as_devices + ascendDevices[dev.CommonWord()] = asDevices } - return ascend_devices + return ascendDevices } -func GetAscendDeviceNames() ([]string){ - cur_config := config.GetConfig() - if cur_config == nil { +func GetAscendDeviceNames() []string { + curConfig := config.GetConfig() + if curConfig == nil { config.InitDevicesConfig(CMName, CMNamespace) - cur_config = config.GetConfig() + curConfig = config.GetConfig() } - deviceNames := make([]string, 0, len(cur_config.VNPUs)) - for _, vnpu := range cur_config.VNPUs { + deviceNames := make([]string, 0, len(curConfig.VNPUs)) + for _, vnpu := range curConfig.VNPUs { deviceNames = append(deviceNames, vnpu.CommonWord) } return deviceNames @@ -223,11 +209,11 @@ func (ads *AscendDevices) HasDeviceRequest(pod *v1.Pod) bool { if !AscendVNPUEnable { return false } - rand_dev, err := ads.getRandomDevice() - if rand_dev == nil || err != nil { + randDev, err := ads.getFirstDevice() + if randDev == nil || err != nil { return false } - var vnpu_config = rand_dev.config + var vnpu_config = randDev.config for _, container := range pod.Spec.Containers { _, ok := container.Resources.Limits[v1.ResourceName(vnpu_config.ResourceName)] if ok { @@ -255,25 +241,25 @@ func (ads *AscendDevices) FilterNode(pod *v1.Pod, policy string) (int, string, e func (ads *AscendDevices) ScoreNode(pod *v1.Pod, policy string) float64 { ads.Policy = policy - pod_devs, err := ads.selectDevices(pod, policy) + podDevs, err := ads.selectDevices(pod, policy) if err != nil { return 0 } score := 0.0 - var used_devs []*AscendDevice - for _, dev := range pod_devs { + var usedDevs []*AscendDevice + for _, dev := range podDevs { dev, ok := ads.Devices[dev[0].UUID] if !ok { return 0 } - used_devs = append(used_devs, dev) + usedDevs = append(usedDevs, dev) score += CalScore(policy, dev.DeviceUsage, dev.DeviceInfo) } - if strings.HasPrefix(ads.Type, Ascend910Prefix) && hasNetworkID(used_devs) { + if strings.HasPrefix(ads.Type, Ascend910Prefix) && hasNetworkID(usedDevs) { klog.V(4).Infof("all devices have NetworkID. device CommonWord %s", ads.Type) cntMap := make(map[int]int) - for _, dev := range used_devs { + for _, dev := range usedDevs { if dev.DeviceInfo.CustomInfo == nil { return 0 } @@ -309,12 +295,12 @@ func (ads *AscendDevices) Allocate(kubeClient kubernetes.Interface, pod *v1.Pod) return errors.Errorf("node %s locked for %s hamivgpu lockname %s", ads.NodeName, pod.Name, err.Error()) } } - pod_devs, err := ads.selectDevices(pod, ads.Policy) + podDevs, err := ads.selectDevices(pod, ads.Policy) if err != nil { return errors.Errorf("failed to select ascend devices for pod %s: %v", pod.Name, err) } annotations := make(map[string]string) - ads.PatchAnnotations(pod, &annotations, pod_devs) + ads.PatchAnnotations(pod, &annotations, podDevs) ads.addResource(annotations, pod) annotations[devices.AssignedNodeAnnotations] = ads.NodeName @@ -338,12 +324,12 @@ func (ads *AscendDevices) Release(kubeClient kubernetes.Interface, pod *v1.Pod) } func (ads *AscendDevices) GetIgnoredDevices() []string { - rand_dev, err := ads.getRandomDevice() - if rand_dev == nil || err != nil { + randDev, err := ads.getFirstDevice() + if randDev == nil || err != nil { return []string{""} } - vnpu_config := rand_dev.config - return []string{vnpu_config.ResourceMemoryName} + vnpuConfig := randDev.config + return []string{vnpuConfig.ResourceMemoryName} } func (ads *AscendDevices) GetStatus() string { @@ -351,48 +337,48 @@ func (ads *AscendDevices) GetStatus() string { } func (ads *AscendDevices) selectDevices(pod *v1.Pod, schedulePolicy string) (devices.PodSingleDevice, error) { - dup_devs := getDeviceSnapshot(ads) - if len(dup_devs) == 0 { + dupDevs := getDeviceSnapshot(ads) + if len(dupDevs) == 0 { return nil, errors.Errorf("no ascend device available") } - for _, dev := range dup_devs { + for _, dev := range dupDevs { dev.Score = CalScore(schedulePolicy, dev.DeviceUsage, dev.DeviceInfo) } - sort.Slice(dup_devs, func(i, j int) bool { - return dup_devs[i].Score > dup_devs[j].Score + sort.Slice(dupDevs, func(i, j int) bool { + return dupDevs[i].Score > dupDevs[j].Score }) needTopology := false - if strings.HasPrefix(ads.Type, Ascend910Prefix) && hasNetworkID(dup_devs) { + if strings.HasPrefix(ads.Type, Ascend910Prefix) && hasNetworkID(dupDevs) { klog.V(4).Infof("all devices have NetworkID. device CommonWord %s", ads.Type) needTopology = true } - reqs := dup_devs[0].ResourceReqs(pod) - var pod_devs devices.PodSingleDevice - used_devs := make([]*AscendDevice, 0) + reqs := dupDevs[0].ResourceReqs(pod) + var podDevs devices.PodSingleDevice + usedDevs := make([]*AscendDevice, 0) for _, req := range reqs { klog.V(5).Infof("req %+v", req) - available_devs := make([]*AscendDevice, 0) - for _, dev := range dup_devs { + availableDevs := make([]*AscendDevice, 0) + for _, dev := range dupDevs { selected := false - for _, used_dev := range used_devs { - if used_dev.DeviceInfo.ID == dev.DeviceInfo.ID { + for _, usedDev := range usedDevs { + if usedDev.DeviceInfo.ID == dev.DeviceInfo.ID { selected = true break } } if !selected { - available_devs = append(available_devs, dev) + availableDevs = append(availableDevs, dev) } } req_nums := req.Nums - selected_devs := make([]*AscendDevice, 0) - for _, dev := range available_devs { + selectedDevs := make([]*AscendDevice, 0) + for _, dev := range availableDevs { klog.V(5).Infof("check fit. req %+v dev_info %+v dev_usage %+v", req, dev.DeviceInfo, dev.DeviceUsage) if fit(&req, dev) == false { klog.V(5).Infof("fit false. dev ID %s", dev.DeviceInfo.ID) continue } - selected_devs = append(selected_devs, dev) + selectedDevs = append(selectedDevs, dev) req_nums -= 1 if req_nums <= 0 && !needTopology { break @@ -403,12 +389,12 @@ func (ads *AscendDevices) selectDevices(pod *v1.Pod, schedulePolicy string) (dev return nil, errors.Errorf("no enough ascend device available") } if needTopology { - selected_devs = selectDevicesWithTopology(int(req.Nums), selected_devs) + selectedDevs = selectDevicesWithTopology(int(req.Nums), selectedDevs) } - used_devs = append(used_devs, selected_devs...) - var con_devs devices.ContainerDevices - for _, dev := range selected_devs { - con_devs = append(con_devs, devices.ContainerDevice{ + usedDevs = append(usedDevs, selectedDevs...) + var conDevs devices.ContainerDevices + for _, dev := range selectedDevs { + conDevs = append(conDevs, devices.ContainerDevice{ UUID: dev.DeviceInfo.ID, Type: ads.Type, Usedmem: req.Memreq, @@ -416,9 +402,9 @@ func (ads *AscendDevices) selectDevices(pod *v1.Pod, schedulePolicy string) (dev CustomInfo: dev.DeviceInfo.CustomInfo, }) } - pod_devs = append(pod_devs, con_devs) + podDevs = append(podDevs, conDevs) } - return pod_devs, nil + return podDevs, nil } func hasNetworkID(devices []*AscendDevice) bool { @@ -437,28 +423,28 @@ func fit(req *devices.ContainerDeviceRequest, dev *AscendDevice) bool { if req.Type != dev.config.CommonWord { return false } - device_usage := dev.DeviceUsage - device_info := dev.DeviceInfo - if device_info.Count < device_usage.Used { + deviceUsage := dev.DeviceUsage + deviceInfo := dev.DeviceInfo + if deviceInfo.Count <= deviceUsage.Used { return false } - if device_info.Devmem-device_usage.Usedmem < req.Memreq { + if deviceInfo.Devmem-deviceUsage.Usedmem < req.Memreq { return false } - if device_info.Devcore-device_usage.Usedcores < req.Coresreq { + if deviceInfo.Devcore-deviceUsage.Usedcores < req.Coresreq { return false } - if device_info.Devcore == 100 && req.Coresreq == 100 && device_usage.Used > 0 { + if deviceInfo.Devcore == 100 && req.Coresreq == 100 && deviceUsage.Used > 0 { return false } - if device_info.Devcore != 0 && device_usage.Usedcores == device_info.Devcore && req.Coresreq == 0 { + if deviceInfo.Devcore != 0 && deviceUsage.Usedcores == deviceInfo.Devcore && req.Coresreq == 0 { return false } return true } func getDeviceSnapshot(ads *AscendDevices) []*AscendDevice { - dup_devs := make([]*AscendDevice, 0, len(ads.Devices)) + dupDevs := make([]*AscendDevice, 0, len(ads.Devices)) for _, dev := range ads.Devices { dup_dev := &AscendDevice{ config: dev.config, @@ -473,19 +459,19 @@ func getDeviceSnapshot(ads *AscendDevices) []*AscendDevice { Usedcores: dev.DeviceUsage.Usedcores, }, } - dup_devs = append(dup_devs, dup_dev) + dupDevs = append(dupDevs, dup_dev) } - return dup_devs + return dupDevs } func selectDevicesWithTopology(req_nums int, selected_devs []*AscendDevice) []*AscendDevice { - network_map := make(map[int][]*AscendDevice) + networkMap := make(map[int][]*AscendDevice) for _, dev := range selected_devs { if dev.DeviceInfo.CustomInfo != nil { if networkID, ok := dev.DeviceInfo.CustomInfo["NetworkID"]; ok { if id, ok := networkID.(float64); ok { - network_map[int(id)] = append(network_map[int(id)], dev) + networkMap[int(id)] = append(networkMap[int(id)], dev) } } } @@ -495,7 +481,7 @@ func selectDevicesWithTopology(req_nums int, selected_devs []*AscendDevice) []*A Count int } var sortedNetworks []NetworkDeviceCount - for networkID, devices := range network_map { + for networkID, devices := range networkMap { sortedNetworks = append(sortedNetworks, NetworkDeviceCount{ NetworkID: networkID, Count: len(devices), @@ -506,7 +492,7 @@ func selectDevicesWithTopology(req_nums int, selected_devs []*AscendDevice) []*A }) devs := make([]*AscendDevice, 0) for _, item := range sortedNetworks { - for _, dev := range network_map[item.NetworkID] { + for _, dev := range networkMap[item.NetworkID] { devs = append(devs, dev) if len(devs) == req_nums { return devs @@ -516,7 +502,7 @@ func selectDevicesWithTopology(req_nums int, selected_devs []*AscendDevice) []*A return devs } -func (ads *AscendDevices) getRandomDevice() (*AscendDevice, error) { +func (ads *AscendDevices) getFirstDevice() (*AscendDevice, error) { if len(ads.Devices) == 0 { return nil, errors.New("no ascend device available") } @@ -651,7 +637,7 @@ func (dev *AscendDevice) ResourceReqs(pod *v1.Pod) []devices.ContainerDeviceRequ } func (ads *AscendDevices) PatchAnnotations(pod *v1.Pod, annoInput *map[string]string, devList devices.PodSingleDevice) map[string]string { - dev, err := ads.getRandomDevice() + dev, err := ads.getFirstDevice() if err != nil { return *annoInput } diff --git a/pkg/scheduler/api/devices/ascend/device_info_test.go b/pkg/scheduler/api/devices/ascend/device_info_test.go index eebb04ee43..f753d5def6 100644 --- a/pkg/scheduler/api/devices/ascend/device_info_test.go +++ b/pkg/scheduler/api/devices/ascend/device_info_test.go @@ -14,20 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package ascend import ( @@ -160,7 +146,6 @@ func Test_trimMemory(t *testing.T) { } func Test_fit(t *testing.T) { - _, err := yamlStringToConfig(config_yaml) conf, err := yamlStringToConfig(config_yaml) assert.Nil(t, err) device_info := &devices.DeviceInfo{ diff --git a/pkg/scheduler/api/devices/device_info.go b/pkg/scheduler/api/devices/device_info.go index be13a22ebc..832d74e0aa 100644 --- a/pkg/scheduler/api/devices/device_info.go +++ b/pkg/scheduler/api/devices/device_info.go @@ -14,20 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package devices import ( @@ -203,7 +189,7 @@ func DecodePodDevices(checklist map[string]string, annos map[string]string) (Pod continue } pd[devID] = make(PodSingleDevice, 0) - for s := range strings.SplitSeq(str, OnePodMultiContainerSplitSymbol) { + for _, s := range strings.Split(str, OnePodMultiContainerSplitSymbol) { cd, err := DecodeContainerDevices(s) if err != nil { return PodDevices{}, nil diff --git a/pkg/scheduler/api/devices/util.go b/pkg/scheduler/api/devices/util.go index e04499e512..a028a08fcd 100644 --- a/pkg/scheduler/api/devices/util.go +++ b/pkg/scheduler/api/devices/util.go @@ -34,7 +34,6 @@ import ( "context" "encoding/json" "fmt" - "time" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -127,17 +126,6 @@ func GetNode(nodename string) (*v1.Node, error) { return n, nil } -func MarkAnnotationsToDelete(devType string, nn string) error { - tmppat := make(map[string]string) - tmppat[devType] = "Deleted_" + time.Now().Format(time.DateTime) - n, err := GetNode(nn) - if err != nil { - klog.Errorln("get node failed", err.Error()) - return err - } - return PatchNodeAnnotations(n, tmppat) -} - func PatchPodAnnotations(kubeClient kubernetes.Interface, pod *v1.Pod, annotations map[string]string) error { type patchMetadata struct { Annotations map[string]string `json:"annotations,omitempty"` diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index 3f9e3ccaf2..494740468a 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -517,10 +517,10 @@ func (ni *NodeInfo) addResource(pod *v1.Pod) { for _, name := range ascend.GetAscendDeviceNames() { if other, exists := ni.Others[name]; exists { if devices, ok := other.(Devices); ok { - devices.AddResource(pod) - } - } - } + devices.AddResource(pod) + } + } + } } // subResource is used to subtract sharable devices @@ -533,10 +533,10 @@ func (ni *NodeInfo) subResource(pod *v1.Pod) { for _, name := range ascend.GetAscendDeviceNames() { if other, exists := ni.Others[name]; exists { if devices, ok := other.(Devices); ok { - devices.SubResource(pod) - } - } - } + devices.SubResource(pod) + } + } + } } // UpdateTask is used to update a task in nodeInfo object. diff --git a/pkg/scheduler/plugins/deviceshare/deviceshare.go b/pkg/scheduler/plugins/deviceshare/deviceshare.go index 80bfa54e97..b1b2451600 100644 --- a/pkg/scheduler/plugins/deviceshare/deviceshare.go +++ b/pkg/scheduler/plugins/deviceshare/deviceshare.go @@ -137,7 +137,7 @@ func getDeviceScore(ctx context.Context, pod *v1.Pod, node *api.NodeInfo, schedu if device.(api.Devices).HasDeviceRequest(pod) { // Only process device types that use NodeOrderFn (vgpu and gpushare) // vnpu devices use BatchNodeOrderFn, skip them here - if deviceType != vnpu.NPUDevices { + if deviceType != vnpu.DeviceName { ns := device.(api.Devices).ScoreNode(pod, schedulePolicy) s += ns } From f9d3dfcfcef8b497e776fbbda5a4be913b43251a Mon Sep 17 00:00:00 2001 From: james Date: Thu, 6 Nov 2025 18:48:31 +0800 Subject: [PATCH 04/33] refactor: rm unused code in test Signed-off-by: james --- pkg/scheduler/api/devices/ascend/device_info_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/scheduler/api/devices/ascend/device_info_test.go b/pkg/scheduler/api/devices/ascend/device_info_test.go index f753d5def6..9393ec95c6 100644 --- a/pkg/scheduler/api/devices/ascend/device_info_test.go +++ b/pkg/scheduler/api/devices/ascend/device_info_test.go @@ -109,7 +109,6 @@ func yamlStringToConfig(yamlStr string) (*config.Config, error) { } func Test_trimMemory(t *testing.T) { - _, err := yamlStringToConfig(config_yaml) conf, err := yamlStringToConfig(config_yaml) assert.Nil(t, err) dev := AscendDevice{ From 702d7237d84675ceaa6f5754824a72d9dfb2efdd Mon Sep 17 00:00:00 2001 From: james Date: Fri, 7 Nov 2025 10:58:44 +0800 Subject: [PATCH 05/33] fix: fix cicd Signed-off-by: james --- .../api/devices/ascend/device_info.go | 3 +-- pkg/scheduler/api/devices/config/config.go | 2 +- pkg/scheduler/api/devices/device_info.go | 26 +++++++++---------- pkg/scheduler/api/shared_device_pool.go | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/pkg/scheduler/api/devices/ascend/device_info.go b/pkg/scheduler/api/devices/ascend/device_info.go index 6af30b4324..3a147cbaf1 100644 --- a/pkg/scheduler/api/devices/ascend/device_info.go +++ b/pkg/scheduler/api/devices/ascend/device_info.go @@ -74,7 +74,6 @@ type RuntimeInfo struct { var ( AscendVNPUEnable bool - configFile string NodeLockEnable bool ) @@ -374,7 +373,7 @@ func (ads *AscendDevices) selectDevices(pod *v1.Pod, schedulePolicy string) (dev selectedDevs := make([]*AscendDevice, 0) for _, dev := range availableDevs { klog.V(5).Infof("check fit. req %+v dev_info %+v dev_usage %+v", req, dev.DeviceInfo, dev.DeviceUsage) - if fit(&req, dev) == false { + if !fit(&req, dev) { klog.V(5).Infof("fit false. dev ID %s", dev.DeviceInfo.ID) continue } diff --git a/pkg/scheduler/api/devices/config/config.go b/pkg/scheduler/api/devices/config/config.go index d5dc56d5dc..d62ecd340b 100644 --- a/pkg/scheduler/api/devices/config/config.go +++ b/pkg/scheduler/api/devices/config/config.go @@ -67,7 +67,7 @@ const ( type Config struct { //NvidiaConfig is used for vGPU feature for nvidia, gpushare is not using this config NvidiaConfig NvidiaConfig `yaml:"nvidia"` - VNPUs []VNPUConfig `yaml:"vnpus"` + VNPUs []VNPUConfig `yaml:"vnpus"` } var ( diff --git a/pkg/scheduler/api/devices/device_info.go b/pkg/scheduler/api/devices/device_info.go index 832d74e0aa..11e9e140d7 100644 --- a/pkg/scheduler/api/devices/device_info.go +++ b/pkg/scheduler/api/devices/device_info.go @@ -28,14 +28,14 @@ import ( ) type DeviceInfo struct { - ID string `json:"id,omitempty"` - Index uint `json:"index,omitempty"` - Count int32 `json:"count,omitempty"` - Devmem int32 `json:"devmem,omitempty"` - Devcore int32 `json:"devcore,omitempty"` - Type string `json:"type,omitempty"` - Numa int `json:"numa,omitempty"` - Mode string `json:"mode,omitempty"` + ID string `json:"id,omitempty"` + Index uint `json:"index,omitempty"` + Count int32 `json:"count,omitempty"` + Devmem int32 `json:"devmem,omitempty"` + Devcore int32 `json:"devcore,omitempty"` + Type string `json:"type,omitempty"` + Numa int `json:"numa,omitempty"` + Mode string `json:"mode,omitempty"` // MIGTemplate []Geometry `json:"migtemplate,omitempty"` Health bool `json:"health,omitempty"` DeviceVendor string `json:"devicevendor,omitempty"` @@ -52,12 +52,12 @@ type DevicePairScore struct { type DeviceUsage struct { // ID string // Index uint - Used int32 + Used int32 // Count int32 - Usedmem int32 + Usedmem int32 // Totalmem int32 // Totalcore int32 - Usedcores int32 + Usedcores int32 // Mode string // MigTemplate []Geometry // MigUsage MigInUse @@ -100,9 +100,9 @@ const ( ) var ( - HandshakeAnnos = map[string]string{} + HandshakeAnnos = map[string]string{} InRequestDevices = map[string]string{} - SupportDevices = map[string]string{} + SupportDevices = map[string]string{} ) type ContainerDevices []ContainerDevice diff --git a/pkg/scheduler/api/shared_device_pool.go b/pkg/scheduler/api/shared_device_pool.go index b96196098f..f65826e7c4 100644 --- a/pkg/scheduler/api/shared_device_pool.go +++ b/pkg/scheduler/api/shared_device_pool.go @@ -22,8 +22,8 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/api/devices/ascend" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/gpushare" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/vgpu" ) From 4461b3fe58d960742e633ad476fc631889a3dada Mon Sep 17 00:00:00 2001 From: james Date: Fri, 7 Nov 2025 11:21:38 +0800 Subject: [PATCH 06/33] fix: fix style Signed-off-by: james --- .../api/devices/ascend/device_info_test.go | 110 +++++++++--------- pkg/scheduler/api/devices/config/vnpu.go | 2 +- pkg/scheduler/api/node_info.go | 2 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/pkg/scheduler/api/devices/ascend/device_info_test.go b/pkg/scheduler/api/devices/ascend/device_info_test.go index 9393ec95c6..52f831e01c 100644 --- a/pkg/scheduler/api/devices/ascend/device_info_test.go +++ b/pkg/scheduler/api/devices/ascend/device_info_test.go @@ -18,10 +18,10 @@ package ascend import ( "fmt" - "testing" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" - "volcano.sh/volcano/pkg/scheduler/api/devices" + "testing" + "volcano.sh/volcano/pkg/scheduler/api/devices" "volcano.sh/volcano/pkg/scheduler/api/devices/config" ) @@ -100,12 +100,12 @@ nvidia: ` func yamlStringToConfig(yamlStr string) (*config.Config, error) { - var config config.Config - err := yaml.Unmarshal([]byte(yamlStr), &config) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal YAML: %v", err) - } - return &config, nil + var config config.Config + err := yaml.Unmarshal([]byte(yamlStr), &config) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal YAML: %v", err) + } + return &config, nil } func Test_trimMemory(t *testing.T) { @@ -115,9 +115,9 @@ func Test_trimMemory(t *testing.T) { config: conf.VNPUs[0], } tests := []struct { - name string - inputMem int64 - wantMem int64 + name string + inputMem int64 + wantMem int64 }{ {"test1", 0, 3072}, {"test2", 1, 3072}, @@ -138,8 +138,8 @@ func Test_trimMemory(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, _ := dev.trimMemory(tt.inputMem) - assert.Equal(t, tt.wantMem, got) + got, _ := dev.trimMemory(tt.inputMem) + assert.Equal(t, tt.wantMem, got) }) } } @@ -148,30 +148,30 @@ func Test_fit(t *testing.T) { conf, err := yamlStringToConfig(config_yaml) assert.Nil(t, err) device_info := &devices.DeviceInfo{ - ID: "68496E64-20E05477-92C31323-6E78030A-BD003019", - Index: 0, - Count: 7, + ID: "68496E64-20E05477-92C31323-6E78030A-BD003019", + Index: 0, + Count: 7, Devcore: 8, - Devmem: 21527, + Devmem: 21527, } tests := []struct { - name string - req *devices.ContainerDeviceRequest - dev *AscendDevice - result bool + name string + req *devices.ContainerDeviceRequest + dev *AscendDevice + result bool }{ { "test1", - &devices.ContainerDeviceRequest { - Nums: 1, - Type: "Ascend310P", + &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", Memreq: 1024, }, - &AscendDevice { - config: conf.VNPUs[0], + &AscendDevice{ + config: conf.VNPUs[0], DeviceInfo: device_info, DeviceUsage: &devices.DeviceUsage{ - Used: 1, + Used: 1, Usedmem: 3072, }, }, @@ -179,16 +179,16 @@ func Test_fit(t *testing.T) { }, { "test2", - &devices.ContainerDeviceRequest { - Nums: 1, - Type: "Ascend310P", + &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", Memreq: 21527, }, - &AscendDevice { - config: conf.VNPUs[0], + &AscendDevice{ + config: conf.VNPUs[0], DeviceInfo: device_info, DeviceUsage: &devices.DeviceUsage{ - Used: 1, + Used: 1, Usedmem: 3072, }, }, @@ -196,16 +196,16 @@ func Test_fit(t *testing.T) { }, { "test3", - &devices.ContainerDeviceRequest { - Nums: 1, - Type: "Ascend310P", + &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", Memreq: 6144, }, - &AscendDevice { - config: conf.VNPUs[0], + &AscendDevice{ + config: conf.VNPUs[0], DeviceInfo: device_info, DeviceUsage: &devices.DeviceUsage{ - Used: 1, + Used: 1, Usedmem: 12288, }, }, @@ -213,16 +213,16 @@ func Test_fit(t *testing.T) { }, { "test4", - &devices.ContainerDeviceRequest { - Nums: 1, - Type: "Ascend310P", + &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", Memreq: 24576, }, - &AscendDevice { - config: conf.VNPUs[0], + &AscendDevice{ + config: conf.VNPUs[0], DeviceInfo: device_info, DeviceUsage: &devices.DeviceUsage{ - Used: 0, + Used: 0, Usedmem: 0, }, }, @@ -230,18 +230,18 @@ func Test_fit(t *testing.T) { }, { "test5_core", - &devices.ContainerDeviceRequest { - Nums: 1, - Type: "Ascend310P", - Memreq: 6144, + &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", + Memreq: 6144, Coresreq: 4, }, - &AscendDevice { - config: conf.VNPUs[0], + &AscendDevice{ + config: conf.VNPUs[0], DeviceInfo: device_info, DeviceUsage: &devices.DeviceUsage{ - Used: 1, - Usedmem: 12288, + Used: 1, + Usedmem: 12288, Usedcores: 6, }, }, @@ -250,8 +250,8 @@ func Test_fit(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ret := fit(tt.req, tt.dev) - assert.Equal(t, tt.result, ret) + ret := fit(tt.req, tt.dev) + assert.Equal(t, tt.result, ret) }) } } diff --git a/pkg/scheduler/api/devices/config/vnpu.go b/pkg/scheduler/api/devices/config/vnpu.go index dc02910352..7b986dbe4f 100644 --- a/pkg/scheduler/api/devices/config/vnpu.go +++ b/pkg/scheduler/api/devices/config/vnpu.go @@ -33,4 +33,4 @@ type VNPUConfig struct { AICore int32 `yaml:"aiCore"` AICPU int32 `yaml:"aiCPU"` Templates []Template `yaml:"templates"` -} \ No newline at end of file +} diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index 494740468a..57d8093463 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -24,10 +24,10 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" fwk "k8s.io/kube-scheduler/framework" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend" "volcano.sh/apis/pkg/apis/scheduling" "volcano.sh/apis/pkg/apis/scheduling/v1beta1" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend" "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/gpushare" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/vgpu" From 13fa9d9ef0a1109b76d63e228420861c3cddfb37 Mon Sep 17 00:00:00 2001 From: james Date: Fri, 7 Nov 2025 11:44:29 +0800 Subject: [PATCH 07/33] fix: fix comment Signed-off-by: james --- .../api/devices/ascend/device_info.go | 2 +- .../api/devices/ascend/device_info_test.go | 121 ++++++++++++++---- pkg/scheduler/api/devices/config/config.go | 32 +++++ pkg/scheduler/api/devices/device_info.go | 33 ++--- 4 files changed, 141 insertions(+), 47 deletions(-) diff --git a/pkg/scheduler/api/devices/ascend/device_info.go b/pkg/scheduler/api/devices/ascend/device_info.go index 3a147cbaf1..718e379b1e 100644 --- a/pkg/scheduler/api/devices/ascend/device_info.go +++ b/pkg/scheduler/api/devices/ascend/device_info.go @@ -26,10 +26,10 @@ import ( "time" "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + "volcano.sh/volcano/pkg/scheduler/api/devices" "volcano.sh/volcano/pkg/scheduler/api/devices/config" "volcano.sh/volcano/pkg/scheduler/plugins/util/nodelock" diff --git a/pkg/scheduler/api/devices/ascend/device_info_test.go b/pkg/scheduler/api/devices/ascend/device_info_test.go index 52f831e01c..60798c9d50 100644 --- a/pkg/scheduler/api/devices/ascend/device_info_test.go +++ b/pkg/scheduler/api/devices/ascend/device_info_test.go @@ -27,28 +27,47 @@ import ( var config_yaml = ` vnpus: -- chipName: 310P3 - - commonWord: Ascend310P - resourceName: huawei.com/Ascend310P - resourceMemoryName: huawei.com/Ascend310P-memory - memoryAllocatable: 21527 - memoryCapacity: 24576 - aiCore: 8 - aiCPU: 7 +- chipName: 910A + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + aiCore: 30 templates: - - name: vir01 - memory: 3072 - aiCore: 1 - aiCPU: 1 - name: vir02 - memory: 6144 + memory: 2184 aiCore: 2 - aiCPU: 2 - name: vir04 - memory: 12288 + memory: 4369 aiCore: 4 - aiCPU: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 +- chipName: 910B2 + commonWord: Ascend910B2 + resourceName: huawei.com/Ascend910B2 + resourceMemoryName: huawei.com/Ascend910B2-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 24 + aiCPU: 6 + templates: + - name: vir03_1c_8g + memory: 8192 + aiCore: 3 + aiCPU: 1 + - name: vir06_1c_16g + memory: 16384 + aiCore: 6 + aiCPU: 1 + - name: vir12_3c_32g + memory: 32768 + aiCore: 12 + aiCPU: 3 - chipName: 910B3 commonWord: Ascend910B3 resourceName: huawei.com/Ascend910B3 @@ -66,6 +85,61 @@ vnpus: memory: 32768 aiCore: 10 aiCPU: 3 +- chipName: 910B4-1 + commonWord: Ascend910B4-1 + resourceName: huawei.com/Ascend910B4-1 + resourceMemoryName: huawei.com/Ascend910B4-1-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 +- chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 +- chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 nvidia: resourceCountName: volcano.sh/vgpu-number resourceMemoryName: volcano.sh/vgpu-memory @@ -112,7 +186,7 @@ func Test_trimMemory(t *testing.T) { conf, err := yamlStringToConfig(config_yaml) assert.Nil(t, err) dev := AscendDevice{ - config: conf.VNPUs[0], + config: conf.VNPUs[len(conf.VNPUs)-1], } tests := []struct { name string @@ -147,6 +221,7 @@ func Test_trimMemory(t *testing.T) { func Test_fit(t *testing.T) { conf, err := yamlStringToConfig(config_yaml) assert.Nil(t, err) + ascend310PConfig := conf.VNPUs[len(conf.VNPUs)-1] device_info := &devices.DeviceInfo{ ID: "68496E64-20E05477-92C31323-6E78030A-BD003019", Index: 0, @@ -168,7 +243,7 @@ func Test_fit(t *testing.T) { Memreq: 1024, }, &AscendDevice{ - config: conf.VNPUs[0], + config: ascend310PConfig, DeviceInfo: device_info, DeviceUsage: &devices.DeviceUsage{ Used: 1, @@ -185,7 +260,7 @@ func Test_fit(t *testing.T) { Memreq: 21527, }, &AscendDevice{ - config: conf.VNPUs[0], + config: ascend310PConfig, DeviceInfo: device_info, DeviceUsage: &devices.DeviceUsage{ Used: 1, @@ -202,7 +277,7 @@ func Test_fit(t *testing.T) { Memreq: 6144, }, &AscendDevice{ - config: conf.VNPUs[0], + config: ascend310PConfig, DeviceInfo: device_info, DeviceUsage: &devices.DeviceUsage{ Used: 1, @@ -219,7 +294,7 @@ func Test_fit(t *testing.T) { Memreq: 24576, }, &AscendDevice{ - config: conf.VNPUs[0], + config: ascend310PConfig, DeviceInfo: device_info, DeviceUsage: &devices.DeviceUsage{ Used: 0, @@ -237,7 +312,7 @@ func Test_fit(t *testing.T) { Coresreq: 4, }, &AscendDevice{ - config: conf.VNPUs[0], + config: ascend310PConfig, DeviceInfo: device_info, DeviceUsage: &devices.DeviceUsage{ Used: 1, diff --git a/pkg/scheduler/api/devices/config/config.go b/pkg/scheduler/api/devices/config/config.go index d62ecd340b..0878e7d6d8 100644 --- a/pkg/scheduler/api/devices/config/config.go +++ b/pkg/scheduler/api/devices/config/config.go @@ -116,6 +116,38 @@ func GetDefaultDevicesConfig() *Config { DeviceCoreScaling: 1, DisableCoreLimit: false, }, + VNPUs: []VNPUConfig{ + { + CommonWord: "Ascend310P", + ChipName: "310P3", + ResourceName: "huawei.com/Ascend310P", + ResourceMemoryName: "huawei.com/Ascend310P-memory", + MemoryAllocatable: 21527, + MemoryCapacity: 24576, + AICore: 8, + AICPU: 7, + Templates: []Template{ + { + Name: "vir01", + Memory: 3072, + AICore: 1, + AICPU: 1, + }, + { + Name: "vir02", + Memory: 6144, + AICore: 2, + AICPU: 2, + }, + { + Name: "vir04", + Memory: 12288, + AICore: 4, + AICPU: 4, + }, + }, + }, + }, } } diff --git a/pkg/scheduler/api/devices/device_info.go b/pkg/scheduler/api/devices/device_info.go index 11e9e140d7..adf51db333 100644 --- a/pkg/scheduler/api/devices/device_info.go +++ b/pkg/scheduler/api/devices/device_info.go @@ -28,15 +28,14 @@ import ( ) type DeviceInfo struct { - ID string `json:"id,omitempty"` - Index uint `json:"index,omitempty"` - Count int32 `json:"count,omitempty"` - Devmem int32 `json:"devmem,omitempty"` - Devcore int32 `json:"devcore,omitempty"` - Type string `json:"type,omitempty"` - Numa int `json:"numa,omitempty"` - Mode string `json:"mode,omitempty"` - // MIGTemplate []Geometry `json:"migtemplate,omitempty"` + ID string `json:"id,omitempty"` + Index uint `json:"index,omitempty"` + Count int32 `json:"count,omitempty"` + Devmem int32 `json:"devmem,omitempty"` + Devcore int32 `json:"devcore,omitempty"` + Type string `json:"type,omitempty"` + Numa int `json:"numa,omitempty"` + Mode string `json:"mode,omitempty"` Health bool `json:"health,omitempty"` DeviceVendor string `json:"devicevendor,omitempty"` CustomInfo map[string]any `json:"custominfo,omitempty"` @@ -50,21 +49,9 @@ type DevicePairScore struct { } type DeviceUsage struct { - // ID string - // Index uint - Used int32 - // Count int32 - Usedmem int32 - // Totalmem int32 - // Totalcore int32 + Used int32 + Usedmem int32 Usedcores int32 - // Mode string - // MigTemplate []Geometry - // MigUsage MigInUse - // Numa int - // Type string - // Health bool - // CustomInfo map[string]any } type ResourceNames struct { From 8502b352ecc3d8f12782f1c342fe46fb6bbe1246 Mon Sep 17 00:00:00 2001 From: james Date: Fri, 7 Nov 2025 14:56:57 +0800 Subject: [PATCH 08/33] docs: add docs Signed-off-by: james --- .../how_to_use_hami_ascend_device_pulgin.md | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 docs/user-guide/how_to_use_hami_ascend_device_pulgin.md diff --git a/docs/user-guide/how_to_use_hami_ascend_device_pulgin.md b/docs/user-guide/how_to_use_hami_ascend_device_pulgin.md new file mode 100644 index 0000000000..282ac3ad9c --- /dev/null +++ b/docs/user-guide/how_to_use_hami_ascend_device_pulgin.md @@ -0,0 +1,74 @@ +# HAMi ascend-device-plugin User Guide + +## Installation + +To enable vGPU scheduling, the following components must be set up based on the selected mode: + + +**Prerequisites**: + +Kubernetes >= 1.16 +Volcano >= 1.14 +[ascend-docker-runtime](https://gitcode.com/Ascend/mind-cluster/tree/master/component/ascend-docker-runtime) + +### Install Volcano: + +Follow instructions in Volcano Installer Guide + + * Follow instructions in [Volcano Installer Guide](https://github.com/volcano-sh/volcano?tab=readme-ov-file#quick-start-guide) + +### Install HAMI ascend-device-plugin + +#### Deploy `hami-scheduler-device` config map + +``` +kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/hami-scheduler-device.yaml +``` + +#### Deploy ascend-device-plugin + +``` +kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-plugin.yaml +``` + +refer https://github.com/Project-HAMi/ascend-device-plugin + +### Scheduler Config Update +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: volcano-scheduler-configmap + namespace: volcano-system +data: + volcano-scheduler.conf: | + actions: "enqueue, allocate, backfill" + tiers: + - plugins: + - name: predicates + - name: deviceshare + arguments: + deviceshare.AscendVNPUEnable: true # enable ascend vnpu + deviceshare.SchedulePolicy: binpack # scheduling policy. binpack / spread +``` + +## Usage + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: ascend-pod +spec: + schedulerName: volcano + containers: + - name: ubuntu-container + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/ascend-pytorch:24.0.RC1-A2-1.11.0-ubuntu20.04 + command: ["sleep"] + args: ["100000"] + resources: + limits: + huawei.com/Ascend310P: "1" + huawei.com/Ascend310P-memory: "4096" + +``` \ No newline at end of file From f8ff3555e504801d5ef77e718b832e08fb5145cf Mon Sep 17 00:00:00 2001 From: james Date: Fri, 7 Nov 2025 15:27:14 +0800 Subject: [PATCH 09/33] fix: fix commit Signed-off-by: james --- .../how_to_use_hami_ascend_device_pulgin.md | 2 ++ pkg/scheduler/api/devices/ascend/device_info.go | 11 ++++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/user-guide/how_to_use_hami_ascend_device_pulgin.md b/docs/user-guide/how_to_use_hami_ascend_device_pulgin.md index 282ac3ad9c..ff5a0b0bdb 100644 --- a/docs/user-guide/how_to_use_hami_ascend_device_pulgin.md +++ b/docs/user-guide/how_to_use_hami_ascend_device_pulgin.md @@ -50,6 +50,8 @@ data: arguments: deviceshare.AscendVNPUEnable: true # enable ascend vnpu deviceshare.SchedulePolicy: binpack # scheduling policy. binpack / spread + deviceshare.KnownGeometriesCMNamespace: kube-system + deviceshare.KnownGeometriesCMName: hami-scheduler-device ``` ## Usage diff --git a/pkg/scheduler/api/devices/ascend/device_info.go b/pkg/scheduler/api/devices/ascend/device_info.go index 718e379b1e..29c2320c28 100644 --- a/pkg/scheduler/api/devices/ascend/device_info.go +++ b/pkg/scheduler/api/devices/ascend/device_info.go @@ -45,8 +45,6 @@ const ( spreadPolicy = "spread" binpackMultiplier = 100 spreadMultiplier = 100 - CMName = "volcano-vgpu-device-config" - CMNamespace = "kube-system" ) type AscendDevice struct { @@ -85,9 +83,8 @@ func NewAscendDevices(name string, node *v1.Node) map[string]*AscendDevices { } curConfig := config.GetConfig() if curConfig == nil { - klog.V(5).InfoS("cur config is null. call InitDevicesConfig") - config.InitDevicesConfig(CMName, CMNamespace) - curConfig = config.GetConfig() + klog.V(5).InfoS("cur config is null. call GetDefaultDevicesConfig") + curConfig = config.GetDefaultDevicesConfig() } devs := InitDevices(curConfig.VNPUs) for _, dev := range devs { @@ -126,8 +123,8 @@ func NewAscendDevices(name string, node *v1.Node) map[string]*AscendDevices { func GetAscendDeviceNames() []string { curConfig := config.GetConfig() if curConfig == nil { - config.InitDevicesConfig(CMName, CMNamespace) - curConfig = config.GetConfig() + klog.V(5).InfoS("cur config is null. call GetDefaultDevicesConfig") + curConfig = config.GetDefaultDevicesConfig() } deviceNames := make([]string, 0, len(curConfig.VNPUs)) for _, vnpu := range curConfig.VNPUs { From d11d22d83b1428bbbe7ae6cca4834bce190f8839 Mon Sep 17 00:00:00 2001 From: james Date: Fri, 7 Nov 2025 18:26:03 +0800 Subject: [PATCH 10/33] fix: fix the comment about log level Signed-off-by: james --- pkg/scheduler/plugins/deviceshare/deviceshare.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/plugins/deviceshare/deviceshare.go b/pkg/scheduler/plugins/deviceshare/deviceshare.go index b1b2451600..1545d44d1b 100644 --- a/pkg/scheduler/plugins/deviceshare/deviceshare.go +++ b/pkg/scheduler/plugins/deviceshare/deviceshare.go @@ -116,7 +116,7 @@ func registerDevices() { once.Do(func() { if ascend.AscendVNPUEnable { for _, vnpu := range config.GetConfig().VNPUs { - klog.Infof("register device %s", vnpu.CommonWord) + klog.V(3).Infof("register device %s", vnpu.CommonWord) api.RegisterDevice(vnpu.CommonWord) } } From 6b81acf119b52f6aba9860ae0a9353acc85f6a24 Mon Sep 17 00:00:00 2001 From: james Date: Mon, 10 Nov 2025 10:43:58 +0800 Subject: [PATCH 11/33] fix: fix comment Signed-off-by: james --- ....md => how_to_use_hami_ascend_device_plugin.md} | 2 +- .../devices/ascend/ascend310p/vnpu/device_info.go | 2 +- .../api/devices/ascend/ascend310p/vnpu/type.go | 2 +- pkg/scheduler/api/devices/ascend/device_info.go | 10 +++++----- pkg/scheduler/api/devices/device_info.go | 2 +- pkg/scheduler/api/shared_device_pool.go | 5 +++++ pkg/scheduler/plugins/deviceshare/deviceshare.go | 14 +++++++------- 7 files changed, 21 insertions(+), 16 deletions(-) rename docs/user-guide/{how_to_use_hami_ascend_device_pulgin.md => how_to_use_hami_ascend_device_plugin.md} (96%) diff --git a/docs/user-guide/how_to_use_hami_ascend_device_pulgin.md b/docs/user-guide/how_to_use_hami_ascend_device_plugin.md similarity index 96% rename from docs/user-guide/how_to_use_hami_ascend_device_pulgin.md rename to docs/user-guide/how_to_use_hami_ascend_device_plugin.md index ff5a0b0bdb..a9c78e3259 100644 --- a/docs/user-guide/how_to_use_hami_ascend_device_pulgin.md +++ b/docs/user-guide/how_to_use_hami_ascend_device_plugin.md @@ -48,7 +48,7 @@ data: - name: predicates - name: deviceshare arguments: - deviceshare.AscendVNPUEnable: true # enable ascend vnpu + deviceshare.AscendHAMiVNPUEnable: true # enable ascend vnpu deviceshare.SchedulePolicy: binpack # scheduling policy. binpack / spread deviceshare.KnownGeometriesCMNamespace: kube-system deviceshare.KnownGeometriesCMName: hami-scheduler-device diff --git a/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/device_info.go b/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/device_info.go index ad6c0eb336..3876fa1bf5 100644 --- a/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/device_info.go +++ b/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/device_info.go @@ -155,7 +155,7 @@ func (ns *NPUDevices) AddQueueResource(pod *v1.Pod) map[string]float64 { } func (ns *NPUDevices) HasDeviceRequest(pod *v1.Pod) bool { - if Ascend310pvNPUEnable && checkVNPUResourcesInPod(pod) { + if AscendMindClusterVNPUEnable && checkVNPUResourcesInPod(pod) { return true } return false diff --git a/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/type.go b/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/type.go index bda9ba3925..13cf285e97 100644 --- a/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/type.go +++ b/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/type.go @@ -9,7 +9,7 @@ import ( "k8s.io/client-go/kubernetes" ) -var Ascend310pvNPUEnable bool +var AscendMindClusterVNPUEnable bool const ( DeviceName = "ascend310p-vNPU" diff --git a/pkg/scheduler/api/devices/ascend/device_info.go b/pkg/scheduler/api/devices/ascend/device_info.go index 29c2320c28..af3e98baf8 100644 --- a/pkg/scheduler/api/devices/ascend/device_info.go +++ b/pkg/scheduler/api/devices/ascend/device_info.go @@ -71,8 +71,8 @@ type RuntimeInfo struct { } var ( - AscendVNPUEnable bool - NodeLockEnable bool + AscendHAMiVNPUEnable bool + NodeLockEnable bool ) func NewAscendDevices(name string, node *v1.Node) map[string]*AscendDevices { @@ -202,7 +202,7 @@ func (ads *AscendDevices) AddQueueResource(pod *v1.Pod) map[string]float64 { } func (ads *AscendDevices) HasDeviceRequest(pod *v1.Pod) bool { - if !AscendVNPUEnable { + if !AscendHAMiVNPUEnable { return false } randDev, err := ads.getFirstDevice() @@ -547,7 +547,7 @@ func InitDevices(config []config.VNPUConfig) []*AscendDevice { } func ParseConfig(fs *flag.FlagSet) { - fs.BoolVar(&AscendVNPUEnable, "AscendVNPUEnable", false, "enable ascend device") + fs.BoolVar(&AscendHAMiVNPUEnable, "AscendHAMiVNPUEnable", false, "enable ascend device") } func (dev *AscendDevice) CommonWord() string { @@ -565,7 +565,7 @@ func (dev *AscendDevice) GetNodeDevices(n v1.Node) ([]*devices.DeviceInfo, error return []*devices.DeviceInfo{}, err } if len(nodeDevices) == 0 { - klog.InfoS("no gpu device found", "node", n.Name, "device annotation", anno) + klog.InfoS("no ascend device found", "node", n.Name, "device annotation", anno) return []*devices.DeviceInfo{}, errors.New("no device found on node") } return nodeDevices, nil diff --git a/pkg/scheduler/api/devices/device_info.go b/pkg/scheduler/api/devices/device_info.go index adf51db333..54f2696cc9 100644 --- a/pkg/scheduler/api/devices/device_info.go +++ b/pkg/scheduler/api/devices/device_info.go @@ -179,7 +179,7 @@ func DecodePodDevices(checklist map[string]string, annos map[string]string) (Pod for _, s := range strings.Split(str, OnePodMultiContainerSplitSymbol) { cd, err := DecodeContainerDevices(s) if err != nil { - return PodDevices{}, nil + return PodDevices{}, err } if len(cd) == 0 { continue diff --git a/pkg/scheduler/api/shared_device_pool.go b/pkg/scheduler/api/shared_device_pool.go index f65826e7c4..cff4abb7f2 100644 --- a/pkg/scheduler/api/shared_device_pool.go +++ b/pkg/scheduler/api/shared_device_pool.go @@ -88,6 +88,11 @@ var RegisteredDevices = []string{ } func RegisterDevice(deviceName string) { + for _, name := range RegisteredDevices { + if name == deviceName { + return + } + } RegisteredDevices = append(RegisteredDevices, deviceName) } diff --git a/pkg/scheduler/plugins/deviceshare/deviceshare.go b/pkg/scheduler/plugins/deviceshare/deviceshare.go index 1545d44d1b..e2d9afe0d4 100644 --- a/pkg/scheduler/plugins/deviceshare/deviceshare.go +++ b/pkg/scheduler/plugins/deviceshare/deviceshare.go @@ -48,8 +48,8 @@ const ( VGPUEnable = "deviceshare.VGPUEnable" - ASCEND310PvGPU = "deviceshare.ASCEND310PVNPUEnable" - AscendVNPUEnable = "deviceshare.AscendVNPUEnable" + ASCEND310PvGPU = "deviceshare.AscendMindClusterVNPUEnable" + AscendHAMiVNPUEnable = "deviceshare.AscendHAMiVNPUEnable" SchedulePolicyArgument = "deviceshare.SchedulePolicy" ScheduleWeight = "deviceshare.ScheduleWeight" @@ -88,8 +88,8 @@ func enablePredicate(dsp *deviceSharePlugin) { args.GetBool(&gpushare.GpuNumberEnable, GPUNumberPredicate) args.GetBool(&nodeLockEnable, NodeLockEnable) args.GetBool(&vgpu.VGPUEnable, VGPUEnable) - args.GetBool(&vnpu.Ascend310pvNPUEnable, ASCEND310PvGPU) - args.GetBool(&ascend.AscendVNPUEnable, AscendVNPUEnable) + args.GetBool(&vnpu.AscendMindClusterVNPUEnable, ASCEND310PvGPU) + args.GetBool(&ascend.AscendHAMiVNPUEnable, AscendHAMiVNPUEnable) gpushare.NodeLockEnable = nodeLockEnable vgpu.NodeLockEnable = nodeLockEnable @@ -114,7 +114,7 @@ func enablePredicate(dsp *deviceSharePlugin) { func registerDevices() { once.Do(func() { - if ascend.AscendVNPUEnable { + if ascend.AscendHAMiVNPUEnable { for _, vnpu := range config.GetConfig().VNPUs { klog.V(3).Infof("register device %s", vnpu.CommonWord) api.RegisterDevice(vnpu.CommonWord) @@ -185,7 +185,7 @@ func initializeDevicesWithSession(ssn *framework.Session) { func initializeDevice(device api.Devices, ssn *framework.Session, nodeInfo *api.NodeInfo) error { switch d := device.(type) { case *vnpu.NPUDevices: - if vnpu.Ascend310pvNPUEnable { + if vnpu.AscendMindClusterVNPUEnable { klog.V(3).Infof("initialize ascend310p device.") return vnpu310p.InitVNPUDevice(d, ssn, nodeInfo) } @@ -213,7 +213,7 @@ func (dp *deviceSharePlugin) OnSessionOpen(ssn *framework.Session) { }) return api.NewFitErrWithStatus(task, node, predicateStatus...) } - klog.V(4).Infof("pod %s/%s did not request device %s on %s, skipping it", task.Pod.Namespace, task.Pod.Name, val, node.Name) + klog.V(4).Infof("device %s is null, skipping it", val) continue } if !dev.HasDeviceRequest(task.Pod) { From c8a7544bff10b1a80b62b6387596692923268795 Mon Sep 17 00:00:00 2001 From: james Date: Mon, 10 Nov 2025 10:47:08 +0800 Subject: [PATCH 12/33] docs: update user guide Signed-off-by: james --- .../how_to_use_hami_ascend_device_plugin.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/user-guide/how_to_use_hami_ascend_device_plugin.md b/docs/user-guide/how_to_use_hami_ascend_device_plugin.md index a9c78e3259..4e6dcb0633 100644 --- a/docs/user-guide/how_to_use_hami_ascend_device_plugin.md +++ b/docs/user-guide/how_to_use_hami_ascend_device_plugin.md @@ -73,4 +73,15 @@ spec: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "4096" -``` \ No newline at end of file +``` + +The supported Ascend chips are shown in the following table: + +| ChipName | ResourceName | ResourceMemoryName | +|-------|-------|-------| +| 910A | huawei.com/Ascend910A | huawei.com/Ascend910A-memory | +| 910B2 | huawei.com/Ascend910B2 | huawei.com/Ascend910B2-memory | +| 910B3 | huawei.com/Ascend910B3 | huawei.com/Ascend910B3-memory | +| 910B4 | huawei.com/Ascend910B4 | huawei.com/Ascend910B4-memory | +| 910B4-1 | huawei.com/Ascend910B4-1 | huawei.com/Ascend910B4-1-memory | +| 310P3 | huawei.com/Ascend310P | huawei.com/Ascend310P-memory | \ No newline at end of file From 20ed06300ec3ad35976999ef4de01dcf7e24e634 Mon Sep 17 00:00:00 2001 From: james Date: Mon, 10 Nov 2025 10:50:55 +0800 Subject: [PATCH 13/33] fix: fix ASCEND310P names Signed-off-by: james --- pkg/scheduler/plugins/deviceshare/deviceshare.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/scheduler/plugins/deviceshare/deviceshare.go b/pkg/scheduler/plugins/deviceshare/deviceshare.go index e2d9afe0d4..0a4019bf01 100644 --- a/pkg/scheduler/plugins/deviceshare/deviceshare.go +++ b/pkg/scheduler/plugins/deviceshare/deviceshare.go @@ -48,8 +48,8 @@ const ( VGPUEnable = "deviceshare.VGPUEnable" - ASCEND310PvGPU = "deviceshare.AscendMindClusterVNPUEnable" - AscendHAMiVNPUEnable = "deviceshare.AscendHAMiVNPUEnable" + AscendMindClusterVNPU = "deviceshare.AscendMindClusterVNPUEnable" + AscendHAMiVNPUEnable = "deviceshare.AscendHAMiVNPUEnable" SchedulePolicyArgument = "deviceshare.SchedulePolicy" ScheduleWeight = "deviceshare.ScheduleWeight" @@ -88,7 +88,7 @@ func enablePredicate(dsp *deviceSharePlugin) { args.GetBool(&gpushare.GpuNumberEnable, GPUNumberPredicate) args.GetBool(&nodeLockEnable, NodeLockEnable) args.GetBool(&vgpu.VGPUEnable, VGPUEnable) - args.GetBool(&vnpu.AscendMindClusterVNPUEnable, ASCEND310PvGPU) + args.GetBool(&vnpu.AscendMindClusterVNPUEnable, AscendMindClusterVNPU) args.GetBool(&ascend.AscendHAMiVNPUEnable, AscendHAMiVNPUEnable) gpushare.NodeLockEnable = nodeLockEnable From 7ac7f2941d5e1ce91b6e88355e844286084c3344 Mon Sep 17 00:00:00 2001 From: limengxuan Date: Mon, 10 Nov 2025 12:04:12 +0800 Subject: [PATCH 14/33] update documents Signed-off-by: limengxuan --- ...nd_device_plugin.md => how_to_use_vnpu.md} | 67 +++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) rename docs/user-guide/{how_to_use_hami_ascend_device_plugin.md => how_to_use_vnpu.md} (56%) diff --git a/docs/user-guide/how_to_use_hami_ascend_device_plugin.md b/docs/user-guide/how_to_use_vnpu.md similarity index 56% rename from docs/user-guide/how_to_use_hami_ascend_device_plugin.md rename to docs/user-guide/how_to_use_vnpu.md index 4e6dcb0633..4b75d24ec5 100644 --- a/docs/user-guide/how_to_use_hami_ascend_device_plugin.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -1,15 +1,44 @@ # HAMi ascend-device-plugin User Guide +## Introduction + +Volcano supports **two vNPU modes** for sharing Ascend devices: + +--- + +### 1. Mindcluster mode + +**Description**: +MindCluster, formerly known as [Ascend/ascend-device-plugin](https://gitee.com/ascend/ascend-device-plugin) is an official device plugin, which supports npu cluster for all Ascend series and vnpu feature for Ascend 310 series. + +**Use case**: +NPU cluster for Ascend 910 series +NPU and vNPU cluster for Ascend 310 series + +--- + +### 2. HAMi mode + +**Description**: +This mode is developed by a third-party community 'HAMi', which is the developer of [volcano-vgpu](./how_to_use_volcano_vgpu.md) feature, It supports vNPU feature for both Ascend 310 and Ascend 910. It also support managing heterogeneous Ascend cluster(Cluster with multiple Ascend types,i.e 910A,910B2,910B3,310p) + +**Use case**: +NPU and vNPU cluster for Ascend 910 series +NPU and vNPU cluster for Ascend 310 series +Heterogeneous Ascend cluster + +--- + ## Installation -To enable vGPU scheduling, the following components must be set up based on the selected mode: +To enable vNPU scheduling, the following components must be set up based on the selected mode: **Prerequisites**: Kubernetes >= 1.16 Volcano >= 1.14 -[ascend-docker-runtime](https://gitcode.com/Ascend/mind-cluster/tree/master/component/ascend-docker-runtime) +[ascend-docker-runtime](https://gitcode.com/Ascend/mind-cluster/tree/master/component/ascend-docker-runtime) (for HAMi Mode) ### Install Volcano: @@ -17,7 +46,21 @@ Follow instructions in Volcano Installer Guide * Follow instructions in [Volcano Installer Guide](https://github.com/volcano-sh/volcano?tab=readme-ov-file#quick-start-guide) -### Install HAMI ascend-device-plugin +### Install ascend-device-plugin + +In this step, you need to select different ascend-device-plugin based on the vNPU mode you selected. + +--- + +#### MindCluster Mode + +``` +Wait for @JackyTYang to fill +``` + +--- + +#### HAMi mode #### Deploy `hami-scheduler-device` config map @@ -54,8 +97,24 @@ data: deviceshare.KnownGeometriesCMName: hami-scheduler-device ``` + **Note:** You may noticed that, 'volcano-vgpu' has its own GeometriesCMName and GeometriesCMNamespace, which means if you want to use both vNPU and vGPU in a same volcano cluster, you need to merge the configMap from both sides and set it here. + ## Usage +Usage is different depending on the mode you selected + +--- + +### MindCluster mode + +``` +Wait for @JackyTYang to fill +``` + +--- + +### HAMi mode + ```yaml apiVersion: v1 kind: Pod @@ -75,7 +134,7 @@ spec: ``` -The supported Ascend chips are shown in the following table: +The supported Ascend chips and their resouruceNames are shown in the following table: | ChipName | ResourceName | ResourceMemoryName | |-------|-------|-------| From 9e4105540a56e751f8f0bd971fa89c8fe91124cd Mon Sep 17 00:00:00 2001 From: limengxuan Date: Mon, 10 Nov 2025 14:59:14 +0800 Subject: [PATCH 15/33] update documents Signed-off-by: limengxuan --- docs/user-guide/how_to_use_vnpu.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/user-guide/how_to_use_vnpu.md b/docs/user-guide/how_to_use_vnpu.md index 4b75d24ec5..3de43e7d83 100644 --- a/docs/user-guide/how_to_use_vnpu.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -9,9 +9,11 @@ Volcano supports **two vNPU modes** for sharing Ascend devices: ### 1. Mindcluster mode **Description**: + MindCluster, formerly known as [Ascend/ascend-device-plugin](https://gitee.com/ascend/ascend-device-plugin) is an official device plugin, which supports npu cluster for all Ascend series and vnpu feature for Ascend 310 series. **Use case**: + NPU cluster for Ascend 910 series NPU and vNPU cluster for Ascend 310 series @@ -20,9 +22,11 @@ NPU and vNPU cluster for Ascend 310 series ### 2. HAMi mode **Description**: + This mode is developed by a third-party community 'HAMi', which is the developer of [volcano-vgpu](./how_to_use_volcano_vgpu.md) feature, It supports vNPU feature for both Ascend 310 and Ascend 910. It also support managing heterogeneous Ascend cluster(Cluster with multiple Ascend types,i.e 910A,910B2,910B3,310p) **Use case**: + NPU and vNPU cluster for Ascend 910 series NPU and vNPU cluster for Ascend 310 series Heterogeneous Ascend cluster @@ -62,13 +66,13 @@ Wait for @JackyTYang to fill #### HAMi mode -#### Deploy `hami-scheduler-device` config map +##### Deploy `hami-scheduler-device` config map ``` kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/hami-scheduler-device.yaml ``` -#### Deploy ascend-device-plugin +##### Deploy ascend-device-plugin ``` kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-plugin.yaml @@ -76,7 +80,7 @@ kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-pl refer https://github.com/Project-HAMi/ascend-device-plugin -### Scheduler Config Update +##### Scheduler Config Update ```yaml kind: ConfigMap apiVersion: v1 From 9778f770cb5bc4541e728b5077d05d82247af13e Mon Sep 17 00:00:00 2001 From: limengxuan Date: Mon, 10 Nov 2025 15:07:50 +0800 Subject: [PATCH 16/33] update documents Signed-off-by: limengxuan --- docs/user-guide/how_to_use_vnpu.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/user-guide/how_to_use_vnpu.md b/docs/user-guide/how_to_use_vnpu.md index 3de43e7d83..08db4cefeb 100644 --- a/docs/user-guide/how_to_use_vnpu.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -14,7 +14,7 @@ MindCluster, formerly known as [Ascend/ascend-device-plugin](https://gitee.com/a **Use case**: -NPU cluster for Ascend 910 series +NPU cluster for Ascend 910 series NPU and vNPU cluster for Ascend 310 series --- @@ -27,8 +27,8 @@ This mode is developed by a third-party community 'HAMi', which is the developer **Use case**: -NPU and vNPU cluster for Ascend 910 series -NPU and vNPU cluster for Ascend 310 series +NPU and vNPU cluster for Ascend 910 series +NPU and vNPU cluster for Ascend 310 series Heterogeneous Ascend cluster --- From 15632818c60bcac1ef25ce57e84e841115678669 Mon Sep 17 00:00:00 2001 From: james Date: Mon, 10 Nov 2025 16:15:36 +0800 Subject: [PATCH 17/33] fix: fix the title of docs Signed-off-by: james --- docs/user-guide/how_to_use_vnpu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/how_to_use_vnpu.md b/docs/user-guide/how_to_use_vnpu.md index 08db4cefeb..6aa4bcc621 100644 --- a/docs/user-guide/how_to_use_vnpu.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -1,4 +1,4 @@ -# HAMi ascend-device-plugin User Guide +# Ascend vNPU User Guide ## Introduction From 6f392a082eb281057fd06e4e22d3788f670f53c3 Mon Sep 17 00:00:00 2001 From: james Date: Mon, 10 Nov 2025 16:21:46 +0800 Subject: [PATCH 18/33] fix: fix typo Signed-off-by: james --- docs/user-guide/how_to_use_vnpu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/how_to_use_vnpu.md b/docs/user-guide/how_to_use_vnpu.md index 6aa4bcc621..1eb2c42c8a 100644 --- a/docs/user-guide/how_to_use_vnpu.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -138,7 +138,7 @@ spec: ``` -The supported Ascend chips and their resouruceNames are shown in the following table: +The supported Ascend chips and their `ResourceNames` are shown in the following table: | ChipName | ResourceName | ResourceMemoryName | |-------|-------|-------| From 4c58fc2e0d3d0554db5317f6a6eb0b846c0f5cae Mon Sep 17 00:00:00 2001 From: james Date: Mon, 10 Nov 2025 19:03:17 +0800 Subject: [PATCH 19/33] docs: update ascend-device-configmap file name Signed-off-by: james --- docs/user-guide/how_to_use_vnpu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/how_to_use_vnpu.md b/docs/user-guide/how_to_use_vnpu.md index 1eb2c42c8a..0a4ae7931e 100644 --- a/docs/user-guide/how_to_use_vnpu.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -69,7 +69,7 @@ Wait for @JackyTYang to fill ##### Deploy `hami-scheduler-device` config map ``` -kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/hami-scheduler-device.yaml +kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-configmap.yaml ``` ##### Deploy ascend-device-plugin From 65d75b4ded06dce2858a4b192acaff30d9b8bc62 Mon Sep 17 00:00:00 2001 From: JackyTYang Date: Tue, 11 Nov 2025 15:54:05 +0800 Subject: [PATCH 20/33] add user guide of mindcluster mode in the doc Signed-off-by: JackyTYang --- docs/user-guide/how_to_use_vnpu.md | 153 +++++++++++++++++- .../devices/ascend/ascend310p/vnpu/type.go | 4 +- 2 files changed, 147 insertions(+), 10 deletions(-) diff --git a/docs/user-guide/how_to_use_vnpu.md b/docs/user-guide/how_to_use_vnpu.md index 1eb2c42c8a..54bc10ff39 100644 --- a/docs/user-guide/how_to_use_vnpu.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -10,12 +10,14 @@ Volcano supports **two vNPU modes** for sharing Ascend devices: **Description**: -MindCluster, formerly known as [Ascend/ascend-device-plugin](https://gitee.com/ascend/ascend-device-plugin) is an official device plugin, which supports npu cluster for all Ascend series and vnpu feature for Ascend 310 series. +The initial version of [MindCluster](https://gitcode.com/Ascend/mind-cluster)—the official Ascend cluster scheduling add-on—required custom modifications and recompilation of Volcano. Furthermore, it was limited to Volcano release1.7 and release1.9, which complicated its use and restricted access to newer Volcano features. + +To address this, we have integrated its core scheduling logic for Ascend vNPU into Volcano's native device-share plugin，which is designed specifically for scheduling and sharing heterogeneous resources like GPUs and NPUs. This integration provides seamless access to vNPU capabilities through the procedure below, while maintaining full compatibility with the latest Volcano features. **Use case**: -NPU cluster for Ascend 910 series -NPU and vNPU cluster for Ascend 310 series +vNPU cluster for Ascend 310 series +with support for more chip types to come --- @@ -50,16 +52,63 @@ Follow instructions in Volcano Installer Guide * Follow instructions in [Volcano Installer Guide](https://github.com/volcano-sh/volcano?tab=readme-ov-file#quick-start-guide) -### Install ascend-device-plugin +### Install ascend-device-plugin and third-party components -In this step, you need to select different ascend-device-plugin based on the vNPU mode you selected. +In this step, you need to select different ascend-device-plugin based on the vNPU mode you selected. MindCluster mode requires additional components from Ascend to be installed. --- #### MindCluster Mode +##### Install Third-Party Components + +Follow the official [Ascend documentation](https://www.hiascend.com/document/detail/zh/mindcluster/72rc1/clustersched/dlug/mxdlug_start_006.html#ZH-CN_TOPIC_0000002470358262__section1837511531098) to install the following components: +- NodeD +- Ascend Device Plugin +- Ascend Docker Runtime +- ClusterD +- Ascend Operator + +> **Note:** Skip the installation of `ascend-volcano` mentioned in the document above, as we have already installed the native Volcano from the Volcano community in the **Prerequisites** part. + +**Configuration Adjustment for Ascend Device Plugin:** + +When installing `ascend-device-plugin`, you must set the `presetVirtualDevice` parameter to `"false"` in the `device-plugin-310P-volcano-v{version}.yaml` file to enable dynamic virtualization of 310p: + +```yaml +... +args: [ + "device-plugin", + "-useAscendDocker=true", + "-volcanoType=true", + "-presetVirtualDevice=false", + "-logFile=/var/log/mindx-dl/devicePlugin/devicePlugin.log", + "-logLevel=0" +] +... ``` -Wait for @JackyTYang to fill +For detailed information, please consult the official [Ascend MindCluster documentation.](https://www.hiascend.com/document/detail/zh/mindcluster/72rc1/clustersched/dlug/cpaug_0020.html) + +##### Scheduler Config Update +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: volcano-scheduler-configmap + namespace: volcano-system +data: + volcano-scheduler.conf: | + actions: "enqueue, allocate, backfill" + tiers: + - plugins: + - name: predicates + - name: deviceshare + arguments: + deviceshare.AscendMindClusterVNPUEnable: true # enable ascend vnpu + configurations: + ... + - name: init-params + arguments: {"grace-over-time":"900","presetVirtualDevice":"false"} # to enable dynamic virtulization，presetVirtualDevice need to be set false ``` --- @@ -111,9 +160,97 @@ Usage is different depending on the mode you selected ### MindCluster mode +```yaml +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + name: mindx-dls + namespace: vnpu + labels: + ring-controller.atlas: ascend-310P +spec: + minAvailable: 1 + schedulerName: volcano + policies: + - event: PodEvicted + action: RestartJob + plugins: + ssh: [] + env: [] + svc: [] + maxRetry: 3 + queue: default + tasks: + - name: "default-test" + replicas: 1 + template: + metadata: + labels: + app: infers + ring-controller.atlas: ascend-310P + vnpu-dvpp: "null" + vnpu-level: low + spec: + schedulerName: volcano + containers: + - name: resnet50infer + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.1.RC1-300I-Duo-py311-openeuler24.03-lts + imagePullPolicy: IfNotPresent + securityContext: + privileged: false + command: ["/bin/bash", "-c", "tail -f /dev/null"] + resources: + requests: + huawei.com/npu-core: 8 + limits: + huawei.com/npu-core: 8 + nodeSelector: + host-arch: huawei-arm + ``` -Wait for @JackyTYang to fill -``` + +The supported Ascend chips and their `ResourceNames` are shown in the following table: + +| ChipName | JobLabel and TaskLabel | ResourceName | +|-------|------------------------------------|-------| +| 310P3 | ring-controller.atlas: ascend-310P | huawei.com/npu-core | + +**Description of Labels in the Virtualization Task YAML** + +| **Key** | **Value** | **Description** | +| ------------------------- | --------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **vnpu-level** | **low** | Low configuration (default). Selects the lowest-configuration "virtualized instance template." | +| | **high** | Performance-first. When cluster resources are sufficient, the scheduler will choose the highest-configured virtualized instance template possible. When most physical NPUs in the cluster are already in use and only a few AI Cores remain on each device, the scheduler will allocate templates that match the remaining AI Core count rather than forcing high-profile templates. For details, refer to the table below. | +| **vnpu-dvpp** | **yes** | The Pod uses DVPP. | +| | **no** | The Pod does not use DVPP. | +| | **null** | Default value. DVPP usage is not considered. | +| **ring-controller.atlas** | **ascend-310P** | Indicates that the task uses products from the Atlas inference series. | + +**Effect of DVPP and Level Configurations** + +| **Product Model** | **Requested AI Core Count** | **vnpu-dvpp** | **vnpu-level** | **Downgrade** | **Selected Template** | +| --------------------------------------- | --------------------------- |---------------| -------------------- | ------------- | --------------------- | +| **Atlas Inference Series (8 AI Cores)** | **1** | `null` | Any value | – | `vir01` | +| | **2** | `null` | `low` / other values | – | `vir02_1c` | +| | **2** | `null` | `high` | No | `vir02` | +| | **2** | `null` | `high` | Yes | `vir02_1c` | +| | **4** | `yes` | `low` / other values | – | `vir04_4c_dvpp` | +| | **4** | `no` | `low` / other values | – | `vir04_3c_ndvpp` | +| | **4** | `null` | `low` / other values | – | `vir04_3c` | +| | **4** | `yes` | `high` | – | `vir04_4c_dvpp` | +| | **4** | `no` | `high` | – | `vir04_3c_ndvpp` | +| | **4** | `null` | `high` | No | `vir04` | +| | **4** | `null` | `high` | Yes | `vir04_3c` | +| | **8 or multiples of 8** | Any value | Any value | – | – | + + +**Notice** + +For **chip virtualization (non–full card usage)**, the value of `vnpu-dvpp` must strictly match the corresponding value listed in the above table. +Any other values will cause the task to fail to be dispatched. + + +For detailed information, please consult the official [Ascend MindCluster documentation.](https://www.hiascend.com/document/detail/zh/mindcluster/72rc1/clustersched/dlug/cpaug_0020.html) --- diff --git a/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/type.go b/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/type.go index 13cf285e97..0e314ec93b 100644 --- a/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/type.go +++ b/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/type.go @@ -15,7 +15,7 @@ const ( DeviceName = "ascend310p-vNPU" ) -type NPUDevice struct { // One VNode/VChip? +type NPUDevice struct { VT VTemplate // Chips map chipID to VChip class Chips map[int]*VChip @@ -42,7 +42,7 @@ type NPUDevice struct { // One VNode/VChip? // for Concurrent task. not same core request task only has one on a node in same time. // nodeName: templateName:taskUID - ConCache map[string]map[types.UID]struct{} //types.UID对应pod.UID + ConCache map[string]map[types.UID]struct{} //types.UID equals to pod.UID } type NodeInf struct { From de55e8100ed99050bf78208e43b2cd2941e449eb Mon Sep 17 00:00:00 2001 From: james Date: Tue, 11 Nov 2025 18:09:50 +0800 Subject: [PATCH 21/33] docs: update docs Signed-off-by: james --- docs/user-guide/how_to_use_vnpu.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/user-guide/how_to_use_vnpu.md b/docs/user-guide/how_to_use_vnpu.md index 5cb7cd1491..9477ff791c 100644 --- a/docs/user-guide/how_to_use_vnpu.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -115,6 +115,12 @@ data: #### HAMi mode +##### Label the Node with `ascend=on` + +``` +kubectl label node {ascend-node} ascend=on +``` + ##### Deploy `hami-scheduler-device` config map ``` @@ -127,7 +133,7 @@ kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-pl kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-plugin.yaml ``` -refer https://github.com/Project-HAMi/ascend-device-plugin +For more information, refer to the [ascend-device-plugin documentation](https://github.com/Project-HAMi/ascend-device-plugin). ##### Scheduler Config Update ```yaml From dfee71885d7c45baddaaadf8e7832234ae3326b9 Mon Sep 17 00:00:00 2001 From: james Date: Thu, 13 Nov 2025 14:51:17 +0800 Subject: [PATCH 22/33] refactor: add hami and mindcluster dir Signed-off-by: james --- docs/user-guide/how_to_use_vnpu.md | 18 +++++++++--------- .../api/devices/ascend/device_info.go | 1 - .../ascend/{ => hami}/device_info_test.go | 6 ++++-- .../ascend310p/vnpu/device_info.go | 0 .../ascend310p/vnpu/device_manage.go | 0 .../{ => mindcluster}/ascend310p/vnpu/type.go | 0 .../{ => mindcluster}/ascend310p/vnpu/utils.go | 0 .../api/devices/{util.go => utils.go} | 0 8 files changed, 13 insertions(+), 12 deletions(-) rename pkg/scheduler/api/devices/ascend/{ => hami}/device_info_test.go (99%) rename pkg/scheduler/api/devices/ascend/{ => mindcluster}/ascend310p/vnpu/device_info.go (100%) rename pkg/scheduler/api/devices/ascend/{ => mindcluster}/ascend310p/vnpu/device_manage.go (100%) rename pkg/scheduler/api/devices/ascend/{ => mindcluster}/ascend310p/vnpu/type.go (100%) rename pkg/scheduler/api/devices/ascend/{ => mindcluster}/ascend310p/vnpu/utils.go (100%) rename pkg/scheduler/api/devices/{util.go => utils.go} (100%) diff --git a/docs/user-guide/how_to_use_vnpu.md b/docs/user-guide/how_to_use_vnpu.md index 9477ff791c..06a6cf0494 100644 --- a/docs/user-guide/how_to_use_vnpu.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -6,13 +6,13 @@ Volcano supports **two vNPU modes** for sharing Ascend devices: --- -### 1. Mindcluster mode +### 1. MindCluster mode **Description**: The initial version of [MindCluster](https://gitcode.com/Ascend/mind-cluster)—the official Ascend cluster scheduling add-on—required custom modifications and recompilation of Volcano. Furthermore, it was limited to Volcano release1.7 and release1.9, which complicated its use and restricted access to newer Volcano features. -To address this, we have integrated its core scheduling logic for Ascend vNPU into Volcano's native device-share plugin，which is designed specifically for scheduling and sharing heterogeneous resources like GPUs and NPUs. This integration provides seamless access to vNPU capabilities through the procedure below, while maintaining full compatibility with the latest Volcano features. +To address this, we have integrated its core scheduling logic for Ascend vNPU into Volcano's native device-share plugin, which is designed specifically for scheduling and sharing heterogeneous resources like GPUs and NPUs. This integration provides seamless access to vNPU capabilities through the procedure below, while maintaining full compatibility with the latest Volcano features. **Use case**: @@ -25,7 +25,7 @@ with support for more chip types to come **Description**: -This mode is developed by a third-party community 'HAMi', which is the developer of [volcano-vgpu](./how_to_use_volcano_vgpu.md) feature, It supports vNPU feature for both Ascend 310 and Ascend 910. It also support managing heterogeneous Ascend cluster(Cluster with multiple Ascend types,i.e 910A,910B2,910B3,310p) +This mode is developed by a third-party community 'HAMi', which is the developer of [volcano-vgpu](./how_to_use_volcano_vgpu.md) feature. It supports vNPU feature for both Ascend 310 and Ascend 910. It also supports managing heterogeneous Ascend cluster(Cluster with multiple Ascend types, i.e. 910A,910B2,910B3,310P) **Use case**: @@ -44,7 +44,7 @@ To enable vNPU scheduling, the following components must be set up based on the Kubernetes >= 1.16 Volcano >= 1.14 -[ascend-docker-runtime](https://gitcode.com/Ascend/mind-cluster/tree/master/component/ascend-docker-runtime) (for HAMi Mode) +[ascend-docker-runtime](https://gitcode.com/Ascend/mind-cluster/tree/master/component/ascend-docker-runtime) (for HAMi Mode) ### Install Volcano: @@ -73,7 +73,7 @@ Follow the official [Ascend documentation](https://www.hiascend.com/document/det **Configuration Adjustment for Ascend Device Plugin:** -When installing `ascend-device-plugin`, you must set the `presetVirtualDevice` parameter to `"false"` in the `device-plugin-310P-volcano-v{version}.yaml` file to enable dynamic virtualization of 310p: +When installing `ascend-device-plugin`, you must set the `presetVirtualDevice` parameter to `"false"` in the `device-plugin-310P-volcano-v{version}.yaml` file to enable dynamic virtualization of 310P: ```yaml ... @@ -108,7 +108,7 @@ data: configurations: ... - name: init-params - arguments: {"grace-over-time":"900","presetVirtualDevice":"false"} # to enable dynamic virtulization，presetVirtualDevice need to be set false + arguments: {"grace-over-time":"900","presetVirtualDevice":"false"} # to enable dynamic virtulization, presetVirtualDevice need to be set false ``` --- @@ -121,7 +121,7 @@ data: kubectl label node {ascend-node} ascend=on ``` -##### Deploy `hami-scheduler-device` config map +##### Deploy `hami-scheduler-device` ConfigMap ``` kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-configmap.yaml @@ -156,7 +156,7 @@ data: deviceshare.KnownGeometriesCMName: hami-scheduler-device ``` - **Note:** You may noticed that, 'volcano-vgpu' has its own GeometriesCMName and GeometriesCMNamespace, which means if you want to use both vNPU and vGPU in a same volcano cluster, you need to merge the configMap from both sides and set it here. + **Note:** You may notice that, 'volcano-vgpu' has its own GeometriesCMName and GeometriesCMNamespace, which means if you want to use both vNPU and vGPU in a same volcano cluster, you need to merge the configMap from both sides and set it here. ## Usage @@ -252,7 +252,7 @@ The supported Ascend chips and their `ResourceNames` are shown in the following **Notice** -For **chip virtualization (non–full card usage)**, the value of `vnpu-dvpp` must strictly match the corresponding value listed in the above table. +For **chip virtualization (non-full card usage)**, the value of `vnpu-dvpp` must strictly match the corresponding value listed in the above table. Any other values will cause the task to fail to be dispatched. diff --git a/pkg/scheduler/api/devices/ascend/device_info.go b/pkg/scheduler/api/devices/ascend/device_info.go index af3e98baf8..3926d22ef3 100644 --- a/pkg/scheduler/api/devices/ascend/device_info.go +++ b/pkg/scheduler/api/devices/ascend/device_info.go @@ -36,7 +36,6 @@ import ( ) const ( - NodeLockAscend = "hami.io/mutex.lock" Ascend910Prefix = "Ascend910" Ascend910NetworkWeight = 10 // binpack means the lower device memory remained after this allocation, the better diff --git a/pkg/scheduler/api/devices/ascend/device_info_test.go b/pkg/scheduler/api/devices/ascend/hami/device_info_test.go similarity index 99% rename from pkg/scheduler/api/devices/ascend/device_info_test.go rename to pkg/scheduler/api/devices/ascend/hami/device_info_test.go index 60798c9d50..a2adfb6c25 100644 --- a/pkg/scheduler/api/devices/ascend/device_info_test.go +++ b/pkg/scheduler/api/devices/ascend/hami/device_info_test.go @@ -18,11 +18,13 @@ package ascend import ( "fmt" - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" "testing" + "volcano.sh/volcano/pkg/scheduler/api/devices" "volcano.sh/volcano/pkg/scheduler/api/devices/config" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" ) var config_yaml = ` diff --git a/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/device_info.go b/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu/device_info.go similarity index 100% rename from pkg/scheduler/api/devices/ascend/ascend310p/vnpu/device_info.go rename to pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu/device_info.go diff --git a/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/device_manage.go b/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu/device_manage.go similarity index 100% rename from pkg/scheduler/api/devices/ascend/ascend310p/vnpu/device_manage.go rename to pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu/device_manage.go diff --git a/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/type.go b/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu/type.go similarity index 100% rename from pkg/scheduler/api/devices/ascend/ascend310p/vnpu/type.go rename to pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu/type.go diff --git a/pkg/scheduler/api/devices/ascend/ascend310p/vnpu/utils.go b/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu/utils.go similarity index 100% rename from pkg/scheduler/api/devices/ascend/ascend310p/vnpu/utils.go rename to pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu/utils.go diff --git a/pkg/scheduler/api/devices/util.go b/pkg/scheduler/api/devices/utils.go similarity index 100% rename from pkg/scheduler/api/devices/util.go rename to pkg/scheduler/api/devices/utils.go From cafdeead1bf1ba979574ca6f4e54cc4ddab6a694 Mon Sep 17 00:00:00 2001 From: james Date: Thu, 13 Nov 2025 14:52:20 +0800 Subject: [PATCH 23/33] refactor: rename ascend/device_info Signed-off-by: james --- pkg/scheduler/api/devices/ascend/{ => hami}/device_info.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/scheduler/api/devices/ascend/{ => hami}/device_info.go (100%) diff --git a/pkg/scheduler/api/devices/ascend/device_info.go b/pkg/scheduler/api/devices/ascend/hami/device_info.go similarity index 100% rename from pkg/scheduler/api/devices/ascend/device_info.go rename to pkg/scheduler/api/devices/ascend/hami/device_info.go From 671309ea3bde5598c5fa9d6582ebc688bd90888c Mon Sep 17 00:00:00 2001 From: james Date: Thu, 13 Nov 2025 15:13:12 +0800 Subject: [PATCH 24/33] fix: fix package name Signed-off-by: james --- pkg/scheduler/api/node_info.go | 4 ++-- pkg/scheduler/api/shared_device_pool.go | 4 ++-- pkg/scheduler/plugins/deviceshare/deviceshare.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index 57d8093463..0b47aa15e7 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -27,8 +27,8 @@ import ( "volcano.sh/apis/pkg/apis/scheduling" "volcano.sh/apis/pkg/apis/scheduling/v1beta1" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/hami" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/gpushare" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/vgpu" ) diff --git a/pkg/scheduler/api/shared_device_pool.go b/pkg/scheduler/api/shared_device_pool.go index cff4abb7f2..2a63d8e904 100644 --- a/pkg/scheduler/api/shared_device_pool.go +++ b/pkg/scheduler/api/shared_device_pool.go @@ -22,8 +22,8 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/hami" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/gpushare" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/vgpu" ) diff --git a/pkg/scheduler/plugins/deviceshare/deviceshare.go b/pkg/scheduler/plugins/deviceshare/deviceshare.go index 0a4019bf01..53b4e38b82 100644 --- a/pkg/scheduler/plugins/deviceshare/deviceshare.go +++ b/pkg/scheduler/plugins/deviceshare/deviceshare.go @@ -29,8 +29,8 @@ import ( "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/api/devices" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" + ascend "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/hami" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/api/devices/config" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/gpushare" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/vgpu" From 77700216698283eae403df72c7f48cab2cd79ae6 Mon Sep 17 00:00:00 2001 From: james Date: Thu, 13 Nov 2025 15:18:54 +0800 Subject: [PATCH 25/33] fix: fix package name Signed-off-by: james --- .../plugins/deviceshare/devices/ascend/310p/vnpu/init_policy.go | 2 +- .../plugins/deviceshare/devices/ascend/310p/vnpu/node.go | 2 +- .../deviceshare/devices/ascend/310p/vnpu/score_policy.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/init_policy.go b/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/init_policy.go index 9ca69f264e..8a6c133adb 100644 --- a/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/init_policy.go +++ b/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/init_policy.go @@ -28,7 +28,7 @@ import ( "k8s.io/klog/v2" "volcano.sh/volcano/pkg/scheduler/api" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/conf" "volcano.sh/volcano/pkg/scheduler/framework" "volcano.sh/volcano/third_party/ascend-for-volcano/common/k8s" diff --git a/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/node.go b/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/node.go index 07b8f9376b..5322d91b94 100644 --- a/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/node.go +++ b/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/node.go @@ -19,7 +19,7 @@ package vnpu310p import ( "sort" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu" "volcano.sh/volcano/third_party/ascend-for-volcano/common/util" ) diff --git a/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/score_policy.go b/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/score_policy.go index a4b048449b..f9bdc3f357 100644 --- a/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/score_policy.go +++ b/pkg/scheduler/plugins/deviceshare/devices/ascend/310p/vnpu/score_policy.go @@ -23,7 +23,7 @@ import ( "k8s.io/klog/v2" "volcano.sh/volcano/pkg/scheduler/api" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu" "volcano.sh/volcano/third_party/ascend-for-volcano/common/util" ) From fc3ea66d457d9b609ef596b3ff6e1192af46742c Mon Sep 17 00:00:00 2001 From: james Date: Thu, 13 Nov 2025 15:23:25 +0800 Subject: [PATCH 26/33] fix: fix node_info_test Signed-off-by: james --- pkg/scheduler/api/node_info_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/api/node_info_test.go b/pkg/scheduler/api/node_info_test.go index 01994a089c..225f003c12 100644 --- a/pkg/scheduler/api/node_info_test.go +++ b/pkg/scheduler/api/node_info_test.go @@ -24,7 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" fwk "k8s.io/kube-scheduler/framework" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/ascend310p/vnpu" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/midcluster/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/gpushare" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/vgpu" ) From 9b9b033e0b94328e9da1d930d73d3dc970894477 Mon Sep 17 00:00:00 2001 From: james Date: Thu, 13 Nov 2025 15:25:31 +0800 Subject: [PATCH 27/33] fix: fix node_info_test Signed-off-by: james --- pkg/scheduler/api/node_info_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/api/node_info_test.go b/pkg/scheduler/api/node_info_test.go index 225f003c12..31ae101b6e 100644 --- a/pkg/scheduler/api/node_info_test.go +++ b/pkg/scheduler/api/node_info_test.go @@ -24,7 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" fwk "k8s.io/kube-scheduler/framework" - "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/midcluster/ascend310p/vnpu" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/gpushare" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/vgpu" ) From 359f08a6fbf158bb93db2c0989f68bedca024a8e Mon Sep 17 00:00:00 2001 From: james Date: Thu, 13 Nov 2025 16:34:58 +0800 Subject: [PATCH 28/33] refactor: rename package Signed-off-by: james --- .../api/devices/ascend/hami/device_info.go | 16 ++++++++-------- .../api/devices/ascend/hami/device_info_test.go | 2 +- pkg/scheduler/api/devices/utils.go | 1 + pkg/scheduler/api/node_info.go | 6 +++--- pkg/scheduler/api/shared_device_pool.go | 2 +- pkg/scheduler/plugins/deviceshare/deviceshare.go | 6 +++--- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/pkg/scheduler/api/devices/ascend/hami/device_info.go b/pkg/scheduler/api/devices/ascend/hami/device_info.go index 3926d22ef3..58cd73a7d9 100644 --- a/pkg/scheduler/api/devices/ascend/hami/device_info.go +++ b/pkg/scheduler/api/devices/ascend/hami/device_info.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ascend +package hami import ( "encoding/json" @@ -287,7 +287,7 @@ func (ads *AscendDevices) Allocate(kubeClient kubernetes.Interface, pod *v1.Pod) nodelock.UseClient(kubeClient) err := nodelock.LockNode(ads.NodeName, ads.Type) if err != nil { - return errors.Errorf("node %s locked for %s hamivgpu lockname %s", ads.NodeName, pod.Name, err.Error()) + return errors.Errorf("node %s locked for %s hami vnpu. lockname %s", ads.NodeName, pod.Name, err.Error()) } } podDevs, err := ads.selectDevices(pod, ads.Policy) @@ -525,18 +525,18 @@ func InitDevices(config []config.VNPUConfig) []*AscendDevice { commonWord := vnpu.CommonWord dev := &AscendDevice{ config: vnpu, - nodeRegisterAnno: fmt.Sprintf("hami.io/node-register-%s", commonWord), - useUUIDAnno: fmt.Sprintf("hami.io/use-%s-uuid", commonWord), - noUseUUIDAnno: fmt.Sprintf("hami.io/no-use-%s-uuid", commonWord), - handshakeAnno: fmt.Sprintf("hami.io/node-handshake-%s", commonWord), + nodeRegisterAnno: fmt.Sprintf("%s/node-register-%s", devices.HAMiAnnotationsPrefix, commonWord), + useUUIDAnno: fmt.Sprintf("%s/use-%s-uuid", devices.HAMiAnnotationsPrefix, commonWord), + noUseUUIDAnno: fmt.Sprintf("%s/no-use-%s-uuid", devices.HAMiAnnotationsPrefix, commonWord), + handshakeAnno: fmt.Sprintf("%s/node-handshake-%s", devices.HAMiAnnotationsPrefix, commonWord), } sort.Slice(dev.config.Templates, func(i, j int) bool { return dev.config.Templates[i].Memory < dev.config.Templates[j].Memory }) _, ok := devices.InRequestDevices[commonWord] if !ok { - devices.InRequestDevices[commonWord] = fmt.Sprintf("hami.io/%s-devices-to-allocate", commonWord) - devices.SupportDevices[commonWord] = fmt.Sprintf("hami.io/%s-devices-allocated", commonWord) + devices.InRequestDevices[commonWord] = fmt.Sprintf("%s/%s-devices-to-allocate", devices.HAMiAnnotationsPrefix, commonWord) + devices.SupportDevices[commonWord] = fmt.Sprintf("%s/%s-devices-allocated", devices.HAMiAnnotationsPrefix, commonWord) // util.HandshakeAnnos[commonWord] = dev.handshakeAnno } devs = append(devs, dev) diff --git a/pkg/scheduler/api/devices/ascend/hami/device_info_test.go b/pkg/scheduler/api/devices/ascend/hami/device_info_test.go index a2adfb6c25..6c8034af9b 100644 --- a/pkg/scheduler/api/devices/ascend/hami/device_info_test.go +++ b/pkg/scheduler/api/devices/ascend/hami/device_info_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ascend +package hami import ( "fmt" diff --git a/pkg/scheduler/api/devices/utils.go b/pkg/scheduler/api/devices/utils.go index a028a08fcd..4ec7e5c119 100644 --- a/pkg/scheduler/api/devices/utils.go +++ b/pkg/scheduler/api/devices/utils.go @@ -68,6 +68,7 @@ const ( ) const ( + HAMiAnnotationsPrefix = "hami.io" AssignedNodeAnnotations = "hami.io/vgpu-node" AssignedTimeAnnotations = "hami.io/vgpu-time" BindTimeAnnotations = "hami.io/bind-time" diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index 0b47aa15e7..71bfb6a496 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -352,7 +352,7 @@ func (ni *NodeInfo) setNodeOthersResource(node *v1.Node) { ni.Others[vgpu.DeviceName] = vgpu.NewGPUDevices(ni.Name, node) ni.Others[vnpu.DeviceName] = vnpu.NewNPUDevices(ni.Name, node) ascend_ignored_list := []string{} - for device_name, devices := range ascend.NewAscendDevices(ni.Name, node) { + for device_name, devices := range hami.NewAscendDevices(ni.Name, node) { ni.Others[device_name] = devices ascend_ignored_list = append(ascend_ignored_list, devices.GetIgnoredDevices()...) } @@ -514,7 +514,7 @@ func (ni *NodeInfo) addResource(pod *v1.Pod) { } ni.Others[vgpu.DeviceName].(Devices).AddResource(pod) ni.Others[vnpu.DeviceName].(Devices).AddResource(pod) - for _, name := range ascend.GetAscendDeviceNames() { + for _, name := range hami.GetAscendDeviceNames() { if other, exists := ni.Others[name]; exists { if devices, ok := other.(Devices); ok { devices.AddResource(pod) @@ -530,7 +530,7 @@ func (ni *NodeInfo) subResource(pod *v1.Pod) { } ni.Others[vgpu.DeviceName].(Devices).SubResource(pod) ni.Others[vnpu.DeviceName].(Devices).SubResource(pod) - for _, name := range ascend.GetAscendDeviceNames() { + for _, name := range hami.GetAscendDeviceNames() { if other, exists := ni.Others[name]; exists { if devices, ok := other.(Devices); ok { devices.SubResource(pod) diff --git a/pkg/scheduler/api/shared_device_pool.go b/pkg/scheduler/api/shared_device_pool.go index 2a63d8e904..27a6a39458 100644 --- a/pkg/scheduler/api/shared_device_pool.go +++ b/pkg/scheduler/api/shared_device_pool.go @@ -79,7 +79,7 @@ type Devices interface { var _ Devices = new(gpushare.GPUDevices) var _ Devices = new(vgpu.GPUDevices) var _ Devices = new(vnpu.NPUDevices) -var _ Devices = new(ascend.AscendDevices) +var _ Devices = new(hami.AscendDevices) var RegisteredDevices = []string{ gpushare.DeviceName, diff --git a/pkg/scheduler/plugins/deviceshare/deviceshare.go b/pkg/scheduler/plugins/deviceshare/deviceshare.go index 53b4e38b82..7325ebe3fd 100644 --- a/pkg/scheduler/plugins/deviceshare/deviceshare.go +++ b/pkg/scheduler/plugins/deviceshare/deviceshare.go @@ -29,7 +29,7 @@ import ( "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/api/devices" - ascend "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/hami" + "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/hami" "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/mindcluster/ascend310p/vnpu" "volcano.sh/volcano/pkg/scheduler/api/devices/config" "volcano.sh/volcano/pkg/scheduler/api/devices/nvidia/gpushare" @@ -89,7 +89,7 @@ func enablePredicate(dsp *deviceSharePlugin) { args.GetBool(&nodeLockEnable, NodeLockEnable) args.GetBool(&vgpu.VGPUEnable, VGPUEnable) args.GetBool(&vnpu.AscendMindClusterVNPUEnable, AscendMindClusterVNPU) - args.GetBool(&ascend.AscendHAMiVNPUEnable, AscendHAMiVNPUEnable) + args.GetBool(&hami.AscendHAMiVNPUEnable, AscendHAMiVNPUEnable) gpushare.NodeLockEnable = nodeLockEnable vgpu.NodeLockEnable = nodeLockEnable @@ -114,7 +114,7 @@ func enablePredicate(dsp *deviceSharePlugin) { func registerDevices() { once.Do(func() { - if ascend.AscendHAMiVNPUEnable { + if hami.AscendHAMiVNPUEnable { for _, vnpu := range config.GetConfig().VNPUs { klog.V(3).Infof("register device %s", vnpu.CommonWord) api.RegisterDevice(vnpu.CommonWord) From 1887e05b572f4de2b04bf97f20cd3b140967e7c2 Mon Sep 17 00:00:00 2001 From: james Date: Thu, 13 Nov 2025 17:25:25 +0800 Subject: [PATCH 29/33] fix: fix style Signed-off-by: james --- pkg/scheduler/api/devices/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/api/devices/utils.go b/pkg/scheduler/api/devices/utils.go index 4ec7e5c119..4c56c2fbca 100644 --- a/pkg/scheduler/api/devices/utils.go +++ b/pkg/scheduler/api/devices/utils.go @@ -68,7 +68,7 @@ const ( ) const ( - HAMiAnnotationsPrefix = "hami.io" + HAMiAnnotationsPrefix = "hami.io" AssignedNodeAnnotations = "hami.io/vgpu-node" AssignedTimeAnnotations = "hami.io/vgpu-time" BindTimeAnnotations = "hami.io/bind-time" From f325340ea85689b3a65a1eb2ea51cfa2d79274cb Mon Sep 17 00:00:00 2001 From: james Date: Thu, 13 Nov 2025 18:05:30 +0800 Subject: [PATCH 30/33] refactor: add ExtractResourceRequest fun Signed-off-by: james --- .../api/devices/ascend/hami/device_info.go | 66 +++------------- pkg/scheduler/api/devices/nvidia/vgpu/type.go | 9 --- .../api/devices/nvidia/vgpu/utils.go | 75 ++----------------- .../api/devices/{utils.go => util.go} | 73 ++++++++++++++++++ 4 files changed, 91 insertions(+), 132 deletions(-) rename pkg/scheduler/api/devices/{utils.go => util.go} (73%) diff --git a/pkg/scheduler/api/devices/ascend/hami/device_info.go b/pkg/scheduler/api/devices/ascend/hami/device_info.go index 58cd73a7d9..983518f7ca 100644 --- a/pkg/scheduler/api/devices/ascend/hami/device_info.go +++ b/pkg/scheduler/api/devices/ascend/hami/device_info.go @@ -570,62 +570,18 @@ func (dev *AscendDevice) GetNodeDevices(n v1.Node) ([]*devices.DeviceInfo, error return nodeDevices, nil } -func (dev *AscendDevice) GenerateResourceRequests(ctr *v1.Container) devices.ContainerDeviceRequest { - ascendResourceCount := v1.ResourceName(dev.config.ResourceName) - ascendResourceMem := v1.ResourceName(dev.config.ResourceMemoryName) - v, ok := ctr.Resources.Limits[ascendResourceCount] - if !ok { - v, ok = ctr.Resources.Requests[ascendResourceCount] - } - if ok { - if n, ok := v.AsInt64(); ok { - memnum := 0 - mem, ok := ctr.Resources.Limits[ascendResourceMem] - if !ok { - mem, ok = ctr.Resources.Requests[ascendResourceMem] - } - if ok { - memnums, ok := mem.AsInt64() - if ok { - m, _ := dev.trimMemory(memnums) - memnum = int(m) - } - klog.V(5).Infof("raw mem %d memnum %d", memnums, memnum) - } - corenum := int32(0) - - mempnum := 0 - if memnum == 0 { - mempnum = 100 - } - - if corenum > 100 { - klog.ErrorS(nil, "core limit can't exceed 100", "device", dev.config.CommonWord) - corenum = 100 - } - if mempnum != 0 && memnum == 0 { - memnum = int(dev.DeviceInfo.Devmem) * mempnum / 100 - klog.V(5).Infof("new memreq %d totalmem %d mempercentage %d", memnum, dev.DeviceInfo.Devmem, mempnum) - } - - return devices.ContainerDeviceRequest{ - Nums: int32(n), - Type: dev.CommonWord(), - Memreq: int32(memnum), - MemPercentagereq: int32(mempnum), - Coresreq: corenum, - } - } - } - return devices.ContainerDeviceRequest{} -} - func (dev *AscendDevice) ResourceReqs(pod *v1.Pod) []devices.ContainerDeviceRequest { - var reqs []devices.ContainerDeviceRequest - for _, ctr := range pod.Spec.Containers { - req := dev.GenerateResourceRequests(&ctr) - if req.Nums > 0 { - reqs = append(reqs, req) + reqs := devices.ExtractResourceRequest(pod, dev.CommonWord(), dev.config.ResourceName, dev.config.ResourceMemoryName, "", "") + for i := range reqs { + req := &reqs[i] + if req.Memreq == 0 && req.MemPercentagereq != 0 { + req.Memreq = int32(dev.DeviceInfo.Devmem * req.MemPercentagereq / 100) + klog.V(5).Infof("new memreq %d totalmem %d mempercentage %d", req.Memreq, dev.DeviceInfo.Devmem, req.MemPercentagereq) + } + if req.Memreq > 0 { + m, _ := dev.trimMemory(int64(req.Memreq)) + klog.V(5).Infof("raw mem %d, trimed mem %d", req.Memreq, m) + req.Memreq = int32(m) } } return reqs diff --git a/pkg/scheduler/api/devices/nvidia/vgpu/type.go b/pkg/scheduler/api/devices/nvidia/vgpu/type.go index 68c62bb3fe..ef202572ff 100644 --- a/pkg/scheduler/api/devices/nvidia/vgpu/type.go +++ b/pkg/scheduler/api/devices/nvidia/vgpu/type.go @@ -60,15 +60,6 @@ var ( NodeLockEnable bool ) -type ContainerDeviceRequest struct { - Nums int32 - // device type, like NVIDIA, MLU - Type string - Memreq uint - MemPercentagereq int32 - Coresreq uint -} - type ContainerDevice struct { UUID string // device type, like NVIDIA, MLU diff --git a/pkg/scheduler/api/devices/nvidia/vgpu/utils.go b/pkg/scheduler/api/devices/nvidia/vgpu/utils.go index 96f0218076..5c98986614 100644 --- a/pkg/scheduler/api/devices/nvidia/vgpu/utils.go +++ b/pkg/scheduler/api/devices/nvidia/vgpu/utils.go @@ -170,73 +170,12 @@ func checkVGPUResourcesInPod(pod *v1.Pod) bool { return false } -func resourcereqs(pod *v1.Pod) []ContainerDeviceRequest { - resourceName := v1.ResourceName(getConfig().ResourceCountName) - resourceMem := v1.ResourceName(getConfig().ResourceMemoryName) - resourceMemPercentage := v1.ResourceName(getConfig().ResourceMemoryPercentageName) - resourceCores := v1.ResourceName(getConfig().ResourceCoreName) - counts := []ContainerDeviceRequest{} - - //Count Nvidia GPU - for i := 0; i < len(pod.Spec.Containers); i++ { - singledevice := false - v, ok := pod.Spec.Containers[i].Resources.Limits[resourceName] - if !ok { - v, ok = pod.Spec.Containers[i].Resources.Limits[resourceMem] - singledevice = true - } - if ok { - n := int64(1) - if !singledevice { - n, _ = v.AsInt64() - } - memnum := uint(0) - mem, ok := pod.Spec.Containers[i].Resources.Limits[resourceMem] - if !ok { - mem, ok = pod.Spec.Containers[i].Resources.Requests[resourceMem] - } - if ok { - memnums, ok := mem.AsInt64() - if ok { - memnum = uint(memnums) - } - } - mempnum := int32(101) - mem, ok = pod.Spec.Containers[i].Resources.Limits[resourceMemPercentage] - if !ok { - mem, ok = pod.Spec.Containers[i].Resources.Requests[resourceMemPercentage] - } - if ok { - mempnums, ok := mem.AsInt64() - if ok { - mempnum = int32(mempnums) - } - } - if mempnum == 101 && memnum == 0 { - mempnum = 100 - } - corenum := uint(0) - core, ok := pod.Spec.Containers[i].Resources.Limits[resourceCores] - if !ok { - core, ok = pod.Spec.Containers[i].Resources.Requests[resourceCores] - } - if ok { - corenums, ok := core.AsInt64() - if ok { - corenum = uint(corenums) - } - } - counts = append(counts, ContainerDeviceRequest{ - Nums: int32(n), - Type: "NVIDIA", - Memreq: memnum, - MemPercentagereq: int32(mempnum), - Coresreq: corenum, - }) - } - } - klog.V(3).Infoln("counts=", counts) - return counts +func resourcereqs(pod *v1.Pod) []devices.ContainerDeviceRequest { + countName := getConfig().ResourceCountName + memoryName := getConfig().ResourceMemoryName + percentageName := getConfig().ResourceMemoryPercentageName + coreName := getConfig().ResourceCoreName + return devices.ExtractResourceRequest(pod, "NVIDIA", countName, memoryName, percentageName, coreName) } func checkGPUtype(annos map[string]string, cardtype string) bool { @@ -273,7 +212,7 @@ func checkGPUtype(annos map[string]string, cardtype string) bool { return true } -func checkType(annos map[string]string, d GPUDevice, n ContainerDeviceRequest) bool { +func checkType(annos map[string]string, d GPUDevice, n devices.ContainerDeviceRequest) bool { //General type check, NVIDIA->NVIDIA MLU->MLU if !strings.Contains(d.Type, n.Type) { return false diff --git a/pkg/scheduler/api/devices/utils.go b/pkg/scheduler/api/devices/util.go similarity index 73% rename from pkg/scheduler/api/devices/utils.go rename to pkg/scheduler/api/devices/util.go index 4c56c2fbca..67664384c0 100644 --- a/pkg/scheduler/api/devices/utils.go +++ b/pkg/scheduler/api/devices/util.go @@ -176,3 +176,76 @@ func PatchNodeAnnotations(node *v1.Node, annotations map[string]string) error { } return err } + +func ExtractResourceRequest(pod *v1.Pod, resourceType, countName, memoryName, percentageName, coreName string) []ContainerDeviceRequest { + resourceName := v1.ResourceName(countName) + resourceMem := v1.ResourceName(memoryName) + counts := []ContainerDeviceRequest{} + + //Count Nvidia GPU + for i := 0; i < len(pod.Spec.Containers); i++ { + singledevice := false + v, ok := pod.Spec.Containers[i].Resources.Limits[resourceName] + if !ok { + v, ok = pod.Spec.Containers[i].Resources.Limits[resourceMem] + singledevice = true + } + if ok { + n := int64(1) + if !singledevice { + n, _ = v.AsInt64() + } + memnum := int32(0) + mem, ok := pod.Spec.Containers[i].Resources.Limits[resourceMem] + if !ok { + mem, ok = pod.Spec.Containers[i].Resources.Requests[resourceMem] + } + if ok { + memnums, ok := mem.AsInt64() + if ok { + memnum = int32(memnums) + } + } + mempnum := int32(101) + if percentageName != "" { + resourceMemPercentage := v1.ResourceName(percentageName) + mem, ok = pod.Spec.Containers[i].Resources.Limits[resourceMemPercentage] + if !ok { + mem, ok = pod.Spec.Containers[i].Resources.Requests[resourceMemPercentage] + } + if ok { + mempnums, ok := mem.AsInt64() + if ok { + mempnum = int32(mempnums) + } + } + if mempnum == 101 && memnum == 0 { + mempnum = 100 + } + } + corenum := int32(0) + if coreName != "" { + resourceCores := v1.ResourceName(coreName) + core, ok := pod.Spec.Containers[i].Resources.Limits[resourceCores] + if !ok { + core, ok = pod.Spec.Containers[i].Resources.Requests[resourceCores] + } + if ok { + corenums, ok := core.AsInt64() + if ok { + corenum = int32(corenums) + } + } + } + counts = append(counts, ContainerDeviceRequest{ + Nums: int32(n), + Type: resourceType, + Memreq: memnum, + MemPercentagereq: int32(mempnum), + Coresreq: corenum, + }) + } + } + klog.V(3).Infoln("counts=", counts) + return counts +} From 6f6c3ee641cad91484b5464f915c0845e5444a7a Mon Sep 17 00:00:00 2001 From: james Date: Thu, 13 Nov 2025 18:20:21 +0800 Subject: [PATCH 31/33] fix: fix compile Signed-off-by: james --- pkg/scheduler/api/devices/nvidia/vgpu/utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/scheduler/api/devices/nvidia/vgpu/utils.go b/pkg/scheduler/api/devices/nvidia/vgpu/utils.go index 5c98986614..d09f87ee61 100644 --- a/pkg/scheduler/api/devices/nvidia/vgpu/utils.go +++ b/pkg/scheduler/api/devices/nvidia/vgpu/utils.go @@ -330,12 +330,12 @@ func checkNodeGPUSharingPredicateAndScore(pod *v1.Pod, gssnap *GPUDevices, repli if val.MemPercentagereq != 101 { memreqForCard = uint(float64(gs.Device[i].Memory) * float64(val.MemPercentagereq) / 100.0) } else { - memreqForCard = val.Memreq + memreqForCard = uint(val.Memreq) } if int(gs.Device[i].Memory)-int(gs.Device[i].UsedMem) < int(memreqForCard) { continue } - if gs.Device[i].UsedCore+val.Coresreq > 100 { + if gs.Device[i].UsedCore+uint(val.Coresreq) > 100 { continue } // Coresreq=100 indicates it want this card exclusively @@ -364,7 +364,7 @@ func checkNodeGPUSharingPredicateAndScore(pod *v1.Pod, gssnap *GPUDevices, repli UUID: uuid, Type: val.Type, Usedmem: memreqForCard, - Usedcores: val.Coresreq, + Usedcores: uint(val.Coresreq), }) score += GPUScore(schedulePolicy, gs.Device[i]) } From 4d07e4b7fe38d9b4836e1e36317801248a42e426 Mon Sep 17 00:00:00 2001 From: james Date: Mon, 17 Nov 2025 14:52:39 +0800 Subject: [PATCH 32/33] refactor: mv the hami definition to hami dir Signed-off-by: james --- pkg/scheduler/api/devices/ascend/hami/device_info.go | 9 +++++++++ pkg/scheduler/api/devices/util.go | 11 ----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pkg/scheduler/api/devices/ascend/hami/device_info.go b/pkg/scheduler/api/devices/ascend/hami/device_info.go index 983518f7ca..b5cca9116d 100644 --- a/pkg/scheduler/api/devices/ascend/hami/device_info.go +++ b/pkg/scheduler/api/devices/ascend/hami/device_info.go @@ -36,6 +36,15 @@ import ( ) const ( + HAMiAnnotationsPrefix = "hami.io" + AssignedNodeAnnotations = "hami.io/vgpu-node" + AssignedTimeAnnotations = "hami.io/vgpu-time" + BindTimeAnnotations = "hami.io/bind-time" + DeviceBindPhase = "hami.io/bind-phase" + DeviceBindAllocating = "allocating" + DeviceBindFailed = "failed" + DeviceBindSuccess = "success" + Ascend910Prefix = "Ascend910" Ascend910NetworkWeight = 10 // binpack means the lower device memory remained after this allocation, the better diff --git a/pkg/scheduler/api/devices/util.go b/pkg/scheduler/api/devices/util.go index 67664384c0..3fe9c039f6 100644 --- a/pkg/scheduler/api/devices/util.go +++ b/pkg/scheduler/api/devices/util.go @@ -67,17 +67,6 @@ const ( Skip ) -const ( - HAMiAnnotationsPrefix = "hami.io" - AssignedNodeAnnotations = "hami.io/vgpu-node" - AssignedTimeAnnotations = "hami.io/vgpu-time" - BindTimeAnnotations = "hami.io/bind-time" - DeviceBindPhase = "hami.io/bind-phase" - DeviceBindAllocating = "allocating" - DeviceBindFailed = "failed" - DeviceBindSuccess = "success" -) - var kubeClient *kubernetes.Clientset func GetClient() kubernetes.Interface { From a7fa2cdbd9b0e8660c4c7d01cb87b216cbeb407e Mon Sep 17 00:00:00 2001 From: james Date: Mon, 17 Nov 2025 15:01:18 +0800 Subject: [PATCH 33/33] fix: fix compile error Signed-off-by: james --- .../api/devices/ascend/hami/device_info.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/scheduler/api/devices/ascend/hami/device_info.go b/pkg/scheduler/api/devices/ascend/hami/device_info.go index b5cca9116d..75e8a3c44d 100644 --- a/pkg/scheduler/api/devices/ascend/hami/device_info.go +++ b/pkg/scheduler/api/devices/ascend/hami/device_info.go @@ -307,10 +307,10 @@ func (ads *AscendDevices) Allocate(kubeClient kubernetes.Interface, pod *v1.Pod) ads.PatchAnnotations(pod, &annotations, podDevs) ads.addResource(annotations, pod) - annotations[devices.AssignedNodeAnnotations] = ads.NodeName - annotations[devices.AssignedTimeAnnotations] = strconv.FormatInt(time.Now().Unix(), 10) - annotations[devices.DeviceBindPhase] = "allocating" - annotations[devices.BindTimeAnnotations] = strconv.FormatInt(time.Now().Unix(), 10) + annotations[AssignedNodeAnnotations] = ads.NodeName + annotations[AssignedTimeAnnotations] = strconv.FormatInt(time.Now().Unix(), 10) + annotations[DeviceBindPhase] = "allocating" + annotations[BindTimeAnnotations] = strconv.FormatInt(time.Now().Unix(), 10) err = devices.PatchPodAnnotations(kubeClient, pod, annotations) if err != nil { @@ -534,18 +534,18 @@ func InitDevices(config []config.VNPUConfig) []*AscendDevice { commonWord := vnpu.CommonWord dev := &AscendDevice{ config: vnpu, - nodeRegisterAnno: fmt.Sprintf("%s/node-register-%s", devices.HAMiAnnotationsPrefix, commonWord), - useUUIDAnno: fmt.Sprintf("%s/use-%s-uuid", devices.HAMiAnnotationsPrefix, commonWord), - noUseUUIDAnno: fmt.Sprintf("%s/no-use-%s-uuid", devices.HAMiAnnotationsPrefix, commonWord), - handshakeAnno: fmt.Sprintf("%s/node-handshake-%s", devices.HAMiAnnotationsPrefix, commonWord), + nodeRegisterAnno: fmt.Sprintf("%s/node-register-%s", HAMiAnnotationsPrefix, commonWord), + useUUIDAnno: fmt.Sprintf("%s/use-%s-uuid", HAMiAnnotationsPrefix, commonWord), + noUseUUIDAnno: fmt.Sprintf("%s/no-use-%s-uuid", HAMiAnnotationsPrefix, commonWord), + handshakeAnno: fmt.Sprintf("%s/node-handshake-%s", HAMiAnnotationsPrefix, commonWord), } sort.Slice(dev.config.Templates, func(i, j int) bool { return dev.config.Templates[i].Memory < dev.config.Templates[j].Memory }) _, ok := devices.InRequestDevices[commonWord] if !ok { - devices.InRequestDevices[commonWord] = fmt.Sprintf("%s/%s-devices-to-allocate", devices.HAMiAnnotationsPrefix, commonWord) - devices.SupportDevices[commonWord] = fmt.Sprintf("%s/%s-devices-allocated", devices.HAMiAnnotationsPrefix, commonWord) + devices.InRequestDevices[commonWord] = fmt.Sprintf("%s/%s-devices-to-allocate", HAMiAnnotationsPrefix, commonWord) + devices.SupportDevices[commonWord] = fmt.Sprintf("%s/%s-devices-allocated", HAMiAnnotationsPrefix, commonWord) // util.HandshakeAnnos[commonWord] = dev.handshakeAnno } devs = append(devs, dev)