HAMi 异构设备虚拟化之海光 DCU 实战：K8s 下的 vDCU 资源调度与纳管

$ hy-smi virtual -show-device-info

Device 0:
 Actual Device: 0
 Compute units: 60
 Global memory: 34342961152 bytes
Device 1:
 Actual Device: 1
 Compute units: 60
 Global memory: 34342961152 bytes

$ hy-smi virtual -create-vdevices 4 -d 0 \
-vdevice-compute-units 5,15,20,20 \
-vdevice-memory-size 4096,8192,8192,8192

The virtual device is created successfully!

$ hy-smi virtual -show-vdevice-info
Virtual Device 0:
 Actual Device: 7
 Compute units: 5
 Global memory: 4294967296 bytes
- 5 -
Virtual Device 1:
 Actual Device: 7
 Compute units: 15
 Global memory: 8589934592 bytes
Virtual Device 2:
 Actual Device: 7
 Compute units: 20
 Global memory: 8589934592 bytes
Virtual Device 3:
 Actual Device: 7
 Compute units: 20
 Global memory: 8589934592 bytes

# -d ${dev_id} 指定物理 DCU ID，不指定则选定所有物理 DCU
# -destroy-vdevice 销毁此物理 DCU 上所有 vDCU
hy-smi virtual -d ${dev_id} -destroy-vdevice

docker run -it --name container_name \ 
--device=/dev/kfd \ 
--device=/dev/dri \ 
--device=/dev/mkfd \ 
-v /etc/vdev/vdev0.conf:/etc/vdev/docker/vdev0.conf:ro \ 
${docker_image:tag} \ 
/bin/bash

# 添加 repo
helm repo add hami-charts https://project-hami.github.io/HAMi/
# 安装
helm install hami hami-charts/hami -n kube-system

export registry=192.168.10.172:5000
# 不指定 tag,会自动使用当前集群同样版本，需要确保 Registry 有该镜像
export kubescheduler=$registry/kube-scheduler

helm upgrade --install hami hami-charts/hami -n kube-system \
--set scheduler.kubeScheduler.image=$kubescheduler \
--set scheduler.extender.image=$registry/projecthami/hami \
--set scheduler.patch.imageNew=$registry/liangjw/kube-webhook-certgen:v1.1.1

$ kubectl -n kube-system get po -l app.kubernetes.io/name=hami
NAME                              READY   STATUS    RESTARTS   AGE
hami-scheduler-6dbdf69644-mz9m5   2/2     Running   0          2m19s

kubectl label nodes {nodeid} dcu=on

# on the dcu node, create these directory:
$ mkdir /etc/vdev

# should change dtk-xx.xx.x to your installed dtk version
$ cp -r /opt/dtk-xx.xx.x /opt/dtk

wget https://raw.githubusercontent.com/Project-HAMi/dcu-vgpu-device-plugin/refs/heads/master/k8s-dcu-plugin.yaml
wget https://raw.githubusercontent.com/Project-HAMi/dcu-vgpu-device-plugin/refs/heads/master/k8s-dcu-rbac.yaml

kubectl apply -f k8s-dcu-rbac.yaml
kubectl apply -f k8s-dcu-plugin.yaml

$ kubectl get po -n kube-system
NAMESPACE     NAME                                   READY   STATUS    RESTARTS   AGE
kube-system   hami-dcu-vgpu-device-plugin-cgwwv      1/1     Running   4          14m
kube-system   hami-dcu-vgpu-device-plugin-lp7gl      1/1     Running   0          8m36s
kube-system   hami-dcu-vgpu-device-plugin-lswbl      1/1     Running   0          8m40s
kube-system   hami-dcu-vgpu-device-plugin-svwl7      1/1     Running   0          8m38s
kube-system   hami-scheduler-bb5586989-q7bvr         2/2     Running   0          54m

             
$ kubectl describe node d41gpucns41 | grep Capacity -A 8   
Capacity:
  cpu:                64
  ephemeral-storage:  934609028Ki
  hugepages-1Gi:      0
  hugepages-2Mi:      0
  hygon.com/dcu:      0
  hygon.com/dcunum:   32
  memory:             1043017856Ki
  pods:               500

apiVersion: v1
kind: Pod
metadata:
  name: alexnet-tf-gpu-pod-mem
  labels:
    purpose: demo-tf-amdgpu
spec:
  containers:
    - name: alexnet-tf-gpu-container
      image: ubuntu:20.04
      workingDir: /root
      command: ["sleep","infinity"]
      resources:
        limits:
          hygon.com/dcunum: 1 # requesting a GPU
          hygon.com/dcumem: 2000 # each dcu require 2000 MiB device memory
          hygon.com/dcucores: 15 # each dcu use 15% of total compute cores

source /opt/hygondriver/env.sh

hy-virtual -show-device-info

Device 0:
        Actual Device: 0
        Compute units: 9
        Global memory: 2097152000 bytes

// Start is an optional interface that could be implemented by plugin.
// If case Start is implemented, it will be executed by Manager after
// plugin instantiation and before its registration to kubelet. This
// method could be used to prepare resources before they are offered
// to Kubernetes.
func (p *Plugin) Start() error {
	var err error

  // 初始化 device info
	dcgm.Init()
	p.devices, err = dcgm.DeviceInfos()
	if err != nil {
		log.Fatalf("dcgm DeviceInfos failed:%s", err.Error())
	}
	for idx := range p.devices {
		p.devices[idx].DevTypeName = fmt.Sprintf("%v-%v", "DCU", p.devices[idx].DevTypeName)
	}
	fmt.Println("infos=", p.devices)

	for idx, val := range p.devices {
		p.coremask[idx][0] = initCoreUsage(int(val.ComputeUnit))
		p.coremask[idx][1] = initCoreUsage(int(val.ComputeUnit))
	}
  
 
	go p.WatchAndRegister()
	return nil
}

	dcgm.Init()
	p.devices, err = dcgm.DeviceInfos()
	if err != nil {
		log.Fatalf("dcgm DeviceInfos failed:%s", err.Error())
	}
	for idx := range p.devices {
		p.devices[idx].DevTypeName = fmt.Sprintf("%v-%v", "DCU", p.devices[idx].DevTypeName)
	}
	fmt.Println("infos=", p.devices)

	for idx, val := range p.devices {
		p.coremask[idx][0] = initCoreUsage(int(val.ComputeUnit))
		p.coremask[idx][1] = initCoreUsage(int(val.ComputeUnit))
	}

// DeviceInfos 获取设备信息列表
// @Summary 获取设备信息列表
// @Description 返回所有设备的详细信息列表
// @Produce json
// @Success 200 {array} DeviceInfo "返回设备信息列表"
// @Failure 500 {object} error "服务器内部错误"
// @Router /DeviceInfos [get]
func DeviceInfos() (deviceInfos []DeviceInfo, err error) {
	numDevices, err := rsmiNumMonitorDevices()
	if err != nil {
		return nil, err
	}
	for i := 0; i < numDevices; i++ {
		bdfid, err := rsmiDevPciIdGet(i)
		if err != nil {
			return nil, err
		}
		// 解析BDFID
		domain := (bdfid >> 32) & 0xffffffff
		bus := (bdfid >> 8) & 0xff
		dev := (bdfid >> 3) & 0x1f
		function := bdfid & 0x7
		// 格式化PCI ID
		pciBusNumber := fmt.Sprintf("%04X:%02X:%02X.%X", domain, bus, dev, function)
		//设备序列号
		deviceId, _ := rsmiDevSerialNumberGet(i)
		//获取设备类型标识id
		devTypeId, _ := rsmiDevIdGet(i)
		devType := fmt.Sprintf("%x", devTypeId)
		//型号名称
		devTypeName := type2name[devType]
		//获取设备内存总量
		memoryTotal, _ := rsmiDevMemoryTotalGet(i, RSMI_MEM_TYPE_FIRST)
		mt, _ := strconv.ParseFloat(fmt.Sprintf("%f", float64(memoryTotal)/1.0), 64)
		glog.Info(" DCU[%v] memory total memory total: %.0f", i, mt)
		//获取设备内存使用量
		memoryUsed, _ := rsmiDevMemoryUsageGet(i, RSMI_MEM_TYPE_FIRST)
		mu, _ := strconv.ParseFloat(fmt.Sprintf("%f", float64(memoryUsed)/1.0), 64)
		glog.Info(" DCU[%v] memory used :%.0f", i, mu)
		computeUnit := computeUnitType[devTypeName]
		glog.Info(" DCU[%v] computeUnit : %.0f", i, computeUnit)
		deviceInfo := DeviceInfo{
			DvInd:        i,
			DeviceId:     deviceId,
			DevType:      devType,
			DevTypeName:  devTypeName,
			PciBusNumber: pciBusNumber,
			MemoryTotal:  mt,
			MemoryUsed:   mu,
			ComputeUnit:  computeUnit,
		}
		deviceInfos = append(deviceInfos, deviceInfo)
	}
	glog.Info("deviceInfos: ", dataToJson(deviceInfos))
	return
}

package dcgm

/*
#cgo CFLAGS: -Wall -I./include
#cgo LDFLAGS: -L./lib -lrocm_smi64 -lhydmi -Wl,--unresolved-symbols=ignore-in-object-files
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <kfd_ioctl.h>
#include <rocm_smi64Config.h>
#include <rocm_smi.h>
#include <dmi_virtual.h>
#include <dmi_error.h>
#include <dmi.h>
#include <dmi_mig.h>
*/
import "C"
import (
	"encoding/json"
	"fmt"
	"os"
	"unsafe"

	"github.com/golang/glog"
)

// rsmiNumMonitorDevices 获取gpu数量 *
func rsmiNumMonitorDevices() (gpuNum int, err error) {
	var p C.uint
	ret := C.rsmi_num_monitor_devices(&p)
	//glog.Info("go_rsmi_num_monitor_devices_ret:", ret)
	if err = errorString(ret); err != nil {
		return 0, fmt.Errorf("Error go_rsmi_num_monitor_devices_ret: %s", err)
	}
	gpuNum = int(p)
	//glog.Info("go_rsmi_num_monitor_devices:", gpuNum)
	return gpuNum, nil
}

// rsmiDevPciIdGet 获取唯一pci设备标识符
func rsmiDevPciIdGet(dvInd int) (bdfid int64, err error) {
	var cbdfid C.uint64_t
	ret := C.rsmi_dev_pci_id_get(C.uint32_t(dvInd), &cbdfid)
	//glog.Infof("rsmi_dev_pci_id_get ret:%v, retStr:%v", ret, errorString(ret))
	if err = errorString(ret); err != nil {
		glog.Errorf("rsmi_dev_pci_id_get err:%v", err.Error())
		return bdfid, err
	}
	bdfid = int64(cbdfid)
	//glog.Infof("rsmiDevPciIdGet bdfid:%v", bdfid)
	return
}

func (r *Plugin) WatchAndRegister() {
	klog.Info("into WatchAndRegister")
	for {
		r.RefreshContainerDevices()
		err := r.RegistrInAnnotation()
		if err != nil {
			klog.Errorf("register error, %v", err)
			time.Sleep(time.Second * 5)
		} else {
			time.Sleep(time.Second * 30)
		}
	}
}

func (p *Plugin) RefreshContainerDevices() error {
	files, err := os.ReadDir("/usr/local/vgpu/dcu")
	if err != nil {
		return err
	}
	idx := 0
	for idx < len(p.devices) {
		p.coremask[idx][0] = initCoreUsage(int(p.devices[idx].ComputeUnit))
		p.coremask[idx][1] = initCoreUsage(int(p.devices[idx].ComputeUnit))
		idx++
	}

	for _, f := range files {
		pods, err := client.GetClient().CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
		if err != nil {
			return err
		}
		found := false
		for _, val := range pods.Items {
			if strings.Contains(f.Name(), string(val.UID)) {
				found = true
				var didx, pid, vdidx int
				tmpstr := strings.Split(f.Name(), "_")
				didx, _ = strconv.Atoi(tmpstr[2])
				pid, _ = strconv.Atoi(tmpstr[3])
				vdidx, _ = strconv.Atoi(tmpstr[4])
				p.coremask[didx][0], _ = addCoreUsage(p.coremask[didx][0], tmpstr[5])
				p.coremask[didx][1], _ = addCoreUsage(p.coremask[didx][1], tmpstr[6])
				p.vidx[vdidx] = true
				p.pipeid[didx][pid] = true
			}
		}
		if !found {
			var didx, pid, vdidx int
			tmpstr := strings.Split(f.Name(), "_")
			didx, _ = strconv.Atoi(tmpstr[2])
			pid, _ = strconv.Atoi(tmpstr[3])
			vdidx, _ = strconv.Atoi(tmpstr[4])
			p.vidx[vdidx] = false
			p.pipeid[didx][pid] = false
			os.RemoveAll("/usr/local/vgpu/dcu/" + f.Name())
			os.Remove(fmt.Sprintf("/etc/vdev/vdev%d.conf", vdidx))
		}
		fmt.Println(f.Name())
	}
	fmt.Println(p.coremask)
	return nil
}

func (r *Plugin) RegistrInAnnotation() error {
	devices := r.apiDevices()
	annos := make(map[string]string)
	if len(util.NodeName) == 0 {
		util.NodeName = os.Getenv(util.NodeNameEnvName)
	}
	node, err := util.GetNode(util.NodeName)
	if err != nil {
		klog.Errorln("get node error", err.Error())
		return err
	}
	encodeddevices := util.EncodeNodeDevices(*devices)
	annos[util.HandshakeAnnosString] = "Reported " + time.Now().String()
	annos[util.RegisterAnnos] = encodeddevices
	klog.Infoln("Reporting devices", encodeddevices, "in", time.Now().String())
	err = util.PatchNodeAnnotations(node, annos)

	if err != nil {
		klog.Errorln("patch node error", err.Error())
	}
	return err
}

// ListAndWatch returns a stream of List of Devices
// Whenever a Device state change or a Device disappears, ListAndWatch
// returns the new list
func (p *Plugin) ListAndWatch(e *kubeletdevicepluginv1beta1.Empty, s kubeletdevicepluginv1beta1.DevicePlugin_ListAndWatchServer) error {
	p.AMDGPUs = amdgpu.GetAMDGPUs()

	devs := make([]*kubeletdevicepluginv1beta1.Device, len(p.AMDGPUs))

	// limit scope for hwloc
	func() {
		var hw hwloc.Hwloc
		hw.Init()
		defer hw.Destroy()

		i := 0
		for id := range p.AMDGPUs {
			dev := &kubeletdevicepluginv1beta1.Device{
				ID:     id,
				Health: kubeletdevicepluginv1beta1.Healthy,
			}
			devs[i] = dev
			i++

			numas, err := hw.GetNUMANodes(id)
			glog.Infof("Watching GPU with bus ID: %s NUMA Node: %+v", id, numas)
			if err != nil {
				glog.Error(err)
				continue
			}

			if len(numas) == 0 {
				glog.Errorf("No NUMA for GPU ID: %s", id)
				continue
			}

			numaNodes := make([]*kubeletdevicepluginv1beta1.NUMANode, len(numas))
			for j, v := range numas {
				numaNodes[j] = &kubeletdevicepluginv1beta1.NUMANode{
					ID: int64(v),
				}
			}

			dev.Topology = &kubeletdevicepluginv1beta1.TopologyInfo{
				Nodes: numaNodes,
			}
		}
	}()

	fakedevs := p.apiDevices()
	s.Send(&kubeletdevicepluginv1beta1.ListAndWatchResponse{Devices: p.generateFakeDevs(fakedevs)})

	for {
		select {
		case <-p.Heartbeat:
			var health = kubeletdevicepluginv1beta1.Unhealthy

			// TODO there are no per device health check currently
			// TODO all devices on a node is used together by kfd
			if simpleHealthCheck() {
				health = kubeletdevicepluginv1beta1.Healthy
			}

			for i := 0; i < len(p.AMDGPUs); i++ {
				devs[i].Health = health
			}
			s.Send(&kubeletdevicepluginv1beta1.ListAndWatchResponse{Devices: p.generateFakeDevs(fakedevs)})
		}
	}
	// returning a value with this function will unregister the plugin from k8s
}

fakedevs := p.apiDevices()
s.Send(&kubeletdevicepluginv1beta1.ListAndWatchResponse{Devices: p.generateFakeDevs(fakedevs)})

for {
    select {
    case <-p.Heartbeat:
       s.Send(&kubeletdevicepluginv1beta1.ListAndWatchResponse{Devices: p.generateFakeDevs(fakedevs)})
    }
}

func (r *Plugin) apiDevices() *[]*api.DeviceInfo {
	res := []*api.DeviceInfo{}
	for idx, val := range r.devices {
		if val.MemoryTotal > 0 {
			res = append(res, &api.DeviceInfo{
				Index:   idx,
				Id:      "DCU-" + fmt.Sprint(idx),
				Count:   4,
				Devmem:  int32(val.MemoryTotal / 1024 / 1024),
				Devcore: 100,
				Numa:    0,
				Type:    val.DevTypeName,
				Health:  true,
			})
		}
	}
	return &res
}

func (p *Plugin) generateFakeDevs(devices *[]*api.DeviceInfo) []*kubeletdevicepluginv1beta1.Device {
	fakedevs := []*kubeletdevicepluginv1beta1.Device{}

	for _, val := range *devices {
		idx := 0
		for idx < int(val.Count) {
			fakedevs = append(fakedevs, &kubeletdevicepluginv1beta1.Device{
				ID:     val.Id + "-fake-" + fmt.Sprint(idx),
				Health: kubeletdevicepluginv1beta1.Healthy,
			})
			idx++
		}
	}
	return fakedevs
}

func (p *Plugin) Allocate(ctx context.Context, reqs *kubeletdevicepluginv1beta1.AllocateRequest) (*kubeletdevicepluginv1beta1.AllocateResponse, error) {
	var car kubeletdevicepluginv1beta1.ContainerAllocateResponse
	var dev *kubeletdevicepluginv1beta1.DeviceSpec
	responses := kubeletdevicepluginv1beta1.AllocateResponse{}
	nodename := util.NodeName
	current, err := hmutil.GetPendingPod(ctx, nodename)
	if err != nil {
		//nodelock.ReleaseNodeLock(nodename, NodeLockDCU, current, false)
		return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
	}
	klog.Infof("Allocate for pod %s/%s uid [%s] \n", current.Namespace, current.Name, current.UID)
	drmCards, drmRenders, err := util.ListDcuDrmDevices()
	if err != nil {
		util.PodAllocationFailed(nodename, current, NodeLockDCU)
		return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
	}
	for idx := range reqs.ContainerRequests {
		currentCtr, devreq, err := util.GetNextDeviceRequest(util.HygonDCUDevice, *current)
		klog.Infoln("deviceAllocateFromAnnotation=", devreq)
		if err != nil {
			util.PodAllocationFailed(nodename, current, NodeLockDCU)
			return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
		}
		if len(devreq) != len(reqs.ContainerRequests[idx].DevicesIDs) {
			util.PodAllocationFailed(nodename, current, NodeLockDCU)
			return &kubeletdevicepluginv1beta1.AllocateResponse{}, errors.New("device number not matched")
		}

		err = util.EraseNextDeviceTypeFromAnnotation(util.HygonDCUDevice, *current)
		if err != nil {
			util.PodAllocationFailed(nodename, current, NodeLockDCU)
			return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
		}

		car = kubeletdevicepluginv1beta1.ContainerAllocateResponse{}
		// Currently, there are only 1 /dev/kfd per nodes regardless of the # of GPU available
		// for compute/rocm/HSA use cases
		dev = new(kubeletdevicepluginv1beta1.DeviceSpec)
		dev.HostPath = "/dev/kfd"
		dev.ContainerPath = "/dev/kfd"
		dev.Permissions = "rwm"
		car.Devices = append(car.Devices, dev)

		dev = new(kubeletdevicepluginv1beta1.DeviceSpec)
		dev.HostPath = "/dev/mkfd"
		dev.ContainerPath = "/dev/mkfd"
		dev.Permissions = "rwm"
		car.Devices = append(car.Devices, dev)

		for _, val := range devreq {
			var devIdx = -1
			klog.Infof("Allocating device ID: %s", val.UUID)
			succeedCount, err := fmt.Sscanf(val.UUID, "DCU-%d", &devIdx)
			if err != nil || succeedCount == 0 || devIdx == -1 {
				klog.Errorf("Invalid request device uuid: %s", val.UUID)
				util.PodAllocationFailed(nodename, current, NodeLockDCU)
				return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("invalid request device uuid %s", val.UUID)
			}

			if devIdx > len(drmCards) || devIdx > len(drmRenders) {
				klog.Errorf("Invalid device index: %d, all devices counts is: %d, all renders count is: %d", devIdx, len(drmCards), len(drmRenders))
				util.PodAllocationFailed(nodename, current, NodeLockDCU)
				return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("can not match dcu dri request %s. cards %d, renders %d", val.UUID, len(drmCards), len(drmRenders))
			}

			drmCardName := drmCards[devIdx]
			klog.Infof("All dcu dri card devs: %v, mapped dri: %s", drmCards, drmCardName)
			devpath := fmt.Sprintf("/dev/dri/%s", drmCardName)
			dev = new(kubeletdevicepluginv1beta1.DeviceSpec)
			dev.HostPath = devpath
			dev.ContainerPath = devpath
			dev.Permissions = "rw"
			car.Devices = append(car.Devices, dev)

			drmRenderName := drmRenders[devIdx]
			klog.Infof("All dcu dri render devs: %v, mapped dri: %s", drmRenders, drmRenderName)
			devpath = fmt.Sprintf("/dev/dri/%s", drmRenderName)
			dev = new(kubeletdevicepluginv1beta1.DeviceSpec)
			dev.HostPath = devpath
			dev.ContainerPath = devpath
			dev.Permissions = "rw"
			car.Devices = append(car.Devices, dev)
		}
		//Create vdev file
		klog.Infoln("devreqs=", len(devreq), "usedmem=", devreq[0].Usedmem, ":", p.devices[0].MemoryTotal/1024/1024)
		if len(devreq) < 2 && devreq[0].Usedmem < int32(p.devices[0].MemoryTotal/1024/1024) {
			filename, err := p.createvdevFiles(current, &currentCtr, devreq)
			if err != nil {
				util.PodAllocationFailed(nodename, current, NodeLockDCU)
				return &responses, err
			}
			if len(filename) > 0 {
				car.Mounts = append(car.Mounts, &kubeletdevicepluginv1beta1.Mount{
					ContainerPath: "/etc/vdev/docker/",
					HostPath:      filename,
					ReadOnly:      false,
				}, &kubeletdevicepluginv1beta1.Mount{
					ContainerPath: "/opt/hygondriver",
					HostPath:      os.Getenv("HYGONPATH"),
					ReadOnly:      false,
				}, &kubeletdevicepluginv1beta1.Mount{
					ContainerPath: "/opt/hyhal",
					HostPath:      "/opt/hyhal",
					ReadOnly:      false,
				})
				car.Mounts = append(car.Mounts)
			}
		}
		responses.ContainerResponses = append(responses.ContainerResponses, &car)
	}
	klog.Infoln("response=", responses)
	util.PodAllocationTrySuccess(nodename, util.HygonDCUDevice, NodeLockDCU, current)
	return &responses, nil
}

func DecodePodDevices(checklist map[string]string, annos map[string]string) (PodDevices, error) {
	klog.V(5).Infof("checklist is [%+v], annos is [%+v]", checklist, annos)
	if len(annos) == 0 {
		return PodDevices{}, nil
	}
	pd := make(PodDevices)
	for devID, devs := range checklist {
		str, ok := annos[devs]
		if !ok {
			continue
		}
		pd[devID] = make(PodSingleDevice, 0)
		for _, s := range strings.Split(str, OnePodMultiContainerSplitSymbol) {
			cd, err := DecodeContainerDevices(s)
			if err != nil {
				return PodDevices{}, nil
			}
			if len(cd) == 0 {
				continue
			}
			pd[devID] = append(pd[devID], cd)
		}
	}
	klog.InfoS("Decoded pod annos", "poddevices", pd)
	return pd, nil
}

			filename, err := p.createvdevFiles(current, &currentCtr, devreq)
			if err != nil {
				util.PodAllocationFailed(nodename, current, NodeLockDCU)
				return &responses, err
			}
			if len(filename) > 0 {
				car.Mounts = append(car.Mounts, &kubeletdevicepluginv1beta1.Mount{
					ContainerPath: "/etc/vdev/docker/",
					HostPath:      filename,
					ReadOnly:      false,
				})
				car.Mounts = append(car.Mounts)
			} 

	dirName := string(current.UID) + "_" + ctr.Name + "_" + fmt.Sprint(devidx) + "_" + fmt.Sprint(pipeid) + "_" + fmt.Sprint(vdevidx) + "_" + fmt.Sprint(coremsk1) + "_" + fmt.Sprint(coremsk2)
	cacheFileHostDirectory := fmt.Sprintf("/usr/local/vgpu/dcu/%s", dirName)
	err = createvdevFile(pcibusId, coremsk1, coremsk2, reqcores, mem, 0, vdevidx, pipeid, cacheFileHostDirectory, "vdev0.conf")
	if err != nil {
		return "", err
	}

func createvdevFile(pcibusId, coremsk1, coremsk2 string, reqcores, mem int32, deviceid, vdevidx, pipeid int, cacheFileHostDirectory, cacheFileName string) error {
	s := ""
	s = fmt.Sprintf("PciBusId: %s\n", pcibusId)
	s = s + fmt.Sprintf("cu_mask: 0x%s\n", coremsk1)
	s = s + fmt.Sprintf("cu_mask: 0x%s\n", coremsk2)
	s = s + fmt.Sprintf("cu_count: %d\n", reqcores)
	s = s + fmt.Sprintf("mem: %d MiB\n", mem)
	s = s + fmt.Sprintf("device_id: %d\n", deviceid)
	s = s + fmt.Sprintf("vdev_id: %d\n", vdevidx)
	s = s + fmt.Sprintf("pipe_id: %d\n", pipeid)
	s = s + fmt.Sprintln("enable: 1")
	klog.Infoln("s=", s)

	_, err := os.Stat(cacheFileHostDirectory)
	if os.IsNotExist(err) {
		err := os.MkdirAll(cacheFileHostDirectory, 0777)
		if err != nil {
			return err
		}
		err = os.Chmod(cacheFileHostDirectory, 0777)
		if err != nil {
			return err
		}
	}

	err = os.WriteFile(fmt.Sprintf("%s/%s", cacheFileHostDirectory, cacheFileName), []byte(s), os.ModePerm)
	if err != nil {
		return err
	}
	return nil
}

$ hy-smi virtual -create-vdevices 4 -d 0 \
-vdevice-compute-units 5,15,20,20 \
-vdevice-memory-size 4096,8192,8192,8192