Kubernetes教程(二二)---在 K8S 中创建 Pod 是如何使用到 GPU 的：device Plugin&nvidia-Container-Toolkit 源码分析

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

func (p *iluvatarDevicePlugin) allocateDevicesByDeviceID(hostminor uint, num int) *pluginapi.DeviceSpec {
	var device pluginapi.DeviceSpec

	hostPathPrefix := "/dev/"
	containerPathPrefix := "/dev/"

	// Expose the device node for iluvatar pod.
	device.HostPath = hostPathPrefix + deviceName + strconv.Itoa(int(hostminor))
	device.ContainerPath = containerPathPrefix + deviceName + strconv.Itoa(num)
	device.Permissions = "rw"

	return &device
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

// https://github.com/NVIDIA/k8s-device-plugin/blob/main/internal/plugin/server.go#L319-L332

// Allocate which return list of devices.
func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
        responses := pluginapi.AllocateResponse{}
        for _, req := range reqs.ContainerRequests {
                if err := plugin.rm.ValidateRequest(req.DevicesIDs); err != nil {
                        return nil, fmt.Errorf("invalid allocation request for %q: %w", plugin.rm.Resource(), err)
                }
                response, err := plugin.getAllocateResponse(req.DevicesIDs)
                if err != nil {
                        return nil, fmt.Errorf("failed to get allocate response: %v", err)
                }
                responses.ContainerResponses = append(responses.ContainerResponses, response)
        }

        return &responses, nil
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

func (plugin *NvidiaDevicePlugin) getAllocateResponse(requestIds []string) (*pluginapi.ContainerAllocateResponse, error) {
	deviceIDs := plugin.deviceIDsFromAnnotatedDeviceIDs(requestIds)

	// Create an empty response that will be updated as required below.
	response := &pluginapi.ContainerAllocateResponse{
		Envs: make(map[string]string),
	}
	if plugin.deviceListStrategies.AnyCDIEnabled() {
		responseID := uuid.New().String()
		if err := plugin.updateResponseForCDI(response, responseID, deviceIDs...); err != nil {
			return nil, fmt.Errorf("failed to get allocate response for CDI: %v", err)
		}
	}
	if plugin.config.Sharing.SharingStrategy() == spec.SharingStrategyMPS {
		plugin.updateResponseForMPS(response)
	}

	// The following modifications are only made if at least one non-CDI device
	// list strategy is selected.
	if plugin.deviceListStrategies.AllCDIEnabled() {
		return response, nil
	}

	if plugin.deviceListStrategies.Includes(spec.DeviceListStrategyEnvvar) {
		plugin.updateResponseForDeviceListEnvvar(response, deviceIDs...)
	}
	if plugin.deviceListStrategies.Includes(spec.DeviceListStrategyVolumeMounts) {
		plugin.updateResponseForDeviceMounts(response, deviceIDs...)
	}
	if *plugin.config.Flags.Plugin.PassDeviceSpecs {
		response.Devices = append(response.Devices, plugin.apiDeviceSpecs(*plugin.config.Flags.NvidiaDevRoot, requestIds)...)
	}
	if *plugin.config.Flags.GDSEnabled {
		response.Envs["NVIDIA_GDS"] = "enabled"
	}
	if *plugin.config.Flags.MOFEDEnabled {
		response.Envs["NVIDIA_MOFED"] = "enabled"
	}
	return response, nil
}

1
2
3
4
5
6
7

// Constants to represent the various device list strategies
const (
	DeviceListStrategyEnvvar         = "envvar"
	DeviceListStrategyVolumeMounts   = "volume-mounts"
	DeviceListStrategyCDIAnnotations = "cdi-annotations"
	DeviceListStrategyCDICRI         = "cdi-cri"
)

1
2
3

if plugin.deviceListStrategies.Includes(spec.DeviceListStrategyEnvvar) {
    plugin.updateResponseForDeviceListEnvvar(response, deviceIDs...)
}

1
2
3
4

// updateResponseForDeviceListEnvvar sets the environment variable for the requested devices.
func (plugin *NvidiaDevicePlugin) updateResponseForDeviceListEnvvar(response *pluginapi.ContainerAllocateResponse, deviceIDs ...string) {
        response.Envs[plugin.deviceListEnvvar] = strings.Join(deviceIDs, ",")
}

1
2
3
4

const (
        DeviceIDStrategyUUID  = "uuid"
        DeviceIDStrategyIndex = "index"
)

1
2
3
4
5

        plugin := NvidiaDevicePlugin{
                deviceListEnvvar:     "NVIDIA_VISIBLE_DEVICES",
                socket:               pluginPath + ".sock",
          // ...
        }

1

NVIDIA_VISIBLE_DEVICES=GPU-03f69c50-207a-2038-9b45-23cac89cb67d

1

NVIDIA_VISIBLE_DEVICES=1,2

1
2
3
4

# --gpus
docker run --gpus device=0 -it tensorflow/tensorflow:latest-gpu bash
# 或者环境变量 NVIDIA_VISIBLE_DEVICES
docker run -e NVIDIA_VISIBLE_DEVICES=0 -it tensorflow/tensorflow:latest-gpu bash

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

// https://github.com/NVIDIA/nvidia-container-toolkit/blob/main/cmd/nvidia-container-runtime/main.go#L9-L15

import (
    "os"

    "github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
)

func main() {
    r := runtime.New()
    err := r.Run(os.Args)
    if err != nil {
       os.Exit(1)
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

// https://github.com/NVIDIA/nvidia-containertoolkit/blob/main/internal/runtime/api.go#L17-L26

type rt struct {
    logger       *Logger
    modeOverride string
}

// Interface is the interface for the runtime library.
type Interface interface {
    Run([]string) error
}
func New(opts ...Option) Interface {
    r := rt{}
    for _, opt := range opts {
       opt(&r)
    }
    if r.logger == nil {
       r.logger = NewLogger()
    }
    return &r
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

// https://github.com/NVIDIA/nvidia-container-toolkit/blob/main/internal/runtime/runtime.go#L34-L91

// Run is an entry point that allows for idiomatic handling of errors
// when calling from the main function.
func (r rt) Run(argv []string) (rerr error) {
    defer func() {
       if rerr != nil {
          r.logger.Errorf("%v", rerr)
       }
    }()

    printVersion := hasVersionFlag(argv)
    if printVersion {
       fmt.Printf("%v version %v\n", "NVIDIA Container Runtime", info.GetVersionString(fmt.Sprintf("spec: %v", specs.Version)))
    }

    cfg, err := config.GetConfig()
    if err != nil {
       return fmt.Errorf("error loading config: %v", err)
    }
    r.logger.Update(
       cfg.NVIDIAContainerRuntimeConfig.DebugFilePath,
       cfg.NVIDIAContainerRuntimeConfig.LogLevel,
       argv,
    )
    defer func() {
       if rerr != nil {
          r.logger.Errorf("%v", rerr)
       }
       if err := r.logger.Reset(); err != nil {
          rerr = errors.Join(rerr, fmt.Errorf("failed to reset logger: %v", err))
       }
    }()

    // We apply some config updates here to ensure that the config is valid in
    // all cases.
    if r.modeOverride != "" {
       cfg.NVIDIAContainerRuntimeConfig.Mode = r.modeOverride
    }
    //nolint:staticcheck  // TODO(elezar): We should swith the nvidia-container-runtime from using nvidia-ctk to using nvidia-cdi-hook.
    cfg.NVIDIACTKConfig.Path = config.ResolveNVIDIACTKPath(&logger.NullLogger{}, cfg.NVIDIACTKConfig.Path)
    cfg.NVIDIAContainerRuntimeHookConfig.Path = config.ResolveNVIDIAContainerRuntimeHookPath(&logger.NullLogger{}, cfg.NVIDIAContainerRuntimeHookConfig.Path)

    // Log the config at Trace to allow for debugging if required.
    r.logger.Tracef("Running with config: %+v", cfg)

    driver := root.New(
       root.WithLogger(r.logger),
       root.WithDriverRoot(cfg.NVIDIAContainerCLIConfig.Root),
    )

    r.logger.Tracef("Command line arguments: %v", argv)
    runtime, err := newNVIDIAContainerRuntime(r.logger, cfg, argv, driver)
    if err != nil {
       return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
    }

    if printVersion {
       fmt.Print("\n")
    }
    return runtime.Exec(argv)
}

1
2
3
4
5
6
7
8
9

runtime, err := newNVIDIAContainerRuntime(r.logger, cfg, argv, driver)
if err != nil {
   return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err)
}

if printVersion {
   fmt.Print("\n")
}
return runtime.Exec(argv)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

// https://github.com/NVIDIA/nvidia-container-toolkit/blob/main/internal/runtime/runtime_factory.go#L32-L62

// newNVIDIAContainerRuntime is a factory method that constructs a runtime based on the selected configuration and specified logger
func newNVIDIAContainerRuntime(logger logger.Interface, cfg *config.Config, argv []string, driver *root.Driver) (oci.Runtime, error) {
    lowLevelRuntime, err := oci.NewLowLevelRuntime(logger, cfg.NVIDIAContainerRuntimeConfig.Runtimes)
    if err != nil {
       return nil, fmt.Errorf("error constructing low-level runtime: %v", err)
    }

    logger.Tracef("Using low-level runtime %v", lowLevelRuntime.String())
    if !oci.HasCreateSubcommand(argv) {
       logger.Tracef("Skipping modifier for non-create subcommand")
       return lowLevelRuntime, nil
    }

    ociSpec, err := oci.NewSpec(logger, argv)
    if err != nil {
       return nil, fmt.Errorf("error constructing OCI specification: %v", err)
    }

    specModifier, err := newSpecModifier(logger, cfg, ociSpec, driver)
    if err != nil {
       return nil, fmt.Errorf("failed to construct OCI spec modifier: %v", err)
    }

    // Create the wrapping runtime with the specified modifier.
    r := oci.NewModifyingRuntimeWrapper(
       logger,
       lowLevelRuntime,
       ociSpec,
       specModifier,
    )

    return r, nil
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

// newSpecModifier is a factory method that creates constructs an OCI spec modifer based on the provided config.
func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Spec, driver *root.Driver) (oci.SpecModifier, error) {
    rawSpec, err := ociSpec.Load()
    if err != nil {
       return nil, fmt.Errorf("failed to load OCI spec: %v", err)
    }

    image, err := image.NewCUDAImageFromSpec(rawSpec)
    if err != nil {
       return nil, err
    }

    mode := info.ResolveAutoMode(logger, cfg.NVIDIAContainerRuntimeConfig.Mode, image)
    modeModifier, err := newModeModifier(logger, mode, cfg, ociSpec, image)
    if err != nil {
       return nil, err
    }
    // For CDI mode we make no additional modifications.
    if mode == "cdi" {
       return modeModifier, nil
    }

    graphicsModifier, err := modifier.NewGraphicsModifier(logger, cfg, image, driver)
    if err != nil {
       return nil, err
    }

    featureModifier, err := modifier.NewFeatureGatedModifier(logger, cfg, image)
    if err != nil {
       return nil, err
    }

    modifiers := modifier.Merge(
       modeModifier,
       graphicsModifier,
       featureModifier,
    )
    return modifiers, nil
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

func newModeModifier(logger logger.Interface, mode string, cfg *config.Config, ociSpec oci.Spec, image image.CUDA) (oci.SpecModifier, error) {
    switch mode {
    case "legacy":
       return modifier.NewStableRuntimeModifier(logger, cfg.NVIDIAContainerRuntimeHookConfig.Path), nil
    case "csv":
       return modifier.NewCSVModifier(logger, cfg, image)
    case "cdi":
       return modifier.NewCDIModifier(logger, cfg, ociSpec)
    }

    return nil, fmt.Errorf("invalid runtime mode: %v", cfg.NVIDIAContainerRuntimeConfig.Mode)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

func (m stableRuntimeModifier) Modify(spec *specs.Spec) error {
    // If an NVIDIA Container Runtime Hook already exists, we don't make any modifications to the spec.
    if spec.Hooks != nil {
       for _, hook := range spec.Hooks.Prestart {
          hook := hook
          if isNVIDIAContainerRuntimeHook(&hook) {
             m.logger.Infof("Existing nvidia prestart hook (%v) found in OCI spec", hook.Path)
             return nil
          }
       }
    }

    path := m.nvidiaContainerRuntimeHookPath
    m.logger.Infof("Using prestart hook path: %v", path)
    args := []string{filepath.Base(path)}
    if spec.Hooks == nil {
       spec.Hooks = &specs.Hooks{}
    }
    spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
       Path: path,
       Args: append(args, "prestart"),
    })

    return nil
}

1
2
3
4
5

path := m.nvidiaContainerRuntimeHookPath
spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
   Path: path,
   Args: append(args, "prestart"),
})

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

switch args[0] {
case "prestart":
    doPrestart()
    os.Exit(0)
case "poststart":
    fallthrough
case "poststop":
    os.Exit(0)
default:
    flag.Usage()
    os.Exit(2)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

func doPrestart() {
    var err error

    defer exit()
    log.SetFlags(0)

    hook, err := getHookConfig()
    if err != nil || hook == nil {
       log.Panicln("error getting hook config:", err)
    }
    cli := hook.NVIDIAContainerCLIConfig

    container := getContainerConfig(*hook)
    nvidia := container.Nvidia
    if nvidia == nil {
       // Not a GPU container, nothing to do.
       return
    }

    if !hook.NVIDIAContainerRuntimeHookConfig.SkipModeDetection && info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntimeConfig.Mode, container.Image) != "legacy" {
       log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
    }

    rootfs := getRootfsPath(container)

    args := []string{getCLIPath(cli)}
    if cli.Root != "" {
       args = append(args, fmt.Sprintf("--root=%s", cli.Root))
    }
    if cli.LoadKmods {
       args = append(args, "--load-kmods")
    }
    if cli.NoPivot {
       args = append(args, "--no-pivot")
    }
    if *debugflag {
       args = append(args, "--debug=/dev/stderr")
    } else if cli.Debug != "" {
       args = append(args, fmt.Sprintf("--debug=%s", cli.Debug))
    }
    if cli.Ldcache != "" {
       args = append(args, fmt.Sprintf("--ldcache=%s", cli.Ldcache))
    }
    if cli.User != "" {
       args = append(args, fmt.Sprintf("--user=%s", cli.User))
    }
    args = append(args, "configure")

    if ldconfigPath := cli.NormalizeLDConfigPath(); ldconfigPath != "" {
       args = append(args, fmt.Sprintf("--ldconfig=%s", ldconfigPath))
    }
    if cli.NoCgroups {
       args = append(args, "--no-cgroups")
    }
    if len(nvidia.Devices) > 0 {
       args = append(args, fmt.Sprintf("--device=%s", nvidia.Devices))
    }
    if len(nvidia.MigConfigDevices) > 0 {
       args = append(args, fmt.Sprintf("--mig-config=%s", nvidia.MigConfigDevices))
    }
    if len(nvidia.MigMonitorDevices) > 0 {
       args = append(args, fmt.Sprintf("--mig-monitor=%s", nvidia.MigMonitorDevices))
    }
    if len(nvidia.ImexChannels) > 0 {
       args = append(args, fmt.Sprintf("--imex-channel=%s", nvidia.ImexChannels))
    }

    for _, cap := range strings.Split(nvidia.DriverCapabilities, ",") {
       if len(cap) == 0 {
          break
       }
       args = append(args, capabilityToCLI(cap))
    }

    for _, req := range nvidia.Requirements {
       args = append(args, fmt.Sprintf("--require=%s", req))
    }

    args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.Pid), 10)))
    args = append(args, rootfs)

    env := append(os.Environ(), cli.Environment...)
    //nolint:gosec // TODO: Can we harden this so that there is less risk of command injection?
    err = syscall.Exec(args[0], args, env)
    log.Panicln("exec failed:", err)
}

1
2
3

args := []string{getCLIPath(cli)}
container := getContainerConfig(*hook)
err = syscall.Exec(args[0], args, env)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

func getContainerConfig(hook HookConfig) (config containerConfig) {
    var h HookState
    d := json.NewDecoder(os.Stdin)
    if err := d.Decode(&h); err != nil {
       log.Panicln("could not decode container state:", err)
    }

    b := h.Bundle
    if len(b) == 0 {
       b = h.BundlePath
    }

    s := loadSpec(path.Join(b, "config.json"))

    image, err := image.New(
       image.WithEnv(s.Process.Env),
       image.WithDisableRequire(hook.DisableRequire),
    )
    if err != nil {
       log.Panicln(err)
    }

    privileged := isPrivileged(s)
    return containerConfig{
       Pid:    h.Pid,
       Rootfs: s.Root.Path,
       Image:  image,
       Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged),
    }
}

1
2
3
4

image, err := image.New(
    image.WithEnv(s.Process.Env),
    image.WithDisableRequire(hook.DisableRequire),
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *nvidiaConfig {
    legacyImage := image.IsLegacy()

    var devices string
    if d := getDevices(hookConfig, image, mounts, privileged); d != nil {
       devices = *d
    } else {
       // 'nil' devices means this is not a GPU container.
       return nil
    }

    var migConfigDevices string
    if d := getMigConfigDevices(image); d != nil {
       migConfigDevices = *d
    }
    if !privileged && migConfigDevices != "" {
       log.Panicln("cannot set MIG_CONFIG_DEVICES in non privileged container")
    }

    var migMonitorDevices string
    if d := getMigMonitorDevices(image); d != nil {
       migMonitorDevices = *d
    }
    if !privileged && migMonitorDevices != "" {
       log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
    }

    var imexChannels string
    if c := getImexChannels(image); c != nil {
       imexChannels = *c
    }

    driverCapabilities := hookConfig.getDriverCapabilities(image, legacyImage).String()

    requirements, err := image.GetRequirements()
    if err != nil {
       log.Panicln("failed to get requirements", err)
    }

    return &nvidiaConfig{
       Devices:            devices,
       MigConfigDevices:   migConfigDevices,
       MigMonitorDevices:  migMonitorDevices,
       ImexChannels:       imexChannels,
       DriverCapabilities: driverCapabilities,
       Requirements:       requirements,
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *string {
    // If enabled, try and get the device list from volume mounts first
    if hookConfig.AcceptDeviceListAsVolumeMounts {
       devices := getDevicesFromMounts(mounts)
       if devices != nil {
          return devices
       }
    }

    // Fallback to reading from the environment variable if privileges are correct
    devices := getDevicesFromEnvvar(image, hookConfig.getSwarmResourceEnvvars())
    if devices == nil {
       return nil
    }
    if privileged || hookConfig.AcceptEnvvarUnprivileged {
       return devices
    }

    configName := hookConfig.getConfigOption("AcceptEnvvarUnprivileged")
    log.Printf("Ignoring devices specified in NVIDIA_VISIBLE_DEVICES (privileged=%v, %v=%v) ", privileged, configName, hookConfig.AcceptEnvvarUnprivileged)

    return nil
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

envNVVisibleDevices     = "NVIDIA_VISIBLE_DEVICES"

func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {
	// We check if the image has at least one of the Swarm resource envvars defined and use this
	// if specified.
	var hasSwarmEnvvar bool
	for _, envvar := range swarmResourceEnvvars {
		if image.HasEnvvar(envvar) {
			hasSwarmEnvvar = true
			break
		}
	}

	var devices []string
	if hasSwarmEnvvar {
		devices = image.DevicesFromEnvvars(swarmResourceEnvvars...).List()
	} else {
		devices = image.DevicesFromEnvvars(envNVVisibleDevices).List()
	}

	if len(devices) == 0 {
		return nil
	}

	devicesString := strings.Join(devices, ",")

	return &devicesString
}

1

devices = image.DevicesFromEnvvars(envNVVisibleDevices).List()

1
2
3
4
5
6

	s := loadSpec(path.Join(b, "config.json"))

	image, err := image.New(
		image.WithEnv(s.Process.Env), // 这里把容器 env 传给了 image 对象
		image.WithDisableRequire(hook.DisableRequire),
	)

1
2
3
4

// Environment variable unset with legacy image: default to "all".
if !isSet && len(devices) == 0 && i.IsLegacy() {
  return NewVisibleDevices("all")
}

1
2
3
4
5
6
7
8

// IsLegacy returns whether the associated CUDA image is a "legacy" image. An
// image is considered legacy if it has a CUDA_VERSION environment variable defined
// and no NVIDIA_REQUIRE_CUDA environment variable defined.
func (i CUDA) IsLegacy() bool {
	legacyCudaVersion := i.env[envCUDAVersion]
	cudaRequire := i.env[envNVRequireCUDA]
	return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0
}

1
2

args := []string{getCLIPath(cli)}
err = syscall.Exec(args[0], args, env)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

func getCLIPath(config config.ContainerCLIConfig) string {
    if config.Path != "" {
       return config.Path
    }

    if err := os.Setenv("PATH", lookup.GetPath(config.Root)); err != nil {
       log.Panicln("couldn't set PATH variable:", err)
    }

    path, err := exec.LookPath("nvidia-container-cli")
    if err != nil {
       log.Panicln("couldn't find binary nvidia-container-cli in", os.Getenv("PATH"), ":", err)
    }
    return path
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

    args := []string{getCLIPath(cli)}
    if cli.Root != "" {
       args = append(args, fmt.Sprintf("--root=%s", cli.Root))
    }
    if cli.LoadKmods {
       args = append(args, "--load-kmods")
    }
    if cli.NoPivot {
       args = append(args, "--no-pivot")
    }
    if *debugflag {
       args = append(args, "--debug=/dev/stderr")
    } else if cli.Debug != "" {
       args = append(args, fmt.Sprintf("--debug=%s", cli.Debug))
    }
    if cli.Ldcache != "" {
       args = append(args, fmt.Sprintf("--ldcache=%s", cli.Ldcache))
    }
    if cli.User != "" {
       args = append(args, fmt.Sprintf("--user=%s", cli.User))
    }
    args = append(args, "configure")

    if ldconfigPath := cli.NormalizeLDConfigPath(); ldconfigPath != "" {
       args = append(args, fmt.Sprintf("--ldconfig=%s", ldconfigPath))
    }
    if cli.NoCgroups {
       args = append(args, "--no-cgroups")
    }
    if len(nvidia.Devices) > 0 {
       args = append(args, fmt.Sprintf("--device=%s", nvidia.Devices))
    }
    if len(nvidia.MigConfigDevices) > 0 {
       args = append(args, fmt.Sprintf("--mig-config=%s", nvidia.MigConfigDevices))
    }
    if len(nvidia.MigMonitorDevices) > 0 {
       args = append(args, fmt.Sprintf("--mig-monitor=%s", nvidia.MigMonitorDevices))
    }
    if len(nvidia.ImexChannels) > 0 {
       args = append(args, fmt.Sprintf("--imex-channel=%s", nvidia.ImexChannels))
    }

    for _, cap := range strings.Split(nvidia.DriverCapabilities, ",") {
       if len(cap) == 0 {
          break
       }
       args = append(args, capabilityToCLI(cap))
    }

    for _, req := range nvidia.Requirements {
       args = append(args, fmt.Sprintf("--require=%s", req))
    }

    args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.Pid), 10)))
    args = append(args, rootfs)

    env := append(os.Environ(), cli.Environment...)