docker: Error response from daemon: error gathering device information while adding custom device

 agent  createDeviceNode

容器侧

kata 虚拟机

Nov 25 11:31:21 pcl-01 kata-runtime[2103926]: time="2020-11-25T11:31:21.185001827+08:00" 
level=info msg="Device has not been passed to the container" arch=arm64 
command=create container=29713c5cffb627585cc9def39792688b68968a831455c94252c16658a2f080ea device=/dev/binder59 name=kata-runtime pid=2103926 source=virtcontainers subsystem=device

 

drivers.NewGenericDevice(&devInfo), nil
}

Nov 25 11:31:21 pcl-01 kata-runtime[2103926]: time="2020-11-25T11:31:21.18526109+08:00" level=info msg="normal attach devices"

 arch=arm64 command=create container=29713c5cffb627585cc9def39792688b68968a831455c94252c16658a2f080ea devices="[{6e2dcf9ae1fc8164 /dev/binder59 -rw------- 0 0}]" 
machine_type=virt name=kata-runtime pid=2103926 sandbox=29713c5cffb627585cc9def39792688b68968a831455c94252c16658a2f080ea source=virtcontainers subsystem=container

func (dm *deviceManager) AttachDevice(id string, dr api.DeviceReceiver) error {
        dm.Lock()
        defer dm.Unlock()

        d, ok := dm.devices[id]
        if !ok {
                return ErrDeviceNotExist
        }

        if err := d.Attach(dr); err != nil {
                return err
        }
        return nil
}

grep ' Attach(' -rn * | grep api.DeviceReceiver
virtcontainers/device/drivers/vhost_user_blk.go:39:func (device *VhostUserBlkDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
virtcontainers/device/drivers/block.go:38:func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
virtcontainers/device/drivers/vhost_user_scsi.go:30:func (device *VhostUserSCSIDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
virtcontainers/device/drivers/generic.go:35:func (device *GenericDevice) Attach(devReceiver api.DeviceReceiver) error {
virtcontainers/device/drivers/vhost_user_net.go:30:func (device *VhostUserNetDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
virtcontainers/device/drivers/vfio.go:59:func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) (retErr error) {
virtcontainers/device/drivers/vhost_user_fs.go:24:func (device *VhostUserFSDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
ubuntu@nshost:/opt/gopath/src/github.com/kata-containers/runtime$ 

// Attach is standard interface of api.Device
func (device *GenericDevice) Attach(devReceiver api.DeviceReceiver) error {
        _, err := device.bumpAttachCount(true)
        return err
}

        devices map[string]api.Device
        sync.RWMutex
}

func deviceLogger() *logrus.Entry {
        return api.DeviceLogger().WithField("subsystem", "device")
}

// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager
func NewDeviceManager(blockDriver string, vhostUserStoreEnabled bool, vhostUserStorePath string, devices []api.Device) api.DeviceManager {
        dm := &deviceManager{
                vhostUserStoreEnabled: vhostUserStoreEnabled,
                vhostUserStorePath:    vhostUserStorePath,
                devices:               make(map[string]api.Device),
        }
        if blockDriver == VirtioMmio {
                dm.blockDriver = VirtioMmio
        } else if blockDriver == VirtioBlock {
                dm.blockDriver = VirtioBlock
        } else if blockDriver == Nvdimm {
                dm.blockDriver = Nvdimm
        } else if blockDriver == VirtioBlockCCW {
                dm.blockDriver = VirtioBlockCCW
        } else {
                dm.blockDriver = VirtioSCSI
        }

        drivers.AllPCIeDevs = make(map[string]bool)

        for _, dev := range devices {
                dm.devices[dev.DeviceID()] = dev
        }
        return dm
}

 

 

root@nshost:/home/ubuntu# docker run -it --runtime=kata-runtime --rm --device /dev/binder:/dev/binder debian /bin/bash
docker: Error response from daemon: error gathering device information while adding custom device "/dev/binder": no such file or directory.
root@nshost:/home/ubuntu# docker run -it --runtime=kata-runtime --rm --device /dev/binder -v /dev/binder debian /bin/bash
docker: Error response from daemon: error gathering device information while adding custom device "/dev/binder": no such file or directory.
root@nshost:/home/ubuntu# docker run -it --runtime=kata-runtime --rm --device /dev/binder -v /dev/binder debian /bin/bash

root@nshost:/home/ubuntu# docker run -it --runtime=kata-runtime --rm --device /dev/vfio/0 -v /dev:/dev debian /bin/bash
Unable to find image 'debian:latest' locally
latest: Pulling from library/debian
22518ad4a7da: Pull complete 
Digest: sha256:e2cc6fb403be437ef8af68bdc3a89fd58e80b4e390c58f14c77c466002391193
Status: Downloaded newer image for debian:latest
docker: Error response from daemon: error gathering device information while adding custom device "/dev/vfio/0": no such file or directory.
root@nshost:/home/ubuntu# 

 hotplugAddBlockDevice

root@nshost:/home/ubuntu# docker run -it --runtime=kata-runtime --rm --device /dev/loop0:/dev/loop0 debian /bin/bash
docker: Error response from daemon: OCI runtime create failed: rpc error: code = DeadlineExceeded desc = Timeout reached after 3s waiting for device 0:0:0:0/block: unknown.
root@nshost:/home/ubuntu# ls /dev/loop0
/dev/loop0
root@nshost:/home/ubuntu# 

 

 

Nov 25 11:24:50 nshost kata-runtime[3126]: time="2020-11-25T11:24:50.098759451+08:00" level=info
 msg="{"arguments":{"driver":"raw","file":{"driver":"file","filename":"/dev/loop0"},"node-name":"drive-8ace5ee3a2698802"},"execute":"blockdev-add"}" 
arch=arm64 command=create container=5755fc7c31f428bd28c8dd6dc76468f897410ca2e3d13ae64d1cf55109ec370f name=kata-runtime pid=3126 source=virtcontainers subsystem=qmp

virtcontainers/hypervisor.go:783:       hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error)
virtcontainers/sandbox.go:1706:                 if _, err := s.hypervisor.hotplugAddDevice(dev, vfioDev); err != nil {
virtcontainers/sandbox.go:1722:         _, err := s.hypervisor.hotplugAddDevice(blockDevice.BlockDrive, blockDev)
virtcontainers/sandbox.go:1729:         _, err := s.hypervisor.hotplugAddDevice(vhostUserBlkDevice.VhostUserDeviceAttrs, vhostuserDev)
virtcontainers/acrn_test.go:147:        _, err := a.hotplugAddDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev)

// HotplugAddDevice is used for add a device to sandbox
// Sandbox implement DeviceReceiver interface from device/api/interface.go
func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) error {
        span, _ := s.trace("HotplugAddDevice")
        defer span.Finish()

        if s.config.SandboxCgroupOnly {
                // We are about to add a device to the hypervisor,
                // the device cgroup MUST be updated since the hypervisor
                // will need access to such device
                hdev := device.GetHostPath()
                if err := s.cgroupMgr.AddDevice(hdev); err != nil {
                        s.Logger().WithError(err).WithField("device", hdev).
                                Warn("Could not add device to cgroup")
                }
        }

        switch devType {
        case config.DeviceVFIO:
                vfioDevices, ok := device.GetDeviceInfo().([]*config.VFIODev)
                if !ok {
                        return fmt.Errorf("device type mismatch, expect device type to be %s", devType)
                }

                // adding a group of VFIO devices
                for _, dev := range vfioDevices {
                        if _, err := s.hypervisor.hotplugAddDevice(dev, vfioDev); err != nil {
                                s.Logger().
                                        WithFields(logrus.Fields{
                                                "sandbox":         s.id,
                                                "vfio-device-ID":  dev.ID,
                                                "vfio-device-BDF": dev.BDF,
                                        }).WithError(err).Error("failed to hotplug VFIO device")
                                return err
                        }
                }
                return nil
        case config.DeviceBlock:
                blockDevice, ok := device.(*drivers.BlockDevice)
                if !ok {
                        return fmt.Errorf("device type mismatch, expect device type to be %s", devType)
                }
                _, err := s.hypervisor.hotplugAddDevice(blockDevice.BlockDrive, blockDev)
                return err
        case config.VhostUserBlk:
                vhostUserBlkDevice, ok := device.(*drivers.VhostUserBlkDevice)
                if !ok {
                        return fmt.Errorf("device type mismatch, expect device type to be %s", devType)
                }
                _, err := s.hypervisor.hotplugAddDevice(vhostUserBlkDevice.VhostUserDeviceAttrs, vhostuserDev)
                return err
        case config.DeviceGeneric:
                // TODO: what?
                return nil
        }
        return nil
}

func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) {
        span, _ := q.trace("hotplugAddDevice")
        defer span.Finish()

        data, err := q.hotplugDevice(devInfo, devType, addDevice)
        if err != nil {
                return data, err
        }

        return data, nil
}

func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) (interface{}, error) {
        switch devType {
        case blockDev:
                drive := devInfo.(*config.BlockDrive)
                return nil, q.hotplugBlockDevice(drive, op)
        case cpuDev:
                vcpus := devInfo.(uint32)
                return q.hotplugCPUs(vcpus, op)
        case vfioDev:
                device := devInfo.(*config.VFIODev)
                return nil, q.hotplugVFIODevice(device, op)
        case memoryDev:
                memdev := devInfo.(*memoryDevice)
                return q.hotplugMemory(memdev, op)
        case netDev:
                device := devInfo.(Endpoint)
                return nil, q.hotplugNetDevice(device, op)
        case vhostuserDev:
                vAttr := devInfo.(*config.VhostUserDeviceAttrs)
                return nil, q.hotplugVhostUserDevice(vAttr, op)
        default:
                return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType)
        }
}

// ExecuteBlockdevAdd sends a blockdev-add to the QEMU instance.  device is the
// path of the device to add, e.g., /dev/rdb0, and blockdevID is an identifier
// used to name the device.  As this identifier will be passed directly to QMP,
// it must obey QMP's naming rules, e,g., it must start with a letter.
func (q *QMP) ExecuteBlockdevAdd(ctx context.Context, device, blockdevID string) error {
        args, _ := q.blockdevAddBaseArgs(device, blockdevID)

        return q.executeCommand(ctx, "blockdev-add", args, nil)
}

// DevicesFromPath computes a list of devices and device permissions from paths (pathOnHost and pathInContainer) and cgroup permissions.
func DevicesFromPath(pathOnHost, pathInContainer, cgroupPermissions string) (devs []specs.LinuxDevice, devPermissions []specs.LinuxDeviceCgroup, err error) {
    resolvedPathOnHost := pathOnHost

    // check if it is a symbolic link
    if src, e := os.Lstat(pathOnHost); e == nil && src.Mode()&os.ModeSymlink == os.ModeSymlink {
        if linkedPathOnHost, e := filepath.EvalSymlinks(pathOnHost); e == nil {
            resolvedPathOnHost = linkedPathOnHost
        }
    }

    device, err := devices.DeviceFromPath(resolvedPathOnHost, cgroupPermissions)
    // if there was no error, return the device
    if err == nil {
        device.Path = pathInContainer
        return append(devs, Device(device)), append(devPermissions, deviceCgroup(device)), nil
    }

    // if the device is not a device node
    // try to see if it's a directory holding many devices
    if err == devices.ErrNotADevice {

        // check if it is a directory
        if src, e := os.Stat(resolvedPathOnHost); e == nil && src.IsDir() {

            // mount the internal devices recursively
            // TODO check if additional errors should be handled or logged
            _ = filepath.Walk(resolvedPathOnHost, func(dpath string, f os.FileInfo, _ error) error {
                childDevice, e := devices.DeviceFromPath(dpath, cgroupPermissions)
                if e != nil {
                    // ignore the device
                    return nil
                }

                // add the device to userSpecified devices
                childDevice.Path = strings.Replace(dpath, resolvedPathOnHost, pathInContainer, 1)
                devs = append(devs, Device(childDevice))
                devPermissions = append(devPermissions, deviceCgroup(childDevice))

                return nil
            })
        }
    }

    if len(devs) > 0 {
        return devs, devPermissions, nil
    }

    return devs, devPermissions, fmt.Errorf("error gathering device information while adding custom device %q: %s", pathOnHost, err)
}

// WithDevices sets the container's devices
func WithDevices(daemon *Daemon, c *container.Container) coci.SpecOpts {
    return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) error {
        // Build lists of devices allowed and created within the container.
        var devs []specs.LinuxDevice
        devPermissions := s.Linux.Resources.Devices

        if c.HostConfig.Privileged && !sys.RunningInUserNS() {
            hostDevices, err := devices.HostDevices()
            if err != nil {
                return err
            }
            for _, d := range hostDevices {
                devs = append(devs, oci.Device(d))
            }

            // adding device mappings in privileged containers
            for _, deviceMapping := range c.HostConfig.Devices {
                // issue a warning that custom cgroup permissions are ignored in privileged mode
                if deviceMapping.CgroupPermissions != "rwm" {
                    logrus.WithField("container", c.ID).Warnf("custom %s permissions for device %s are ignored in privileged mode", deviceMapping.CgroupPermissions, deviceMapping.PathOnHost)
                }
                // issue a warning that the device path already exists via /dev mounting in privileged mode
                if deviceMapping.PathOnHost == deviceMapping.PathInContainer {
                    logrus.WithField("container", c.ID).Warnf("path in container %s already exists in privileged mode", deviceMapping.PathInContainer)
                    continue
                }
                d, _, err := oci.DevicesFromPath(deviceMapping.PathOnHost, deviceMapping.PathInContainer, "rwm")
                if err != nil {
                    return err
                }
                devs = append(devs, d...)
            }

            devPermissions = []specs.LinuxDeviceCgroup{
                {
                    Allow:  true,
                    Access: "rwm",
                },
            }
        } else {
            for _, deviceMapping := range c.HostConfig.Devices {
                d, dPermissions, err := oci.DevicesFromPath(deviceMapping.PathOnHost, deviceMapping.PathInContainer, deviceMapping.CgroupPermissions)
                if err != nil {
                    return err
                }
                devs = append(devs, d...)
                devPermissions = append(devPermissions, dPermissions...)
            }

            var err error
            devPermissions, err = oci.AppendDevicePermissionsFromCgroupRules(devPermissions, c.HostConfig.DeviceCgroupRules)
            if err != nil {
                return err
            }
        }

        s.Linux.Devices = append(s.Linux.Devices, devs...)
        s.Linux.Resources.Devices = devPermissions

        for _, req := range c.HostConfig.DeviceRequests {
            if err := daemon.handleDevice(req, s); err != nil {
                return err
            }
        }
        return nil
    }
}

func (daemon *Daemon) handleDevice(req container.DeviceRequest, spec *specs.Spec) error {
    if req.Driver == "" {
        for _, dd := range deviceDrivers {
            if selected := dd.capset.Match(req.Capabilities); selected != nil {
                return dd.updateSpec(spec, &deviceInstance{req: req, selectedCaps: selected})
            }
        }
    } else if dd := deviceDrivers[req.Driver]; dd != nil {
        if selected := dd.capset.Match(req.Capabilities); selected != nil {
            return dd.updateSpec(spec, &deviceInstance{req: req, selectedCaps: selected})
        }
    }
    return incompatibleDeviceRequest{req.Driver, req.Capabilities}
}