https://wohin.me/kata-containerstao-yi-yan-jiu/
https://blog.csdn.net/zhonglinzhang/article/details/99458561
Nov 19 20:11:51 pcl-01 kata-runtime[3435871]: time="2020-11-19T20:11:51.628452777+08:00" level=error msg="failed to launch qemu: exit status 1, error messages from qemu log: qemu-system-aarch64: -device usb-ehci -device usb-tablet -device usb-kbd -device usb-mouse -usb: invalid option
" arch=arm64 command=create container=094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9 name=kata-runtime pid=3435871 source=runtime
Nov 19 20:11:51 pcl-01 kata-runtime[3435871]: time="2020-11-19T20:11:51.519068847+08:00" level=error msg="failed to launch qemu: qemu-system-aarch64: -device usb-ehci -device usb-tablet -device usb-kbd -device usb-mouse -usb: invalid option
" arch=arm64 command=create container=094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9 error="exit status 1" name=kata-runtime pid=3435871 source=virtcontainers subsystem=qemu
Nov 19 20:11:51 pcl-01 kata-runtime[3435871]: time="2020-11-19T20:11:51.518976207+08:00" level=error msg="qemu-system-aarch64: -device usb-ehci -device usb-tablet -device usb-kbd -device usb-mouse -usb: invalid option
" arch=arm64 command=create container=094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9 name=kata-runtime pid=3435871 source=virtcontainers subsystem=qmp
Nov 19 20:11:51 pcl-01 kata-runtime[3435871]: time="2020-11-19T20:11:51.478740137+08:00" level=info msg="launching /usr/local/bin/qemu-system-aarch64 with: [-name sandbox-094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9 -uuid 9a921885-b67b-40e2-aaca-e0f381a56f0a -machine virt,usb=off,accel=kvm,gic-version=host -cpu host,pmu=off -qmp unix:/run/vc/vm/094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9/qmp.sock,server,nowait -m 2048M,slots=10,maxmem=514710M -device pcie-pci-bridge,bus=pcie.0,id=pcie-bridge-0,addr=2,romfile= -device virtio-serial-pci,disable-modern=false,id=serial0,romfile= -device virtconsole,chardev=charconsole0,id=console0 -chardev socket,id=charconsole0,path=/run/vc/vm/094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9/console.sock,server,nowait -device virtio-blk-pci,disable-modern=false,drive=image-f992fb770ed45ca2,scsi=off,config-wce=off,romfile=,share-rw=on -drive id=image-f992fb770ed45ca2,file=/home/pcl/kata-containers-ubuntu-latest-osbuilder-88dbc2c-agent-fc6fcf2.img,aio=threads,format=raw,if=none,readonly -device virtio-scsi-pci,id=scsi0,disable-modern=false,romfile= -object rng-random,id=rng0,filename=/dev/urandom -device virtio-rng-pci,rng=rng0,romfile= -device virtserialport,chardev=charch0,id=channel0,name=agent.channel.0 -chardev socket,id=charch0,path=/run/vc/vm/094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9/kata.sock,server,nowait -device virtio-9p-pci,disable-modern=false,fsdev=extra-9p-kataShared,mount_tag=kataShared,romfile= -fsdev local,id=extra-9p-kataShared,path=/run/kata-containers/shared/sandboxes/094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9/shared,security_model=none -netdev tap,id=network-0,vhost=on,vhostfds=3,fds=4 -device driver=virtio-net-pci,netdev=network-0,mac=02:42:ac:11:00:03,disable-modern=false,mq=on,vectors=4,romfile= -device usb-ehci -device usb-tablet -device usb-kbd -device usb-mouse -usb -rtc base=utc,driftfix=slew,clock=host -global kvm-pit.lost_tick_policy=discard -vga none -no-user-config -nodefaults -nographic --no-reboot -daemonize -kernel /home/pcl/vmlinuz-5.4.60-89 -append console=hvc0 console=hvc1 iommu.passthrough=0 root=/dev/vda1 rootflags=data=ordered,errors=remount-ro ro rootfstype=ext4 debug systemd.show_status=true systemd.log_level=debug panic=1 nr_cpus=64 agent.use_vsock=false systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket scsi_mod.scan=none agent.log=debug agent.debug_console -pidfile /run/vc/vm/094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9/pid -D /run/vc/vm/094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9/qemu.log -smp 1,cores=1,threads=1,sockets=64,maxcpus=64]" arch=arm64 command=create container=094c542a5125e0890e3a10cefe7ac06a62941952f77d9b6551ca838cb0f08cf9 name=kata-runtime pid=3435871 source=virtcontainers subsystem=qmp
type Config struct { // Path is the qemu binary path. Path string // Ctx is the context used when launching qemu. Ctx context.Context // Name is the qemu guest name Name string // UUID is the qemu process UUID. UUID string // CPUModel is the CPU model to be used by qemu. CPUModel string // Machine Machine Machine // QMPSockets is a slice of QMP socket description. QMPSockets []QMPSocket // Devices is a list of devices for qemu to create and drive. Devices []Device // RTC is the qemu Real Time Clock configuration RTC RTC // VGA is the qemu VGA mode. VGA string // Kernel is the guest kernel configuration. Kernel Kernel // Memory is the guest memory configuration. Memory Memory // SMP is the quest multi processors configuration. SMP SMP // GlobalParam is the -global parameter. GlobalParam string // Knobs is a set of qemu boolean settings. Knobs Knobs // Bios is the -bios parameter Bios string // PFlash specifies the parallel flash images (-pflash parameter) PFlash []string // Incoming controls migration source preparation Incoming Incoming // fds is a list of open file descriptors to be passed to the spawned qemu process fds []*os.File // FwCfg is the -fw_cfg parameter FwCfg []FwCfg IOThreads []IOThread // PidFile is the -pidfile parameter PidFile string // LogFile is the -D parameter LogFile string qemuParams []string }
// QemuParams returns the qemu parameters built out of this Object device. func (object Object) QemuParams(config *Config) []string { var objectParams []string var deviceParams []string var qemuParams []string deviceParams = append(deviceParams, string(object.Driver)) deviceParams = append(deviceParams, fmt.Sprintf(",id=%s", object.DeviceID)) switch object.Type { case MemoryBackendFile: objectParams = append(objectParams, string(object.Type)) objectParams = append(objectParams, fmt.Sprintf(",id=%s", object.ID)) objectParams = append(objectParams, fmt.Sprintf(",mem-path=%s", object.MemPath)) objectParams = append(objectParams, fmt.Sprintf(",size=%d", object.Size)) deviceParams = append(deviceParams, fmt.Sprintf(",memdev=%s", object.ID)) } qemuParams = append(qemuParams, "-device") qemuParams = append(qemuParams, strings.Join(deviceParams, "")) qemuParams = append(qemuParams, "-object") qemuParams = append(qemuParams, strings.Join(objectParams, "")) return qemuParams }
创建虚拟机
root@ubuntu:/opt/gopath/src/github.com/kata-containers/runtime# grep LaunchCustomQemu -rn * vendor/github.com/intel/govmm/qemu/qemu.go:2605: return LaunchCustomQemu(ctx, config.Path, config.qemuParams, vendor/github.com/intel/govmm/qemu/qemu.go:2609:// LaunchCustomQemu can be used to launch a new qemu instance. vendor/github.com/intel/govmm/qemu/qemu.go:2624:func LaunchCustomQemu(ctx context.Context, path string, params []string, fds []*os.File, 启动cloudhypervisor root@ubuntu:/opt/gopath/src/github.com/kata-containers/runtime# grep LaunchCl -rn * Binary file containerd-shim-kata-v2 matches Binary file kata-runtime matches virtcontainers/clh.go:368: strErr, pid, err := clh.LaunchClh() virtcontainers/clh.go:934:func (clh *cloudHypervisor) LaunchClh() (string, int, error) { root@ubuntu:/opt/gopath/src/github.com/kata-containers/runtime#
virtcontainers/clh.go
func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error { apiSocketPath, err := clh.apiSocketPath(id) if err != nil { clh.Logger().Info("Invalid api socket path for cloud-hypervisor") return nil } clh.state.apiSocket = apiSocketPath clh.virtiofsd = &virtiofsd{ path: clh.config.VirtioFSDaemon, sourcePath: filepath.Join(getSharePath(clh.id)), socketPath: virtiofsdSocketPath, extraArgs: clh.config.VirtioFSExtraArgs, debug: clh.config.Debug, cache: clh.config.VirtioFSCache, }
step console
// set the serial console to the cloud hypervisor if clh.config.Debug { clh.vmconfig.Serial = chclient.ConsoleConfig{ Mode: cctTTY, } } else { clh.vmconfig.Serial = chclient.ConsoleConfig{ Mode: cctNULL, } }
func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error {
func (q *qemu) getSandboxConsole(id string) (string, error) { span, _ := q.trace("getSandboxConsole") defer span.Finish() return utils.BuildSocketPath(q.store.RunVMStoragePath(), id, consoleSocket) }
virtiofsd.Start
func (clh *cloudHypervisor) startSandbox(timeout int) error { if clh.config.SharedFS == config.VirtioFS { clh.Logger().WithField("function", "startSandbox").Info("Starting virtiofsd") pid, err := clh.virtiofsd.Start(ctx) if err != nil { return err } clh.state.VirtiofsdPID = pid }
clh.bootVM
if err := clh.bootVM(ctx); err != nil { return err }
func (clh *cloudHypervisor) bootVM(ctx context.Context) error {
/* CreateVM Create the cloud-hypervisor Virtual Machine (VM) instance. The instance is not booted, only created. * @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). * @param vmConfig The VM configuration */ func (a *DefaultApiService) CreateVM(ctx _context.Context, vmConfig VmConfig) (*_nethttp.Response, error) { var (
agent CreateSandbox
可以看到，Docker引擎向kata-runtime下发create指令，然后，kata-runtime通过调用virtcontainers的CreateSandbox来启动具体的容器创建过程。接着，virtcontainers承担起主要职责，调用Hypervisor提供的服务去创建网络、启动虚拟机。
我们重点关注virtcontainers向agent发起的CreateSandbox调用，从这里开始，virtcontainers与agent连续两次请求响应，是容器创建过程中最核心的部分，也是CVE-2020-2026漏洞存在的地方：
virtcontainers --- CreateSandbox ---> agent
virtcontainers <-- Sandbox Created -- agent
virtcontainers -- CreateContainer --> agent
virtcontainers <--Container Created-- agent
这里的Sandbox与Container有什么不同呢？Sandbox是一个统一、基本的隔离空间，一个虚拟机中只有一个Sandbox，但是该Sandbox内可以有多个容器，这就对应了Kubernetes Pod的模型；对于Docker来说，一般一个Sandbox内只运行一个Container。无论是哪种情况，Sandbox的ID与内部第一个容器的ID相同。
在上面这两来两往的过程中，容器即创建完成。我们知道，容器是由镜像创建而来，那么kata-runtime是如何将镜像内容传递给虚拟机内部kata-agent的呢？答案是，将根文件目录（rootfs）挂载到宿主机与虚拟机的共享目录中。
首先，runtime/virtcontainers/kata_agent.go的startSandbox函数向kata-agent发起gRPC调用：
storages := setupStorages(sandbox)
kmodules := setupKernelModules(k.kmodules)
req := &grpc.CreateSandboxRequest{
Hostname: hostname,
Dns: dns,
Storages: storages,
SandboxPidns: sandbox.sharePidNs,
SandboxId: sandbox.id,
GuestHookPath: sandbox.config.HypervisorConfig.GuestHookPath,
KernelModules: kmodules,
}
可以看到，其中带有SandboxId和Storages参数。其中，Storages的值来自setupStorages函数，这个函数用于配置共享目录的存储驱动、文件系统类型和挂载点等。Storages内的元素定义如下（setupStorages函数）：
sharedVolume := &grpc.Storage{
Driver: kataVirtioFSDevType,
Source: mountGuestTag,
MountPoint: kataGuestSharedDir(),
Fstype: typeVirtioFS,
Options: sharedDirVirtioFSOptions,
}
其中，kataGuestSharedDir函数会返回共享目录在虚拟机内部的路径，也就是MountPoint的值：/run/kata-containers/shared/containers/。
OK，切换到kata-agent侧。当它收到gRPC调用请求后，内部的CreateSandbox函数开始执行（位于agent/grpc.go）。具体如下（我们省略了内核模块加载、命名空间创建等代码逻辑）：
func (a *agentGRPC) CreateSandbox(ctx context.Context, req *pb.CreateSandboxRequest) (*gpb.Empty, error) {
if a.sandbox.running {
return emptyResp, grpcStatus.Error(codes.AlreadyExists, "Sandbox already started, impossible to start again")
}
// 省略...
if req.SandboxId != "" {
a.sandbox.id = req.SandboxId
agentLog = agentLog.WithField("sandbox", a.sandbox.id)
}
// 省略...
mountList, err := addStorages(ctx, req.Storages, a.sandbox)
if err != nil {
return emptyResp, err
}
a.sandbox.mounts = mountList
if err := setupDNS(a.sandbox.network.dns); err != nil {
return emptyResp, err
}
return emptyResp, nil
}
可以看到，在收到请求后，kata-agent会调用addStorages函数去根据kata-runtime的指令挂载共享目录，经过深入，该函数最终会调用mountStorage函数执行挂载操作：
// mountStorage performs the mount described by the storage structure.
func mountStorage(storage pb.Storage) error {
flags, options := parseMountFlagsAndOptions(storage.Options)
return mount(storage.Source, storage.MountPoint, storage.Fstype, flags, options)
}
这里的MountPoint即是来自kata-runtime的/run/kata-containers/shared/containers/。至此，宿主机与虚拟机的共享目录已经挂载到了虚拟机内。
最后，CreateSandbox执行完成，kata-runtime收到回复。
虚拟机root
root@25a725e7599e:/# ls bin dev home lost+found mnt proc run srv tmp var boot etc lib media opt root sbin sys usr root@25a725e7599e:/# ls lib/ aarch64-linux-gnu ld-linux-aarch64.so.1 modprobe.d terminfo init lsb systemd udev root@25a725e7599e:/# ls bin dev home lost+found mnt proc run srv tmp var boot etc lib media opt root sbin sys usr root@25a725e7599e:/# ls bin/ps bin/ps root@25a725e7599e:/# ps -elf | grep nginx 4 S root 71 50 0 80 0 - 2069 arm64_ Oct31 ? 00:00:00 nginx: master process nginx -g daemon off; 5 S systemd+ 99 71 0 80 0 - 2164 ep_pol Oct31 ? 00:00:00 nginx: worker process 0 S root 201 57 0 80 0 - 676 pipe_w 04:54 hvc0 00:00:00 grep --color=auto nginx root@25a725e7599e:/# ls run/ kata-containers kata1.txt libcontainer lock mount sandbox-ns systemd root@25a725e7599e:/#
容器 rootfs
root@fa55c7478feb:/# ls bin/ps ls: cannot access 'bin/ps': No such file or directory root@fa55c7478feb:/# ls bin docker-entrypoint.d home mnt root srv usr boot docker-entrypoint.sh lib opt run sys var dev etc media proc sbin tmp root@fa55c7478feb:/# ps -elf | grep nginx bash: ps: command not found root@fa55c7478feb:/# ls run/ lock nginx.pid utmp root@fa55c7478feb:/#
那么，kata-runtime什么时候会向共享目录中挂载呢？如下图所示，发送完CreateSandobx请求后，kata-runtme在bindMountContainerRootfs中开始挂载容器根文件系统：
runtime/virtcontainers/mount.go:278:func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string, readonly bool) error
代码如下：
func bindMountContainerRootfs(ctx context.Context, sharedDir, sandboxID, cID, cRootFs string, readonly bool) error {
span, _ := trace(ctx, "bindMountContainerRootfs")
defer span.Finish()
rootfsDest := filepath.Join(sharedDir, sandboxID, cID, rootfsDir)
return bindMount(ctx, cRootFs, rootfsDest, readonly)
}
其中，rootfsDest是宿主机上共享目录中容器根文件系统的位置。它的形式是/run/kata-containers/shared/sandboxes/sandbox_id/container_id/rootfs，其中sandbox_id与container_id分别是Sandbox和容器的ID。如前所述，对于只运行一个容器的情况来说，这两个ID是一致的；cRootFs是根文件系统在虚拟机内部共享目录中的挂载位置，形式为/run/kata-containers/shared/containers/sandbox_id/rootfs。
在函数的末尾，bindMount函数执行实际的绑定挂载任务：
func bindMount(ctx context.Context, source, destination string, readonly bool) error {
// 省略...
absSource, err := filepath.EvalSymlinks(source) // 重点！！！
if err != nil {
return fmt.Errorf("Could not resolve symlink for source %v", source)
}
// 省略...
if err := syscall.Mount(absSource, destination, "bind", syscall.MS_BIND, ""); err != nil {
return fmt.Errorf("Could not bind mount %v to %v: %v", absSource, destination, err)
}
// 省略...
return nil
}
agent 启动 proxyfunc (k *kataAgent) startSandbox(sandbox *Sandbox) error { span, _ := k.trace("startSandbox") defer span.Finish() err := k.startProxy(sandbox) if err != nil { return err } defer func() { if err != nil { k.proxy.stop(k.state.ProxyPid) }
agent prepareAndStartShim
virtcontainers/kata_agent.go:590: return prepareAndStartShim(sandbox, k.shim, c.id, req.ExecId, virtcontainers/kata_agent.go:1524: return prepareAndStartShim(sandbox, k.shim, c.id, req.ExecId, virtcontainers/shim.go:151:func prepareAndStartShim(sandbox *Sandbox, shim shim, cid, token, url, consoleURL string, cmd types.Cmd,
Sandbox Create
CreateSandbox
createSandboxFromConfig
createSandbox
createAssets
newSandbox
newAgent
newHypervisor
newNetwork
globalSandboxList.addSandbox
sandbox.storage.createAllResources
sandbox.hypervisor.init
qemu.init
newQemuArch
RunningOnVMM
sandbox.hypervisor.createSandbox
qemu.createSandbox
q.getQemuMachine
q.qemuConfig = qemuConfig
newAgentConfig
sandbox.agent.init
kata_agent.init
kata_agent.generateVMSocket
newProxy
newShim
sandbox.storage.fetchAgentState
sandbox.storage.fetchSandboxNetwork
sandbox.storage.fetchSandboxDevices
deviceManager.NewDeviceManager
sandbox.storage.fetchSandboxState
sandbox.agent.createSandbox
kata_agent.createSandbox
kata_agent.configure
if config != nil kata_agent.generateVMSocket
hypervisor.addDevice (agent socket)
hypervisor.addDevice (shared volume for all containers bundles)
sandbox.setSandboxState
sandbox.createNetwork
sandbox.network.add
default_network.add
createEndpointsFromScan
doNetNS
endpoint.HotAttach
OR
endpoint.Attach
sandbox.storage.storeSandboxNetwork
sandbox.startVM
sandbox.hypervisor.startSandbox
qemu.startSandbox
govmmQemu.LaunchQemu
sandbox.hypervisor.waitSandbox
qemu.waitSandbox
govmmQemu.QMPStart
sandbox.agent.startSandbox
kata_agent.startSandbox
kata_agent.startProxy
kata_agent.agentURL
kata_agent.proxy.start
kata_proxy.start
exec.Command
kata_agent.setProxy
kata_agent.check
kata_agent.sendReq(&grpc.CheckRequest{})
kata_agent.updateInterfaces
kata_agent.updateRoutes
sandbox.createContainers
for each container from the config: createContainer
newContainer
container.createContainersDirs
if containerc.checkBlockDeviceSupport
container.hotplugDrive
container.attachDevices
container.addResources
container.getSystemMountInfo
container.storeDevices
sandbox.agent.createContainer
kata_agent.createContainer
kata_agent.buildContainerRootfs
container.mountSharedDirMounts
kata_agent.handleEphemeralStorage
kata_agent.appendDevices
kata_agent.handleBlockVolumes
kata_agent.replaceOCIMountsForStorages
kata_agent.handlePidNamespace
kata_agent.handleShm
kata_agent.sendReq
kata_agent.sendReq(&grpc.CreateContainerRequest)
prepareAndStartShim
shim.start
newShimConfig
startShim
exec.Command
cmd.Start
container.GetAnnotations
if container is sandbox
sandbox.setSandboxPid
container.storeProcess
container.setContainerState
for each container from the config: sandbox.addContainer
sandbox.storeSandbox
Sandbox Start
StartSandbox
fetchSandbox
sandbox.Start
sandbox.setSandboxState
container.start (For all containers)
container.checkSandboxRunning
sandbox.agent.startContainer
kata_agent.startContainer()
kata_agent.sendReq(grpc.StartContainerRequest)
container.setContainerState
Sandbox Run
RunSandbox
createSandboxFromConfig
sandbox.Start
sandbox.setSandboxState
container.start (For all containers)
container.checkSandboxRunning
sandbox.agent.startContainer
kata_agent.sendReq(grpc.StartContainerRequest)
container.setContainerState
Container Create
CreateContainer
fetchSandbox
sandbox.CreateContainer
createContainer
newContainer
container.createContainersDirs
if containerc.checkBlockDeviceSupport
container.hotplugDrive
container.attachDevices
container.addResources
container.getSystemMountInfo
container.storeDevices
sandbox.agent.createContainer
kata_agent.createContainer
kata_agent.buildContainerRootfs
container.mountSharedDirMounts
kata_agent.handleEphemeralStorage
kata_agent.appendDevices
kata_agent.handleBlockVolumes
kata_agent.replaceOCIMountsForStorages
kata_agent.handlePidNamespace
kata_agent.handleShm
kata_agent.sendReq
kata_agent.sendReq(&grpc.CreateContainerRequest)
prepareAndStartShim
shim.start
newShimConfig
startShim
exec.Command
cmd.Start
container.GetAnnotations
if container is sandbox
sandbox.setSandboxPid
container.storeProcess
container.setContainerState
sandbox.addContainer
container.storeContainer
sandbox.storage.storeSandboxResource
sandbox.setupCgroups
Container Start
StartContainer
fetchSandbox
sandox.StartContainer
sandbox.findContainer
container.start
container.checkSandboxRunning
sandbox.agent.startContainer
kata_agent.sendReq(grpc.StartContainerRequest)
container.setContainerState
virtcontainers/api.go CreateSandbox---->createSandboxFromConfig---->createContainers func (s *Sandbox) createContainers() error { span, _ := s.trace("createContainers") defer span.Finish() for _, contConfig := range s.config.Containers { c, err := newContainer(s, &contConfig) if err != nil { return err } if err := c.create(); err != nil { return err } if err := s.addContainer(c); err != nil { return err } // Update resources after having added containers to the sandbox, since // container status is requiered to know if more resources should be added. if err := s.updateResources(); err != nil { return err } if err := s.cgroupsUpdate(); err != nil { return err } if err := s.storeSandbox(); err != nil { return err } return nil } virtcontainers/api.go:547: c, process, err := s.EnterContainer(containerID, cmd) // EnterContainer is the virtcontainers container command execution entry point. // EnterContainer enters an already running container and runs a given command. func (s *Sandbox) EnterContainer(containerID string, cmd types.Cmd) (VCContainer, *Process, error) { // Fetch the container. c, err := s.findContainer(containerID) if err != nil { return nil, nil, err } // Enter it. process, err := c.enter(cmd) if err != nil { return nil, nil, err } return c, process, nil } func (c *Container) enter(cmd types.Cmd) (*Process, error) { if err := c.checkSandboxRunning("enter"); err != nil { return nil, err } if c.state.State != types.StateReady && c.state.State != types.StateRunning { return nil, fmt.Errorf("Container not ready or running, " + "impossible to enter") } process, err := c.sandbox.agent.exec(c.sandbox, *c, cmd) if err != nil { return nil, err } return process, nil } c.sandbox.agent.exec func (k *kataAgent) exec(sandbox *Sandbox, c Container, cmd types.Cmd) (*Process, error) { return prepareAndStartShim(sandbox, k.shim, c.id, req.ExecId, k.state.URL, "", cmd, []ns.NSType{}, enterNSList) } 调用 prepareAndStartShim 为每一个容器启动一个 shim /usr/libexec/kata-containers/kata-shim -agent unix:///run/vc/sbs/9c14119f6bf0d55c049b2ddceffe87beb45a731ac289e61cc5e069e08e081818/proxy.sock -container 9c14119f6bf0d55c049b2ddceffe87beb45a731ac289e61cc5e069e08e081818 -exec-id 9c14119f6bf0d55c049b2ddceffe87beb45a731ac289e61cc5e069e08e081818 -terminal -log debug
func (k *kataAgent) sendReq(request interface{}) (interface{}, error)
发送 CreateSandboxRequest 请求创建 req := &grpc.CreateSandboxRequest{ Hostname: hostname, Storages: storages, SandboxPidns: sandbox.sharePidNs, SandboxId: sandbox.id, GuestHookPath: sandbox.config.HypervisorConfig.GuestHookPath, } _, err = k.sendReq(req) if err != nil { return err
/go/src/github.com/kata-containers/runtime/virtcontainers/kata_agent.go:1716 +0x94 github.com/kata-containers/runtime/virtcontainers.(*kataAgent).sendReq(0xc00047d200, 0xeb1320, 0xc000160540, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/kata-containers/runtime/virtcontainers/kata_agent.go:1812 +0x60f github.com/kata-containers/runtime/virtcontainers.(*kataAgent).createContainer(0xc00047d200, 0xc0000ac840, 0xc0002fa820, 0x0, 0x0, 0x0) /go/src/github.com/kata-containers/runtime/virtcontainers/kata_agent.go:1242 +0xa7f github.com/kata-containers/runtime/virtcontainers.(*Container).create(0xc0002fa820, 0x0, 0x0) /go/src/github.com/kata-containers/runtime/virtcontainers/container.go:860 +0xf1 ithub.com/kata-containers/runtime/virtcontainers.(*Sandbox).CreateContainer(0xc0000ac840, 0xc00044a380, 0x40, 0xc000036050, 0xa, 0x0, 0x0, 0xc0002bc390, 0x4, 0x0, ...) /go/src/github.com/kata-containers/runtime/virtcontainers/sandbox.go:1146 +0x14c github.com/kata-containers/runtime/pkg/katautils.CreateContainer(0x10840e0, 0xc000504d80, 0x10a22e0, 0x19f9720, 0x10a3220, 0xc0000ac840, 0xc0002bc410, 0x9, 0x0, 0xc00042cf40, ...) /go/src/github.com/kata-containers/runtime/pkg/katautils/create.go:214 +0x2fc github.com/kata-containers/runtime/containerd-shim-v2.create(0x1084020, 0xc0000358c0, 0xc00047d800, 0xc0000330e0, 0x0, 0x0, 0x0) /go/src/github.com/kata-containers/runtime/containerd-shim-v2/create.go:114 +0x998 github.com/kata-containers/runtime/containerd-shim-v2.(*service).Create(0xc00047d800, 0x1084020, 0xc0000358c0, 0xc0000330e0, 0x0, 0x0, 0x0) /go/src/github.com/kata-containers/runtime/containerd-shim-v2/service.go:339 +0xce github.com/kata-containers/runtime/vendor/github.com/containerd/containerd/runtime/v2/task.RegisterTaskService.func2(0x1084020, 0xc0000358c0, 0xc00042cc00, 0xc0002bc380, 0x6, 0xc0003e60e0, 0x0)