https://segmentfault.com/a/1190000017576314
root@ubuntu:/# ls /run/libcontainer/ -al total 0 drwx------ 3 root root 60 Dec 4 06:37 . drwxr-xr-x 11 root root 220 Dec 4 06:37 .. drwx------ 3 root root 60 Dec 4 06:37 9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be root@ubuntu:/# ls /run/libcontainer/*/ 9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be root@ubuntu:/# ls /run/libcontainer/*/* state.json root@ubuntu:/# ls /run/libcontainer/*/state.json ls: cannot access '/run/libcontainer/*/state.json': No such file or directory root@ubuntu:/# ls /run/libcontainer/*/ 9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be root@ubuntu:/# ls /run/libcontainer/*/*/state.json /run/libcontainer/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be/state.json root@ubuntu:/# cat /run/libcontainer/*/*/state.json {"id":"9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be","init_process_pid":79,
"init_process_start":183,"created":"2020-12-04T06:37:22.27285124Z","config":{"no_pivot_root":false,
"parent_death_signal":0,"rootfs":"/run/kata-containers/shared/containers/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be/rootfs",
"readonlyfs":false,"rootPropagation":0,"mounts":[{"source":"proc","destination":"/proc","device":"proc","flags":14,"propagation_flags":null,"data":"","relabel":"",
"extensions":0,"premount_cmds":null,"postmount_cmds":null},{"source":"tmpfs","destination":"/dev","device":"tmpfs","flags":16777218,"propagation_flags":null,"data":"mode=755,size=65536k",
"relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},{"source":"devpts","destination":"/dev/pts","device":"devpts","flags":10,"propagation_flags":null,
"data":"newinstance,ptmxmode=0666,mode=0620,gid=5","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},{"source":"sysfs","destination":"/sys",
"device":"sysfs","flags":15,"propagation_flags":null,"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},
{"source":"cgroup","destination":"/sys/fs/cgroup","device":"cgroup","flags":15,"propagation_flags":null,"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},
{"source":"mqueue","destination":"/dev/mqueue","device":"mqueue","flags":14,"propagation_flags":null,"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},
{"source":"/run/kata-containers/shared/containers/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be-d7b1133d2132078e-resolv.conf","destination":"/etc/resolv.conf","device":"bind","flags":20480,"propagation_flags":[278528],
"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},
{"source":"/run/kata-containers/shared/containers/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be-164d7e3912cd20b1-hostname","destination":"/etc/hostname","device":"bind","flags":20480,"propagation_flags":[278528],
"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},{"source":"/run/kata-containers/shared/containers/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be-c27b5092d21c0407-hosts","destination":"/etc/hosts",
"device":"bind","flags":20480,"propagation_flags":[278528],"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},{"source":"/run/kata-containers/sandbox/shm","destination":"/dev/shm","device":"bind","flags":20480,
"propagation_flags":null,"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null}],"devices":[{"type":99,"path":"/dev/null","major":1,"minor":3,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},
{"type":99,"path":"/dev/random","major":1,"minor":8,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},{"type":99,"path":"/dev/full","major":1,"minor":7,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},
{"type":99,"path":"/dev/tty","major":5,"minor":0,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},{"type":99,"path":"/dev/zero","major":1,"minor":5,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},
{"type":99,"path":"/dev/urandom","major":1,"minor":9,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},{"type":99,"path":"/dev/fuse","major":10,"minor":229,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},
{"type":99,"path":"/dev/binder","major":10,"minor":60,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},{"type":99,"path":"/dev/ashmem","major":10,"minor":61,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},
{"type":99,"path":"/dev/input/mice","major":13,"minor":63,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false}],"mount_label":"","hostname":"9ddf50af4f9d","namespaces":[{"type":"NEWNS","path":""},{"type":"NEWUTS","path":"/var/run/sandbox-ns/uts"},
{"type":"NEWIPC","path":"/var/run/sandbox-ns/ipc"},{"type":"NEWPID","path":""}],"capabilities":{"Bounding":["CAP_CHOWN","CAP_DAC_OVERRIDE","CAP_FSETID","CAP_FOWNER","CAP_MKNOD","CAP_NET_RAW","CAP_SETGID","CAP_SETUID","CAP_SETFCAP","CAP_SETPCAP",
"CAP_NET_BIND_SERVICE","CAP_SYS_CHROOT","CAP_KILL","CAP_AUDIT_WRITE"],"Effective":["CAP_CHOWN","CAP_DAC_OVERRIDE","CAP_FSETID","CAP_FOWNER","CAP_MKNOD","CAP_NET_RAW","CAP_SETGID","CAP_SETUID","CAP_SETFCAP","CAP_SETPCAP","CAP_NET_BIND_SERVICE",
"CAP_SYS_CHROOT","CAP_KILL","CAP_AUDIT_WRITE"],"Inheritable":["CAP_CHOWN","CAP_DAC_OVERRIDE","CAP_FSETID","CAP_FOWNER","CAP_MKNOD","CAP_NET_RAW","CAP_SETGID","CAP_SETUID","CAP_SETFCAP","CAP_SETPCAP","CAP_NET_BIND_SERVICE","CAP_SYS_CHROOT",
"CAP_KILL","CAP_AUDIT_WRITE"],"Permitted":["CAP_CHOWN","CAP_DAC_OVERRIDE","CAP_FSETID","CAP_FOWNER","CAP_MKNOD","CAP_NET_RAW","CAP_SETGID","CAP_SETUID","CAP_SETFCAP","CAP_SETPCAP","CAP_NET_BIND_SERVICE","CAP_SYS_CHROOT","CAP_KILL","CAP_AUDIT_WRITE"],
"Ambient":[]},"networks":null,"routes":null,"cgroups":{"path":"/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be","scope_prefix":"","Paths":null,"allowed_devices":[{"type":99,"path":"","major":-1,"minor":-1,
"permissions":"m","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":98,"path":"","major":-1,"minor":-1,"permissions":"m","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/null","major":1,"minor":3,"permissions":
"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/random","major":1,"minor":8,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/full","major":1,"minor":7,"permissions":
"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/tty","major":5,"minor":0,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/zero","major":1,"minor":5,"permissions":"rwm",
"file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/urandom","major":1,"minor":9,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/console","major":5,"minor":1,"permissions":"rwm",
"file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"","major":136,"minor":-1,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"","major":5,"minor":2,"permissions":"rwm","file_mode":0,"uid":0,
"gid":0,"allow":true},{"type":99,"path":"","major":10,"minor":200,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true}],"devices":[{"type":98,"path":"","major":254,"minor":1,"permissions":"rw","file_mode":0,"uid":0,"gid":0,
"allow":false},{"type":99,"path":"","major":-1,"minor":-1,"permissions":"m","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":98,"path":"","major":-1,"minor":-1,"permissions":"m","file_mode":0,"uid":0,"gid":0,"allow":true},
{"type":99,"path":"/dev/null","major":1,"minor":3,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/random","major":1,"minor":8,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},
{"type":99,"path":"/dev/full","major":1,"minor":7,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/tty","major":5,"minor":0,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},
{"type":99,"path":"/dev/zero","major":1,"minor":5,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/urandom","major":1,"minor":9,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},
{"type":99,"path":"/dev/console","major":5,"minor":1,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"","major":136,"minor":-1,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},
{"type":99,"path":"","major":5,"minor":2,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"","major":10,"minor":200,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true}],"memory":0,"memory_reservation":0,
"memory_swap":0,"kernel_memory":0,"kernel_memory_tcp":0,"cpu_shares":0,"cpu_quota":0,"cpu_period":0,"cpu_rt_quota":0,"cpu_rt_period":0,"cpuset_cpus":"","cpuset_mems":"","pids_limit":0,"blkio_weight":0,"blkio_leaf_weight":0,"blkio_weight_device":null,
"blkio_throttle_read_bps_device":null,"blkio_throttle_write_bps_device":null,"blkio_throttle_read_iops_device":null,"blkio_throttle_write_iops_device":null,"freezer":"","hugetlb_limit":null,"oom_kill_disable":false,"memory_swappiness":0,
"net_prio_ifpriomap":null,"net_cls_classid_u":0,"cpu_weight":0,"cpu_max":""},"oom_score_adj":0,"uid_mappings":null,"gid_mappings":null,"mask_paths":["/proc/asound","/proc/acpi","/proc/kcore","/proc/keys","/proc/latency_stats",
"/proc/timer_list","/proc/timer_stats","/proc/sched_debug","/proc/scsi","/sys/firmware"],"readonly_paths":["/proc/bus","/proc/fs","/proc/irq","/proc/sys","/proc/sysrq-trigger"],"sysctl":{},"seccomp":null,
"Hooks":{"poststart":null,"poststop":null,"prestart":null},"version":"1.0.1-dev","labels":["bundle=/"],"no_new_keyring":true},
"rootless":false,"cgroup_paths":{"blkio":"/sys/fs/cgroup/blkio/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"cpu":"/sys/fs/cgroup/cpu,cpuacct/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"cpuacct":"/sys/fs/cgroup/cpu,cpuacct/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be","
cpuset":"/sys/fs/cgroup/cpuset/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"devices":"/sys/fs/cgroup/devices/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"freezer":"/sys/fs/cgroup/freezer/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"memory":"/sys/fs/cgroup/memory/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"name=systemd":"/sys/fs/cgroup/systemd/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"net_cls":"/sys/fs/cgroup/net_cls,net_prio/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"net_prio":"/sys/fs/cgroup/net_cls,net_prio/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"perf_event":"/sys/fs/cgroup/perf_event/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"pids":"/sys/fs/cgroup/pids/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be"},
"namespace_paths":{"NEWCGROUP":"/proc/79/ns/cgroup","NEWIPC":"/proc/79/ns/ipc","NEWNET":"/proc/79/ns/net","NEWNS":"/proc/79/ns/mnt","NEWPID":"/proc/79/ns/pid","NEWUSER":"/proc/79/ns/user","NEWUTS":"/proc/79/ns/uts"},
"external_descriptors":["/dev/null","/dev/null","/dev/null"],"intel_rdt_path":""}root@ubuntu:/#
root@ubuntu:/# ps -e -o pid,cmd PID CMD 1 /sbin/init 2 [kthreadd] 3 [rcu_gp] 4 [rcu_par_gp] 6 [kworker/0:0H-kb] 7 [kworker/u2:0-ev] 8 [mm_percpu_wq] 9 [ksoftirqd/0] 10 [rcu_sched] 11 [migration/0] 12 [cpuhp/0] 13 [kdevtmpfs] 14 [netns] 15 [oom_reaper] 16 [writeback] 17 [kcompactd0] 18 [kblockd] 19 [blkcg_punt_bio] 21 [kswapd0] 22 [xfsalloc] 23 [xfs_mru_cache] 24 [kthrotld] 26 [khvcd] 27 [hwrng] 29 [scsi_eh_0] 30 [scsi_tmf_0] 31 [uas] 32 [usbip_event] 33 [ipv6_addrconf] 34 [jbd2/vda1-8] 35 [ext4-rsv-conver] 51 /lib/systemd/systemd-journald 55 /usr/bin/Xvfb :8 -ac -screen 0 720x1280x24 56 /usr/bin/kata-agent 57 /opt/openvmi/bin/openvmi session-manager --run-multiple 59 [kworker/u2:2-ev] 65 /bin/bash 79 /bin/bash 105 [kworker/0:1H-kb] 2814 [kworker/0:1-cgr] 2837 [kworker/0:2-eve] 2850 ps -e -o pid,cmd
root@ubuntu:/# ls -al /proc/79/ns/ total 0 dr-x--x--x 2 root root 0 Dec 4 07:55 . dr-xr-xr-x 9 root root 0 Dec 4 06:37 .. lrwxrwxrwx 1 root root 0 Dec 4 07:55 cgroup -> 'cgroup:[4026531835]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 ipc -> 'ipc:[4026532171]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 mnt -> 'mnt:[4026532173]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 net -> 'net:[4026531897]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 pid -> 'pid:[4026532174]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 pid_for_children -> 'pid:[4026532174]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 user -> 'user:[4026531837]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 uts -> 'uts:[4026532172]' root@ubuntu:/#
docker执行ls,kata vm虚拟机接收请求的是pid=56
time="2020-12-04T07:59:07.02983572Z" level=debug msg="request end" debug_console=true duration=44.3329ms name=kata-agent pid=56 request=/grpc.AgentService/ReadStdout resp="data:"]" " sandbox=9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be source=agent
time="2020-12-04T07:59:08.73918658Z" level=debug msg="request end" debug_console=true duration=191.4736ms name=kata-agent pid=56 request=/grpc.AgentService/ReadStdout resp="data:"bin boot dev\tetc home lib\tmedia mnt opt proc root run sbin\tsrv sys tmp usr var\r\n" " sandbox=9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be source=agent
root@ubuntu:/# ps -e -o pid,cmd,ppid PID CMD PPID 1 /sbin/init 0 2 [kthreadd] 0 3 [rcu_gp] 2 4 [rcu_par_gp] 2 6 [kworker/0:0H-kb] 2 7 [kworker/u2:0-ev] 2 8 [mm_percpu_wq] 2 9 [ksoftirqd/0] 2 10 [rcu_sched] 2 11 [migration/0] 2 12 [cpuhp/0] 2 13 [kdevtmpfs] 2 14 [netns] 2 15 [oom_reaper] 2 16 [writeback] 2 17 [kcompactd0] 2 18 [kblockd] 2 19 [blkcg_punt_bio] 2 21 [kswapd0] 2 22 [xfsalloc] 2 23 [xfs_mru_cache] 2 24 [kthrotld] 2 26 [khvcd] 2 27 [hwrng] 2 29 [scsi_eh_0] 2 30 [scsi_tmf_0] 2 31 [uas] 2 32 [usbip_event] 2 33 [ipv6_addrconf] 2 34 [jbd2/vda1-8] 2 35 [ext4-rsv-conver] 2 51 /lib/systemd/systemd-journa 1 55 /usr/bin/Xvfb :8 -ac -scree 1 56 /usr/bin/kata-agent 1 57 /opt/openvmi/bin/openvmi se 1 59 [kworker/u2:2-ev] 2 65 /bin/bash 56 79 /bin/bash 56 105 [kworker/0:1H-kb] 2 2814 [kworker/0:1-cgr] 2 2837 [kworker/0:2-eve] 2 2859 ps -e -o pid,cmd,ppid 65
kata-agent有两个子进程
agent创建进程
libcontainerPath = "/run/libcontainer"
func (a *agentGRPC) finishCreateContainer(ctr *container, req *pb.CreateContainerRequest, config *configs.Config) (resp *gpb.Empty, err error) { containerPath := filepath.Join(libcontainerPath, a.sandbox.id) factory, err := libcontainer.New(containerPath, libcontainer.Cgroupfs) if err != nil { return emptyResp, err } ctr.container, err = factory.Create(req.ContainerId, config) if err != nil { return emptyResp, err } ctr.config = *config ctr.initProcess, err = buildProcess(req.OCI.Process, req.ExecId, true) ------------------------- init 进程 standard init if err != nil { return emptyResp, err }
func (a *agentGRPC) ExecProcess(ctx context.Context, req *pb.ExecProcessRequest) (*gpb.Empty, error) { ctr, err := a.getContainer(req.ContainerId) if err != nil { return emptyResp, err } status, err := ctr.container.Status() if err != nil { return nil, err } if status == libcontainer.Stopped { return nil, grpcStatus.Errorf(codes.FailedPrecondition, "Cannot exec in stopped container %s", req.ContainerId) } proc, err := buildProcess(req.Process, req.ExecId, false) if err != nil { return emptyResp, err } if err := a.execProcess(ctr, proc, false); err != nil { return emptyResp, err } return emptyResp, a.postExecProcess(ctr, proc) }
// Shared function between CreateContainer and ExecProcess, because those expect // a process to be run. func (a *agentGRPC) execProcess(ctr *container, proc *process, createContainer bool) (err error) { if ctr == nil { return grpcStatus.Error(codes.InvalidArgument, "Container cannot be nil") } if proc == nil { return grpcStatus.Error(codes.InvalidArgument, "Process cannot be nil") } // This lock is very important to avoid any race with reaper.reap(). // Indeed, if we don't lock this here, we could potentially get the // SIGCHLD signal before the channel has been created, meaning we will // miss the opportunity to get the exit code, leading WaitProcess() to // wait forever on the new channel. // This lock has to be taken before we run the new process. a.sandbox.subreaper.lock() defer a.sandbox.subreaper.unlock() if createContainer { err = ctr.container.Start(&proc.process) } else { err = ctr.container.Run(&(proc.process)) } if err != nil { return grpcStatus.Errorf(codes.Internal, "Could not run process: %v", err) } // Get process PID pid, err := proc.process.Pid() if err != nil { return err } proc.exitCodeCh = make(chan int, 1) // Create process channel to allow WaitProcess to wait on it. // This channel is buffered so that reaper.reap() will not // block until WaitProcess listen onto this channel. a.sandbox.subreaper.setExitCodeCh(pid, proc.exitCodeCh) return nil }
buildProcess init 参数确定是调用container run还是 container start
func buildProcess(agentProcess *pb.Process, procID string, init bool) (*process, error) { user := agentProcess.User.Username if user == "" { // We can specify the user and the group separated by ":" user = fmt.Sprintf("%d:%d", agentProcess.User.UID, agentProcess.User.GID) } additionalGids := []string{} for _, gid := range agentProcess.User.AdditionalGids { additionalGids = append(additionalGids, fmt.Sprintf("%d", gid)) } proc := &process{ id: procID, process: libcontainer.Process{ Cwd: agentProcess.Cwd, Args: agentProcess.Args, Env: agentProcess.Env, User: user, AdditionalGroups: additionalGids, Init: init, }, } if agentProcess.Terminal { parentSock, childSock, err := utils.NewSockPair("console") if err != nil { return nil, err } proc.process.ConsoleSocket = childSock proc.consoleSock = parentSock epoller, err := newEpoller() if err != nil { return nil, err } proc.epoller = epoller return proc, nil } rStdin, wStdin, err := os.Pipe() if err != nil { return nil, err } rStdout, wStdout, err := createExtendedPipe() if err != nil { return nil, err } rStderr, wStderr, err := createExtendedPipe() if err != nil { return nil, err } proc.process.Stdin = rStdin proc.process.Stdout = wStdout proc.process.Stderr = wStderr proc.stdin = wStdin proc.stdout = rStdout proc.stderr = rStderr
创建子进程
func (c *linuxContainer) start(process *Process) error { parent, err := c.newParentProcess(process) /* 1. 创建parentProcess */ err := parent.start(); /* 2. 启动这个parentProcess */ ......
/grpc.AgentService/ExecProcess 执行container.run
docker exec -it fa2d3b3aec76 bash OCI runtime exec failed: rpc error: code = Internal desc = Could not run process:
container_linux.go:349: starting container process caused "exec: "bash": executable file not found in $PATH": unknown
start是产生container init进程和docker exec -it xxx xxx的入口
func (c *linuxContainer) start(process *Process) error {
err := c.newParentProcess(process) //产生一个setnsprocess
if err != nil {
return newSystemErrorWithCause(err, "creating new parent process")
}
parent.forwardChildLogs()
if err := parent.start(); err != nil {
// terminate the process to ensure that it properly is reaped.
if err := ignoreTerminateErrors(parent.terminate()); err != nil {
logrus.Warn(err)
}
return newSystemErrorWithCause(err, "starting container process")
}
虚拟机日志
root@18b8dcedaa17:/# time="2020-12-04T08:20:33.4474552Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333"
exec_id:"80125fa6-c99c-4030-9561-9de98ab2f39d" process:<Terminal:true User:<> Args:"bash" Env:"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" Env:"HOSTNAME=fa2d3b3aec76" Env:"TERM=xterm" Cwd:"/" > "
request=/grpc.AgentService/ExecProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent
time="2020-12-04T08:20:33.4685562Z" level=debug msg="nsexec:601 nsexec started"
time="2020-12-04T08:20:33.49172604Z" level=debug msg="log pipe has been closed: EOF"
time="2020-12-04T08:20:33.4920517Z" level=debug msg="request end" debug_console=true duration=44.40578ms name=kata-agent pid=53 request=/grpc.AgentService/ExecProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent
time="2020-12-04T08:20:33.4923135Z" level=info msg="ignoring unexpected signal" debug_console=true name=kata-agent pid=53 sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 signal="child exited" source=agent
time="2020-12-04T08:20:33.49250004Z" level=info msg="ignoring unexpected signal" debug_console=true name=kata-agent pid=53 sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 signal="child exited" source=agent
time="2020-12-04T08:20:33.4926396Z" level=info msg="ignoring unexpected signal" debug_console=true name=kata-agent pid=53 sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 signal="child exited" source=agent
func (a *agentGRPC) ExecProcess(ctx context.Context, req *pb.ExecProcessRequest) (*gpb.Empty, error) { ctr, err := a.getContainer(req.ContainerId) if err != nil { return emptyResp, err } status, err := ctr.container.Status() if err != nil { return nil, err } if status == libcontainer.Stopped { return nil, grpcStatus.Errorf(codes.FailedPrecondition, "Cannot exec in stopped container %s", req.ContainerId) } proc, err := buildProcess(req.Process, req.ExecId, false) if err != nil { return emptyResp, err } if err := a.execProcess(ctr, proc, false); err != nil { return emptyResp, err } return emptyResp, a.postExecProcess(ctr, proc) }
void nsexec(void) { int pipenum; jmp_buf env; int sync_child_pipe[2], sync_grandchild_pipe[2]; struct nlconfig_t config = { 0 }; /* * Setup a pipe to send logs to the parent. This should happen * first, because bail will use that pipe. */ setup_logpipe(); /* * If we don't have an init pipe, just return to the go routine. * We'll only get an init pipe for start or exec. */ pipenum = initpipe(); if (pipenum == -1) return; /* * We need to re-exec if we are not in a cloned binary. This is necessary * to ensure that containers won't be able to access the host binary * through /proc/self/exe. See CVE-2019-5736. */ if (ensure_cloned_binary() < 0) bail("could not ensure we are a cloned binary"); write_log(DEBUG, "nsexec started");
msg="log pipe has been closed: EOF"
func ForwardLogs(logPipe io.Reader) { lineReader := bufio.NewReader(logPipe) for { line, err := lineReader.ReadBytes(' ') if len(line) > 0 { processEntry(line) } if err == io.EOF { logrus.Debugf("log pipe has been closed: %+v", err) return } if err != nil { logrus.Errorf("log pipe read error: %+v", err) } } }
func (p *setnsProcess) forwardChildLogs() {
go logs.ForwardLogs(p.logFilePair.parent)
}
func (c *linuxContainer) start(process *Process) error { parent, err := c.newParentProcess(process) if err != nil { return newSystemErrorWithCause(err, "creating new parent process") } parent.forwardChildLogs()
"SIGCHLD", "child exited"},
runc进程
/* utils_linux.go */ func startContainer(context *cli.Context, spec *specs.Spec, action CtAct, criuOpts *libcontainer.CriuOpts) (int, error) { id := context.Args().First() container, err := createContainer(context, id, spec) r := &runner{ container: container, action: action, init: true, ...... } return r.run(spec.Process) }
func (r *runner) run(config *specs.Process) (int, error) { ...... process, err := newProcess(*config, r.init) /* 第1部分 */ ...... switch r.action { case CT_ACT_CREATE: err = r.container.Start(process) /* runc start */ /* 第2部分 */ case CT_ACT_RESTORE: err = r.container.Restore(process, r.criuOpts) /* runc restore */ case CT_ACT_RUN: err = r.container.Run(process) /* runc run */ default: panic("Unknown action") } ...... return status, err }
创建子进程
func (c *linuxContainer) start(process *Process) error { parent, err := c.newParentProcess(process) /* 1. 创建parentProcess */ err := parent.start(); /* 2. 启动这个parentProcess */ ......
newContainerInit
StartInitialization is an internal API to libcontainer used during the reexec
func newContainerInit(t initType, pipe *os.File, consoleSocket *os.File, fifoFd int) (initer, error) { var config *initConfig if err := json.NewDecoder(pipe).Decode(&config); err != nil { return nil, err } if err := populateProcessEnvironment(config.Env); err != nil { return nil, err } switch t { case initSetns: return &linuxSetnsInit{ pipe: pipe, consoleSocket: consoleSocket, config: config, }, nil case initStandard: return &linuxStandardInit{ pipe: pipe, consoleSocket: consoleSocket, parentPid: unix.Getppid(), config: config, fifoFd: fifoFd, }, nil } return nil, fmt.Errorf("unknown init type %q", t) }
setnsProcess
func (c *linuxContainer) newParentProcess(p *Process) (parentProcess, error) { parentInitPipe, childInitPipe, err := utils.NewSockPair("init") if err != nil { return nil, newSystemErrorWithCause(err, "creating new init pipe") } messageSockPair := filePair{parentInitPipe, childInitPipe} parentLogPipe, childLogPipe, err := os.Pipe() if err != nil { return nil, fmt.Errorf("Unable to create the log pipe: %s", err) } logFilePair := filePair{parentLogPipe, childLogPipe} cmd, err := c.commandTemplate(p, childInitPipe, childLogPipe) if err != nil { return nil, newSystemErrorWithCause(err, "creating new command template") } if !p.Init { return c.newSetnsProcess(p, cmd, messageSockPair, logFilePair) } // We only set up fifoFd if we're not doing a `runc exec`. The historic // reason for this is that previously we would pass a dirfd that allowed // for container rootfs escape (and not doing it in `runc exec` avoided // that problem), but we no longer do that. However, there's no need to do // this for `runc exec` so we just keep it this way to be safe. if err := c.includeExecFifo(cmd); err != nil { return nil, newSystemErrorWithCause(err, "including execfifo in cmd.Exec setup") } return c.newInitProcess(p, cmd, messageSockPair, logFilePair) }
func (p *setnsProcess) start() (err error) { defer p.messageSockPair.parent.Close() err = p.cmd.Start() // close the write-side of the pipes (controlled by child) p.messageSockPair.child.Close() p.logFilePair.child.Close() if err != nil { return newSystemErrorWithCause(err, "starting setns process") } if p.bootstrapData != nil { if _, err := io.Copy(p.messageSockPair.parent, p.bootstrapData); err != nil { return newSystemErrorWithCause(err, "copying bootstrap data to pipe") } } if err = p.execSetns(); err != nil { return newSystemErrorWithCause(err, "executing setns process") } if len(p.cgroupPaths) > 0 { if err := cgroups.EnterPid(p.cgroupPaths, p.pid()); err != nil && !p.rootlessCgroups { return newSystemErrorWithCausef(err, "adding pid %d to cgroups", p.pid()) } } if p.intelRdtPath != "" { // if Intel RDT "resource control" filesystem path exists _, err := os.Stat(p.intelRdtPath) if err == nil { if err := intelrdt.WriteIntelRdtTasks(p.intelRdtPath, p.pid()); err != nil { return newSystemErrorWithCausef(err, "adding pid %d to Intel RDT resource control filesystem", p.pid()) } } }
root@ubuntu:/opt/gopath/src/github.com/kata-containers/osbuilder/image-builder# docker exec -it fa2d3b3aec76 sh / # xuy sh: xuy: not found / #
root@18b8dcedaa17:/# time="2020-12-04T09:23:19.51302612Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" data:"x" " request=/grpc.AgentService/WriteStdin sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:19.51334756Z" level=debug msg="request end" debug_console=true duration=1m15.9069783s name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"x" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:19.51361074Z" level=debug msg="request end" debug_console=true duration="348.78µs" name=kata-agent pid=53 request=/grpc.AgentService/WriteStdin resp="len:1 " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:19.51449044Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:19.8648261Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" data:"u" " request=/grpc.AgentService/WriteStdin sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:19.86506808Z" level=debug msg="request end" debug_console=true duration=350.43514ms name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"u" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:19.8652743Z" level=debug msg="request end" debug_console=true duration="301.02µs" name=kata-agent pid=53 request=/grpc.AgentService/WriteStdin resp="len:1 " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:19.86563104Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:19.86596286Z" level=debug msg="request end" debug_console=true duration="197.98µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:19.86634324Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.18408504Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.18431734Z" level=debug msg="request end" debug_console=true duration="91µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.18449314Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" data:"y" " request=/grpc.AgentService/WriteStdin sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.18462954Z" level=debug msg="request end" debug_console=true duration="7.9µs" name=kata-agent pid=53 request=/grpc.AgentService/WriteStdin resp="len:1 " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.18489352Z" level=debug msg="request end" debug_console=true duration=318.41182ms name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"y" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.18572276Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.1859757Z" level=debug msg="request end" debug_console=true duration="109.12µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.18633482Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.53632508Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.53657206Z" level=debug msg="request end" debug_console=true duration="101.72µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.5367555Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" data:"\r" " request=/grpc.AgentService/WriteStdin sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.536957Z" level=debug msg="request end" debug_console=true duration="70.92µs" name=kata-agent pid=53 request=/grpc.AgentService/WriteStdin resp="len:1 " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.53828124Z" level=debug msg="request end" debug_console=true duration=351.75706ms name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"\r\n" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.53917278Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.54349172Z" level=debug msg="request end" debug_console=true duration=4.09394ms name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"sh: xuy: not found\r\n" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.54420054Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.54509254Z" level=debug msg="request end" debug_console=true duration="738.76µs" name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"/ # " " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:23:20.54574606Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent
/ # ls bin dev etc home proc root sys tmp usr var / # ls bin dev etc home proc root sys tmp usr var
docker exec -it fa2d3b3aec76 sh
root@18b8dcedaa17:/# time="2020-12-04T09:22:03.54754226Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" process:<Terminal:true User:<> Args:"sh" Env:"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" Env:"HOSTNAME=fa2d3b3aec76" Env:"TERM=xterm" Cwd:"/" > " request=/grpc.AgentService/ExecProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:22:03.56826312Z" level=debug msg="nsexec:601 nsexec started" time="2020-12-04T09:22:03.58762078Z" level=debug msg="log pipe has been closed: EOF" time="2020-12-04T09:22:03.5879807Z" level=debug msg="request end" debug_console=true duration=40.21832ms name=kata-agent pid=53 request=/grpc.AgentService/ExecProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:22:03.58826668Z" level=info msg="ignoring unexpected signal" debug_console=true name=kata-agent pid=53 sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 signal="child exited" source=agent time="2020-12-04T09:22:03.58844564Z" level=info msg="ignoring unexpected signal" debug_console=true name=kata-agent pid=53 sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 signal="child exited" source=agent time="2020-12-04T09:22:03.60393984Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:22:03.60417806Z" level=debug msg="request end" debug_console=true duration="67.22µs" name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"/ # " " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:22:03.60481176Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" row:68 column:252 " request=/grpc.AgentService/TtyWinResize sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:22:03.6049623Z" level=debug msg="request end" debug_console=true duration="12.6µs" name=kata-agent pid=53 request=/grpc.AgentService/TtyWinResize resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:22:03.60542124Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:22:03.6056203Z" level=debug msg="request end" debug_console=true duration="43.54µs" name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"\r/ # \033[J" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:22:03.60622234Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"84a1a25d-a11a-4beb-b0ca-611eeeb0c46d" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent
退出
docker exec -it fa2d3b3aec76 sh
/ # exit
root@18b8dcedaa17:/# time="2020-12-04T09:18:45.76305396Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:45.7633844Z" level=debug msg="request end" debug_console=true duration="162.5µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:45.76358678Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" data:"e" " request=/grpc.AgentService/WriteStdin sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:45.76375774Z" level=debug msg="request end" debug_console=true duration="31.98µs" name=kata-agent pid=53 request=/grpc.AgentService/WriteStdin resp="len:1 " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:45.76408524Z" level=debug msg="request end" debug_console=true duration=31.5602787s name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"e" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:45.76473818Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.12692556Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" data:"x" " request=/grpc.AgentService/WriteStdin sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.12707716Z" level=debug msg="request end" debug_console=true duration="8.5µs" name=kata-agent pid=53 request=/grpc.AgentService/WriteStdin resp="len:1 " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.12735044Z" level=debug msg="request end" debug_console=true duration=362.40738ms name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"x" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.12765788Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.12835694Z" level=debug msg="request end" debug_console=true duration="142.76µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.1288451Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.1292482Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.12946872Z" level=debug msg="request end" debug_console=true duration="81.96µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.35423358Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.35446256Z" level=debug msg="request end" debug_console=true duration="94.36µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.35463972Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" data:"i" " request=/grpc.AgentService/WriteStdin sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.3548014Z" level=debug msg="request end" debug_console=true duration="22.68µs" name=kata-agent pid=53 request=/grpc.AgentService/WriteStdin resp="len:1 " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.35510566Z" level=debug msg="request end" debug_console=true duration=226.1052ms name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"i" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.35634962Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.5427709Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.5430257Z" level=debug msg="request end" debug_console=true duration="107.82µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.54321534Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" data:"t" " request=/grpc.AgentService/WriteStdin sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.54337258Z" level=debug msg="request end" debug_console=true duration="11.98µs" name=kata-agent pid=53 request=/grpc.AgentService/WriteStdin resp="len:1 " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.54364966Z" level=debug msg="request end" debug_console=true duration=187.14902ms name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"t" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.54460772Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.54497516Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.54522954Z" level=debug msg="request end" debug_console=true duration="102.02µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.74553508Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" signal:23 " request=/grpc.AgentService/SignalProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.74584552Z" level=debug msg="request end" debug_console=true duration="137.42µs" name=kata-agent pid=53 request=/grpc.AgentService/SignalProcess resp="&Empty{XXX_unrecognized:[],}" sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.74605258Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" data:"\r" " request=/grpc.AgentService/WriteStdin sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.74620478Z" level=debug msg="request end" debug_console=true duration="16.04µs" name=kata-agent pid=53 request=/grpc.AgentService/WriteStdin resp="len:1 " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.74640278Z" level=debug msg="request end" debug_console=true duration=201.63198ms name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp="data:"\r\n" " sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.74797294Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.74911146Z" level=debug msg="request end" debug_console=true duration="907.28µs" name=kata-agent pid=53 request=/grpc.AgentService/ReadStdout resp= sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.7494895Z" level=debug msg="process exited" debug_console=true name=kata-agent pid=609 sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent status=0 time="2020-12-04T09:18:46.74962264Z" level=info msg="ignoring unexpected signal" debug_console=true name=kata-agent pid=53 sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 signal="child exited" source=agent time="2020-12-04T09:18:46.75011404Z" level=debug msg="new request" debug_console=true name=kata-agent pid=53 req="container_id:"fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333" exec_id:"27cfb0d7-fd94-49d1-8b08-e0f0b417840b" " request=/grpc.AgentService/WaitProcess sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent time="2020-12-04T09:18:46.7503451Z" level=debug msg="request end" debug_console=true duration="50.46µs" name=kata-agent pid=53 request=/grpc.AgentService/WaitProcess resp= sandbox=fa2d3b3aec7624d7472d9b6aa8a7bccd15e6212a12adda08b72f61b995ccf333 source=agent
管道
https://juejin.cn/post/6844903941860245512
func (c *linuxContainer) newParentProcess(p *Process) (parentProcess, error) { parentInitPipe, childInitPipe, err := utils.NewSockPair("init") if err != nil { return nil, newSystemErrorWithCause(err, "creating new init pipe") } messageSockPair := filePair{parentInitPipe, childInitPipe} parentLogPipe, childLogPipe, err := os.Pipe() if err != nil { return nil, fmt.Errorf("Unable to create the log pipe: %s", err) } logFilePair := filePair{parentLogPipe, childLogPipe} cmd, err := c.commandTemplate(p, childInitPipe, childLogPipe) if err != nil { return nil, newSystemErrorWithCause(err, "creating new command template") } if !p.Init { return c.newSetnsProcess(p, cmd, messageSockPair, logFilePair) } // We only set up fifoFd if we're not doing a `runc exec`. The historic // reason for this is that previously we would pass a dirfd that allowed // for container rootfs escape (and not doing it in `runc exec` avoided // that problem), but we no longer do that. However, there's no need to do // this for `runc exec` so we just keep it this way to be safe. if err := c.includeExecFifo(cmd); err != nil { return nil, newSystemErrorWithCause(err, "including execfifo in cmd.Exec setup") } return c.newInitProcess(p, cmd, messageSockPair, logFilePair) }
// CloseExecFrom applies O_CLOEXEC to all file descriptors currently open for // the process (except for those below the given fd value). func CloseExecFrom(minFd int) error { fdDir, err := os.Open("/proc/self/fd") if err != nil { return err } defer fdDir.Close() if err := EnsureProcHandle(fdDir); err != nil { return err } fdList, err := fdDir.Readdirnames(-1) if err != nil { return err } for _, fdStr := range fdList { fd, err := strconv.Atoi(fdStr) // Ignore non-numeric file names. if err != nil { continue } // Ignore descriptors lower than our specified minimum. if fd < minFd { continue } // Intentionally ignore errors from unix.CloseOnExec -- the cases where // this might fail are basically file descriptors that have already // been closed (including and especially the one that was created when // ioutil.ReadDir did the "opendir" syscall). unix.CloseOnExec(fd) } return nil } // NewSockPair returns a new unix socket pair func NewSockPair(name string) (parent *os.File, child *os.File, err error) { fds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) if err != nil { return nil, nil, err } return os.NewFile(uintptr(fds[1]), name+"-p"), os.NewFile(uintptr(fds[0]), name+"-c"), nil }
- 上面这段 C 代码中，initpipe() 从环境中读取父进程之前设置的pipe(_LIBCONTAINER_INITPIPE记录的的文件描述符)，然后调用 nl_parse 从这个管道中读取配置到变量 config ，那么谁会往这个管道写配置呢 ? 当然就是runc create父进程了。父进程通过这个pipe，将新建容器的配置发给子进程，
- 发送的具体数据在 linuxContainer 的 bootstrapData() 函数中封装成netlink msg格式的消息。忽略大部分配置，要创建哪些类型的namespace，这些都是源自最初的config.json文件。
func (l *linuxStandardInit) Init() error { ...... name, err := exec.LookPath(l.config.Args[0]) syscall.Exec(name, l.config.Args[0:], os.Environ()) }