https://github.com/haslm/toolchain/tree/a29fb3909141887df1d898181010f71ac011876f/runc
/ finalizeRootfs sets anything to ro if necessary. You must call // prepareRootfs first. func finalizeRootfs(config *configs.Config) (err error) { // remount dev as ro if specified for _, m := range config.Mounts { if libcontainerUtils.CleanPath(m.Destination) == "/dev" { if m.Flags&unix.MS_RDONLY == unix.MS_RDONLY { if err := remountReadonly(m); err != nil { return newSystemErrorWithCausef(err, "remounting %q as readonly", m.Destination) } } break } } // set rootfs ( / ) as readonly if config.Readonlyfs { if err := setReadonly(); err != nil { return newSystemErrorWithCause(err, "setting rootfs as readonly") } } unix.Umask(0022) return nil }
docker: Error response from daemon: OCI runtime create failed: rpc error: code = Internal desc = Could not run process: container_linux.go:370: starting container process caused
"process_linux.go:470: container init caused "rootfs_linux.go:121: chmod \"/run/kata-containers/shared/containers/21e0278cb1557bb003c9496db4bdb7291ef12ea1f8cc66dccffd8809e668ca88
docker: Error response from daemon: OCI runtime create failed: rpc error: code = Internal desc = Could not run process:
container_linux.go:370: starting container process caused "process_linux.go:470: container init caused "rootfs_linux.go:121:
chmod \"/run/kata-containers/shared/containers/21e0278cb1557bb003c9496db4bdb7291ef12ea1f8cc66dccffd8809e668ca88/rootfs/dev/openvmi/
sockets/qemu_pipe\"
fail caused \"chmod /run/kata-containers/shared/containers/21e0278cb1557bb003c9496db4bdb7291ef12ea1f8cc66dccffd8809e668ca88/rootfs
/dev/openvmi/sockets/qemu_pipe: no such file or directory\""": unknown.
// initialises the labeling system selinux.GetEnabled() if err := prepareRootfs(l.pipe, l.config); err != nil { return err } prepareRootfs失败了，不会发syncParentReady // Tell our parent that we're ready to Execv. This must be done before the // Seccomp rules have been applied, because we need to be able to read and // write to a socket. if err := syncParentReady(l.pipe); err != nil { return errors.Wrap(err, "sync ready") }
go get github.com/opencontainers/runc cd $GOPATH/src/github.com/opencontainers/runc ls make -j48 mkdir /mycontainer cd /mycontainer mkdir rootfs docker export $(docker create busybox) | tar -C rootfs -xvf -
runc spec
cat config.json
$GOPATH/src/github.com/opencontainers/runc/runc run mycontaineridroot@cloud:/mycontainer# cat config.json { "ociVersion": "1.0.2-dev", "process": { "terminal": true, "user": { "uid": 0, "gid": 0 }, "args": [ "sh" ], "env": [ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "TERM=xterm" ], "cwd": "/", "capabilities": { "bounding": [ "CAP_AUDIT_WRITE", "CAP_KILL", "CAP_NET_BIND_SERVICE" ], "effective": [ "CAP_AUDIT_WRITE", "CAP_KILL", "CAP_NET_BIND_SERVICE" ], "inheritable": [ "CAP_AUDIT_WRITE", "CAP_KILL", "CAP_NET_BIND_SERVICE" ], "permitted": [ "CAP_AUDIT_WRITE", "CAP_KILL", "CAP_NET_BIND_SERVICE" ], "ambient": [ "CAP_AUDIT_WRITE", "CAP_KILL", "CAP_NET_BIND_SERVICE" ] }, "rlimits": [ { "type": "RLIMIT_NOFILE", "hard": 1024, "soft": 1024 } ], "noNewPrivileges": true }, "root": { "path": "rootfs", "readonly": true }, "hostname": "runc", "mounts": [ { "destination": "/proc", "type": "proc", "source": "proc" }, { "destination": "/dev", "type": "tmpfs", "source": "tmpfs", "options": [ "nosuid", "strictatime", "mode=755", "size=65536k" ] }, { "destination": "/dev/pts", "type": "devpts", "source": "devpts", "options": [ "nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620", "gid=5" ] }, { "destination": "/dev/shm", "type": "tmpfs", "source": "shm", "options": [ "nosuid", "noexec", "nodev", "mode=1777", "size=65536k" ] }, { "destination": "/dev/mqueue", "type": "mqueue", "source": "mqueue", "options": [ "nosuid", "noexec", "nodev" ] }, { "destination": "/sys", "type": "sysfs", "source": "sysfs", "options": [ "nosuid", "noexec", "nodev", "ro" ] }, { "destination": "/sys/fs/cgroup", "type": "cgroup", "source": "cgroup", "options": [ "nosuid", "noexec", "nodev", "relatime", "ro" ] } ], "linux": { "resources": { "devices": [ { "allow": false, "access": "rwm" } ] }, "namespaces": [ { "type": "pid" }, { "type": "network" }, { "type": "ipc" }, { "type": "uts" }, { "type": "mount" } ], "maskedPaths": [ "/proc/acpi", "/proc/asound", "/proc/kcore", "/proc/keys", "/proc/latency_stats", "/proc/timer_list", "/proc/timer_stats", "/proc/sched_debug", "/sys/firmware", "/proc/scsi" ], "readonlyPaths": [ "/proc/bus", "/proc/fs", "/proc/irq", "/proc/sys", "/proc/sysrq-trigger" ] } }
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt spec File config.json exists. Remove it first root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mycontainerid / # ls bin dev etc home proc root sys tmp usr var / # exit root@cloud:/mycontainer# l;s config.json log.txt rootfs/ s: command not found root@cloud:/mycontainer# ls config.json log.txt rootfs root@cloud:/mycontainer# cat log.txt time="2020-12-04T20:13:48+08:00" level=error msg="File config.json exists. Remove it first" time="2020-12-04T20:14:02+08:00" level=debug msg="nsexec:601 nsexec started" time="2020-12-04T20:14:02+08:00" level=debug msg="child process in init()" time="2020-12-04T20:14:02+08:00" level=debug msg="logging has already been configured" time="2020-12-04T20:14:02+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-04T20:14:02+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-04T20:14:02+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-04T20:14:06+08:00" level=debug msg="process exited" pid=12741 status=0 root@cloud:/mycontainer#
// forward handles the main signal event loop forwarding, resizing, or reaping depending // on the signal received. func (h *signalHandler) forward(process *libcontainer.Process, tty *tty, detach bool) (int, error) { // make sure we know the pid of our main process so that we can return // after it dies. if detach && h.notifySocket == nil { return 0, nil } pid1, err := process.Pid() if err != nil { return -1, err } if h.notifySocket != nil { if detach { h.notifySocket.run(pid1) return 0, nil } h.notifySocket.run(os.Getpid()) go h.notifySocket.run(0) } // Perform the initial tty resize. Always ignore errors resizing because // stdout might have disappeared (due to races with when SIGHUP is sent). _ = tty.resize() // Handle and forward signals. for s := range h.signals { switch s { case unix.SIGWINCH: // Ignore errors resizing, as above. _ = tty.resize() case unix.SIGCHLD: exits, err := h.reap() if err != nil { logrus.Error(err) } for _, e := range exits { logrus.WithFields(logrus.Fields{ "pid": e.pid, "status": e.status, }).Debug("process exited") if e.pid == pid1 { // call Wait() on the process even though we already have the exit // status because we must ensure that any of the go specific process // fun such as flushing pipes are complete before we return. process.Wait() return e.status, nil } } default: logrus.Debugf("sending signal to process %s", s) if err := unix.Kill(pid1, s.(unix.Signal)); err != nil { logrus.Error(err) } } } return -1, nil }
root@cloud:/mycontainer# cat log.txt time="2020-12-04T20:13:48+08:00" level=error msg="File config.json exists. Remove it first" time="2020-12-04T20:14:02+08:00" level=debug msg="nsexec:601 nsexec started" time="2020-12-04T20:14:02+08:00" level=debug msg="child process in init()" time="2020-12-04T20:14:02+08:00" level=debug msg="logging has already been configured" time="2020-12-04T20:14:02+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-04T20:14:02+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-04T20:14:02+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-04T20:14:06+08:00" level=debug msg="process exited" pid=12741 status=0 time="2020-12-05T10:58:32+08:00" level=debug msg="nsexec:602 nsexec started" time="2020-12-05T10:58:32+08:00" level=debug msg="child process in init()" time="2020-12-05T10:58:32+08:00" level=debug msg="logging has already been configured" time="2020-12-05T10:58:32+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-05T10:58:32+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T10:58:32+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T10:59:56+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T10:59:56+08:00" level=debug msg="sending signal to process urgent I/O condition" root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc list ID PID STATUS BUNDLE CREATED OWNER mycontainerid 14506 running /mycontainer 2020-12-05T02:58:32.931468305Z root root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc exec mycontainerid ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever root@cloud:/mycontainer# cat log.txt time="2020-12-04T20:13:48+08:00" level=error msg="File config.json exists. Remove it first" time="2020-12-04T20:14:02+08:00" level=debug msg="nsexec:601 nsexec started" time="2020-12-04T20:14:02+08:00" level=debug msg="child process in init()" time="2020-12-04T20:14:02+08:00" level=debug msg="logging has already been configured" time="2020-12-04T20:14:02+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-04T20:14:02+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-04T20:14:02+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-04T20:14:06+08:00" level=debug msg="process exited" pid=12741 status=0 time="2020-12-05T10:58:32+08:00" level=debug msg="nsexec:602 nsexec started" time="2020-12-05T10:58:32+08:00" level=debug msg="child process in init()" time="2020-12-05T10:58:32+08:00" level=debug msg="logging has already been configured" time="2020-12-05T10:58:32+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-05T10:58:32+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T10:58:32+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T10:59:56+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T10:59:56+08:00" level=debug msg="sending signal to process urgent I/O condition" root@cloud:/mycontainer#
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mycontainerid container_linux.go:367: starting container process caused: exec: "sleep 5": executable file not found in $PATH root@cloud:/mycontainer#
func (c *linuxContainer) start(process *Process) error { parent, err := c.newParentProcess(process) if err != nil { return newSystemErrorWithCause(err, "creating new parent process") } parent.forwardChildLogs() if err := parent.start(); err != nil { return newSystemErrorWithCause(err, "starting container process") } if process.Init { if c.config.Hooks != nil { s, err := c.currentOCIState() if err != nil { return err } if err := c.config.Hooks[configs.Poststart].RunHooks(s); err != nil { if err := ignoreTerminateErrors(parent.terminate()); err != nil { logrus.Warn(errorsf.Wrapf(err, "Running Poststart hook")) } return err } }
nginx
root@cloud:/mycontainer/nginx# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mynginx
standard_init_linux.go:218: ** exec user process /bin/sh and Args[0:] sh caused:
root@cloud:/mycontainer/nginx# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mynginx panic from initialization: runtime error: index out of range [1] with length 1, goroutine 1 [running, locked to thread]: runtime/debug.Stack(0x40002aadc8, 0xaaaacb026f60, 0x4000152200) /usr/local/go/src/runtime/debug/stack.go:24 +0x88 github.com/opencontainers/runc/libcontainer.(*LinuxFactory).StartInitialization.func2(0x40002ab4a0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/factory_linux.go:386 +0x4c panic(0xaaaacb026f60, 0x4000152200) /usr/local/go/src/runtime/panic.go:975 +0x370 github.com/opencontainers/runc/libcontainer.(*linuxStandardInit).Init(0x4000282420, 0x0, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/standard_init_linux.go:218 +0x115c github.com/opencontainers/runc/libcontainer.(*LinuxFactory).StartInitialization(0x4000240120, 0x0, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/factory_linux.go:396 +0x2c4 main.glob..func6(0x40001c49a0, 0xaaaacb02a100, 0x0) /opt/gopath/src/github.com/opencontainers/runc/init.go:43 +0x40 github.com/urfave/cli.HandleAction(0xaaaacafac800, 0xaaaacb061978, 0x40001c49a0, 0x40001c49a0, 0x0) /opt/gopath/src/github.com/opencontainers/runc/vendor/github.com/urfave/cli/app.go:523 +0x140 github.com/urfave/cli.Command.Run(0xaaaacae78f4c, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0xaaaacae970db, 0x51, 0x0, ...) /opt/gopath/src/github.com/opencontainers/runc/vendor/github.com/urfave/cli/command.go:174 +0x410 github.com/urfave/cli.(*App).Run(0x40001be8c0, 0x400000e080, 0x2, 0x2, 0x0, 0x0) /opt/gopath/src/github.com/opencontainers/runc/vendor/github.com/urfave/cli/app.go:276 +0x584 main.main() /opt/gopath/src/github.com/opencontainers/runc/main.go:160 +0xa28
docker export $(docker create nginx/unit:latest) | tar -C rootfs -xvf -
root@cloud:/mycontainer/nginx# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mynginx standard_init_linux.go:220: exec user process caused: exec format error root@cloud:/mycontainer/nginx#
root@cloud:/mycontainer# runc exec mybusy lsns ERRO[0000] exec failed: container_linux.go:349: starting container process caused "exec: "lsns": executable file not found in $PATH" exec failed: container_linux.go:349: starting container process caused "exec: "lsns": executable file not found in $PATH" root@cloud:/mycontainer# runc exec mybusy lsns
root@cloud:/mycontainer# runc exec mybusy lsns ERRO[0000] exec failed: container_linux.go:349: starting container process caused "exec: "lsns": executable file not found in $PATH" exec failed: container_linux.go:349: starting container process caused "exec: "lsns": executable file not found in $PATH" root@cloud:/mycontainer#
busy box
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mybusy / # ls bin dev etc home proc root sys tmp usr var / # ps -elf PID USER TIME COMMAND 1 root 0:00 sh 8 root 0:00 ps -elf / #
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log2.txt exec mybusy ps PID USER TIME COMMAND 1 root 0:00 sh 21 root 0:00 ps root@cloud:/mycontainer# cat log2.txt time="2020-12-05T12:01:37+08:00" level=debug msg="nsexec:602 nsexec started" time="2020-12-05T12:01:37+08:00" level=debug msg="nsexec:702 jump parent" time="2020-12-05T12:01:37+08:00" level=debug msg="nsexec:831 jump child" time="2020-12-05T12:01:37+08:00" level=debug msg="nsexec:970 jump init" time="2020-12-05T12:01:37+08:00" level=debug msg="child process in init()" time="2020-12-05T12:01:37+08:00" level=debug msg="logging has already been configured" time="2020-12-05T12:01:37+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-05T12:01:38+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T12:01:38+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T12:01:38+08:00" level=debug msg="process exited" pid=18005 status=0 root@cloud:/mycontainer# ls config.json exec.sh log2.txt log.txt nginx rootfs root@cloud:/mycontainer# cat log.txt time="2020-12-05T11:54:50+08:00" level=debug msg="nsexec:602 nsexec started" time="2020-12-05T11:54:50+08:00" level=debug msg="nsexec:702 jump parent" time="2020-12-05T11:54:50+08:00" level=debug msg="nsexec:831 jump child" time="2020-12-05T11:54:50+08:00" level=debug msg="nsexec:970 jump init" time="2020-12-05T11:54:50+08:00" level=debug msg="child process in init()" time="2020-12-05T11:54:50+08:00" level=debug msg="logging has already been configured" time="2020-12-05T11:54:50+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-05T11:54:50+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T11:54:50+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T11:54:52+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T11:54:52+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T11:55:19+08:00" level=debug msg="sending signal to process urgent I/O condition" root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log2.txt exec mybusy ls bin dev etc home proc root sys tmp usr var root@cloud:/mycontainer# cat log2.txt time="2020-12-05T12:01:37+08:00" level=debug msg="nsexec:602 nsexec started" time="2020-12-05T12:01:37+08:00" level=debug msg="nsexec:702 jump parent" time="2020-12-05T12:01:37+08:00" level=debug msg="nsexec:831 jump child" time="2020-12-05T12:01:37+08:00" level=debug msg="nsexec:970 jump init" time="2020-12-05T12:01:37+08:00" level=debug msg="child process in init()" time="2020-12-05T12:01:37+08:00" level=debug msg="logging has already been configured" time="2020-12-05T12:01:37+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-05T12:01:38+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T12:01:38+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T12:01:38+08:00" level=debug msg="process exited" pid=18005 status=0 time="2020-12-05T12:02:13+08:00" level=debug msg="nsexec:602 nsexec started" time="2020-12-05T12:02:13+08:00" level=debug msg="nsexec:702 jump parent" time="2020-12-05T12:02:13+08:00" level=debug msg="nsexec:831 jump child" time="2020-12-05T12:02:13+08:00" level=debug msg="nsexec:970 jump init" time="2020-12-05T12:02:13+08:00" level=debug msg="child process in init()" time="2020-12-05T12:02:13+08:00" level=debug msg="logging has already been configured" time="2020-12-05T12:02:13+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-05T12:02:13+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-05T12:02:13+08:00" level=debug msg="process exited" pid=18036 status=0 root@cloud:/mycontainer#
join_namespaces
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log3.txt exec mybusy ls bin dev etc home proc root sys tmp usr var
root@cloud:/mycontainer# cat log3.txt time="2020-12-06T12:34:52+08:00" level=error msg="setnsprocess cmd /proc/self/exe, Dir /mycontainer/rootfs ,argv[0],argv[1] /opt/gopath/src/github.com/opencontainers/runc/runc, init" time="2020-12-06T12:34:52+08:00" level=debug msg="nsexec:603 nsexec started" time="2020-12-06T12:34:52+08:00" level=debug msg="nsexec:703 jump parent" time="2020-12-06T12:34:52+08:00" level=debug msg="nsexec:832 jump child" time="2020-12-06T12:34:52+08:00" level=debug msg="join_namespaces:540 join namespace /proc/24439/ns/ipc" time="2020-12-06T12:34:52+08:00" level=debug msg="join_namespaces:540 join namespace /proc/24439/ns/uts" time="2020-12-06T12:34:52+08:00" level=debug msg="join_namespaces:540 join namespace /proc/24439/ns/net" time="2020-12-06T12:34:52+08:00" level=debug msg="join_namespaces:540 join namespace /proc/24439/ns/pid" time="2020-12-06T12:34:52+08:00" level=debug msg="join_namespaces:540 join namespace /proc/24439/ns/mnt" time="2020-12-06T12:34:52+08:00" level=debug msg="nsexec:971 jump init" time="2020-12-06T12:34:52+08:00" level=debug msg="child process in init()" time="2020-12-06T12:34:52+08:00" level=debug msg="logging has already been configured" time="2020-12-06T12:34:52+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-06T12:34:52+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T12:34:52+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T12:34:52+08:00" level=debug msg="process exited" pid=24577 status=0 root@cloud:/mycontainer# ps -elf | grep 24439 4 S root 24439 24422 0 80 0 - 343 select 12:32 pts/0 00:00:00 sh 0 S root 24595 24537 0 80 0 - 1418 pipe_r 12:36 pts/3 00:00:00 grep --color=auto 24439 root@cloud:/mycontainer# ps -elf | grep 24422 0 S root 24422 24316 0 80 0 - 345490 futex_ 12:32 pts/2 00:00:00 /opt/gopath/src/github.com/opencontainers/runc/runc --debug --log log2.txt run mybusy 4 S root 24439 24422 0 80 0 - 343 select 12:32 pts/0 00:00:00 sh 0 S root 24597 24537 0 80 0 - 1418 pipe_r 12:36 pts/3 00:00:00 grep --color=auto 24422 root@cloud:/mycontainer#
$GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mybusy panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0xaaaaea168ca4] goroutine 1 [running]: github.com/opencontainers/runc/libcontainer.(*initProcess).pid(0x4000254000, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:267 +0xc github.com/opencontainers/runc/libcontainer.(*linuxContainer).start(0x40002be1e0, 0x40001b7040, 0x0, 0xaaaaea3bc700) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/container_linux.go:366 +0x8c github.com/opencontainers/runc/libcontainer.(*linuxContainer).Start(0x40002be1e0, 0x40001b7040, 0x0, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/container_linux.go:263 +0xa4 github.com/opencontainers/runc/libcontainer.(*linuxContainer).Run(0x40002be1e0, 0x40001b7040, 0x0, 0x1) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/container_linux.go:273 +0x30 main.(*runner).run(0x40001dd408, 0x40002be000, 0x0, 0x0, 0x0) /opt/gopath/src/github.com/opencontainers/runc/utils_linux.go:325 +0x8e8 main.startContainer(0x4000296160, 0x40001b2300, 0x2, 0x0, 0x0, 0x4, 0x40001fa710) /opt/gopath/src/github.com/opencontainers/runc/utils_linux.go:464 +0x318 main.glob..func13(0x4000296160, 0xaaaaea384e00, 0x0) /opt/gopath/src/github.com/opencontainers/runc/run.go:76 +0x88 github.com/urfave/cli.HandleAction(0xaaaaea307560, 0xaaaaea3bc670, 0x4000296160, 0x4000296160, 0x0) /opt/gopath/src/github.com/opencontainers/runc/vendor/github.com/urfave/cli/app.go:523 +0x140 github.com/urfave/cli.Command.Run(0xaaaaea1d2f8a, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0xaaaaea1e2261, 0x1a, 0x0, ...) /opt/gopath/src/github.com/opencontainers/runc/vendor/github.com/urfave/cli/command.go:174 +0x410 github.com/urfave/cli.(*App).Run(0x4000280000, 0x40000201e0, 0x6, 0x6, 0x0, 0x0) /opt/gopath/src/github.com/opencontainers/runc/vendor/github.com/urfave/cli/app.go:276 +0x584 main.main() /opt/gopath/src/github.com/opencontainers/runc/main.go:160 +0xa28
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log2.txt exec mybusy ls runtime: goroutine stack exceeds 1000000000-byte limit runtime: sp=0x4020560360 stack=[0x4020560000, 0x4040560000] fatal error: stack overflow runtime stack: runtime.throw(0xaaaaea05e41c, 0xe) /usr/local/go/src/runtime/panic.go:1116 +0x54 runtime.newstack() /usr/local/go/src/runtime/stack.go:1042 +0x590 runtime.morestack() /usr/local/go/src/runtime/asm_arm64.s:310 +0x70 goroutine 1 [running]: github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:236 +0xb8 fp=0x4020560360 sp=0x4020560360 pc=0xaaaae9febca0 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205603c0 sp=0x4020560360 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560420 sp=0x40205603c0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560480 sp=0x4020560420 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205604e0 sp=0x4020560480 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560540 sp=0x40205604e0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205605a0 sp=0x4020560540 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560600 sp=0x40205605a0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560660 sp=0x4020560600 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205606c0 sp=0x4020560660 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560720 sp=0x40205606c0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560780 sp=0x4020560720 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205607e0 sp=0x4020560780 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560840 sp=0x40205607e0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205608a0 sp=0x4020560840 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560900 sp=0x40205608a0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560960 sp=0x4020560900 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205609c0 sp=0x4020560960 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560a20 sp=0x40205609c0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560a80 sp=0x4020560a20 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560ae0 sp=0x4020560a80 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560b40 sp=0x4020560ae0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560ba0 sp=0x4020560b40 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560c00 sp=0x4020560ba0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560c60 sp=0x4020560c00 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560cc0 sp=0x4020560c60 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560d20 sp=0x4020560cc0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560d80 sp=0x4020560d20 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560de0 sp=0x4020560d80 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560e40 sp=0x4020560de0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560ea0 sp=0x4020560e40 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560f00 sp=0x4020560ea0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560f60 sp=0x4020560f00 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020560fc0 sp=0x4020560f60 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561020 sp=0x4020560fc0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561080 sp=0x4020561020 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205610e0 sp=0x4020561080 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561140 sp=0x40205610e0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205611a0 sp=0x4020561140 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561200 sp=0x40205611a0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561260 sp=0x4020561200 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205612c0 sp=0x4020561260 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561320 sp=0x40205612c0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561380 sp=0x4020561320 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205613e0 sp=0x4020561380 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561440 sp=0x40205613e0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205614a0 sp=0x4020561440 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561500 sp=0x40205614a0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561560 sp=0x4020561500 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205615c0 sp=0x4020561560 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561620 sp=0x40205615c0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561680 sp=0x4020561620 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205616e0 sp=0x4020561680 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561740 sp=0x40205616e0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205617a0 sp=0x4020561740 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561800 sp=0x40205617a0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561860 sp=0x4020561800 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205618c0 sp=0x4020561860 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561920 sp=0x40205618c0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561980 sp=0x4020561920 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205619e0 sp=0x4020561980 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561a40 sp=0x40205619e0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561aa0 sp=0x4020561a40 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561b00 sp=0x4020561aa0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561b60 sp=0x4020561b00 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561bc0 sp=0x4020561b60 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561c20 sp=0x4020561bc0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561c80 sp=0x4020561c20 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561ce0 sp=0x4020561c80 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561d40 sp=0x4020561ce0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561da0 sp=0x4020561d40 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561e00 sp=0x4020561da0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561e60 sp=0x4020561e00 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561ec0 sp=0x4020561e60 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561f20 sp=0x4020561ec0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561f80 sp=0x4020561f20 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020561fe0 sp=0x4020561f80 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562040 sp=0x4020561fe0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205620a0 sp=0x4020562040 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562100 sp=0x40205620a0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562160 sp=0x4020562100 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205621c0 sp=0x4020562160 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562220 sp=0x40205621c0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562280 sp=0x4020562220 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205622e0 sp=0x4020562280 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562340 sp=0x40205622e0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205623a0 sp=0x4020562340 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562400 sp=0x40205623a0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562460 sp=0x4020562400 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205624c0 sp=0x4020562460 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562520 sp=0x40205624c0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562580 sp=0x4020562520 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205625e0 sp=0x4020562580 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562640 sp=0x40205625e0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205626a0 sp=0x4020562640 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562700 sp=0x40205626a0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562760 sp=0x4020562700 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x40205627c0 sp=0x4020562760 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562820 sp=0x40205627c0 pc=0xaaaae9febc10 github.com/opencontainers/runc/libcontainer.(*setnsProcess).pid(0x40002fcf30, 0x0) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:237 +0x28 fp=0x4020562880 sp=0x4020562820 pc=0xaaaae9febc10 ...additional frames elided... goroutine 5 [syscall]: os/signal.signal_recv(0xaaaaea251f20) /usr/local/go/src/runtime/sigqueue.go:147 +0xc4 os/signal.loop() /usr/local/go/src/os/signal/signal_unix.go:23 +0x20 created by os/signal.Notify.func1 /usr/local/go/src/os/signal/signal.go:127 +0x44 goroutine 7 [syscall]: syscall.Syscall(0x3f, 0x0, 0x4000260000, 0x8000, 0x40000dae08, 0xaaaae9cf80b8, 0x800000) /usr/local/go/src/syscall/asm_linux_arm64.s:9 +0x10 syscall.read(0x0, 0x4000260000, 0x8000, 0x8000, 0x0, 0xaaaaea131401, 0x40000dae01) /usr/local/go/src/syscall/zsyscall_linux_arm64.go:686 +0x4c syscall.Read(...) /usr/local/go/src/syscall/syscall_unix.go:189 internal/poll.(*FD).Read(0x40001b6000, 0x4000260000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:165 +0xfc os.(*File).read(...) /usr/local/go/src/os/file_unix.go:263 os.(*File).Read(0x40001b4000, 0x4000260000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/os/file.go:116 +0x60 io.copyBuffer(0xaaaaea24bd40, 0x40000110e8, 0xaaaaea24bd20, 0x40001b4000, 0x4000260000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/io/io.go:405 +0xd4 io.Copy(...) /usr/local/go/src/io/io.go:364 main.setupProcessPipes.func1(0x40002d3b30) /opt/gopath/src/github.com/opencontainers/runc/tty.go:59 +0x78 created by main.setupProcessPipes /opt/gopath/src/github.com/opencontainers/runc/tty.go:58 +0x290 goroutine 8 [syscall]: syscall.Syscall(0x3f, 0x9, 0x400030e000, 0x8000, 0x40000e6dc8, 0xaaaae9cf80b8, 0xffff5ce54e60) /usr/local/go/src/syscall/asm_linux_arm64.s:9 +0x10 syscall.read(0x9, 0x400030e000, 0x8000, 0x8000, 0x0, 0x1, 0x1) /usr/local/go/src/syscall/zsyscall_linux_arm64.go:686 +0x4c syscall.Read(...) /usr/local/go/src/syscall/syscall_unix.go:189 internal/poll.(*FD).Read(0x4000105320, 0x400030e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:165 +0xfc os.(*File).read(...) /usr/local/go/src/os/file_unix.go:263 os.(*File).Read(0x40000110f0, 0x400030e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/os/file.go:116 +0x60 io.copyBuffer(0xaaaaea24bd40, 0x40001b4008, 0xaaaaea24bd20, 0x40000110f0, 0x400030e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/io/io.go:405 +0xd4 io.Copy(...) /usr/local/go/src/io/io.go:364 main.(*tty).copyIO(0x40001559d0, 0xaaaaea24bd40, 0x40001b4008, 0xaaaaea251ae0, 0x40000110f0) /opt/gopath/src/github.com/opencontainers/runc/tty.go:30 +0x98 created by main.setupProcessPipes /opt/gopath/src/github.com/opencontainers/runc/tty.go:63 +0x2f8 goroutine 9 [syscall]: syscall.Syscall(0x3f, 0xb, 0x400038e000, 0x8000, 0x40000dbdc8, 0xaaaae9cf80b8, 0xffff5ce54ca0) /usr/local/go/src/syscall/asm_linux_arm64.s:9 +0x10 syscall.read(0xb, 0x400038e000, 0x8000, 0x8000, 0x0, 0x1, 0x1) /usr/local/go/src/syscall/zsyscall_linux_arm64.go:686 +0x4c syscall.Read(...) /usr/local/go/src/syscall/syscall_unix.go:189 internal/poll.(*FD).Read(0x40001053e0, 0x400038e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:165 +0xfc os.(*File).read(...) /usr/local/go/src/os/file_unix.go:263 os.(*File).Read(0x4000011100, 0x400038e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/os/file.go:116 +0x60 io.copyBuffer(0xaaaaea24bd40, 0x40001b4010, 0xaaaaea24bd20, 0x4000011100, 0x400038e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/io/io.go:405 +0xd4 io.Copy(...) /usr/local/go/src/io/io.go:364 main.(*tty).copyIO(0x40001559d0, 0xaaaaea24bd40, 0x40001b4010, 0xaaaaea251ae0, 0x4000011100) /opt/gopath/src/github.com/opencontainers/runc/tty.go:30 +0x98 created by main.setupProcessPipes /opt/gopath/src/github.com/opencontainers/runc/tty.go:64 +0x348 goroutine 10 [IO wait]: internal/poll.runtime_pollWait(0xffff5ce54ae0, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x44 internal/poll.(*pollDesc).wait(0x4000105578, 0x72, 0x1001, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x38 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0x4000105560, 0x400048e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x158 os.(*File).read(...) /usr/local/go/src/os/file_unix.go:263 os.(*File).Read(0x4000011120, 0x400048e000, 0x1000, 0x1000, 0x40001d1d01, 0x4000322090, 0x40001d1d88) /usr/local/go/src/os/file.go:116 +0x60 bufio.(*Reader).fill(0x40001d1f60) /usr/local/go/src/bufio/bufio.go:100 +0xf8 bufio.(*Reader).ReadSlice(0x40001d1f60, 0x400032400a, 0x400040a688, 0xaaaae9fc93c8, 0x40001b8cb0, 0x5, 0x40003200f0) /usr/local/go/src/bufio/bufio.go:359 +0x38 bufio.(*Reader).ReadBytes(0x40001d1f60, 0xa, 0x5c, 0x5c, 0x5c, 0x0, 0x0) /usr/local/go/src/bufio/bufio.go:438 +0x5c github.com/opencontainers/runc/libcontainer/logs.ForwardLogs(0xaaaaea24bd20, 0x4000011120) /opt/gopath/src/github.com/opencontainers/runc/libcontainer/logs/logs.go:32 +0x68 created by github.com/opencontainers/runc/libcontainer.(*setnsProcess).forwardChildLogs /opt/gopath/src/github.com/opencontainers/runc/libcontainer/process_linux.go:250 +0x4c
host
root@cloud:/mycontainer# ps -elf | grep 27291 4 S root 27291 27275 0 80 0 - 343 select 13:14 pts/0 00:00:00 sh 0 S root 27348 24537 0 80 0 - 1418 pipe_r 13:18 pts/3 00:00:00 grep --color=auto 27291 root@cloud:/mycontainer# ps -elf | grep 27275 0 S root 27275 24316 0 80 0 - 363923 futex_ 13:14 pts/2 00:00:00 /opt/gopath/src/github.com/opencontainers/runc/runc --debug --log log.txt run mybusy 4 S root 27291 27275 0 80 0 - 343 select 13:14 pts/0 00:00:00 sh 0 S root 27351 24537 0 80 0 - 1418 pipe_r 13:18 pts/3 00:00:00 grep --color=auto 27275 root@cloud:/mycontainer#
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log2.txt exec mybusy ls bin dev etc home proc root sys tmp usr var root@cloud:/mycontainer# ls config.json exec.sh log2.txt log.txt nginx rootfs root@cloud:/mycontainer# cat log2.txt time="2020-12-06T13:14:50+08:00" level=error msg="newsetnsprocess call newInitConfig " time="2020-12-06T13:14:50+08:00" level=error msg="call newInitConfig by new init or new setns process Args[0] ls" time="2020-12-06T13:14:50+08:00" level=error msg="parent pid is 27307 " time="2020-12-06T13:14:50+08:00" level=error msg="setnsprocess cmd /proc/self/exe, Dir /mycontainer/rootfs ,argv[0],argv[1] /opt/gopath/src/github.com/opencontainers/runc/runc, init" time="2020-12-06T13:14:50+08:00" level=debug msg="nsexec:603 nsexec started 27317" time="2020-12-06T13:14:50+08:00" level=debug msg="nsexec:703 jump parent" time="2020-12-06T13:14:50+08:00" level=debug msg="nsexec:832 jump child" time="2020-12-06T13:14:50+08:00" level=debug msg="join_namespaces:540 join namespace /proc/27291/ns/ipc" time="2020-12-06T13:14:50+08:00" level=debug msg="join_namespaces:540 join namespace /proc/27291/ns/uts" time="2020-12-06T13:14:50+08:00" level=debug msg="join_namespaces:540 join namespace /proc/27291/ns/net" time="2020-12-06T13:14:50+08:00" level=debug msg="join_namespaces:540 join namespace /proc/27291/ns/pid" time="2020-12-06T13:14:50+08:00" level=debug msg="join_namespaces:540 join namespace /proc/27291/ns/mnt" time="2020-12-06T13:14:50+08:00" level=debug msg="nsexec:971 jump init" time="2020-12-06T13:14:50+08:00" level=debug msg="child process in init() and child pid is 7" time="2020-12-06T13:14:50+08:00" level=debug msg="logging has already been configured" time="2020-12-06T13:14:50+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-06T13:14:50+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T13:14:50+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T13:14:50+08:00" level=debug msg="process exited" pid=27320 status=0
$GOPATH/src/github.com/opencontainers/runc/runc --debug --log log3.txt exec mybusy ps -elf PID USER TIME COMMAND 1 root 0:00 sh 19 root 0:00 ps -elf
root@cloud:/mycontainer# cat log3.txt time="2020-12-06T13:21:04+08:00" level=error msg="newsetnsprocess call newInitConfig " time="2020-12-06T13:21:04+08:00" level=error msg="call newInitConfig by new init or new setns process Args[0] ps" time="2020-12-06T13:21:04+08:00" level=error msg="parent pid is 27372 " time="2020-12-06T13:21:04+08:00" level=error msg="setnsprocess cmd /proc/self/exe, Dir /mycontainer/rootfs ,argv[0],argv[1] /opt/gopath/src/github.com/opencontainers/runc/runc, init" time="2020-12-06T13:21:04+08:00" level=debug msg="nsexec:603 nsexec started 27382" time="2020-12-06T13:21:04+08:00" level=debug msg="nsexec:703 jump parent" time="2020-12-06T13:21:04+08:00" level=debug msg="nsexec:832 jump child" time="2020-12-06T13:21:04+08:00" level=debug msg="join_namespaces:540 join namespace /proc/27291/ns/ipc" time="2020-12-06T13:21:04+08:00" level=debug msg="join_namespaces:540 join namespace /proc/27291/ns/uts" time="2020-12-06T13:21:04+08:00" level=debug msg="join_namespaces:540 join namespace /proc/27291/ns/net" time="2020-12-06T13:21:04+08:00" level=debug msg="join_namespaces:540 join namespace /proc/27291/ns/pid" time="2020-12-06T13:21:04+08:00" level=debug msg="join_namespaces:540 join namespace /proc/27291/ns/mnt" time="2020-12-06T13:21:04+08:00" level=debug msg="nsexec:971 jump init" time="2020-12-06T13:21:04+08:00" level=debug msg="child process in init() and child pid is 19"
19 root 0:00 ps -elf
time="2020-12-06T13:21:04+08:00" level=debug msg="logging has already been configured" time="2020-12-06T13:21:04+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-06T13:21:04+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T13:21:04+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T13:21:04+08:00" level=debug msg="process exited" pid=27385 status=0
$GOPATH/src/github.com/opencontainers/runc/runc --debug --log log4.txt exec mycontainerid sleep 30
$GOPATH/src/github.com/opencontainers/runc/runc --debug --log log4.txt exec mycontainerid sleep 30
root@cloud:/mycontainer# ps -elf | grep 31150 0 S root 31150 28015 0 80 0 - 345554 futex_ 18:35 pts/2 00:00:00 /opt/gopath/src/github.com/opencontainers/runc/runc --debug --log log.txt run mycontainerid 4 S root 31167 31150 0 80 0 - 343 select 18:35 pts/0 00:00:00 sh 0 S root 31189 27922 0 80 0 - 1418 pipe_r 18:38 pts/1 00:00:00 grep --color=auto 31150 root@cloud:/mycontainer# ps -elf | grep 31158 0 S root 31192 27922 0 80 0 - 1418 pipe_r 18:39 pts/1 00:00:00 grep --color=auto 31158
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mycontainerid / # ls bin dev etc home proc root sys tmp usr var / # sleep 50
host上也能看到
root@cloud:/mycontainer# ps -elf | grep 31167 4 S root 31167 31150 0 80 0 - 343 select 18:35 pts/0 00:00:00 sh 0 S root 31595 31551 0 80 0 - 1418 pipe_r 19:04 pts/3 00:00:00 grep --color=auto 31167 root@cloud:/mycontainer# ps -elf | grep 31167 4 S root 31167 31150 0 80 0 - 344 wait 18:35 pts/0 00:00:00 sh 0 S root 31603 31167 0 80 0 - 339 hrtime 19:06 pts/0 00:00:00 sleep 50
namespace
没有独立的namesapce pids
"namespaces": [ { "type": "network" }, { "type": "ipc" }, { "type": "uts" }, { "type": "mount" } ],
root@cloud:/mycontainer# ps -elf | grep 35291 0 S root 35291 32537 0 80 0 - 363924 futex_ 21:58 pts/1 00:00:00 /opt/gopath/src/github.com/opencontainers/runc/runc --debug --log log.txt run mycontainerid 4 S root 35308 35291 0 80 0 - 343 select 21:58 pts/0 00:00:00 sh 0 S root 35348 32677 0 80 0 - 1418 pipe_r 22:03 pts/2 00:00:00 grep --color=auto 35291 root@cloud:/mycontainer# cat log.txt time="2020-12-06T21:58:59+08:00" level=error msg="newinitsprocess call newInitConfig" time="2020-12-06T21:58:59+08:00" level=error msg="call newInitConfig by new init or new setns process Args[0] sh" time="2020-12-06T21:58:59+08:00" level=error msg="parent pid is 35291 " time="2020-12-06T21:58:59+08:00" level=error msg="initprocess cmd /proc/self/exe, Dir /mycontainer/rootfs ,argv[0],argv[1] /opt/gopath/src/github.com/opencontainers/runc/runc, init" time="2020-12-06T21:58:59+08:00" level=error msg="initProcess manager.Apply cgroup pid 35299 " time="2020-12-06T21:58:59+08:00" level=debug msg="nsexec:603 nsexec started 35299" time="2020-12-06T21:58:59+08:00" level=debug msg="nsexec:703 jump parent" time="2020-12-06T21:58:59+08:00" level=debug msg="nsexec:832 jump child" time="2020-12-06T21:58:59+08:00" level=debug msg="nsexec:971 jump init" time="2020-12-06T21:58:59+08:00" level=error msg="initProcess child pid 35308" ---sh进程 time="2020-12-06T21:58:59+08:00" level=debug msg="child process in init() and child pid is 35308" time="2020-12-06T21:58:59+08:00" level=debug msg="logging has already been configured" time="2020-12-06T21:58:59+08:00" level=error msg="newContainerInit pid 35308" time="2020-12-06T21:58:59+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-06T21:58:59+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T21:58:59+08:00" level=debug msg="sending signal to process urgent I/O condition"
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mycontainerid / # ps -elf | grep sleep 35387 root 0:00 /opt/gopath/src/github.com/opencontainers/runc/runc --debug --log log2.txt exec mycontainerid sleep 30 35400 root 0:00 sleep 30 35407 root 0:00 grep sleep / #
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log2.txt exec mycontainerid sleep 30 root@cloud:/mycontainer# ls config.json config.json.bak exec.sh log2.txt log.txt nginx rootfs root@cloud:/mycontainer# cat log2.txt time="2020-12-06T22:06:00+08:00" level=error msg="newsetnsprocess call newInitConfig " time="2020-12-06T22:06:00+08:00" level=error msg="call newInitConfig by new init or new setns process Args[0] sleep" time="2020-12-06T22:06:00+08:00" level=error msg="parent pid is 35387 " time="2020-12-06T22:06:00+08:00" level=error msg="setnsprocess cmd /proc/self/exe, Dir /mycontainer/rootfs ,argv[0],argv[1] /opt/gopath/src/github.com/opencontainers/runc/runc, init" time="2020-12-06T22:06:00+08:00" level=error msg="setnsProcess execns and apply cgroup pid 35397" time="2020-12-06T22:06:00+08:00" level=debug msg="nsexec:603 nsexec started 35397" time="2020-12-06T22:06:00+08:00" level=debug msg="nsexec:703 jump parent" time="2020-12-06T22:06:00+08:00" level=debug msg="nsexec:832 jump child"
------------------没有pid ns
time="2020-12-06T22:06:00+08:00" level=debug msg="join_namespaces:540 join namespace /proc/35308/ns/ipc" time="2020-12-06T22:06:00+08:00" level=debug msg="join_namespaces:540 join namespace /proc/35308/ns/uts" time="2020-12-06T22:06:00+08:00" level=debug msg="join_namespaces:540 join namespace /proc/35308/ns/net" time="2020-12-06T22:06:00+08:00" level=debug msg="join_namespaces:540 join namespace /proc/35308/ns/mnt" time="2020-12-06T22:06:00+08:00" level=debug msg="nsexec:971 jump init" time="2020-12-06T22:06:00+08:00" level=debug msg="child process in init() and child pid is 35400" time="2020-12-06T22:06:00+08:00" level=debug msg="logging has already been configured" time="2020-12-06T22:06:00+08:00" level=error msg="newContainerInit pid 35400" time="2020-12-06T22:06:00+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-06T22:06:00+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T22:06:00+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T22:06:30+08:00" level=debug msg="process exited" pid=35400 status=0
nsexec.c
write_log(DEBUG, "unshare config.cloneflags include %x %x %x", CLONE_NEWPID &(config.cloneflags & ~CLONE_NEWCGROUP) ,CLONE_NEWPID & config.cloneflags , CLONE_NEWCGROUP & config.cloneflags); if (unshare(config.cloneflags & ~CLONE_NEWCGROUP) < 0) bail("failed to unshare namespaces");
可以看到host的namespace
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mycontainerid / # ps -elf | more PID USER TIME COMMAND 1 root 0:30 {systemd} /sbin/init 2 root 0:01 [kthreadd] 3 root 0:00 [rcu_gp] 4 root 0:00 [rcu_par_gp] 6 root 0:00 [kworker/0:0H-kb] 9 root 0:00 [mm_percpu_wq] 10 root 0:00 [ksoftirqd/0] 11 root 6:39 [rcu_sched] 12 root 0:04 [migration/0] 13 root 0:00 [idle_inject/0] 15 root 0:00 [cpuhp/0] 16 root 0:00 [cpuhp/1] 17 root 0:00 [idle_inject/1] 18 root 0:03 [migration/1] 19 root 0:00 [ksoftirqd/1] 21 root 0:00 [kworker/1:0H-kb] 22 root 0:00 [cpuhp/2] 23 root 0:00 [idle_inject/2] 24 root 0:03 [migration/2] 25 root 0:00 [ksoftirqd/2] 27 root 0:00 [kworker/2:0H-kb] 28 root 0:00 [cpuhp/3] 29 root 0:00 [idle_inject/3] 30 root 0:03 [migration/3] 31 root 0:00 [ksoftirqd/3] 33 root 0:00 [kworker/3:0H-kb] 34 root 0:00 [cpuhp/4] 35 root 0:00 [idle_inject/4] 36 root 0:03 [migration/4] 37 root 0:00 [ksoftirqd/4]
root@cloud:/mycontainer# cat log.txt time="2020-12-06T22:58:25+08:00" level=error msg="newinitsprocess call newInitConfig and contains NEWPID yes or no : false" time="2020-12-06T22:58:25+08:00" level=error msg="call newInitConfig by new init or new setns process Args[0] sh" time="2020-12-06T22:58:25+08:00" level=error msg="parent pid is 37787 " time="2020-12-06T22:58:25+08:00" level=error msg="initprocess cmd /proc/self/exe, Dir /mycontainer/rootfs ,argv[0],argv[1] /opt/gopath/src/github.com/opencontainers/runc/runc, init" time="2020-12-06T22:58:25+08:00" level=error msg="initProcess manager.Apply cgroup pid 37795 " time="2020-12-06T22:58:25+08:00" level=debug msg="nsexec:603 nsexec started 37795" time="2020-12-06T22:58:25+08:00" level=debug msg="nsexec:703 jump parent" time="2020-12-06T22:58:25+08:00" level=debug msg="nsexec:832 jump child" time="2020-12-06T22:58:25+08:00" level=debug msg="nsexec:916 unshare config.cloneflags include 0 0 0" time="2020-12-06T22:58:25+08:00" level=debug msg="nsexec:972 jump init" time="2020-12-06T22:58:25+08:00" level=error msg="initProcess child pid 37803" time="2020-12-06T22:58:25+08:00" level=debug msg="child process in init() and child pid is 37803" time="2020-12-06T22:58:25+08:00" level=debug msg="logging has already been configured" time="2020-12-06T22:58:25+08:00" level=error msg="newContainerInit pid 37803" time="2020-12-06T22:58:25+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-06T22:58:25+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T22:58:25+08:00" level=debug msg="sending signal to process urgent I/O condition"
独立的namesapce pids
time="2020-12-06T22:56:20+08:00" level=error msg="newinitsprocess call newInitConfig and contains NEWPID yes or no : true" time="2020-12-06T22:56:20+08:00" level=error msg="call newInitConfig by new init or new setns process Args[0] sh" time="2020-12-06T22:56:20+08:00" level=error msg="parent pid is 37708 " time="2020-12-06T22:56:20+08:00" level=error msg="initprocess cmd /proc/self/exe, Dir /mycontainer/rootfs ,argv[0],argv[1] /opt/gopath/src/github.com/opencontainers/runc/runc, init" time="2020-12-06T22:56:20+08:00" level=error msg="initProcess manager.Apply cgroup pid 37716 " time="2020-12-06T22:56:20+08:00" level=debug msg="nsexec:603 nsexec started 37716" time="2020-12-06T22:56:20+08:00" level=debug msg="nsexec:703 jump parent" time="2020-12-06T22:56:20+08:00" level=debug msg="nsexec:832 jump child" time="2020-12-06T22:56:20+08:00" level=debug msg="nsexec:916 unshare config.cloneflags include 20000000 20000000 0" time="2020-12-06T22:56:20+08:00" level=debug msg="nsexec:972 jump init" time="2020-12-06T22:56:20+08:00" level=error msg="initProcess child pid 37724" time="2020-12-06T22:56:20+08:00" level=debug msg="child process in init() and child pid is 1" time="2020-12-06T22:56:20+08:00" level=debug msg="logging has already been configured" time="2020-12-06T22:56:20+08:00" level=error msg="newContainerInit pid 1" time="2020-12-06T22:56:20+08:00" level=debug msg="sending signal to process urgent I/O condition" time="2020-12-06T22:56:20+08:00" level=debug msg="log pipe has been closed: EOF"
configs.NEWPID
func (c *linuxContainer) newInitProcess(p *Process, cmd *exec.Cmd, messageSockPair, logFilePair filePair) (*initProcess, error) { cmd.Env = append(cmd.Env, "_LIBCONTAINER_INITTYPE="+string(initStandard)) nsMaps := make(map[configs.NamespaceType]string) for _, ns := range c.config.Namespaces { if ns.Path != "" { nsMaps[ns.Type] = ns.Path } } _, sharePidns := nsMaps[configs.NEWPID] data, err := c.bootstrapData(c.config.Namespaces.CloneFlags(), nsMaps) if err != nil { return nil, err } logrus.Errorf("newinitsprocess call newInitConfig and contains NEWPID yes or no : %t", c.config.Namespaces.Contains(configs.NEWPID))
Namespace API提供了三种系统调用接口：
● clone()：创建新的进程
● setns()：允许指定进程加入特定的namespace
● unshare()：将指定进程移除指定的namespace
nsexec.c分别使用这三种接口，对于一般新建命名空间，使用unshare()实现；对于已有的命名空间，使用setns()实现。
nsexec.c 里面的nsexec()函数在包被导入时候就开始执行，等待通过管道传输ns数据过来
nsexec()首先通过环境变量_LIBCONTAINER_INITPIPE 获取初始管道的的文件描述符。该管道是父进程打开并保持打开状态，跨越初始进程和nsenec clone出来的进程。
如下图是nsexec.c 流程图：
关键是unshare(config.cloneflags），改config.cloneflags默认值值是（CLONE_NEWNET|CLONE_NEWPID|CLONE_NEWIPC|CLONE_NEWUTS|CLONE_NEWNS） 唯独没有CLONE_NEWUSER。
然后这个unshare出来的命名空间在改进程child clone出来的init 进程体现出来，init 进程查看自己的pid：1 ，ppid：0.
cgroup
用go语言实现通过cgroup限制容器的资源 下面我们在上一节的容器的基础上加上cgroup的限制，下面这个demo实现了限制容器的内存的功能： package main import ( "os/exec" "path" "os" "fmt" "io/ioutil" "syscall" "strconv" ) const cgroupMemoryHierarchyMount = "/sys/fs/cgroup/memory" func main() { if os.Args[0] == "/proc/self/exe" { //容器进程 fmt.Printf("current pid %d", syscall.Getpid()) fmt.Println() cmd := exec.Command("sh", "-c", `stress --vm-bytes 200m --vm-keep -m 1`) cmd.SysProcAttr = &syscall.SysProcAttr{ } cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { fmt.Println(err) os.Exit(1) } } cmd := exec.Command("/proc/self/exe") cmd.SysProcAttr = &syscall.SysProcAttr{ Cloneflags: syscall.CLONE_NEWUTS | syscall.CLONE_NEWPID | syscall.CLONE_NEWNS, } cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { fmt.Println("ERROR", err) os.Exit(1) } else { //得到fork出来进程映射在外部命名空间的pid fmt.Printf("%v", cmd.Process.Pid) // 在系统默认创建挂载了memory subsystem的Hierarchy上创建cgroup os.Mkdir(path.Join(cgroupMemoryHierarchyMount, "testmemorylimit"), 0755) // 将容器进程加入到这个cgroup中 ioutil.WriteFile(path.Join(cgroupMemoryHierarchyMount, "testmemorylimit", "tasks") , []byte(strconv.Itoa(cmd.Process.Pid)), 0644) // 限制cgroup进程使用 ioutil.WriteFile(path.Join(cgroupMemoryHierarchyMount, "testmemorylimit", "memory.limit_in_bytes") , []byte("100m"), 0644) } cmd.Process.Wait() } 通过对Cgroups虚拟文件系统的配置，我们让容器中的把stress进程的内存占用限制到了100m。 PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 10861 root 20 0 212284 102464 212 R 6.2 5.0 0:01.13 stress
不挂载proc
{
"destination": "/proc",
"type": "proc",
"source": "proc"
},
root@cloud:/mycontainer# $GOPATH/src/github.com/opencontainers/runc/runc --debug --log log.txt run mycontainerid container_linux.go:370: starting container process caused: close exec fds: open /proc/self/fd: no such file or directory root@cloud:/mycontainer# cat log.txt time="2020-12-06T21:45:43+08:00" level=error msg="newinitsprocess call newInitConfig" time="2020-12-06T21:45:43+08:00" level=error msg="call newInitConfig by new init or new setns process Args[0] sh" time="2020-12-06T21:45:43+08:00" level=error msg="parent pid is 35124 " time="2020-12-06T21:45:43+08:00" level=error msg="initprocess cmd /proc/self/exe, Dir /mycontainer/rootfs ,argv[0],argv[1] /opt/gopath/src/github.com/opencontainers/runc/runc, init" time="2020-12-06T21:45:43+08:00" level=error msg="initProcess manager.Apply cgroup pid 35132 " time="2020-12-06T21:45:43+08:00" level=debug msg="nsexec:603 nsexec started 35132" time="2020-12-06T21:45:43+08:00" level=debug msg="nsexec:703 jump parent" time="2020-12-06T21:45:43+08:00" level=debug msg="nsexec:832 jump child" time="2020-12-06T21:45:43+08:00" level=debug msg="nsexec:971 jump init" time="2020-12-06T21:45:43+08:00" level=error msg="initProcess child pid 35142" time="2020-12-06T21:45:43+08:00" level=debug msg="child process in init() and child pid is 1" time="2020-12-06T21:45:43+08:00" level=debug msg="logging has already been configured" time="2020-12-06T21:45:43+08:00" level=error msg="newContainerInit pid 1" time="2020-12-06T21:45:43+08:00" level=debug msg="log pipe has been closed: EOF" time="2020-12-06T21:45:43+08:00" level=error msg="signal to process 35142" time="2020-12-06T21:45:43+08:00" level=error msg="container_linux.go:370: starting container process caused: close exec fds: open /proc/self/fd: no such file or directory"
func (raw *cgroupData) path(subsystem string) (string, error) {
// If the cgroup name/path is absolute do not look relative to the cgroup of the init process.
if filepath.IsAbs(raw.innerPath) {
mnt, err := cgroups.FindCgroupMountpoint(raw.root, subsystem)
// If we didn't mount the subsystem, there is no point we make the path.
if err != nil {
return "", err
}
// Sometimes subsystems can be mounted together as 'cpu,cpuacct'.
return filepath.Join(raw.root, filepath.Base(mnt), raw.innerPath), nil
}
// Use GetOwnCgroupPath instead of GetInitCgroupPath, because the creating
// process could in container and shared pid namespace with host, and
// /proc/1/cgroup could point to whole other world of cgroups.
parentPath, err := cgroups.GetOwnCgroupPath(subsystem)
if err != nil {
return "", err
}
return filepath.Join(parentPath, raw.innerPath), nil
}
standard_init_linux.go:212: chmod "/dev/openvmi/sockets/qemu_pipe" fail caused "chmod /dev/openvmi/sockets/qemu_pipe: no such file or directory" ERRO[0000] init failed error="standard_init_linux.go:212: chmod "/dev/openvmi/sockets/qemu_pipe" fail caused "chmod /dev/openvmi/sockets/qemu_pipe: no such file or directory"" name=kata-agent pid=1 source=agent panic: --this line should have never been executed, congratulations-- goroutine 1 [running, locked to thread]: main.init.0() /opt/gopath/src/github.com/kata-containers/agent/agent.go:1508 +0x1f4
root@cloud:/mycontainer# unshare --fork --pid --mount --mount-proc bash root@cloud:/mycontainer# mount --bind rootfs / root@cloud:/mycontainer# ls dev ls: cannot access 'dev': No such file or directory root@cloud:/mycontainer# ls config.json config.json.bak exec.sh log.txt nginx root rootfs root@cloud:/mycontainer# ls rootfs/ bin dev etc home proc root sys tmp usr var root@cloud:/mycontainer# cd rootfs root@cloud:/mycontainer/rootfs# chroot . chroot: failed to run command ‘/bin/bash’: No such file or directory root@cloud:/mycontainer/rootfs# chroot . bin/bash chroot: failed to run command ‘bin/bash’: No such file or directory root@cloud:/mycontainer/rootfs# chroot . /bin/bash chroot: failed to run command ‘/bin/bash’: No such file or directory root@cloud:/mycontainer/rootfs# pwd /mycontainer/rootfs root@cloud:/mycontainer/rootfs# ls bin dev etc home proc root sys tmp usr var root@cloud:/mycontainer/rootfs# ls bin/ root@cloud:/mycontainer/rootfs# chroot . bin/sh / # pwd / / # ls bin dev etc home proc root sys tmp usr var / # cd dev/ /dev # ls console hello.txt pts shm /dev # touch hello2.txt /dev #
root@cloud:/mycontainer/rootfs/dev# ls
console hello2.txt hello.txt pts shm
root@cloud:/mycontainer# unshare --fork --pid --mount --mount-proc bash root@cloud:/mycontainer# mount --make-private rootfs / mount: /: /mycontainer/rootfs is not a block device. root@cloud:/mycontainer# ls config.json config.json.bak exec.sh log.txt nginx root rootfs root@cloud:/mycontainer# mount --move rootfs / mount: /: wrong fs type, bad option, bad superblock on /mycontainer/rootfs, missing codepage or helper program, or other error. root@cloud:/mycontainer# mount --bind --make-private rootfs / root@cloud:/mycontainer# chroot rootfs /bin/sh / # ls bin dev etc home proc root sys tmp usr var / # ls dev/ console hello.txt hello2.txt pts shm / # touch hello3.txt / #
看不到hello3.txt
root@cloud:/mycontainer/rootfs/dev# ls
console hello2.txt hello.txt pts shm
root@cloud:/mycontainer/rootfs/dev#
vendor/github.com/opencontainers/runc/libcontainer/specconv/spec_linux.go:740: "private": unix.MS_PRIVATE,
vendor/github.com/opencontainers/runc/libcontainer/specconv/spec_linux.go:744: "rprivate": unix.MS_PRIVATE | unix.MS_REC,
docker-runc –log /run/docker/libcontainerd/containerd/e41e4aa7a42c363ae019af5ad519d20d1d302380c9e72c02f6cb06b3d562637e/init/log.json –log-format json restore -d –image-path /var/local/p.haul-fs/rst-gk4dpK-17.08.16-20.45/img/1/mysql_checkpoint –work-path /var/local/p.haul-fs/rst-gk4dpK-17.08.16-20.45/img/1/mysql_checkpoint/criu.work/restore-2017-08-16T20:46:10-07:00 –tcp-established –ext-unix-sk –empty-ns network –pid-file /run/docker/libcontainerd/containerd/e41e4aa7a42c363ae019af5ad519d20d1d302380c9e72c02f6cb06b3d562637e/init/pid e41e4aa7a42c363ae019af5ad519d20d1d302380c9e72c02f6cb06b3d562637e ***