# mkdir rootfs # docker export $(docker create busybox) | tar -C rootfs -xvf - # runc spec
查不到mount信息
root@ubuntu:~# mount | grep 'container/rootfs' root@ubuntu:~#
root@ubuntu:~# ps -elf | grep mycontainer 4 S root 34589 13451 0 80 0 - 132473 futex_ 14:00 pts/0 00:00:00 docker-runc run mycontainer 0 S root 34972 14029 0 80 0 - 1097 pipe_w 14:01 pts/1 00:00:00 grep --color=auto mycontainer root@ubuntu:~# lsof -p 34589 COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME docker-ru 34589 root cwd DIR 8,35 4096 3806963 /root/docker/container docker-ru 34589 root rtd DIR 8,35 4096 2 / docker-ru 34589 root txt REG 8,35 5505840 9314261 /usr/bin/docker-runc docker-ru 34589 root mem REG 8,35 1345176 7083151 /lib/aarch64-linux-gnu/libc-2.27.so docker-ru 34589 root mem REG 8,35 18544 7083154 /lib/aarch64-linux-gnu/libdl-2.27.so docker-ru 34589 root mem REG 8,35 280648 7078009 /lib/aarch64-linux-gnu/libseccomp.so.2.4.1 docker-ru 34589 root mem REG 8,35 137848 7083165 /lib/aarch64-linux-gnu/libpthread-2.27.so docker-ru 34589 root mem REG 8,35 59864 7077901 /lib/aarch64-linux-gnu/libapparmor.so.1.4.2 docker-ru 34589 root mem REG 8,35 125896 7083147 /lib/aarch64-linux-gnu/ld-2.27.so docker-ru 34589 root 0u CHR 136,0 0t0 3 /dev/pts/0 docker-ru 34589 root 1u CHR 136,0 0t0 3 /dev/pts/0 docker-ru 34589 root 2u CHR 136,0 0t0 3 /dev/pts/0 docker-ru 34589 root 3w CHR 1,3 0t0 6 /dev/null docker-ru 34589 root 4u a_inode 0,13 0 9285 [eventpoll] docker-ru 34589 root 5u CHR 5,2 0t0 88 /dev/ptmx root@ubuntu:~# docker-runc state mycontainer { "ociVersion": "1.0.0-rc2-dev", "id": "mycontainer", "pid": 34600, "status": "running", "bundle": "/root/docker/container", "rootfs": "/root/docker/container/rootfs", "created": "2020-10-17T06:00:32.147160143Z" }root@ubuntu:~#
- runtime规定了如下内容
- state
ociVersion：创建容器时的OCI版本- id：容器唯一的ID
- status：容器的runtime状态，可以为如下值
- creating：容器正在被创建（lifecycle的第2步）
- created：容器完成创建，但没有返回错误且没有执行用户程序（lifecycle的第2步之后）
- running：容器正在执行用户程序且没有返回错误（lifecycle的第5步之后）
- stoped：容器进程退出（lifecycle的第7步）
- pid：host上看到的容器进程
- bundle：host上容器bundle目录的绝对路径
- annotation：容器相关的标注，可选
- state
由于runc实现了OCI runtime，使用runc state查看上述 state相关的信息
root@ubuntu:~# docker-runc state mycontainer { "ociVersion": "1.0.0-rc2-dev", "id": "mycontainer", "pid": 34600, "status": "running", "bundle": "/root/docker/container", "rootfs": "/root/docker/container/rootfs", "created": "2020-10-17T06:00:32.147160143Z" }root@ubuntu:~#
root@ubuntu:~/docker/container# ctr run --runtime io.containerd.runc.v1 -t --rm docker.io/library/busybox:latest hello sh
/ #
root@ubuntu:~# ctr c ls | grep busybox hello docker.io/library/busybox:latest io.containerd.runc.v1 root@ubuntu:~# ctr ns ls NAME LABELS default demo k8s.gcr.io k8s.io moby root@ubuntu:~# ctr -n default c ls | grep busybox hello docker.io/library/busybox:latest io.containerd.runc.v1 root@ubuntu:~#
root@ubuntu:~# ps -elf | grep hello 0 S root 42310 13451 0 80 0 - 308244 futex_ 14:26 pts/0 00:00:00 ctr run --runtime io.containerd.runc.v1 -t --rm docker.io/library/busybox:latest hello sh 0 S root 42334 1 0 80 0 - 27806 futex_ 14:26 ? 00:00:00 /usr/bin/containerd-shim-runc-v1 -namespace default -id hello -address /run/containerd/containerd.sock 0 S root 43783 14029 0 80 0 - 1097 pipe_w 14:31 pts/1 00:00:00 grep --color=auto hello root@ubuntu:~#
root@ubuntu:~# lsof -p 42310 COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME ctr 42310 root cwd DIR 8,35 4096 3806963 /root/docker/container ctr 42310 root rtd DIR 8,35 4096 2 / ctr 42310 root txt REG 8,35 21756192 9309549 /usr/bin/ctr ctr 42310 root mem REG 8,35 1345176 7083151 /lib/aarch64-linux-gnu/libc-2.27.so ctr 42310 root mem REG 8,35 137848 7083165 /lib/aarch64-linux-gnu/libpthread-2.27.so ctr 42310 root mem REG 8,35 125896 7083147 /lib/aarch64-linux-gnu/ld-2.27.so ctr 42310 root 0u CHR 136,0 0t0 3 /dev/pts/0 ctr 42310 root 1u CHR 136,0 0t0 3 /dev/pts/0 ctr 42310 root 2u CHR 136,0 0t0 3 /dev/pts/0 ctr 42310 root 3u unix 0xffff80af95828800 0t0 11504561 type=STREAM ctr 42310 root 4u a_inode 0,13 0 9285 [eventpoll] ctr 42310 root 5u FIFO 0,25 0t0 830 /run/containerd/fifo/485221562/hello-stdin ctr 42310 root 6u FIFO 0,25 0t0 1251 /run/containerd/fifo/485221562/hello-stdout ctr 42310 root 7w FIFO 0,25 0t0 830 /run/containerd/fifo/485221562/hello-stdin ctr 42310 root 8r FIFO 0,25 0t0 1251 /run/containerd/fifo/485221562/hello-stdout root@ubuntu:~# lsof -p 42334 COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME container 42334 root cwd DIR 0,25 240 1275 /run/containerd/io.containerd.runtime.v2.task/default/hello container 42334 root rtd DIR 8,35 4096 2 / container 42334 root txt REG 8,35 8335160 9309547 /usr/bin/containerd-shim-runc-v1 container 42334 root 0r CHR 1,3 0t0 6 /dev/null container 42334 root 1w CHR 1,3 0t0 6 /dev/null container 42334 root 2w CHR 1,3 0t0 6 /dev/null container 42334 root 4u unix 0xffff809f9cb8a000 0t0 11627923 type=STREAM container 42334 root 5u a_inode 0,13 0 9285 [eventpoll] container 42334 root 6u a_inode 0,13 0 9285 [eventpoll] container 42334 root 7u a_inode 0,13 0 9285 [eventpoll] container 42334 root 8u FIFO 0,25 0t0 1279 /run/containerd/io.containerd.runtime.v2.task/default/hello/log container 42334 root 9w FIFO 0,25 0t0 1279 /run/containerd/io.containerd.runtime.v2.task/default/hello/log container 42334 root 10u unix 0xffff809ed0552c00 0t0 11638948 @/containerd-shim/a159efe0307130e2366f05bca00c1cb2cb9e40375e35d351e4efe9f2c468c427.sock@ type=STREAM container 42334 root 11u unix 0xffff809cdf1bc800 0t0 11627926 @/containerd-shim/a159efe0307130e2366f05bca00c1cb2cb9e40375e35d351e4efe9f2c468c427.sock@ type=STREAM container 42334 root 13u FIFO 0,25 0t0 830 /run/containerd/fifo/485221562/hello-stdin container 42334 root 14w FIFO 0,25 0t0 830 /run/containerd/fifo/485221562/hello-stdin container 42334 root 15u CHR 5,2 0t0 2 /dev/pts/ptmx container 42334 root 16u FIFO 0,25 0t0 830 /run/containerd/fifo/485221562/hello-stdin container 42334 root 17u FIFO 0,25 0t0 1251 /run/containerd/fifo/485221562/hello-stdout container 42334 root 18w FIFO 0,25 0t0 1251 /run/containerd/fifo/485221562/hello-stdout container 42334 root 19u FIFO 0,25 0t0 1251 /run/containerd/fifo/485221562/hello-stdout container 42334 root 20r FIFO 0,25 0t0 1251 /run/containerd/fifo/485221562/hello-stdout container 42334 root 21r FIFO 0,25 0t0 830 /run/containerd/fifo/485221562/hello-stdin container 42334 root 22u a_inode 0,13 0 9285 [eventfd] root@ubuntu:~#
root@ubuntu:~# ctr -n k8s.io c ls | grep busy 6a91f780eda7a3b382bc79acec45e8905c2ee63243ba74300145a217e617bf10 docker.io/library/busybox:latest io.containerd.kata.v2 root@ubuntu:~# mount | grep overlay |grep 6a91f780eda7a3b382bc79acec45e8905c2ee63243ba74300145a217e617bf10 overlay on /run/containerd/io.containerd.runtime.v2.task/k8s.io/6a91f780eda7a3b382bc79acec45e8905c2ee63243ba74300145a217e617bf10/rootfs type overlay (rw,relatime,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/361/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/362/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/362/work,xino=off) overlay on /run/kata-containers/shared/sandboxes/64abb3931350d2762e805122d39ecbcee47e59a40331d1e09a5823dd0d11fa27/mounts/6a91f780eda7a3b382bc79acec45e8905c2ee63243ba74300145a217e617bf10/rootfs type overlay (rw,relatime,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/361/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/362/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/362/work,xino=off) overlay on /run/kata-containers/shared/sandboxes/64abb3931350d2762e805122d39ecbcee47e59a40331d1e09a5823dd0d11fa27/shared/6a91f780eda7a3b382bc79acec45e8905c2ee63243ba74300145a217e617bf10/rootfs type overlay (rw,relatime,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/361/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/362/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/362/work,xino=off) root@ubuntu:~# kata-runtime list | grep 64abb3931350d2762e805122d39ecbcee47e59a40331d1e09a5823dd0d11fa27 64abb3931350d2762e805122d39ecbcee47e59a40331d1e09a5823dd0d11fa27 -1 running /run/containerd/io.containerd.runtime.v2.task/k8s.io/64abb3931350d2762e805122d39ecbcee47e59a40331d1e09a5823dd0d11fa27 2020-10-17T02:13:37.762824405Z #0 root@ubuntu:~# kata-runtime list ID PID STATUS BUNDLE CREATED OWNER 64abb3931350d2762e805122d39ecbcee47e59a40331d1e09a5823dd0d11fa27 -1 running /run/containerd/io.containerd.runtime.v2.task/k8s.io/64abb3931350d2762e805122d39ecbcee47e59a40331d1e09a5823dd0d11fa27 2020-10-17T02:13:37.762824405Z #0 6a91f780eda7a3b382bc79acec45e8905c2ee63243ba74300145a217e617bf10 -1 running /run/containerd/io.containerd.runtime.v2.task/k8s.io/6a91f780eda7a3b382bc79acec45e8905c2ee63243ba74300145a217e617bf10 2020-10-17T02:14:04.602169815Z #0 96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146 -1 running /run/containerd/io.containerd.runtime.v2.task/k8s.io/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146 2020-10-16T09:17:50.17411532Z #0 d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9 -1 running /run/containerd/io.containerd.runtime.v2.task/k8s.io/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9 2020-10-16T09:18:08.084499379Z #0 root@ubuntu:~#
root@ubuntu:/home/ubuntu/containerddemo# find /run/kata-containers/shared/sandboxes/ -name nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/usr/sbin/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/usr/lib/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/usr/share/doc/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/usr/share/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/etc/logrotate.d/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/etc/default/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/etc/init.d/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/etc/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/var/cache/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/var/log/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/usr/sbin/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/usr/lib/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/usr/share/doc/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/usr/share/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/etc/logrotate.d/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/etc/default/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/etc/init.d/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/etc/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/var/cache/nginx /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs/var/log/nginx root@ubuntu:/home/ubuntu/containerddemo#
root@ubuntu:/home/ubuntu/containerddemo# mount | grep overlay | grep 96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146 overlay on /run/containerd/io.containerd.runtime.v2.task/k8s.io/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/rootfs type overlay (rw,relatime,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/87/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/348/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/348/work,xino=off) overlay on /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/rootfs type overlay (rw,relatime,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/87/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/348/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/348/work,xino=off) overlay on /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/rootfs type overlay (rw,relatime,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/87/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/348/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/348/work,xino=off) overlay on /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/mounts/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs type overlay (rw,relatime,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/353/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/352/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/351/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/350/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/349/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/354/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/354/work,xino=off) overlay on /run/kata-containers/shared/sandboxes/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/shared/d811e8590aecb55091918cefc11cef16665250c878a0646fb04039f7f67d1cd9/rootfs type overlay (rw,relatime,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/353/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/352/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/351/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/350/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/349/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/354/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/354/work,xino=off) root@ubuntu:/home/ubuntu/containerddemo#
root@ubuntu:/home/ubuntu/containerddemo# cd /run/containerd/io.containerd.runtime.v2.task/k8s.io/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/rootfs root@ubuntu:/run/containerd/io.containerd.runtime.v2.task/k8s.io/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/rootfs# find ./ -name nginx root@ubuntu:/run/containerd/io.containerd.runtime.v2.task/k8s.io/96a36a40169dd6add5bdf3367a4313bec828a598fb001a45bac7d01bec1f0146/rootfs#
docker overlay2
root@ubuntu:/home/ubuntu# ps -elf | grep qemu | grep 4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92 7 S root 1620 1559 1 80 0 - 755962 sys_po 09:53 ? 00:00:01 /usr/share/kata-containers/binary/qemu-system-aarch64 -name sandbox-4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92 -uuid 0ac9369f-b23d-4a4b-a434-aa7d77d77343 -machine virt,usb=off,accel=kvm,gic-version=host -cpu host,pmu=off -qmp unix:/run/vc/vm/4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92/qmp.sock,server,nowait -m 2048M,slots=10,maxmem=257176M -device pcie-pci-bridge,bus=pcie.0,id=pcie-bridge-0,addr=2,romfile= -device virtio-serial-pci,disable-modern=false,id=serial0,romfile= -device virtconsole,chardev=charconsole0,id=console0 -chardev socket,id=charconsole0,path=/run/vc/vm/4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92/console.sock,server,nowait -device virtio-blk-pci,disable-modern=false,drive=image-5dd09490599834cb,scsi=off,config-wce=off,romfile=,share-rw=on -drive id=image-5dd09490599834cb,file=/usr/share/kata-containers/binary/kata-containers.img,aio=threads,format=raw,if=none,readonly -device virtio-scsi-pci,id=scsi0,disable-modern=false,romfile= -object rng-random,id=rng0,filename=/dev/urandom -device virtio-rng-pci,rng=rng0,romfile= -device virtserialport,chardev=charch0,id=channel0,name=agent.channel.0 -chardev socket,id=charch0,path=/run/vc/vm/4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92/kata.sock,server,nowait -device virtio-9p-pci,disable-modern=false,fsdev=extra-9p-kataShared,mount_tag=kataShared,romfile= -fsdev local,id=extra-9p-kataShared,path=/run/kata-containers/shared/sandboxes/4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92/shared,security_model=none -netdev tap,id=network-0,vhost=on,vhostfds=3,fds=4 -device driver=virtio-net-pci,netdev=network-0,mac=02:42:ac:11:00:02,disable-modern=false,mq=on,vectors=4,romfile= -rtc base=utc,driftfix=slew,clock=host -global kvm-pit.lost_tick_policy=discard -vga none -no-user-config -nodefaults -nographic --no-reboot -daemonize -kernel /usr/share/kata-containers/binary/vmlinuz-5.4.34-88 -append console=hvc0 console=hvc1 iommu.passthrough=0 root=/dev/vda1 rootflags=data=ordered,errors=remount-ro ro rootfstype=ext4 debug systemd.show_status=true systemd.log_level=debug panic=1 nr_cpus=64 agent.use_vsock=false systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket scsi_mod.scan=none agent.log=debug agent.debug_console -pidfile /run/vc/vm/4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92/pid -D /run/vc/vm/4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92/qemu.log -smp 1,cores=1,threads=1,sockets=64,maxcpus=64 root@ubuntu:/home/ubuntu# mount | grep overlay | grep 4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92 overlay on /run/kata-containers/shared/sandboxes/4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92/mounts/4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92/rootfs type overlay (rw,relatime,lowerdir=/var/lib/docker/overlay2/l/NBZRP6DEJWSRWZENXGSWMBVXNJ:/var/lib/docker/overlay2/l/7YK2UOEWIEZ7ADIHF3XEXJNGVJ:/var/lib/docker/overlay2/l/NFDWMDPDYJ7R44SB7C3EJY3RPO:/var/lib/docker/overlay2/l/J5E7PT7ZX67W4Q2VMC4R2CGNNP,upperdir=/var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/diff,workdir=/var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/work,xino=off) overlay on /run/kata-containers/shared/sandboxes/4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92/shared/4f38d6f4dd9f4f8ac68e56a6d919fefc4c9482858c404b6f63f41fd644dcde92/rootfs type overlay (rw,relatime,lowerdir=/var/lib/docker/overlay2/l/NBZRP6DEJWSRWZENXGSWMBVXNJ:/var/lib/docker/overlay2/l/7YK2UOEWIEZ7ADIHF3XEXJNGVJ:/var/lib/docker/overlay2/l/NFDWMDPDYJ7R44SB7C3EJY3RPO:/var/lib/docker/overlay2/l/J5E7PT7ZX67W4Q2VMC4R2CGNNP,upperdir=/var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/diff,workdir=/var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/work,xino=off) root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/l/NBZRP6DEJWSRWZENXGSWMBVXNJ dev etc root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/l/7YK2UOEWIEZ7ADIHF3XEXJNGVJ run root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/l/NFDWMDPDYJ7R44SB7C3EJY3RPO etc usr var root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/l/J5E7PT7ZX67W4Q2VMC4R2CGNNP bin boot dev etc home lib media mnt opt proc root run sbin srv sys tmp usr var root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0 diff link lower merged work root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/diff/ root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/work work root@ubuntu:/home/ubuntu#
在容器内创建一个文件
docker run --rm -it --runtime kata-runtime ubuntu 再创建一个容器
root@ubuntu:/home/ubuntu# mount | grep c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28 tmpfs on /run/kata-containers/shared/sandboxes/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/shared type tmpfs (ro,relatime,size=26334908k,mode=755) overlay on /run/kata-containers/shared/sandboxes/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/mounts/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/rootfs type overlay (rw,relatime,lowerdir=/var/lib/docker/overlay2/l/ZKFJRXVQIBUAMRAOZB4OWQ6CVC:/var/lib/docker/overlay2/l/7YK2UOEWIEZ7ADIHF3XEXJNGVJ:/var/lib/docker/overlay2/l/NFDWMDPDYJ7R44SB7C3EJY3RPO:/var/lib/docker/overlay2/l/J5E7PT7ZX67W4Q2VMC4R2CGNNP,upperdir=/var/lib/docker/overlay2/6c51b552b6e1f54442b84b5a2a1e0a3458b32b18f7702ae3d80c31059d720ae4/diff,workdir=/var/lib/docker/overlay2/6c51b552b6e1f54442b84b5a2a1e0a3458b32b18f7702ae3d80c31059d720ae4/work,xino=off) overlay on /run/kata-containers/shared/sandboxes/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/shared/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/rootfs type overlay (rw,relatime,lowerdir=/var/lib/docker/overlay2/l/ZKFJRXVQIBUAMRAOZB4OWQ6CVC:/var/lib/docker/overlay2/l/7YK2UOEWIEZ7ADIHF3XEXJNGVJ:/var/lib/docker/overlay2/l/NFDWMDPDYJ7R44SB7C3EJY3RPO:/var/lib/docker/overlay2/l/J5E7PT7ZX67W4Q2VMC4R2CGNNP,upperdir=/var/lib/docker/overlay2/6c51b552b6e1f54442b84b5a2a1e0a3458b32b18f7702ae3d80c31059d720ae4/diff,workdir=/var/lib/docker/overlay2/6c51b552b6e1f54442b84b5a2a1e0a3458b32b18f7702ae3d80c31059d720ae4/work,xino=off) /dev/sdc3 on /run/kata-containers/shared/sandboxes/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/mounts/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28-d4904454f19eb5a9-resolv.conf type ext4 (rw,relatime,errors=remount-ro,stripe=64) /dev/sdc3 on /run/kata-containers/shared/sandboxes/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/shared/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28-d4904454f19eb5a9-resolv.conf type ext4 (rw,relatime,errors=remount-ro,stripe=64) /dev/sdc3 on /run/kata-containers/shared/sandboxes/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/mounts/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28-0214a654c0ee667d-hostname type ext4 (rw,relatime,errors=remount-ro,stripe=64) /dev/sdc3 on /run/kata-containers/shared/sandboxes/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/shared/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28-0214a654c0ee667d-hostname type ext4 (rw,relatime,errors=remount-ro,stripe=64) /dev/sdc3 on /run/kata-containers/shared/sandboxes/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/mounts/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28-4fbe7e6e9e125687-hosts type ext4 (rw,relatime,errors=remount-ro,stripe=64) /dev/sdc3 on /run/kata-containers/shared/sandboxes/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28/shared/c1d7b1292cf086ffea4df996c718ad454b15c8c0e31293b17ba2e59397817e28-4fbe7e6e9e125687-hosts type ext4 (rw,relatime,errors=remount-ro,stripe=64) root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/6c51b552b6e1f54442b84b5a2a1e0a3458b32b18f7702ae3d80c31059d720ae4/diff root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/6c51b552b6e1f54442b84b5a2a1e0a3458b32b18f7702ae3d80c31059d720ae4/merged/ -li total 60 6946820 lrwxrwxrwx 1 root root 7 Sep 25 09:23 bin -> usr/bin 6946821 drwxr-xr-x 2 root root 4096 Apr 15 2020 boot 8389141 drwxr-xr-x 1 root root 4096 Oct 29 10:12 dev 8389144 drwxr-xr-x 1 root root 4096 Oct 29 10:12 etc 6946988 drwxr-xr-x 2 root root 4096 Apr 15 2020 home 6946989 lrwxrwxrwx 1 root root 7 Sep 25 09:23 lib -> usr/lib 6946990 drwxr-xr-x 2 root root 4096 Sep 25 09:23 media 6946991 drwxr-xr-x 2 root root 4096 Sep 25 09:23 mnt 6946992 drwxr-xr-x 2 root root 4096 Sep 25 09:23 opt 6946993 drwxr-xr-x 2 root root 4096 Apr 15 2020 proc 6946994 drwx------ 2 root root 4096 Sep 25 09:46 root 6950091 drwxr-xr-x 1 root root 4096 Sep 26 06:48 run 6947001 lrwxrwxrwx 1 root root 8 Sep 25 09:23 sbin -> usr/sbin 6947002 drwxr-xr-x 2 root root 4096 Sep 25 09:23 srv 6947003 drwxr-xr-x 2 root root 4096 Apr 15 2020 sys 6947004 drwxrwxrwt 2 root root 4096 Sep 25 09:47 tmp 6950076 drwxr-xr-x 1 root root 4096 Sep 25 09:23 usr 6950080 drwxr-xr-x 1 root root 4096 Sep 25 09:46 var root@ubuntu:/home/ubuntu#
ls /var/lib/docker/overlay2/6c51b552b6e1f54442b84b5a2a1e0a3458b32b18f7702ae3d80c31059d720ae4/merged/ -li
和ls /var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/merged/ -i -l
的inode一样
ubuntu1
root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/ -i -l total 20 8389125 drwxr-xr-x 3 root root 4096 Oct 29 09:53 diff 8389126 -rw-r--r-- 1 root root 26 Oct 29 09:53 link 8389129 -rw-r--r-- 1 root root 115 Oct 29 09:53 lower 8389125 drwxr-xr-x 1 root root 4096 Oct 29 09:53 merged 8389127 drwx------ 3 root root 4096 Oct 29 09:53 work root@ubuntu:/home/ubuntu#
lower
root@ubuntu:/home/ubuntu# cat /var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/lower l/NBZRP6DEJWSRWZENXGSWMBVXNJ:l/7YK2UOEWIEZ7ADIHF3XEXJNGVJ:l/NFDWMDPDYJ7R44SB7C3EJY3RPO:l/J5E7PT7ZX67W4Q2VMC4R2CGNNProot@ubuntu:/home/ubuntu# root@ubuntu:/home/ubuntu# root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/lower -li 8389129 -rw-r--r-- 1 root root 115 Oct 29 09:53 /var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/lower root@ubuntu:/home/ubuntu#
ubuntu2
root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/3b09015cb6eb5549a5b0fc1236bc733445d7dc4b517f0a1e9f1b47c7cbe4e9b0/ -i -l total 20 8389125 drwxr-xr-x 3 root root 4096 Oct 29 09:53 diff 8389126 -rw-r--r-- 1 root root 26 Oct 29 09:53 link 8389129 -rw-r--r-- 1 root root 115 Oct 29 09:53 lower 8389125 drwxr-xr-x 1 root root 4096 Oct 29 09:53 merged 8389127 drwx------ 3 root root 4096 Oct 29 09:53 work root@ubuntu:/home/ubuntu#
lower
root@ubuntu:/home/ubuntu# ls /var/lib/docker/overlay2/6c51b552b6e1f54442b84b5a2a1e0a3458b32b18f7702ae3d80c31059d720ae4/lower -li 8389155 -rw-r--r-- 1 root root 115 Oct 29 10:12 /var/lib/docker/overlay2/6c51b552b6e1f54442b84b5a2a1e0a3458b32b18f7702ae3d80c31059d720ae4/lower root@ubuntu:/home/ubuntu# cat /var/lib/docker/overlay2/6c51b552b6e1f54442b84b5a2a1e0a3458b32b18f7702ae3d80c31059d720ae4/lower l/ZKFJRXVQIBUAMRAOZB4OWQ6CVC:l/7YK2UOEWIEZ7ADIHF3XEXJNGVJ:l/NFDWMDPDYJ7R44SB7C3EJY3RPO:l/J5E7PT7ZX67W4Q2VMC4R2CGNNProot@ubuntu:/home/ubuntu#
work层是一个中间层
https://juejin.im/post/6844903574137208839