bindMountContainerRootfs(

func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, err error) {
        span, _ := k.trace("createContainer")
        defer span.Finish()

        var ctrStorages []*grpc.Storage
        var ctrDevices []*grpc.Device
        var rootfs *grpc.Storage

        // This is the guest absolute root path for that container.
        rootPathParent := filepath.Join(kataGuestSharedDir(), c.id)
        rootPath := filepath.Join(rootPathParent, c.rootfsSuffix)

        // In case the container creation fails, the following defer statement
        // takes care of rolling back actions previously performed.
        defer func() {
                if err != nil {
                        k.Logger().WithError(err).Error("createContainer failed")
                        k.rollbackFailingContainerCreation(c)
                }
        }()

        if rootfs, err = k.buildContainerRootfs(sandbox, c, rootPathParent); err != nil {
                return nil, err
        } else if rootfs != nil {
                // Add rootfs to the list of container storage.
                // We only need to do this for block based rootfs, as we
                // want the agent to mount it into the right location
                // (kataGuestSharedDir/ctrID/
                ctrStorages = append(ctrStorages, rootfs)
        }

        ociSpec := c.GetPatchedOCISpec()
        if ociSpec == nil {
                return nil, errorMissingOCISpec
        }

        // Handle container mounts
        newMounts, ignoredMounts, err := c.mountSharedDirMounts(getMountPath(sandbox.id), kataGuestSharedDir())
        if err != nil {
                return nil, err
        }

        k.handleShm(ociSpec.Mounts, sandbox)

        epheStorages := k.handleEphemeralStorage(ociSpec.Mounts)
        ctrStorages = append(ctrStorages, epheStorages...)

        localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
        ctrStorages = append(ctrStorages, localStorages...)

ls /var/lib/docker/volumes/data1/
_data
root@p:/home/pcl# ls /var/lib/docker/volumes/data1/_data/

find  /var/lib/docker/overlay2/0302ff6f07ca65ebbf82884a33e0dcec795e0de1d5af7006d5473402028099eb/  -name data1
/var/lib/docker/overlay2/0302ff6f07ca65ebbf82884a33e0dcec795e0de1d5af7006d5473402028099eb/merged/data1
/var/lib/docker/overlay2/0302ff6f07ca65ebbf82884a33e0dcec795e0de1d5af7006d5473402028099eb/diff/data

func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPathParent string) (*grpc.Storage, error) {
        if c.state.Fstype != "" && c.state.BlockDeviceID != "" {
                // The rootfs storage volume represents the container rootfs
                // mount point inside the guest.
                // It can be a block based device (when using block based container
                // overlay on the host) mount or a 9pfs one (for all other overlay
                // implementations).
                rootfs := &grpc.Storage{}

                // This is a block based device rootfs.
                device := sandbox.devManager.GetDeviceByID(c.state.BlockDeviceID)
                if device == nil {
                        k.Logger().WithField("device", c.state.BlockDeviceID).Error("failed to find device by id")
                        return nil, fmt.Errorf("failed to find device by id %q", c.state.BlockDeviceID)
                }

                blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive)
                if !ok || blockDrive == nil {
                        k.Logger().Error("malformed block drive")
                        return nil, fmt.Errorf("malformed block drive")
                }
                switch {
                case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio:
                        rootfs.Driver = kataMmioBlkDevType
                        rootfs.Source = blockDrive.VirtPath
                case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlockCCW:
                        rootfs.Driver = kataBlkCCWDevType
                        rootfs.Source = blockDrive.DevNo
                case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock:
                        rootfs.Driver = kataBlkDevType
                        if blockDrive.PCIAddr == "" {
                                rootfs.Source = blockDrive.VirtPath
                        } else {
                                rootfs.Source = blockDrive.PCIAddr
                        }

                case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI:

                        rootfs.Driver = kataSCSIDevType
                        rootfs.Source = blockDrive.SCSIAddr
                default:
                        return nil, fmt.Errorf("Unknown block device driver: %s", sandbox.config.HypervisorConfig.BlockDeviceDriver)
                }

                rootfs.MountPoint = rootPathParent
                rootfs.Fstype = c.state.Fstype

                if c.state.Fstype == "xfs" {
                        rootfs.Options = []string{"nouuid"}
                }

                // Ensure container mount destination exists
                // TODO: remove dependency on shared fs path. shared fs is just one kind of storage sources.
                // we should not always use shared fs path for all kinds of storage. Stead, all storage
                // should be bind mounted to a tmpfs path for containers to use.
                if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
                        return nil, err
                }
                return rootfs, nil
        }
        // This is not a block based device rootfs.
        // We are going to bind mount it into the 9pfs
        // shared drive between the host and the guest.
        // With 9pfs we don't need to ask the agent to
        // mount the rootfs as the shared directory
        // (kataGuestSharedDir) is already mounted in the
        // guest. We only need to mount the rootfs from
        // the host and it will show up in the guest.
        if err := bindMountContainerRootfs(k.ctx, getMountPath(sandbox.id), c.id, c.rootFs.Target, false); err != nil {
                return nil, err
        }

        return nil, nil
}

// bindMountContainerRootfs bind mounts a container rootfs into a 9pfs shared
// directory between the guest and the host.
func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string, readonly bool) error {
        span, _ := trace(ctx, "bindMountContainerRootfs")
        defer span.Finish()

        rootfsDest := filepath.Join(shareDir, cid, rootfsDir)

        return bindMount(ctx, cRootFs, rootfsDest, readonly, "private")
}

// bindMount bind mounts a source in to a destination. This will
// do some bookkeeping:
// * evaluate all symlinks
// * ensure the source exists
// * recursively create the destination
// pgtypes stands for propagation types, which are shared, private, slave, and ubind.
func bindMount(ctx context.Context, source, destination string, readonly bool, pgtypes string) error {
        span, _ := trace(ctx, "bindMount")
        defer span.Finish()

        if source == "" {
                return fmt.Errorf("source must be specified")
        }
        if destination == "" {
                return fmt.Errorf("destination must be specified")
        }

        absSource, err := filepath.EvalSymlinks(source)
        if err != nil {
                return fmt.Errorf("Could not resolve symlink for source %v", source)
        }

        if err := ensureDestinationExists(absSource, destination); err != nil {
                return fmt.Errorf("Could not create destination mount point %v: %v", destination, err)
        }

        if err := syscall.Mount(absSource, destination, "bind", syscall.MS_BIND, ""); err != nil {
                return fmt.Errorf("Could not bind mount %v to %v: %v", absSource, destination, err)
        }

        if pgtype, exist := propagationTypes[pgtypes]; exist {
                if err := syscall.Mount("none", destination, "", pgtype, ""); err != nil {
                        return fmt.Errorf("Could not make mount point %v %s: %v", destination, pgtypes, err)
                }
        } else {
                return fmt.Errorf("Wrong propagation type %s", pgtypes)
        }

        // For readonly bind mounts, we need to remount with the readonly flag.
        // This is needed as only very recent versions of libmount/util-linux support "bind,ro"
        if readonly {
                return syscall.Mount(absSource, destination, "bind", uintptr(syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY), "")
        }

        return nil
}

func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, rootfsSuffix string) []*grpc.Storage {
        var localStorages []*grpc.Storage
        for idx, mnt := range mounts {
                if mnt.Type == KataLocalDevType {
                        // Set the mount source path to a the desired directory point in the VM.
                        // In this case it is located in the sandbox directory.
                        // We rely on the fact that the first container in the VM has the same ID as the sandbox ID.
                        // In Kubernetes, this is usually the pause container and we depend on it existing for
                        // local directories to work.
                        mounts[idx].Source = filepath.Join(kataGuestSharedDir(), sandboxID, rootfsSuffix, KataLocalDevType, filepath.Base(mnt.Source))

                        // Create a storage struct so that the kata agent is able to create the
                        // directory inside the VM.
                        localStorage := &grpc.Storage{
                                Driver:     KataLocalDevType,
                                Source:     KataLocalDevType,
                                Fstype:     KataLocalDevType,
                                MountPoint: mounts[idx].Source,
                                Options:    localDirOptions,
                        }
                        localStorages = append(localStorages, localStorage)
                }
        }
        return localStorages
}

docker run -it --runtime=kata-runtime -v data1:/data1 --rm debian /bin/bash
root@0e13f5a22fca:/#

    "Mounts": [
            {
                "Type": "volume",
                "Name": "data1",
                "Source": "/var/lib/docker/volumes/data1/_data",
                "Destination": "/data1",
                "Driver": "local",
                "Mode": "z",
                "RW": true,
                "Propagation": ""
            }
        ],

docker  inspect 0e13f5a22fca
[
    {
        "Id": "0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e",
        "Created": "2020-12-04T00:57:07.930022529Z",
        "Path": "/bin/bash",
        "Args": [],
        "State": {
            "Status": "running",
            "Running": true,
            "Paused": false,
            "Restarting": false,
            "OOMKilled": false,
            "Dead": false,
            "Pid": 3105787,
            "ExitCode": 0,
            "Error": "",
            "StartedAt": "2020-12-04T00:57:10.936012588Z",
            "FinishedAt": "0001-01-01T00:00:00Z"
        },
        "Image": "sha256:2aa621c7db398cbad23b51b63348fedf0a779542d52f71acd4f24072cb872170",
        "ResolvConfPath": "/var/lib/docker/containers/0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e/hostname",
        "HostsPath": "/var/lib/docker/containers/0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e/hosts",
        "LogPath": "/var/lib/docker/containers/0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e/0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e-json.log",
        "Name": "/distracted_haibt",
        "RestartCount": 0,
        "Driver": "overlay2",
        "Platform": "linux",
        "MountLabel": "",
        "ProcessLabel": "",
        "AppArmorProfile": "docker-default",
        "ExecIDs": null,
        "HostConfig": {
            "Binds": [
                "data1:/data1"
            ],
            "ContainerIDFile": "",
            "LogConfig": {
                "Type": "json-file",
                "Config": {
                    "max-size": "100m"
                }
            },
            "NetworkMode": "default",
            "PortBindings": {},
            "RestartPolicy": {
                "Name": "no",
                "MaximumRetryCount": 0
            },
            "AutoRemove": true,
            "VolumeDriver": "",
            "VolumesFrom": null,
            "CapAdd": null,
            "CapDrop": null,
            "Dns": [],
            "DnsOptions": [],
            "DnsSearch": [],
            "ExtraHosts": null,
            "GroupAdd": null,
            "IpcMode": "shareable",
            "Cgroup": "",
            "Links": null,
            "OomScoreAdj": 0,
            "PidMode": "",
            "Privileged": false,
            "PublishAllPorts": false,
            "ReadonlyRootfs": false,
            "SecurityOpt": null,
            "UTSMode": "",
            "UsernsMode": "",
            "ShmSize": 67108864,
            "Runtime": "kata-runtime",
            "ConsoleSize": [
                0,
                0
            ],
            "Isolation": "",
            "CpuShares": 0,
            "Memory": 0,
            "NanoCpus": 0,
            "CgroupParent": "",
            "BlkioWeight": 0,
            "BlkioWeightDevice": [],
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,
            "CpuPeriod": 0,
            "CpuQuota": 0,
            "CpuRealtimePeriod": 0,
            "CpuRealtimeRuntime": 0,
            "CpusetCpus": "",
            "CpusetMems": "",
            "Devices": [],
            "DeviceCgroupRules": null,
            "DiskQuota": 0,
            "KernelMemory": 0,
            "MemoryReservation": 0,
            "MemorySwap": 0,
            "MemorySwappiness": null,
            "OomKillDisable": false,
            "PidsLimit": 0,
            "Ulimits": null,
            "CpuCount": 0,
            "CpuPercent": 0,
            "IOMaximumIOps": 0,
            "IOMaximumBandwidth": 0,
            "MaskedPaths": [
                "/proc/asound",
                "/proc/acpi",
                "/proc/kcore",
                "/proc/keys",
                "/proc/latency_stats",
                "/proc/timer_list",
                "/proc/timer_stats",
                "/proc/sched_debug",
                "/proc/scsi",
                "/sys/firmware"
            ],
            "ReadonlyPaths": [
                "/proc/bus",
                "/proc/fs",
                "/proc/irq",
                "/proc/sys",
                "/proc/sysrq-trigger"
            ]
        },
        "GraphDriver": {
            "Data": {
                "LowerDir": "/var/lib/docker/overlay2/e17f8f08ca2d3e8dd4dc29d9167fca9856beeb25cac37104bd43573d628c849d-init/diff:/var/lib/docker/overlay2/53cfc5f6a35592815e1df5bcdb226c096e7bdbd59fb0c58253d82c055c2b8435/diff",
                "MergedDir": "/var/lib/docker/overlay2/e17f8f08ca2d3e8dd4dc29d9167fca9856beeb25cac37104bd43573d628c849d/merged",
                "UpperDir": "/var/lib/docker/overlay2/e17f8f08ca2d3e8dd4dc29d9167fca9856beeb25cac37104bd43573d628c849d/diff",
                "WorkDir": "/var/lib/docker/overlay2/e17f8f08ca2d3e8dd4dc29d9167fca9856beeb25cac37104bd43573d628c849d/work"
            },
            "Name": "overlay2"
        },
        "Mounts": [
            {
                "Type": "volume",
                "Name": "data1",
                "Source": "/var/lib/docker/volumes/data1/_data",
                "Destination": "/data1",
                "Driver": "local",
                "Mode": "z",
                "RW": true,
                "Propagation": ""
            }
        ],
        "Config": {
            "Hostname": "0e13f5a22fca",
            "Domainname": "",
            "User": "",
            "AttachStdin": true,
            "AttachStdout": true,
            "AttachStderr": true,
            "Tty": true,
            "OpenStdin": true,
            "StdinOnce": true,
            "Env": [
                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
            ],
            "Cmd": [
                "/bin/bash"
            ],
            "Image": "debian",
            "Volumes": null,
            "WorkingDir": "",
            "Entrypoint": null,
            "OnBuild": null,
            "Labels": {}
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "b7dab902309c0390682b0f572832ecb996f8cb1fe3fb942ebe4a1a2cd8301fe6",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": {},
            "SandboxKey": "/var/run/docker/netns/b7dab902309c",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "e15b8cdb018fac7f63bf18cbafd3904cfbdf7c5102d3542ad5d0f6c7f51cbe27",
            "Gateway": "172.17.0.1",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "172.17.0.3",
            "IPPrefixLen": 16,
            "IPv6Gateway": "",
            "MacAddress": "02:42:ac:11:00:03",
            "Networks": {
                "bridge": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": null,
                    "NetworkID": "46f664be20d1ea20abc7661d495d3f7f5283588f940faa72b9c0ab5d077b076a",
                    "EndpointID": "e15b8cdb018fac7f63bf18cbafd3904cfbdf7c5102d3542ad5d0f6c7f51cbe27",
                    "Gateway": "172.17.0.1",
                    "IPAddress": "172.17.0.3",
                    "IPPrefixLen": 16,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "02:42:ac:11:00:03",
                    "DriverOpts": null
                }
            }
        }
    }
]

docker run -it --runtime=kata-runtime  -v data1:/data1 --rm  debian /bin/bash
root@0e13f5a22fca:/# ls data1/
root@0e13f5a22fca:/# 

time="2020-12-04T01:01:45.94077978Z" level=debug msg="new request" debug_console=true name=kata-agent pid=56 req="container_id:"0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e" exec_id:"0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e" len:32768 " request=/grpc.AgentService/ReadStdout sandbox=0e13f5a22fcafdca480f16ef9bae2e6e5da5d14f8d50b712a51ba1e51a68233e source=agent

ls: cannot access 'data': No such file or directory
root@ubuntu:/# ls data
ls: cannot access 'data': No such file or directory
root@ubuntu:/# 

kata虚拟机看不到

root@ubuntu:/# ls data
ls: cannot access 'data': No such file or directory
root@ubuntu:/# ls
bin   dev  home  lost+found  mnt  proc  run   srv  tmp  var
boot  etc  lib   media       opt  root  sbin  sys  usr  vmi.sh
root@ubuntu:/#