libcontainer AF_UNIX套接字 + SCM_RIGHTS

func RecvFd(socket *os.File) (*os.File, error) {
        // For some reason, unix.Recvmsg uses the length rather than the capacity
        // when passing the msg_controllen and other attributes to recvmsg.  So we
        // have to actually set the length.
        name := make([]byte, MaxNameLen)
        oob := make([]byte, oobSpace)

        sockfd := socket.Fd()
        n, oobn, _, _, err := unix.Recvmsg(int(sockfd), name, oob, 0)
        if err != nil {
                return nil, err
        }

        if n >= MaxNameLen || oobn != oobSpace {
                return nil, fmt.Errorf("recvfd: incorrect number of bytes read (n=%d oobn=%d)", n, oobn)
        }

        // Truncate.
        name = name[:n]
        oob = oob[:oobn]

        scms, err := unix.ParseSocketControlMessage(oob)
        if err != nil {
                return nil, err
        }
        if len(scms) != 1 {
                return nil, fmt.Errorf("recvfd: number of SCMs is not 1: %d", len(scms))
        }
        scm := scms[0]

        fds, err := unix.ParseUnixRights(&scm)
        if err != nil {
                return nil, err
        }
        if len(fds) != 1 {
                return nil, fmt.Errorf("recvfd: number of fds is not 1: %d", len(fds))
        }
        fd := uintptr(fds[0])

        return os.NewFile(fd, string(name)), nil
}

// SendFd sends a file descriptor over the given AF_UNIX socket. In
// addition, the file.Name() of the given file will also be sent as
// non-auxiliary data in the same payload (allowing to send contextual
// information for a file descriptor).
func SendFd(socket *os.File, name string, fd uintptr) error {
        if len(name) >= MaxNameLen {
                return fmt.Errorf("sendfd: filename too long: %s", name)
        }
        oob := unix.UnixRights(int(fd))
        return unix.Sendmsg(int(socket.Fd()), []byte(name), oob, nil, 0)
}

/ RecvFd waits for a file descriptor to be sent over the given AF_UNIX
// socket. The file name of the remote file descriptor will be recreated
// locally (it is sent as non-auxiliary data in the same payload).
func RecvFd(socket *os.File) (*os.File, error) {
    // For some reason, unix.Recvmsg uses the length rather than the capacity
    // when passing the msg_controllen and other attributes to recvmsg.  So we
    // have to actually set the length.
    name := make([]byte, MaxNameLen)
    oob := make([]byte, oobSpace)

    sockfd := socket.Fd()
    n, oobn, _, _, err := unix.Recvmsg(int(sockfd), name, oob, 0)
    if err != nil {
        return nil, err
    }

    if n >= MaxNameLen || oobn != oobSpace {
        return nil, fmt.Errorf("recvfd: incorrect number of bytes read (n=%d oobn=%d)", n, oobn)
    }

    // Truncate.
    name = name[:n]
    oob = oob[:oobn]

    scms, err := unix.ParseSocketControlMessage(oob)
    if err != nil {
        return nil, err
    }
    if len(scms) != 1 {
        return nil, fmt.Errorf("recvfd: number of SCMs is not 1: %d", len(scms))
    }
    scm := scms[0]

    fds, err := unix.ParseUnixRights(&scm)
    if err != nil {
        return nil, err
    }
    if len(fds) != 1 {
        return nil, fmt.Errorf("recvfd: number of fds is not 1: %d", len(fds))
    }
    fd := uintptr(fds[0])

    return os.NewFile(fd, string(name)), nil
}

// SendFd sends a file descriptor over the given AF_UNIX socket. In
// addition, the file.Name() of the given file will also be sent as
// non-auxiliary data in the same payload (allowing to send contextual
// information for a file descriptor).
func SendFd(socket *os.File, name string, fd uintptr) error {
    if len(name) >= MaxNameLen {
        return fmt.Errorf("sendfd: filename too long: %s", name)
    }
    oob := unix.UnixRights(int(fd))
    return unix.Sendmsg(int(socket.Fd()), []byte(name), oob, nil, 0)
}

// NewAgentClient creates a new agent gRPC client and handles both unix and vsock addresses.
//
// Supported sock address formats are:
//   - unix://<unix socket path>
//   - vsock://<cid>:<port>
//   - <unix socket path>
//   - hvsock://<path>:<port>. Firecracker implements the virtio-vsock device
//     model, and mediates communication between AF_UNIX sockets (on the host end)
//     and AF_VSOCK sockets (on the guest end).
func NewAgentClient(ctx context.Context, sock string, enableYamux bool) (*AgentClient, error) {
        grpcAddr, parsedAddr, err := parse(sock)
        if err != nil {
                return nil, err
        }
        dialOpts := []grpc.DialOption{grpc.WithInsecure(), grpc.WithBlock()}
        dialOpts = append(dialOpts, grpc.WithDialer(agentDialer(parsedAddr, enableYamux)))

        var tracer opentracing.Tracer

        span := opentracing.SpanFromContext(ctx)

        // If the context contains a trace span, trace all client comms
        if span != nil {
                tracer = span.Tracer()

                dialOpts = append(dialOpts,
                        grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(tracer)))
                dialOpts = append(dialOpts,
                        grpc.WithStreamInterceptor(otgrpc.OpenTracingStreamClientInterceptor(tracer)))
        }

        ctx, cancel := context.WithTimeout(ctx, defaultDialTimeout)
        defer cancel()
        conn, err := grpc.DialContext(ctx, grpcAddr, dialOpts...)
        if err != nil {
                return nil, err
        }

        return &AgentClient{
                AgentServiceClient: agentgrpc.NewAgentServiceClient(conn),
                HealthClient:       agentgrpc.NewHealthClient(conn),
                conn:               conn,
        }, nil
}

fd 可以通过 Unix domain socket 传出去，可以参考 runc 的： https://github.com/opencontainers/runc/blob/master/libcontainer/utils/cmsg.go

 https://blog.cloudflare.com/know-your-scm_rights/

// UnixRights encodes a set of open file descriptors into a socket
// control message for sending to another process.
func UnixRights(fds ...int) []byte {
    datalen := len(fds) * 4
    b := make([]byte, CmsgSpace(datalen))
    h := (*Cmsghdr)(unsafe.Pointer(&b[0]))
    h.Level = SOL_SOCKET
    h.Type = SCM_RIGHTS
    h.SetLen(CmsgLen(datalen))
    for i, fd := range fds {
        *(*int32)(h.data(4 * uintptr(i))) = int32(fd)
    }
    return b
}

// ParseUnixRights decodes a socket control message that contains an
// integer array of open file descriptors from another process.
func ParseUnixRights(m *SocketControlMessage) ([]int, error) {
    if m.Header.Level != SOL_SOCKET {
        return nil, EINVAL
    }
    if m.Header.Type != SCM_RIGHTS {
        return nil, EINVAL
    }
    fds := make([]int, len(m.Data)>>2)
    for i, j := 0, 0; i < len(m.Data); i += 4 {
        fds[j] = int(*(*int32)(unsafe.Pointer(&m.Data[i])))
        j++
    }
    return fds, nil
}

遇到这样一个需求：一个进程将自己的标准输入、标准输出和标准错误输出映射到另外一个进程相应的位置。带着对 Unix Domain Socket 的朦胧认识，写了一个简单的实现原型：

其中，outlet 是“贡献”标准输入输出的进程，inner 是“抢占” outlet 标准输入输出的进程。整体上代码还比较好理解，只是如果需要发文文件描述符，需要注意 send_fds函数中 cmsg->cmsg_type = SCM_RIGHTS; 一行，必须制定 cmsg_type为 SCM_RIGHTS。

分别编译两个程序文件：

先执行 outlet，先输出 “write from outlet” 后阻塞，等待 inner 进程的连接：

然后再启动 inner 进程，inner 进程输出“4 5 6”，并在 outlet 的标准输出输出 “write from inner“：

也有 “write from inner“：

这样就大功告成了。

inner 之所以输出 ”4 5 6“ 是比较好奇传递过来的文件描述符到底是长啥样的，这行打印看代码可以知道是打印了接收到的文件描述符的值，也就是创建了几个新的文件描述符么——就像侦探推理一样，没有揭开谜底前各种好奇，一旦揭开谜底了反倒趣味全无了。

P.S. unix(7)中其实交代了传递文件描述符的效果跟 dup2 差不多的，还怪自己平常没有好好 RTFM 啊：