简单IOCP例子

使用IOCP模型编程的优点
① 帮助维持重复使用的内存池。(与重叠I/O技术有关)
② 去除删除线程创建/终结负担。
③ 利于管理，分配线程，控制并发，最小化的线程上下文切换。
④ 优化线程调度，提高CPU和内存缓冲的命中率。
服务器：

// IOCP_TCPIP_Socket_Server.cpp

#include <WinSock2.h>
#include <Windows.h>
#include <vector>
#include <iostream>


using namespace std;

#pragma comment(lib, "Ws2_32.lib")      // Socket编程需用的动态链接库
#pragma comment(lib, "Kernel32.lib")    // IOCP需要用到的动态链接库

/**
* 结构体名称：PER_IO_DATA
* 结构体功能：重叠I/O需要用到的结构体，临时记录IO数据
**/
const int DataBuffSize = 2 * 1024;
typedef struct
{
    OVERLAPPED overlapped;
    WSABUF databuff;
    char buffer[DataBuffSize];
    int BufferLen;
    int operationType;
}PER_IO_OPERATEION_DATA, *LPPER_IO_OPERATION_DATA, *LPPER_IO_DATA, PER_IO_DATA;

/**
* 结构体名称：PER_HANDLE_DATA
* 结构体存储：记录单个套接字的数据，包括了套接字的变量及套接字的对应的客户端的地址。
* 结构体作用：当服务器连接上客户端时，信息存储到该结构体中，知道客户端的地址以便于回访。
**/
typedef struct
{
    SOCKET socket;
    SOCKADDR_STORAGE ClientAddr;
}PER_HANDLE_DATA, *LPPER_HANDLE_DATA;

// 定义全局变量
const int DefaultPort = 5000;
vector < PER_HANDLE_DATA* > clientGroup;        // 记录客户端的向量组
vector<LPPER_IO_OPERATION_DATA> IOOperationDataGroup;

HANDLE hMutex = CreateMutex(NULL, FALSE, NULL);
DWORD WINAPI ServerWorkThread(LPVOID CompletionPortID);
DWORD WINAPI ServerSendThread(LPVOID IpParam);

// 开始主函数
int main()
{
    // 加载socket动态链接库
    WORD wVersionRequested = MAKEWORD(2, 2); // 请求2.2版本的WinSock库
    WSADATA wsaData;    // 接收Windows Socket的结构信息
    DWORD err = WSAStartup(wVersionRequested, &wsaData);

    if (0 != err) { // 检查套接字库是否申请成功
        cerr << "Request Windows Socket Library Error!
";
        system("pause");
        return -1;
    }
    if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2) {// 检查是否申请了所需版本的套接字库
        WSACleanup();
        cerr << "Request Windows Socket Version 2.2 Error!
";
        system("pause");
        return -1;
    }

    // 创建IOCP的内核对象
    /**
    * 需要用到的函数的原型：
    * HANDLE WINAPI CreateIoCompletionPort(
    *    __in   HANDLE FileHandle,      // 已经打开的文件句柄或者空句柄，一般是客户端的句柄
    *    __in   HANDLE ExistingCompletionPort,  // 已经存在的IOCP句柄
    *    __in   ULONG_PTR CompletionKey,    // 完成键，包含了指定I/O完成包的指定文件
    *    __in   DWORD NumberOfConcurrentThreads // 真正并发同时执行最大线程数，一般推介是CPU核心数*2
    * );
    **/
    HANDLE completionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
    if (NULL == completionPort) {   // 创建IO内核对象失败
        cerr << "CreateIoCompletionPort failed. Error:" << GetLastError() << endl;
        system("pause");
        return -1;
    }

    // 创建IOCP线程--线程里面创建线程池

    // 确定处理器的核心数量
    SYSTEM_INFO mySysInfo;
    GetSystemInfo(&mySysInfo);

    // 基于处理器的核心数量创建线程
    for (DWORD i = 0; i < (mySysInfo.dwNumberOfProcessors * 2); ++i) {
        // 创建服务器工作器线程，并将完成端口传递到该线程
        HANDLE ThreadHandle = CreateThread(NULL, 0, ServerWorkThread, completionPort, 0, NULL);//第一NULL代表默认安全选项，第一个0，代表线程占用资源大小，第二个0，代表线程创建后立即执行
        if (NULL == ThreadHandle) {
            cerr << "Create Thread Handle failed. Error:" << GetLastError() << endl;
            system("pause");
            return -1;
        }
        CloseHandle(ThreadHandle);
    }

    // 建立流式套接字
    SOCKET srvSocket = socket(AF_INET, SOCK_STREAM, 0);

    // 绑定SOCKET到本机
    SOCKADDR_IN srvAddr;
    srvAddr.sin_addr.S_un.S_addr = htonl(INADDR_ANY);
    srvAddr.sin_family = AF_INET;
    srvAddr.sin_port = htons(DefaultPort);
    int bindResult = bind(srvSocket, (SOCKADDR*)&srvAddr, sizeof(SOCKADDR));
    if (SOCKET_ERROR == bindResult) {
        cerr << "Bind failed. Error:" << GetLastError() << endl;
        system("pause");
        return -1;
    }

    // 将SOCKET设置为监听模式
    int listenResult = listen(srvSocket, 10);
    if (SOCKET_ERROR == listenResult) {
        cerr << "Listen failed. Error: " << GetLastError() << endl;
        system("pause");
        return -1;
    }

    // 开始处理IO数据
    cout << "本服务器已准备就绪，正在等待客户端的接入...
";

    //// 创建用于发送数据的线程
    //HANDLE sendThread = CreateThread(NULL, 0, ServerSendThread, 0, 0, NULL);//第二个0，代表回掉函数参数为0

    while (true) {
        PER_HANDLE_DATA * PerHandleData = NULL;
        SOCKADDR_IN saRemote;
        int RemoteLen;
        SOCKET acceptSocket;

        // 接收连接，并分配完成端，这儿可以用AcceptEx()
        RemoteLen = sizeof(saRemote);
        acceptSocket = accept(srvSocket, (SOCKADDR*)&saRemote, &RemoteLen);
        if (SOCKET_ERROR == acceptSocket) { // 接收客户端失败
            cerr << "Accept Socket Error: " << GetLastError() << endl;
            system("pause");
            return -1;
        }

        // 创建用来和套接字关联的单句柄数据信息结构
        PerHandleData = (LPPER_HANDLE_DATA)GlobalAlloc(GPTR, sizeof(PER_HANDLE_DATA));  // 在堆中为这个PerHandleData申请指定大小的内存
        PerHandleData->socket = acceptSocket;
        memcpy(&PerHandleData->ClientAddr, &saRemote, RemoteLen);
        clientGroup.push_back(PerHandleData);       // 将单个客户端数据指针放到客户端组中

                                                    // 将接受套接字和完成端口关联
        CreateIoCompletionPort((HANDLE)(PerHandleData->socket), completionPort, (DWORD)PerHandleData, 0);


        // 开始在接受套接字上处理I/O使用重叠I/O机制
        // 在新建的套接字上投递一个或多个异步
        // WSARecv或WSASend请求，这些I/O请求完成后，工作者线程会为I/O请求提供服务    
        // 单I/O操作数据(I/O重叠)
        LPPER_IO_OPERATION_DATA PerIoData = NULL;
        PerIoData = (LPPER_IO_OPERATION_DATA)GlobalAlloc(GPTR, sizeof(PER_IO_OPERATEION_DATA));
        ZeroMemory(&(PerIoData->overlapped), sizeof(OVERLAPPED));
        PerIoData->databuff.len = 1024;
        PerIoData->databuff.buf = PerIoData->buffer;
        PerIoData->operationType = 0;   // read

        IOOperationDataGroup.push_back(PerIoData);

        DWORD RecvBytes;
        DWORD Flags = 0;  //WSARecv中的1，代表缓冲区lpBuffers只包含一个WSABUF，Flags代表接收普通数据
        WSARecv(PerHandleData->socket, &(PerIoData->databuff), 1, &RecvBytes, &Flags, &(PerIoData->overlapped), NULL);//PerIoData->overlapped就是CONTAINING_RECORD的第一个变量
    }


    for (auto it = IOOperationDataGroup.begin(); it != IOOperationDataGroup.end(); it++)
    {
        if (*it != NULL)
            GlobalFree(*it);
    }

    system("pause");
    return 0;
}

// 开始服务工作线程函数
DWORD WINAPI ServerWorkThread(LPVOID IpParam)
{
    HANDLE CompletionPort = (HANDLE)IpParam;
    DWORD BytesTransferred;
    LPOVERLAPPED IpOverlapped;
    LPPER_HANDLE_DATA PerHandleData = NULL;
    LPPER_IO_DATA PerIoData = NULL;
    DWORD RecvBytes;
    DWORD Flags = 0;
    BOOL bRet = false;

    while (true) {
        bRet = GetQueuedCompletionStatus(CompletionPort, &BytesTransferred, (PULONG_PTR)&PerHandleData, (LPOVERLAPPED*)&IpOverlapped, INFINITE);//此处可以将IpOverlapped换为PerIoData，然后将下面CONTAINING_RECORD注释掉
        if (bRet == 0) {
            cerr << "GetQueuedCompletionStatus Error: " << GetLastError() << endl;
            return -1;
        }
        PerIoData = (LPPER_IO_DATA)CONTAINING_RECORD(IpOverlapped, PER_IO_DATA, overlapped);
        //这个宏的作用是：根据一个结构体实例中的成员的地址，取到整个结构体实例的地址
        //PER_IO_DATA的成员overlapped的地址为&IpOverlapped，结果就可以获得PER_IO_DATA的地址

        // 检查在套接字上是否有错误发生
        if (0 == BytesTransferred) {
            closesocket(PerHandleData->socket);
            GlobalFree(PerHandleData);
            GlobalFree(PerIoData);
            continue;
        }

        // 开始数据处理，接收来自客户端的数据
        WaitForSingleObject(hMutex, INFINITE);
        cout << "A Client says: " << PerIoData->databuff.buf << endl;
        ReleaseMutex(hMutex);

        // 为下一个重叠调用建立单I/O操作数据
        ZeroMemory(&(PerIoData->overlapped), sizeof(OVERLAPPED)); // 清空内存
        PerIoData->databuff.len = 1024;
        PerIoData->databuff.buf = PerIoData->buffer;//buf是个指针，这一过程会清空buffer的内容
        PerIoData->operationType = 0;   // read
        WSARecv(PerHandleData->socket, &(PerIoData->databuff), 1, &RecvBytes, &Flags, &(PerIoData->overlapped), NULL);
    }

    return 0;
}


// 发送信息的线程执行函数
DWORD WINAPI ServerSendThread(LPVOID IpParam)
{
    while (1) {
        if (clientGroup.empty())
        {
            Sleep(5000);
            continue;
        }

        char talk[200];
        cin.get(talk,200);
        int len;
        for (len = 0; talk[len] != ''; ++len) {
            // 找出这个字符组的长度
        }
        talk[len] = '
';
        talk[++len] = '';
        printf("I Say:");
        cout << talk;
        WaitForSingleObject(hMutex, INFINITE);
        for (int i = 0; i < clientGroup.size(); ++i) {
            send(clientGroup[i]->socket, talk, 200, 0); // 发送信息
        }
        ReleaseMutex(hMutex);
    }
    return 0;
}

服务器2：将信息原样返回去

#include <iostream>
using namespace std;

#include <cstdio>
#include <WINSOCK2.H>
#include <windows.h>
#include <stdio.h>

#pragma comment(lib, "Ws2_32.lib")      // Socket编程需用的动态链接库
#pragma comment(lib, "Kernel32.lib")    // IOCP需要用到的动态链接库

#define PORT 5150
#define DATA_BUFSIZE 8192

DWORD WINAPI ServerWorkerThread(LPVOID ComlpetionPortID);


typedef struct
{
    OVERLAPPED OVerlapped;
    WSABUF DATABuf;
    CHAR Buffer[DATA_BUFSIZE];
    DWORD BytesSend, BytesRecv;
}PER_IO_OPERATION_DATA, *LPPER_IO_OPERATION_DATA;


typedef struct
{
    SOCKET Socket;
}PER_HANDLE_DATA, *LPPER_HANDLE_DATA;


DWORD WINAPI ServerWorkerThread(LPVOID ComlpetionPortID);


int main(int argc, char* argv[])
{
    SOCKADDR_IN InternetAddr;
    SOCKET  Listen, Accept;
    HANDLE  CompetionPort;
    SYSTEM_INFO SystenInfo;
    LPPER_HANDLE_DATA   PerHandleData;
    LPPER_IO_OPERATION_DATA PerIOData;
    int i;
    DWORD   RecvBytes;
    DWORD   Flags;
    DWORD   ThreadID;
    WSADATA     wsadata;
    DWORD   Ret;
    if (Ret = WSAStartup(0x2020, &wsadata) != 0)
    {
        printf("WSAStartup failed with error %d/n", Ret);
        return 0;
    }

    //打开一个空的完成端口
    if ((CompetionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0)) == NULL)
    {
        printf("CreateIoCompletionPort failed with error %d/n", GetLastError());
        return 0;
    }

    GetSystemInfo(&SystenInfo);
    // 开启cpu个数的2倍个的线程
    for (i = 0; i < SystenInfo.dwNumberOfProcessors * 2; i++)
    {
        HANDLE  ThreadHandle;

        //创建服务器工作线程，并且向线程传送完成端口
        if ((ThreadHandle = CreateThread(NULL, 0, ServerWorkerThread, CompetionPort, 0, &ThreadID)) == NULL)
        {
            printf("CreateThread failed with error %d/n", GetLastError());
            return 0;
        }
        CloseHandle(ThreadHandle);
    }


    //打开一个服务器socket
    if ((Listen = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED)) == INVALID_SOCKET)
    {
        printf("WSASocket()failed with error %d/n", WSAGetLastError());
        return 0;
    }
    InternetAddr.sin_family = AF_INET;
    InternetAddr.sin_addr.S_un.S_addr = htonl(INADDR_ANY);
    InternetAddr.sin_port = htons(PORT);

    if (bind(Listen, (LPSOCKADDR)&InternetAddr, sizeof(InternetAddr)) == SOCKET_ERROR)
    {
        printf("bind failed with error %d/n", WSAGetLastError());
        return 0;
    }

    if (listen(Listen, 5) == SOCKET_ERROR)
    {
        printf("listen failed with error %d/n", WSAGetLastError());
        return 0;
    }

    //接收连接并且分发给完成端口
    while (TRUE)
    {
        if ((Accept = WSAAccept(Listen, NULL, NULL, NULL, 0)) == SOCKET_ERROR)
        {
            printf("WSAAccept failed with error %d/n", WSAGetLastError());
            return 0;
        }

        //创建与套接字相关的套接字信息结构
        if ((PerHandleData = (LPPER_HANDLE_DATA)GlobalAlloc(GPTR, sizeof(PER_HANDLE_DATA))) == NULL)
        {
            printf("GlobalAlloc failed with error %d/n", GetLastError());
            return 0;
        }

        // Associate the accepted socket with the original completion port.
        printf("Socket number %d connected/n", Accept);

        PerHandleData->Socket = Accept;//结构中存入接收的套接字
                                       //与我们的创建的那个完成端口关联起来,将关键项也与指定的一个完成端口关联
        if ((CreateIoCompletionPort((HANDLE)Accept, CompetionPort, (DWORD)PerHandleData, 0)) == NULL)
        {
            printf("CreateIoCompletionPort failed with error%d/n", GetLastError());
            return 0;
        }
        // 创建同下面的WSARecv调用相关的IO套接字信息结构体
        if ((PerIOData = (LPPER_IO_OPERATION_DATA)GlobalAlloc(GPTR, sizeof(PER_IO_OPERATION_DATA))) == NULL)
        {
            printf("GlobalAloc failed with error %d/n", GetLastError());
            return 0;
        }

        ZeroMemory(&(PerIOData->OVerlapped), sizeof(OVERLAPPED));
        PerIOData->BytesRecv = 0;
        PerIOData->BytesSend = 0;
        PerIOData->DATABuf.len = DATA_BUFSIZE;
        PerIOData->DATABuf.buf = PerIOData->Buffer;
        Flags = 0;

        if (WSARecv(Accept, &(PerIOData->DATABuf), 1, &RecvBytes, &Flags, &(PerIOData->OVerlapped), NULL) == SOCKET_ERROR)
        {
            if (WSAGetLastError() != ERROR_IO_PENDING)
            {
                printf("WSARecv()failed with error %d/n", WSAGetLastError());
                return 0;
            }
        }
    }

    return 0;
}

//工作线程
DWORD WINAPI ServerWorkerThread(LPVOID ComlpetionPortID)
{
    HANDLE  ComplectionPort = (HANDLE)ComlpetionPortID;
    DWORD   BytesTransferred;
    LPOVERLAPPED Overlapped;
    LPPER_HANDLE_DATA   PerHandleData;
    LPPER_IO_OPERATION_DATA PerIOData;
    DWORD   SendBytes, RecvBytes;

    DWORD   Flags;

    while (TRUE)
    {
        if (GetQueuedCompletionStatus(ComplectionPort, &BytesTransferred, (LPDWORD)&PerHandleData, (LPOVERLAPPED*)&PerIOData, INFINITE) == 0)
        {
            printf("GetQueuedCompletionStatus failed with error%d/n", GetLastError());
            return 0;
        }

        //首先检查套接字上是否发生错误，如果发生了则关闭套接字并且清除同套节字相关的SOCKET_INFORATION 结构体
        if (BytesTransferred == 0)
        {
            printf("Closing Socket %d/n", PerHandleData->Socket);
            if (closesocket(PerHandleData->Socket) == SOCKET_ERROR)
            {
                printf("closesocket failed with error %d/n", WSAGetLastError());
                return 0;
            }
            GlobalFree(PerHandleData);
            GlobalFree(PerIOData);
            continue;
        }

        //检查BytesRecv域是否等于0，如果是，说明WSARecv调用刚刚完成，可以用从己完成的WSARecv调用返回的BytesTransferred值更新BytesRecv域
        if (PerIOData->BytesRecv == 0)
        {
            PerIOData->BytesRecv = BytesTransferred;
            PerIOData->BytesSend = 0;
        }
        else
        {
            PerIOData->BytesRecv += BytesTransferred;
        }

        //
        if (PerIOData->BytesRecv > PerIOData->BytesSend)//收到数据比发送的多了，就回发出去
        {
            //发布另一个WSASend()请求，因为WSASendi 不能确保发送了请的所有字节，继续WSASend调用直至发送完所有收到的字节
            ZeroMemory(&(PerIOData->OVerlapped), sizeof(OVERLAPPED));
            PerIOData->DATABuf.buf = PerIOData->Buffer + PerIOData->BytesSend;
            PerIOData->DATABuf.len = PerIOData->BytesRecv - PerIOData->BytesSend;

            if (WSASend(PerHandleData->Socket, &(PerIOData->DATABuf), 1, &SendBytes, 0, &(PerIOData->OVerlapped), NULL) == SOCKET_ERROR)
            {
                if (WSAGetLastError() != ERROR_IO_PENDING)
                {
                    printf("WSASend()fialed with error %d/n", WSAGetLastError());
                    return 0;
                }
            }
        }
        else
        {
            PerIOData->BytesRecv = 0;
            //Now that is no more bytes to send post another WSARecv()request
            //现在己经发送完成
            Flags = 0;
            ZeroMemory(&(PerIOData->OVerlapped), sizeof(OVERLAPPED));
            PerIOData->DATABuf.buf = PerIOData->Buffer;
            PerIOData->DATABuf.len = DATA_BUFSIZE;
            if (WSARecv(PerHandleData->Socket, &(PerIOData->DATABuf), 1, &RecvBytes, &Flags, &(PerIOData->OVerlapped), NULL) == SOCKET_ERROR)
            {
                if (WSAGetLastError() != ERROR_IO_PENDING)
                {
                    printf("WSARecv()failed with error %d/n", WSAGetLastError());
                    return 0;
                }
            }
        }
    }
}