epoll 边界触发模式1,2的实现

本篇贴出在上篇文章中介绍的模式1的实现代码.

首先,因为是多线程的程序,必须防止某一资源在一个线程中使用的时候,却在另一个线程中释放了.
其中最主要的便是socket_t结构,为了杜绝这个问题,对应用层来说,应该根本不知道socket_t的存在,
仅仅提供一个HANDLE给应用层就足够了.

#ifndef _KENDYNET_H
#define _KENDYNET_H
#include "MsgQueue.h"
typedef struct list_node
{
    struct list_node *next;
}list_node;

#define LIST_NODE list_node node;

/*IO请求和完成队列使用的结构*/
typedef struct
{
    LIST_NODE;
    struct iovec *iovec;
    int    iovec_count;
    int    bytes_transfer;
    int    error_code;
}st_io;

enum
{    
    //完成模式,当套接口从不可读/写变为激活态时, 如果有IO请求,则完成请求,并将完成通告发送给工作线程
    MODE_COMPLETE = 0,
    //poll模式,当套接口从不可读/写变为激活态时, 如果有IO请求,将请求发送给工作线程,由工作线程完成请求
    MODE_POLL,
};

//初始化网络系统
int      InitNetSystem();

typedef int HANDLE;
struct block_queue;

struct block_queue* CreateEventQ();
void DestroyEventQ(struct block_queue**);

HANDLE   CreateEngine(char);
void     CloseEngine(HANDLE);
int      EngineRun(HANDLE);

int     Bind2Engine(HANDLE,HANDLE);

//获取和投递事件到engine的队列中
//int      GetQueueEvent(HANDLE,struct block_queue*,st_io **,int timeout);
int      GetQueueEvent(HANDLE,MsgQueue_t,st_io **,int timeout);
int      PutQueueEvent(HANDLE,st_io *);

/*
*   发送和接收,如果操作立即完成返回完成的字节数
*   否则返回0,当IO异步完成后会通告到事件队列中.
*   返回-1表示套接口出错或断开
*   notify:当IO立即可完成时,是否向事件队
*   列提交一个完成通告,设置为1会提交,0为不提交.
*/
int WSASend(HANDLE,st_io*,int notify);
int WSARecv(HANDLE,st_io*,int notify);

#endif

这里定义的st_io结构,相当与IOCP里的OVERLAPPED结构,其中,第一个成员是list_node,这样
可以很方便的将st_io放在队列中.

然后是GetQueueEvent函数,这个函数的作用相当于IOCP里的GetCompleteStatus,用于从队列
中获取事件,不同的是,这个队列是由用户提供的,也就是参数中的block_queue*,如果调用
GetQueueEvent的时候,engine的buffering_event_queue中没有事件,就会将block_queue
放到一个等待队列中(block_thread_queue),当其它地方调用PutQueueEvent的时候,首先
会看下是否有线程在等待,有就把它的block_queue弹出push一个事件进去,然后唤醒阻塞
的线程.

#include "KendyNet.h"
#include "Engine.h"
#include "Socket.h"
#include "link_list.h"
#include "HandleMgr.h"
#include <assert.h>
#include "MsgQueue.h"

int InitNetSystem()
{
    return InitHandleMgr();
}

struct block_queue* CreateEventQ()
{
    return BLOCK_QUEUE_CREATE();
}

void DestroyEventQ(struct block_queue** eq)
{
    BLOCK_QUEUE_DESTROY(eq);
}

int EngineRun(HANDLE engine)
{
    engine_t e = GetEngineByHandle(engine);
    if(!e)
        return -1;
    e->Loop(e);    
    return 0;
}

HANDLE CreateEngine(char mode)
{
    HANDLE engine = NewEngine();
    if(engine >= 0)
    {
        engine_t e = GetEngineByHandle(engine);
        e->mode = mode;
        LIST_CLEAR(e->buffering_event_queue);
        LIST_CLEAR(e->block_thread_queue);
        if(0 != e->Init(e))
        {
            CloseEngine(engine);
            engine = -1;
        }
    }
    return engine;
}

void CloseEngine(HANDLE handle)
{
    ReleaseEngine(handle);
}

int Bind2Engine(HANDLE e,HANDLE s)
{
    engine_t engine = GetEngineByHandle(e);
    socket_t sock   = GetSocketByHandle(s);
    if(!engine || ! sock)
        return -1;
    if(engine->Register(engine,sock) == 0)
    {
        sock->engine = engine;
        return 0;
    }
    return -1;
}

//int    GetQueueEvent(HANDLE handle, struct block_queue *EventQ,st_io **io ,int timeout)
int    GetQueueEvent(HANDLE handle, MsgQueue_t EventQ,st_io **io ,int timeout)
{
    assert(EventQ);assert(io);
/*    
    engine_t e = GetEngineByHandle(handle);
    if(!e)
        return -1;
    spin_lock(e->mtx,0);//mutex_lock(e->mtx);
    if(e->status == 0)
    {
        spin_unlock(e->mtx);//mutex_unlock(e->mtx);
        return -1;
    }
    
    if(*io = LIST_POP(st_io*,e->buffering_event_queue))
    {
        spin_unlock(e->mtx);//mutex_unlock(e->mtx);
        return 0;
    }
    //插入到等待队列中，阻塞
    LIST_PUSH_FRONT(e->block_thread_queue,EventQ);
    spin_unlock(e->mtx);//mutex_unlock(e->mtx);

    if(POP_FORCE_WAKE_UP == BLOCK_QUEUE_POP(EventQ,io,timeout))
        return -1;//因为engine的释放而被强制唤醒
    */
    engine_t e = GetEngineByHandle(handle);
    if(!e)
        return -1;
    mutex_lock(e->mtx);
    if(e->status == 0)
    {
        mutex_unlock(e->mtx);
        return -1;
    }
    
    if(*io = LIST_POP(st_io*,e->buffering_event_queue))
    {
        mutex_unlock(e->mtx);
        return 0;
    }
    //插入到等待队列中，阻塞
    LIST_PUSH_FRONT(e->block_thread_queue,EventQ);
    mutex_unlock(e->mtx);

    GetMsg(EventQ,io,sizeof(*io),0);
    if(*io == 0)
        return -1;
    
    return 0;    
}

int    PutQueueEvent(HANDLE handle,st_io *io)
{
    assert(io);
    engine_t e = GetEngineByHandle(handle);
    if(!e)
        return -1;
    
    return put_event(e,io);
}

extern int put_event(engine_t e,st_io *io);

int WSASend(HANDLE sock,st_io *io,int notify)
{
    assert(io);
    socket_t s = GetSocketByHandle(sock);
    if(!s)
        return -1;
    
    int active_send_count = -1;
    int ret = 0;
    
    mutex_lock(s->send_mtx);
    //为保证执行顺序与请求的顺序一致,先将请求插入队列尾部,再弹出队列首元素
    LIST_PUSH_BACK(s->pending_send,io);
    io = 0;
    if(s->writeable)
    {
        io = LIST_POP(st_io*,s->pending_send);
        active_send_count = s->active_write_count;
    }
    mutex_unlock(s->send_mtx);
    
    if(s->engine->mode == MODE_POLL)
    {
        if(io)
        {
            //当前处于激活态
            if(notify)
                //不直接处理,将处理交给完成线程 
                put_event(s->engine,io);
            else
                ret = _send(s,io,active_send_count,notify);
        }
    }
    else
    {        
        if(io)
            ret = _send(s,io,active_send_count,notify);
    }
    return ret;
}

int WSARecv(HANDLE sock,st_io *io,int notify)
{
    assert(io);
    socket_t s = GetSocketByHandle(sock);
    if(!s)
        return -1;

    int active_recv_count = -1;
    int ret = 0;
    
    mutex_lock(s->recv_mtx);
    //为保证执行顺序与请求的顺序一致,先将请求插入队列尾部,再弹出队列首元素
    LIST_PUSH_BACK(s->pending_recv,io);
    io  = 0;
    if(s->readable)
    {
        io = LIST_POP(st_io*,s->pending_recv);
        active_recv_count = s->active_read_count;
    }
    mutex_unlock(s->recv_mtx);
    
    if(s->engine->mode == MODE_POLL)
    {
        if(io)
        {
            //当前处于激活态
            if(notify)
                //不直接处理,将处理交给完成线程 
                put_event(s->engine,io);
            else
                ret =  _recv(s,io,active_recv_count,notify);
        }
    }
    else
    {    
        if(io)
            ret = _recv(s,io,active_recv_count,notify);
    }
    return ret;
}

然后是engine接口:

#ifndef _ENGINE_H
#define _ENGINE_H

#include "sync.h"
//#include "thread.h"
#include "link_list.h"

struct socket_wrapper;
typedef struct engine
{
    int  (*Init)(struct engine*);
    void (*Loop)(struct engine*);
    int  (*Register)(struct engine*,struct socket_wrapper*);
    int  (*UnRegister)(struct engine*,struct socket_wrapper*);
    
    mutex_t mtx;
    volatile int status; /*0:关闭状态,1:开启状态*/
    int poller_fd;
    
    //完成事件相关成员
    struct link_list   *buffering_event_queue;    //当没有线程等待时，事件将被缓存到这个队列中
    struct link_list   *block_thread_queue;    //正在等待完成消息的线程
    
    //thread_t     engine_thread;         //运行engine的线程
    
}*engine_t;

engine_t create_engine();
void   free_engine(engine_t *);
void   stop_engine(engine_t);


#endif

engine接口提供了4个函数指针,可根据不同的系统实现相应的函数,在这里使用的是linux的epoll.

#include "Engine.h"
#include "link_list.h"
#include "KendyNet.h"
#include <stdlib.h>
#include "epoll.h"
#include <assert.h>

engine_t create_engine()
{
    engine_t e = malloc(sizeof(*e));
    if(e)
    {
        e->mtx = mutex_create();
        e->status = 0;
        e->buffering_event_queue = LIST_CREATE();
        e->block_thread_queue = LIST_CREATE();
        e->Init = epoll_init;
        e->Loop = epoll_loop;
        e->Register = epoll_register;
        e->UnRegister = epoll_unregister;
        //e->engine_thread = 0;
    }
    return e;
}

void   free_engine(engine_t *e)
{
    assert(e);
    assert(*e);
    mutex_destroy(&(*e)->mtx);
    LIST_DESTROY(&(*e)->buffering_event_queue);
    LIST_DESTROY(&(*e)->block_thread_queue);
    //if((*e)->engine_thread)
    //    destroy_thread(&(*e)->engine_thread);
    free(*e);
    *e = 0;
}

int put_event(engine_t e,st_io *io)
{
    mutex_lock(e->mtx);
    struct block_queue *EventQ = LIST_POP(struct block_queue*,e->block_thread_queue);
    if(!EventQ)
    {
        //没有等待的线程，先缓冲事件
        LIST_PUSH_BACK(e->buffering_event_queue,io);
        io = 0;
    }
    mutex_unlock(e->mtx);    
    if(io)
        BLOCK_QUEUE_PUSH(EventQ,io);
    return 0;
}

void   stop_engine(engine_t e)
{
    mutex_lock(e->mtx);
    e->status = 0;
    //join(e->engine_thread);
    //强制唤醒所有等待在完成队列上的线程 
    struct block_queue *queue = 0;
    /*唤醒所有等待在完成队列的线程*/
    while(queue = LIST_POP(struct block_queue *,e->block_thread_queue))
    {
        BLOCK_QUEUE_FORCE_WAKEUP(queue);
    }    
    mutex_unlock(e->mtx);
}

epoll的实现：

#include "epoll.h"
#include "Socket.h"
#include "SocketWrapper.h"
#include "HandleMgr.h"


int  epoll_init(engine_t e);
{
    e->poller_fd = TEMP_FAILURE_RETRY(epoll_create(MAX_SOCKET));
    return     e->poller_fd > 0 ? 0:-1; 
}

int epoll_register(engine_t e, socket_t s)
{
    int ret = -1;    
    struct epoll_event ev;    
    ev.data.ptr = s;
    ev.events = EV_IN | EV_OUT | EV_ET;
    TEMP_FAILURE_RETRY(ret = epoll_ctl(e->poller_fd,EPOLL_CTL_ADD,s->fd,&ev));
    return ret;
}


inline int epoll_unregister(engine_t e,socket_t s)
{
    struct epoll_event ev;int ret;
    TEMP_FAILURE_RETRY(ret = epoll_ctl(s->poller_fd,EPOLL_CTL_DEL,s->fd,&ev));
    return ret;
}

void epoll_loop(engine_t n)
{
    struct epoll_event events[MAX_SOCKET];
    memset(events,0,sizeof(events));
    while(1)
    {
        int nfds = TEMP_FAILURE_RETRY(epoll_wait(n->poller_fd,events,MAX_SOCKET,-1));
        if(nfds < 0)
        {
            break;
        }
        int i;
        for(i = 0 ; i < nfds ; ++i)
        {    
            socket_t sock = (socket_t)events[i].data.ptr;
            if(sock)
            {
                //套接口可读
                if(events[i].events & EPOLLIN)
                { 
                    on_read_active(sock);
                }

                //套接口可写
                if(events[i].events & EPOLLOUT)
                {
                    on_write_active(sock);
                }        
            
                if(events[i].events & EPOLLERR)
                {
                    //套接口异常
                }
            }
        }
    }
}

最后是socket_t

#ifndef _SOCKETWRAPPER_H
#define _SOCKETWRAPPER_H

#include "Engine.h"

typedef struct socket_wrapper
{
    mutex_t  mtx;//保证ReleaseSocketWrapper只会被正常执行一次
    volatile int status;//0:未开启;1:正常;
    engine_t  *engine;
        
    volatile int readable;
    volatile int writeable;
    volatile int active_read_count;
    volatile int active_write_count;
    int fd;
    
    //当发送/接收无法立即完成时,把请求投入下面两个队列中,
    //当套接口可读/可写时再完成请求
    struct link_list *pending_send;//尚未处理的发请求
    struct link_list *pending_recv;//尚未处理的读请求
    
    mutex_t   recv_mtx;
    mutex_t   send_mtx;
    
    
}*socket_t;


void on_read_active(socket_t);
void on_write_active(socket_t);
socket_t create_socket();
void free_socket(socket_t*);

int _recv(socket_t,st_io*,int,int notify);
int _send(socket_t,st_io*,int,int notify);


#endif

socket_t.c

#include "Socket.h"
#include <assert.h>
#include <stdlib.h>
#include <errno.h>
#include "SocketWrapper.h"
#include "KendyNet.h"

socket_t create_socket()
{
    socket_t s = malloc(sizeof(*s));
    if(s)
    {
        s->mtx = mutex_create();
        s->recv_mtx = mutex_create();
        s->send_mtx = mutex_create();
        s->pending_send = LIST_CREATE();
        s->pending_recv = LIST_CREATE();
        s->status = 0;
        s->engine = 0;
    }
    return s;
}

void free_socket(socket_t *s)
{
    assert(s);assert(*s);
    mutex_destroy(&(*s)->mtx);
    mutex_destroy(&(*s)->recv_mtx);
    mutex_destroy(&(*s)->send_mtx);
    destroy_list(&(*s)->pending_send);
    destroy_list(&(*s)->pending_recv);
    free(*s);
    *s = 0;
}

void on_read_active(socket_t s)
{
    assert(s);
    mutex_lock(s->recv_mtx);
    s->readable = 1;
    int active_recv_count = ++s->active_read_count;
    st_io *req = LIST_POP(st_io*,s->pending_recv);
    mutex_unlock(s->recv_mtx);
    
    if(req)
    {
        if(s->engine->mode == MODE_COMPLETE)
            _recv(s,req,active_recv_count,1);
        else
            put_event(s->engine,req);
    }
}

void on_write_active(socket_t s)
{
    assert(s);
    mutex_lock(s->send_mtx);
    s->writeable = 1;
    int active_send_count = ++s->active_write_count;
    st_io *req = LIST_POP(st_io*,s->pending_send);
    mutex_unlock(s->send_mtx);
    if(req)
    {
        if(s->engine->mode == MODE_COMPLETE)
            _send(s,req,active_send_count,1);
        else
            put_event(s->engine,req);
    }        
}

int _recv(socket_t s,st_io* io_req,int active_recv_count,int notify)
{
    assert(s);assert(io_req);
    while(1)
    {
        int retry = 0;
        int bytes_transfer = io_req->bytes_transfer = TEMP_FAILURE_RETRY(readv(s->fd,io_req->iovec,io_req->iovec_count));
        io_req->error_code = 0;
        if(bytes_transfer < 0)
        {
            switch(errno)
            {
                case EAGAIN:
                {
                    mutex_lock(s->recv_mtx);
                    if(active_recv_count != s->active_read_count)
                    {
                        active_recv_count = s->active_read_count;
                        retry = 1;
                    }
                    else
                    {
                        s->readable = 0;
                        LIST_PUSH_FRONT(s->pending_recv,io_req);
                    }
                    mutex_unlock(s->recv_mtx);
                    
                    if(retry)
                        continue;
                    return 0;
                }
                break;
                default://连接出错
                {
                    io_req->error_code = errno;
                }
                break;
            }
        }
        else if(bytes_transfer == 0)
            bytes_transfer = -1;
        if(notify)
            put_event(s->engine,io_req);    
        return bytes_transfer;    
    }    
}

int _send(socket_t s,st_io* io_req,int active_send_count,int notify)
{
    assert(s);assert(io_req);
    while(1)
    {
        int retry = 0;
        int bytes_transfer = io_req->bytes_transfer = TEMP_FAILURE_RETRY(writev(s->fd,io_req->iovec,io_req->iovec_count));
        io_req->error_code = 0;    
        if(bytes_transfer < 0)
        {
            switch(errno)
            {
                case EAGAIN:
                {
                    mutex_lock(s->send_mtx);
                    if(active_send_count != s->active_write_count)
                    {
                        active_send_count = s->active_write_count;
                        retry = 1;
                    }
                    else
                    {
                        s->writeable = 0;
                        //将请求重新放回到队列
                        LIST_PUSH_FRONT(s->pending_send,io_req);
                    }
                    mutex_unlock(s->send_mtx);
                    
                    if(retry)
                        continue;
                    return 0;
                }
                break;
                default://连接出错
                {
                    io_req->error_code = errno;
                }
                break;
            }
        }
        else if(bytes_transfer == 0)
            bytes_transfer = -1;
        if(notify)
            put_event(s->engine,io_req);    
        return bytes_transfer;    
    }
}

用户提供的完成例程代码大概如下:

st_io *io = 0;
block_queue *EventQ;
while(GetQueueEvent(engine,EventQ,&io ,0) == 0)
{
    if(io)
    {
        HANDLE sock = GetHandle(io);
        OnRecv(io);
        int ret;
        //发起新的读,直到套接字变为不可读
        while((ret = WSARecv(sock,io,0)) > 0)
        {
            OnRecv(io);
        }
        
        if(ret < 0)
        {
            //连接断开
            CloseSocket(sock);
        }
    }
}

对于快速的发送方，完成例程接收完并处理之后，套接口上可能马上又有数据可读了，这样在epoll线程中执行recv的机会比较小。
但如果发送方全都是慢速的，例如每一秒发送一次，则会导致几乎所有的recv操作都在epoll线程中执行，当连接数巨大时没法发挥

多核处理器的优势，所以就有了模型2,epoll线程完全不处理recv,当套接字变为激活时，将请求投递到队列中，交给complete例程

去执行recv.

(2012.4.16修改,增加对模型2的支持，CreateEngine时可选择传入参数MODE_COMPLETE或MODE_POLL,

当传入MODE_POLL时，GetQueueEvent返回的事件表明套接口可读/写，且有读/写请求,传入MODE_COMPLETE

GetQueueEvent返回的事件表明一个读/写请求已经完成)

测试程序如下,注意IORoutinePoll和IORoutine的区别:

test.c

#include <stdio.h>
#include <stdlib.h>
#include "KendyNet.h"
#include "thread.h"
#include "SocketWrapper.h"
#include "atomic.h"
enum
{
    RECV_FINISH = 0,
    SEND_FINISH,
};

typedef struct IoContext
{
    st_io m_ioStruct;
    unsigned char    m_opType;
    void             *ud;
}IoContext;

typedef struct _Socket
{
    char send_buf[64];
    char recv_buf[64];
    struct iovec recv_iovec;
    struct iovec send_iovec;
    IoContext m_IORecvComp;
    IoContext m_IOSendComp;
    HANDLE    m_sock;
}_Socket;

HANDLE engine;
const char *ip;
long port;

void *ListerRoutine(void *arg)
{
    thread_t thread = (thread_t)CUSTOM_ARG(arg);
    struct sockaddr_in servaddr;
    HANDLE listerfd;
    if((listerfd = Tcp_Listen(ip,port,&servaddr,5)) == 0)
    {
        while(!is_terminate(thread))
        {
            struct sockaddr sa;
            socklen_t salen;
            HANDLE sock = Accept(listerfd,&sa,&salen);
            if(sock >= 0)
            {
                setNonblock(sock);
                _Socket *_sock = malloc(sizeof(*_sock));
                _sock->m_sock = sock;
                _sock->m_IORecvComp.m_opType = RECV_FINISH;
                _sock->m_IOSendComp.m_opType = SEND_FINISH;
                _sock->m_IORecvComp.ud = _sock->m_IOSendComp.ud = _sock;
                _sock->recv_iovec.iov_base = _sock->recv_buf;
                _sock->send_iovec.iov_base = _sock->send_buf;
                _sock->m_IORecvComp.m_ioStruct.iovec = &_sock->recv_iovec;
                _sock->m_IOSendComp.m_ioStruct.iovec = &_sock->send_iovec;
                _sock->m_IOSendComp.m_ioStruct.iovec_count = _sock->m_IORecvComp.m_ioStruct.iovec_count = 1;
                //printf("接到一个连接\n");
                if(0 != Bind2Engine(engine,sock))
                {
                    printf("bind出错\n");
                    CloseSocket(sock);
                    free(_sock);
                }
                else
                {
                    //发起第一个读请求
                    _sock->m_IORecvComp.m_ioStruct.iovec->iov_len = 64;
                    WSARecv(sock,(st_io*)&(_sock->m_IORecvComp),1);
                }
            }
            else
            {
                printf("accept出错\n");
            }
        }
        printf("listener 终止\n");
    }
    return 0;
}

//COMPLETE MODE
void *IORoutine(void *arg)
{
    st_io* ioComp;
    //struct block_queue *EventQ = CreateEventQ();
    MsgQueue_t EventQ = CreateMsgQ();
    while(1)
    {
        ioComp = 0;
        if(0 > GetQueueEvent(engine,EventQ,&ioComp,-1))
        {
            printf("poller终止\n");
            break;
        }
        //printf("唤醒咯\n");
        if(ioComp)
        {
            if(ioComp->bytes_transfer <= 0)
            {
                //printf("套接口断开\n");
                _Socket *sock = (_Socket*)(((IoContext*)ioComp)->ud);
                //连接关闭
                CloseSocket(sock->m_sock);
                free(sock);
            }
            else
            {
                IoContext *context = (IoContext*)ioComp;
                _Socket *sock = (_Socket*)context->ud;
                if(context->m_opType == RECV_FINISH)
                {
                    int byteTransfer = 0;
                    //把数据回发给客户端
                    memcpy(sock->send_buf,sock->recv_buf,ioComp->bytes_transfer);
                    sock->m_IOSendComp.m_ioStruct.iovec->iov_len = ioComp->bytes_transfer;
                    WSASend(sock->m_sock,(st_io*)&(sock->m_IOSendComp),0);
                    
                    //继续读
                    while((byteTransfer = WSARecv(sock->m_sock,(st_io*)context,0)) > 0)
                    {
                        memcpy(sock->send_buf,sock->recv_buf,byteTransfer);
                        sock->m_IOSendComp.m_ioStruct.iovec->iov_len = byteTransfer;
                        byteTransfer = WSASend(sock->m_sock,(st_io*)&(sock->m_IOSendComp),0);
                        if(byteTransfer == 0)
                            printf("can't write\n");
                        if(byteTransfer < 0)
                        {
                            printf("close\n");
                            break;
                        }
                    }

                    if(byteTransfer < 0)
                    {
                        //printf("套接口断开\n");
                        CloseSocket(sock->m_sock);
                        free(sock);
                    }
                }
            }
        }
    }
    printf("IO end\n");
    //DestroyEventQ(&EventQ);
    DestroyMsgQ(&EventQ);
    return 0;
}
//POLL MODE
void *IORoutinePoll(void *arg)
{
    st_io* ioComp;
    MsgQueue_t EventQ = CreateMsgQ();
    while(1)
    {
        ioComp = 0;
        if(0 > GetQueueEvent(engine,EventQ,&ioComp,-1))
        {
            printf("poller终止\n");
            break;
        }
        if(ioComp)
        {

            IoContext *context = (IoContext*)ioComp;
            _Socket *sock = (_Socket*)context->ud;
            if(context->m_opType == RECV_FINISH)
            {
                int byteTransfer = 0;
                while((byteTransfer = WSARecv(sock->m_sock,(st_io*)context,0)) > 0)
                {
                    memcpy(sock->send_buf,sock->recv_buf,byteTransfer);
                    sock->m_IOSendComp.m_ioStruct.iovec->iov_len = byteTransfer;
                    byteTransfer = WSASend(sock->m_sock,(st_io*)&(sock->m_IOSendComp),0);
                    if(byteTransfer == 0)
                        printf("can't write1\n");
                    if(byteTransfer < 0)
                    {
                        printf("close\n");
                        break;
                    }
                }

                if(byteTransfer < 0)
                {
                    //printf("connection close\n");
                    CloseSocket(sock->m_sock);
                    free(sock);
                }
            }
            else
                WSASend(sock->m_sock,(st_io*)&(sock->m_IOSendComp),0);

        }
    }
    printf("IO end\n");
    DestroyMsgQ(&EventQ);
    return 0;
}
void *EngineRoutine(void *arg)
{
    EngineRun(engine);
    printf("Engine stop\n");
    return 0;
}

int main(int argc,char **argv)
{
    ip = "127.0.0.1";//argv[];
    port = atoi(argv[1]);
    signal(SIGPIPE,SIG_IGN);
    if(InitNetSystem() != 0)
    {
        printf("Init error\n");
        return 0;
    }

    //engine = CreateEngine(MODE_POLL);
    engine = CreateEngine(MODE_COMPLETE);
    thread_t listener = create_thread(0);
    start_run(listener,ListerRoutine,listener);
    
    int complete_count = atoi(argv[2]);
    int i = 0;
    for( ; i < complete_count; ++i)
    {
        thread_t complete_t = create_thread(0);
        
        //start_run(complete_t,IORoutinePoll,0);
        start_run(complete_t,IORoutine,0);
    }
    thread_t engine_thread = create_thread(1);
    start_run(engine_thread,EngineRoutine,0);
    getchar();
    CloseEngine(engine);
    join(engine_thread);
    return 0;
}