由浅入深了解Thrift之客户端连接池化

一、问题描述                                                                                        

    在上一篇《由浅入深了解Thrift之服务模型和序列化机制》文章中，我们已经了解了thrift的基本架构和网络服务模型的优缺点。如今的互联网圈中，RPC服务化的思想如火如荼。我们又该如何将thrift服务化应用到我们的项目中哪？实现thrift服务化前，我们先想想这几个问题：服务注册、服务发现、服务健康检测、服务“Load Balance”、隐藏client和server端的交互细节、服务调用端的对象池化。

• 服务的注册、发现和健康检测，我们使用zookeeper可以很好的解决
• 服务“Load Balance"，我们可以使用简单的算法“权重+随机”，当然也可以使用成熟复杂的算法
• 服务调用端的对象池化，我们可以使用common pool，使用简单又可以满足我们的需求   

二、实现思路                                                                                        

    1、thrift server端启动时，每个实例向zk集群以临时节点方式注册（这样，遍历zk上/server下有多少个临时节点就知道有哪些server实例）

        thrift server端可以单机多端口多实例或多机部署多实例方式运行。

   2、服务调用方实现一个连接池，连接池初始化时，通过zk将在线的server实例信息同步到本地并缓存，同时监听zk下的节点变化。

   3、服务调用方与Server通讯时，从连接池中取一个可用的连接，用它实现RPC调用。

                 

三、具体实现                                                                                       

   1、thrift server端   

      thrift server端,向zk中注册server address

package com.wy.thriftpool.commzkpool;

import java.lang.instrument.IllegalClassFormatException;
import java.lang.reflect.Constructor;

import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TBinaryProtocol.Factory;
import org.apache.thrift.server.TServer;
import org.apache.thrift.server.TThreadedSelectorServer;
import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TNonblockingServerSocket;
import org.springframework.beans.factory.InitializingBean;

import com.wy.thrift.service.UserService.Processor;
import com.wy.thriftpool.commzkpool.support.ThriftServerAddressReporter;
import com.wy.thriftpool.commzkpool.support.ThriftServerIpTransfer;
import com.wy.thriftpool.commzkpool.support.impl.LocalNetworkIpTransfer;

/**
 * thrift server端,向zk中注册server address
 * 
 * @author wy
 * 
 */
public class ThriftServiceServerFactory implements InitializingBean {

    // thrift server 服务端口
    private Integer port;
    // default 权重
    private Integer priority = 1;
    // service实现类
    private Object service;
    // thrift server 注册路径
    private String configPath;

    private ThriftServerIpTransfer ipTransfer;
    // thrift server注册类
    private ThriftServerAddressReporter addressReporter;
    // thrift server开启服务
    private ServerThread serverThread;

    @Override
    public void afterPropertiesSet() throws Exception {
        if (ipTransfer == null) {
            ipTransfer = new LocalNetworkIpTransfer();
        }
        String ip = ipTransfer.getIp();
        if (ip == null) {
            throw new NullPointerException("cant find server ip...");
        }
        String hostname = ip + ":" + port + ":" + priority;
        Class<? extends Object> serviceClass = service.getClass();
        ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
        Class<?>[] interfaces = serviceClass.getInterfaces();
        if (interfaces.length == 0) {
            throw new IllegalClassFormatException("service-class should implements Iface");
        }

        // reflect,load "Processor";
        Processor<?> processor = null;
        for (Class<?> clazz : interfaces) {
            String cname = clazz.getSimpleName();
            if (!cname.equals("Iface")) {
                continue;
            }
            String pname = clazz.getEnclosingClass().getName() + "$Processor";
            try {
                Class<?> pclass = classLoader.loadClass(pname);
                if (!pclass.isAssignableFrom(Processor.class)) {
                    continue;
                }
                Constructor<?> constructor = pclass.getConstructor(clazz);
                processor = (Processor<?>) constructor.newInstance(service);
                break;
            } catch (Exception e) {
                // TODO
            }
        }

        if (processor == null) {
            throw new IllegalClassFormatException("service-class should implements Iface");
        }
        // 需要单独的线程,因为serve方法是阻塞的.
        serverThread = new ServerThread(processor, port);
        serverThread.start();
        // report
        if (addressReporter != null) {
            addressReporter.report(configPath, hostname);
        }
    }

    class ServerThread extends Thread {
        private TServer server;

        ServerThread(Processor<?> processor, int port) throws Exception {
            // 设置传输通道
            TNonblockingServerSocket serverTransport = new TNonblockingServerSocket(port);
            // 设置二进制协议
            Factory protocolFactory = new TBinaryProtocol.Factory();
            
            TThreadedSelectorServer.Args tArgs = new TThreadedSelectorServer.Args(serverTransport);
            tArgs.processor(processor);
            tArgs.transportFactory(new TFramedTransport.Factory());
            tArgs.protocolFactory(protocolFactory);
            int num = Runtime.getRuntime().availableProcessors() * 2 + 1;
            tArgs.selectorThreads(num);
            tArgs.workerThreads(num * 10);
            
            // 网络服务模型
            server = new TThreadedSelectorServer(tArgs);
            
        }

        @Override
        public void run() {
            try {
                server.serve();
            } catch (Exception e) {
                //TODO
            }
        }

        public void stopServer() {
            server.stop();
        }
    }

    public void close() {
        serverThread.stopServer();
    }

    public void setService(Object service) {
        this.service = service;
    }

    public void setPriority(Integer priority) {
        this.priority = priority;
    }

    public void setPort(Integer port) {
        this.port = port;
    }

    public void setIpTransfer(ThriftServerIpTransfer ipTransfer) {
        this.ipTransfer = ipTransfer;
    }

    public void setAddressReporter(ThriftServerAddressReporter addressReporter) {
        this.addressReporter = addressReporter;
    }

    public void setConfigPath(String configPath) {
        this.configPath = configPath;
    }
}

      thrift server address注册到zk   

package com.wy.thriftpool.commzkpool.support.impl;

import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.zookeeper.CreateMode;

import com.wy.thriftpool.commzkpool.support.ThriftServerAddressReporter;

/**
 * thrift server address注册到zk
 * 
 * @author wy
 *
 */
public class DynamicAddressReporter implements ThriftServerAddressReporter {

    private CuratorFramework zookeeper;

    public DynamicAddressReporter() {
    }

    public DynamicAddressReporter(CuratorFramework zookeeper) {
        this.zookeeper = zookeeper;
    }

    public void setZookeeper(CuratorFramework zookeeper) {
        this.zookeeper = zookeeper;
    }

    @Override
    public void report(String service, String address) throws Exception {
        if (zookeeper.getState() == CuratorFrameworkState.LATENT) {
            zookeeper.start();
            zookeeper.newNamespaceAwareEnsurePath(service);
        }
        zookeeper.create().creatingParentsIfNeeded()
                .withMode(CreateMode.EPHEMERAL_SEQUENTIAL)
                .forPath(service + "/i_", address.getBytes("utf-8"));
    }

    public void close() {
        zookeeper.close();
    }

}

      。。。

      spring配置文件

<!-- zookeeper -->
    <bean id="thriftZookeeper" class="com.wy.thriftpool.commzkpool.zookeeper.ZookeeperFactory" destroy-method="close">
        <property name="connectString" value="127.0.0.1:2181"></property>
        <property name="namespace" value="thrift/thrift-service"></property>
    </bean>
    
    <bean id="serviceAddressReporter" class="com.wy.thriftpool.commzkpool.support.impl.DynamicAddressReporter" destroy-method="close">
        <property name="zookeeper" ref="thriftZookeeper"></property>
    </bean>
    
    <bean id="userService" class="com.wy.thrift.service.UserServiceImpl"/>
    
    <bean class="com.wy.thriftpool.commzkpool.ThriftServiceServerFactory" destroy-method="close">
        <property name="service" ref="userService"></property>
        <property name="configPath" value="UserServiceImpl"></property>
        <property name="port" value="9090"></property>
        <property name="addressReporter" ref="serviceAddressReporter"></property>
    </bean>

   2、服务调用端

      连接池实现  

     杯了个具，为啥就不能提交。代码在评论中。

　　 连接池工厂,负责与Thrift server通信 

package com.wy.thriftpool.commzkconnpool;

import java.net.InetSocketAddress;

import org.apache.commons.pool.PoolableObjectFactory;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.wy.thriftpool.commzkpool.support.ThriftServerAddressProvider;

/**
 * 连接池工厂,负责与Thrift server通信
 * 
 * @author wy
 *
 */
public class ThriftPoolFactory implements PoolableObjectFactory<TTransport> {
    private final Logger logger = LoggerFactory.getLogger(getClass());
    // 超时设置 
    public int timeOut;
    
    private final ThriftServerAddressProvider addressProvider;
    private PoolOperationCallBack callback;
    
    public ThriftPoolFactory(ThriftServerAddressProvider addressProvider, PoolOperationCallBack callback) {
        super();
        this.addressProvider = addressProvider;
        this.callback = callback;
    }
    
    public ThriftPoolFactory(ThriftServerAddressProvider addressProvider, PoolOperationCallBack callback, int timeOut) {
        super();
        this.addressProvider = addressProvider;
        this.callback = callback;
        this.timeOut = timeOut;
    }

    /**
     * 创建对象
     */
    @Override
    public TTransport makeObject() throws Exception {
        try {
            InetSocketAddress address = addressProvider.selector();
            TTransport transport = new TSocket(address.getHostName(), address.getPort(), this.timeOut);
            transport.open();
            if (callback != null) {
                callback.make(transport);
            }
            return transport;
        } catch (Exception e) {
            logger.error("creat transport error:", e);
            throw new RuntimeException(e);
        }
    }

    /**
     * 销毁对象
     */
    @Override
    public void destroyObject(TTransport transport) throws Exception {
        if (transport != null && transport.isOpen()) {
            transport.close();
        }
    }

    /**
     * 检验对象是否可以由pool安全返回
     */
    @Override
    public boolean validateObject(TTransport transport) {
        try {
            if (transport != null && transport instanceof TSocket) {
                TSocket thriftSocket = (TSocket) transport;
                if (thriftSocket.isOpen()) {
                    return true;
                } else {
                    return false;
                }
            } else {
                return false;
            }
        } catch (Exception e) {
            return false;
        }
    }

    @Override
    public void activateObject(TTransport obj) throws Exception {
        // TODO Auto-generated method stub
        
    }

    @Override
    public void passivateObject(TTransport obj) throws Exception {
        // TODO Auto-generated method stub
        
    }

    public static interface PoolOperationCallBack {
        // 创建成功是执行
        void make(TTransport transport);

        // 销毁之前执行
        void destroy(TTransport transport);
    }
}

      连接池管理 

package com.wy.thriftpool.commzkconnpool;

import org.apache.thrift.transport.TSocket;  
import org.slf4j.Logger;  
import org.slf4j.LoggerFactory;  
import org.springframework.beans.factory.annotation.Autowired;  
import org.springframework.stereotype.Service;  

  
/**
 * 连接池管理
 *   
 * @author wy
 *
 */
@Service  
public class ConnectionManager {  
    private final Logger logger = LoggerFactory.getLogger(getClass());  
    // 保存local对象 
    ThreadLocal<TSocket> socketThreadSafe = new ThreadLocal<TSocket>();  
  
    // 连接提供池   
    @Autowired  
    private ConnectionProvider connectionProvider;  
  
    public TSocket getSocket() {  
        TSocket socket = null;  
        try {  
            socket = connectionProvider.getConnection();  
            socketThreadSafe.set(socket);  
            return socketThreadSafe.get();  
        } catch (Exception e) {  
            logger.error("error ConnectionManager.invoke()", e);  
        } finally {  
            connectionProvider.returnCon(socket);  
            socketThreadSafe.remove();  
        }  
        return socket;  
    }  
  
}  

      spring配置文件  

<!-- zookeeper -->
    <bean id="thriftZookeeper" class="com.wy.thriftpool.commzkpool.zookeeper.ZookeeperFactory" destroy-method="close">
        <property name="connectString" value="127.0.0.1:2181" />
        <property name="namespace" value="thrift/thrift-service" />
    </bean>
    <bean id="connectionProvider" class="com.wy.thriftpool.commzkconnpool.impl.ConnectionProviderImpl">
        <property name="maxActive" value="10" />
        <property name="maxIdle" value="10" />
        <property name="conTimeOut" value="2000" />
        <property name="testOnBorrow" value="true" />
        <property name="testOnReturn" value="true" />
        <property name="testWhileIdle" value="true" />
        
        <property name="addressProvider">
            <bean class="com.wy.thriftpool.commzkpool.support.impl.DynamicAddressProvider">
                <property name="configPath" value="UserServiceImpl" />
                <property name="zookeeper" ref="thriftZookeeper" />
            </bean>
        </property>
    </bean>

参考：http://www.cnblogs.com/mumuxinfei/p/3876187.html

由于本人经验有限，文章中难免会有错误，请浏览文章的您指正或有不同的观点共同探讨！