HBase的java客户端测试(二)---DML操作

测试准备

【首先同步时间：】

for node in CloudDeskTop master01 master02 slave01 slave02 slave03;do ssh $node "date -s '2017-12-30 21:32:30'";done

【slave各节点启动zookeeper集群：】

cd /software/zookeeper-3.4.10/bin/ && ./zkServer.sh start && cd - && jps

【master01启动HDFS集群：】

cd /software/ && start-dfs.sh && jps

【master01启动HBase：】

cd /software/hbase-1.2.6/bin && start-hbase.sh && jps

【master02上启动HBase:】

cd /software/hbase-1.2.6/bin && hbase-daemon.sh start master && jps

如有节点启动出现故障：
单独启动master：
cd /software/hbase-1.2.6/bin && hbase-daemon.sh start master && jps
单独启动regionserver：
cd /software/hbase-1.2.6/bin && hbase-daemon.sh start regionserver && jps
通过命令终端查看：
hbase(main):009:0> status
通过web终端查看：
http://master01的IP地址:16010/

【在主机CloudDeskTop导入java客户端开发所需jar包：】HBase1.2.6-All.zip

测试目标： 

 运用java代码编写程序操作HBase数据库，本次测试致力于对DML语法的java客户端操作；

测试代码：大数据学习交流QQ群：217770236 

package com.mmzs.bigdata.hbase.dml;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;

/**
 * @author hadoop
 * 
 * [HTable].put|delete|get|getScanner([Scan].addColumn|setStartRow|setStopRow|setFilter([FilterList].addFilter))
 *
 */
public class DMLMain {
    /**
     * 操作HBase集群的客户端
     */
    private static Admin admin;
    
    private static Connection conn;
    
    static{
        //创建HBase配置
        Configuration conf=HBaseConfiguration.create();
        conf.set("hbase.zookeeper.quorum", "slave01:2181,slave02:2181,slave03:2181");
        try {
            //根据HBase配置获取HBase集群连接
            conn=ConnectionFactory.createConnection(conf);
            admin=conn.getAdmin();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) {
        //这种添加方式的读数据在后面查找时容易出IO异常，因为中间涉及了二进制转换
//        put("mmzs:myuser","008","base","userName","ligang00");
//        put("mmzs:myuser","008","base","userAge","18");
//        put("mmzs:myuser","008","base","gender","woman");
        
/*        Date d=new Date();
        System.out.println(d);
        byte[] b=getBytes(d);
        System.out.println(b);
        Object obj=getObject(b);
        System.out.println(obj);*/
    
        //添加数据    
/*        Map<String,Map<String,Map<String,Object>>> dataMap=new HashMap<String,Map<String,Map<String,Object>>>();
        
        Map<String,Map<String,Object>> familyMap=new HashMap<String,Map<String,Object>>();
//        dataMap.put("ligang+28+1.67", familyMap);
//        dataMap.put("zhanghua+28+1.67", familyMap);
//        dataMap.put("zhanghua+29+1.67", familyMap);
//        dataMap.put("ligang+28+1.72", familyMap);
//        dataMap.put("wangwu+29+1.82", familyMap);
        dataMap.put("001", familyMap);
        dataMap.put("002", familyMap);
        dataMap.put("003", familyMap);
        dataMap.put("004", familyMap);
        dataMap.put("005", familyMap);
        dataMap.put("006", familyMap);
        
        Map<String,Object> keyValMap=new HashMap<String,Object>();
        keyValMap.put("height", 1.68);
        keyValMap.put("weight", 75.5);
        keyValMap.put("gender", "women");
        keyValMap.put("username", "zhangsan");
        
        familyMap.put("extra", keyValMap);
        
        puts("mmzs:myuser",dataMap);*/
        
        
//        delete("mmzs:myuser","005");
//        delete("mmzs:myuser","002","extra","username");
//        deletes("mmzs:myuser","001","002","003","004","005","006","007","008");
        
//        get("mmzs:myuser","005");
        
//        scan("mmzs:myuser");
        
//        scanByCondition("mmzs:myuser");
    }
    
    /**
     * 将二进制流读到内存通过反序列化重构生成对象
     * @param object
     * @return
     */
    private static Object getObject(byte[] b){
        ObjectInputStream ois=null;
        try {
            ByteArrayInputStream bais=new ByteArrayInputStream(b);
            ois=new ObjectInputStream(bais);
            return ois.readObject();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }finally{
            try{
                if(null!=ois)ois.close();
            }catch(IOException e){
                e.printStackTrace();
            }
        }
        return null;
    }
    
    /**
     * 将对象序列化成二进制数组
     * @param object
     * @return
     */
    private static byte[] getBytes(Object object){
        ObjectOutputStream oos=null;
        try {
            ByteArrayOutputStream baos=new ByteArrayOutputStream();
            oos=new ObjectOutputStream(baos);
            oos.writeObject(object);
            oos.flush();
            return baos.toByteArray();
        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            try{
                if(null!=oos)oos.close();
            }catch(IOException e){
                e.printStackTrace();
            }
        }
        return null;
    }
    
    /**
     * 打印结果集
     * @param ress
     */
    private static void printResult(ResultScanner rss){
        //遍历结果集,每一个Result对应一行记录(即对应一个RowKey)
        for(Result res:rss){
            String rowKey=new String(res.getRow());
            List<Cell> cellList=res.listCells();
            for(Cell cell:cellList){
                //获取当前键值对所在的列族名称
                String familyName=new String(CellUtil.cloneFamily(cell));
                //获取当前键值对的键(Key)
                String key=new String(CellUtil.cloneQualifier(cell));
                //获取当前键值对的值(Value)
                String value=getObject(CellUtil.cloneValue(cell)).toString();
                
                System.out.println(rowKey+"	"+familyName+"	"+key+":"+value);
            }
        }
    }
    
    /**
     * 条件查询
     * @param tabNameStr
     * 
     * PrefixFilter和RowFilter都是基于行键(RowKey)的过滤器
     */
    public static void scanByCondition(String tabNameStr){
        TableName tabName=TableName.valueOf(tabNameStr);
        Scan scan=new Scan();
        
        //前缀过滤器,行键前缀是ligang的
        PrefixFilter prefixFilter=new PrefixFilter("ligang".getBytes());
        
        //子串过滤,行键中包含1.72的
        RowFilter rowFilter=new RowFilter(CompareOp.EQUAL,new SubstringComparator("28"));
        
        //行键过滤器，列族 列名 比较操作 值,显示不包含满足条件的
        SingleColumnValueFilter scvFilter=new SingleColumnValueFilter("base".getBytes(),"userName".getBytes(),CompareOp.EQUAL,new SubstringComparator("ligang"));
        
        //FilterList.Operator.MUST_PASS_ALL相当于and,FilterList.Operator.MUST_PASS_ONE相当于or
        FilterList filterList=new FilterList(FilterList.Operator.MUST_PASS_ALL);
        //添加使用的过滤器
//        filterList.addFilter(prefixFilter);
//        filterList.addFilter(rowFilter);
        filterList.addFilter(scvFilter);
        
//        scan.setFilter(prefixFilter);
        
        scan.setFilter(filterList);//设置过滤
        try {
            Table table=conn.getTable(tabName);
            ResultScanner ress=table.getScanner(scan);
            printResult(ress);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    
    /**
     * 查询多条记录
     * @param tabNameStr
     * @param rowKey
     */
    public static void scan(String tabNameStr){
        TableName tabName=TableName.valueOf(tabNameStr);
        
        Scan scan=new Scan();
        
        //过滤查询结果集中的字段
        scan.addColumn("extra".getBytes(), "height".getBytes());
        scan.addColumn("extra".getBytes(), "weight".getBytes());
        
        //设置查询的起始和结束行索引(通过行键RowKey指定)
        scan.setStartRow("002".getBytes());
        scan.setStopRow("006".getBytes());
        
        try {
            Table table=conn.getTable(tabName);
            //查询多行返回一个结果集
            ResultScanner rss=table.getScanner(scan);
            //遍历结果集,每一个Result对应一行记录(即对应一个RowKey)
            printResult(rss);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    
    /**
     * 查询单条记录
     * @param rowKey
     * @param tabNameStr
     */
    public static void get(String tabNameStr,String rowKey){
        TableName tabName=TableName.valueOf(tabNameStr);
        
        Get get=new Get(rowKey.getBytes());
        //相当于select..字段列表...from...,如果没有下面的addColumn方法调用则相当于select *...
        get.addColumn("base".getBytes(), "height".getBytes());
        get.addColumn("base".getBytes(), "gender".getBytes());
        try {
            Table table=conn.getTable(tabName);
            Result result=table.get(get);
            
            //获取行键
            String rowKeyStr=new String(result.getRow());
            System.out.println("行键:"+rowKeyStr);
            
            //获取键所对应的值
            byte[] byteName=result.getValue("base".getBytes(), "gender".getBytes());
            String gender=getObject(byteName).toString();
            System.out.println("gender:"+gender);
            
            //获取当前行中的所有键值对
            List<Cell> cellList=result.listCells();
            for(Cell cell:cellList){
                //获取当前键值对所在的列族名称
                String familyName=new String(CellUtil.cloneFamily(cell));
                //获取当前键值对的键(Key)
                String key=new String(CellUtil.cloneQualifier(cell));
                //获取当前键值对的值(Value)
                byte[] byteValue=CellUtil.cloneValue(cell);
                String value=getObject(byteValue).toString();
                
                System.out.println(rowKey+"	"+familyName+"	"+key+":"+value);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    
    /**
     * 批量删除多行
     * @param tabNameStr
     * @param rowKey
     */
    public static void deletes(String tabNameStr,String... rowKeys){
        if(rowKeys.length==0)return;
        TableName tabName=TableName.valueOf(tabNameStr);
        
        List<Delete> deleteList=new ArrayList<Delete>();
        for(String rowKey:rowKeys)deleteList.add(new Delete(rowKey.getBytes()));
        
        try {
            Table table=conn.getTable(tabName);
            table.delete(deleteList);
            System.out.println("删除完成！");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    
    /**
     * 删除行中的键值对
     * @param tabNameStr
     * @param rowKey
     * @param key
     */
    public static void delete(String tabNameStr,String rowKey,String family,String key){
        TableName tabName=TableName.valueOf(tabNameStr);
        try {
            Table table=conn.getTable(tabName);
            Delete delete=new Delete(rowKey.getBytes());
            delete.addColumn(family.getBytes(), key.getBytes());
            table.delete(delete);
            System.out.println("删除完成！");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    
    /**
     * 删除整行
     * @param tabNameStr
     * @param rowKey
     */
    public static void delete(String tabNameStr,String rowKey){
        TableName tabName=TableName.valueOf(tabNameStr);
        try {
            Table table=conn.getTable(tabName);
            Delete delete=new Delete(rowKey.getBytes());
            table.delete(delete);
            System.out.println("删除完成！");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    
    /**
     * 增加或修改数据(表名、行键、列族、列、值)
     */
    public static void put(String tabNameStr,String rowKey,String family,String key,String value){
        TableName tabName=TableName.valueOf(tabNameStr);
        try {
            Table table=conn.getTable(tabName);
            Put put=new Put(rowKey.getBytes());
            put.addColumn(family.getBytes(), key.getBytes(), value.getBytes());
            table.put(put);
            System.out.println("操作完成！");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    
    /**
     * 批量插入或修改数据
     * @param tabNameStr
     * @param dataMap
     */
    public static void puts(String tabNameStr,Map<String,Map<String,Map<String,Object>>> dataMap){
        List<Put> putList=new ArrayList<Put>();
        Set<Entry<String, Map<String, Map<String, Object>>>> entrys=dataMap.entrySet();
        for(Entry<String, Map<String, Map<String, Object>>> entry:entrys){
            //获取行的rowKey
            byte[] rowKey=entry.getKey().getBytes();
            Put put=new Put(rowKey);
            putList.add(put);
            //获取行的所有列族
            Map<String, Map<String, Object>> familyMap=entry.getValue();
            Set<Entry<String, Map<String, Object>>> familyEntrys=familyMap.entrySet();
            for(Entry<String, Map<String, Object>> familyEntry:familyEntrys){
                //获取列族名称
                byte[] familyName=familyEntry.getKey().getBytes();
                //获取列族下左右的键值对
                Map<String, Object> keyVals=familyEntry.getValue();
                Set<Entry<String, Object>> keyValEntrys=keyVals.entrySet();
                for(Entry<String, Object> keyValEntry:keyValEntrys){
                    byte[] key=keyValEntry.getKey().getBytes();
                    byte[] value=getBytes(keyValEntry.getValue());
                    put.addColumn(familyName, key, value);
                }
            }
        }
        
        TableName tabName=TableName.valueOf(tabNameStr);
        try {
            Table table=conn.getTable(tabName);
            table.put(putList);
            System.out.println("操作完成！");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

测试结果：

  在命令端查看，查看方式，可参考：http://www.cnblogs.com/mmzs/p/8135327.html

测试小结：

　　HBase是分布式的、可扩展的、且基于Hadop的大数据NOSQL存储数据库(号称十亿行、百万列),该数据库的每一行也是保存一条记录,但是与关系数据库不同的是,该数据库无需事先定义字段和表结构,只需要指定表名和列族;每一行由指定的一到多个族组成,每个族上可以保存若干个键值对(每一行的每个族上的键值对数量可以是任意的),即每一行的单元格数量不一定相等,每一个单元格中保存同一个键名但不同版本值的若干个键值对(即同一个字段名中有不同版本的字段值),当需要查询某一个字段值时需要指定的坐标是:
　　表名(table name)—>行健(row key)—>列族(column family)—>字段名(column name)—>版本号(version code)