zoukankan      html  css  js  c++  java

  • spring-hadoop之操作hbase

    　　Srping对于属于java web技术的程序员都不会陌生，jdbcTemplate更是用的熟之又熟，下面我们来认识一下Spring大家庭的新成员：Spring-data-hadoop项目。Spring-hadoop这个项目应该是在Spring Data项目的一部分（Srping data其余还包括把Spring和JDBC，REST，主流的NoSQL结合起来了）。其实再一想，Spring和Hadoop结合会发生什么呢，其实就是把Hadoop组件的配置，任务部署之类的东西都统一到Spring的bean管理里去了。

    具体操作如下：

    1、下载并导入需要的jar包：

    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/commons-configuration-1.6.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/commons-lang-2.5.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/commons-logging-1.1.1.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/dom4j-1.6.1.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/guava-11.0.2.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/hadoop-auth-2.0.0-cdh4.4.0.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/hadoop-common-2.0.0-cdh4.4.0.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/hadoop-mapreduce-client-core-2.0.0-cdh4.4.0.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/hbase-0.94.6-cdh4.4.0-security.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/log4j-1.2.17.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/protobuf-java-2.4.0a.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/slf4j-api-1.6.1.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/slf4j-log4j12-1.6.1.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-aop-4.0.3.RELEASE.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-aspects-4.0.3.RELEASE.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-beans-4.0.3.RELEASE.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-context-4.0.3.RELEASE.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-context-support-4.0.3.RELEASE.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-core-4.0.3.RELEASE.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-data-hadoop-2.0.0.RC2-cdh4.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-data-hadoop-batch-2.0.0.RC2-cdh4.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-data-hadoop-core-2.0.0.RC2-cdh4.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-data-hadoop-store-2.0.0.RC2-cdh4.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-expression-4.0.3.RELEASE.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-tx-4.0.3.RELEASE.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-web-4.0.3.RELEASE.jar"/>
    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/zookeeper-3.4.5-cdh4.4.0.jar"/>

    2、配置web.xml：

     1 <?xml version="1.0" encoding="UTF-8"?>
 2 <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/j2ee"
 3     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4     xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee http://java.sun.com/xml/ns/j2ee/web-app_2_5.xsd">
 5     <context-param>
 6         <param-name>contextConfigLocation</param-name>
 7         <param-value>classpath:beans.xml</param-value>
 8     </context-param>
 9     
10     
11     <listener>
12         <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
13     </listener>
14     <!-- Spring 刷新Introspector防止内存泄露 -->
15     <listener>
16         <listener-class>org.springframework.web.util.IntrospectorCleanupListener</listener-class>
17     </listener>
18     
19     <welcome-file-list>
20         <welcome-file>login.jsp</welcome-file>
21     </welcome-file-list>
22 </web-app>

    3、配置beans.xml:

     1 <?xml version="1.0" encoding="UTF-8"?>
 2 <beans xmlns="http://www.springframework.org/schema/beans"
 3        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4         xmlns:context="http://www.springframework.org/schema/context"
 5         xmlns:p="http://www.springframework.org/schema/p"
 6         xmlns:aop="http://www.springframework.org/schema/aop"
 7         xmlns:hdp="http://www.springframework.org/schema/hadoop"
 8         xsi:schemaLocation="
 9             http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
10             http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-4.0.xsd
11             http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-4.0.xsd
12             http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop-2.0.xsd
13             ">
14     
15     <!-- 自动扫描beans+注解功能注册 -->
16     <context:component-scan base-package="net.xinhong"/>
17     
18     <hdp:configuration resources="classpath:/hbase-site.xml" />
19     
20     <hdp:hbase-configuration configuration-ref="hadoopConfiguration" />
21     
22     <bean id="hbaseTemplate" class="org.springframework.data.hadoop.hbase.HbaseTemplate" p:configuration-ref="hbaseConfiguration"/>
23     
24 </beans>

    4、将hbase-site.xml配置文件拷贝到src目录下，参考内容如下：

     1 <?xml version="1.0" encoding="UTF-8"?>
 2 <configuration>
 3   <property>
 4     <name>hbase.rootdir</name>
 5     <value>hdfs://nameservice1/hbase</value>
 6   </property>
 7   <property>
 8     <name>hbase.client.write.buffer</name>
 9     <value>62914560</value>
10   </property>
11   <property>
12     <name>hbase.client.pause</name>
13     <value>1000</value>
14   </property>
15   <property>
16     <name>hbase.client.retries.number</name>
17     <value>10</value>
18   </property>
19   <property>
20     <name>hbase.client.scanner.caching</name>
21     <value>1</value>
22   </property>
23   <property>
24     <name>hbase.client.keyvalue.maxsize</name>
25     <value>62914560</value>
26   </property>
27   <property>
28     <name>hbase.rpc.timeout</name>
29     <value>60000</value>
30   </property>
31   <property>
32     <name>hbase.security.authentication</name>
33     <value>simple</value>
34   </property>
35   <property>
36     <name>zookeeper.session.timeout</name>
37     <value>60000</value>
38   </property>
39   <property>
40     <name>zookeeper.znode.parent</name>
41     <value>/hbase</value>
42   </property>
43   <property>
44     <name>zookeeper.znode.rootserver</name>
45     <value>root-region-server</value>
46   </property>
47   <property>
48     <name>hbase.zookeeper.quorum</name>
49     <value>xinhong-hadoop-56,xinhong-hadoop-52,xinhong-hadoop-53</value>
50   </property>
51   <property>
52     <name>hbase.zookeeper.property.clientPort</name>
53     <value>2181</value>
54   </property>
55 </configuration>

    5、hbase模板的demo:

    查询列族下的列的数据：

    1 public List<String> find(String tableName,String family,String cloumn){
2         List<String> rows = hbaseTemplate.find(tableName, family,cloumn, new RowMapper<String>() {
3             public String mapRow(Result result, int rowNum) throws Exception {
4                 return Bytes.toString(result.getRow());
5             }
6         });
7         return rows;
8     }

    查询指定行健的一列数据：

    1 public String get(String tableName,String family,String cloumn,String rowKey){
2         String context = hbaseTemplate.get(tableName, "NCEP_p_wa_2014032212_tp_006.nc", family, cloumn, new RowMapper<String>() {
3             public String mapRow(Result result, int rowNum) throws Exception {
4                 return Bytes.toString(result.value());
5               }
6             });
7         return context;
8     }

    PS:其中RowMapper回调函数的使用方法可以参考http://www.cnblogs.com/zhangyukun/p/3685369.html来使用，用法大致相同。

    转载请注明出处，期待共同进步...
  • 相关阅读:
    uboot的配置文件在哪里
    uboot的flash sectore的大小由哪个宏指定
    openwrt中的append-ubi定义在哪里
    vi在行首插入注释符号#
    openwrt设置uboot环境变量在flash上的存储地址
    git重命名分支名
    ifconfig设置ip时出现提示 ifconfig: SIOCSIFFLAGS: Address not available
    jquery checkbox选中、改变状态、change和click事件
    jQuery操作复选框checkbox技巧总结 ---- 设置选中、取消选中、获取被选中的值、判断是否选中等
    checkbox选择根据后台List数据进行回显
  • 原文地址：https://www.cnblogs.com/zhangyukun/p/3685468.html
Copyright © 2011-2022 走看看