zoukankan      html  css  js  c++  java
  • hadoop yarn HA集群搭建

    可先完成hadoop namenode HA的搭建:http://www.cnblogs.com/kisf/p/7458519.html

    搭建yarnde HA只需要在namenode HA配置基础上修改yarn-site.xml。

    1. 修改yarn-site.xml  (yarn另一台机器上一定要将yarn.resourcemanager.ha.id改成rm1)

    <configuration>
        <!-- Site specific YARN configuration properties -->
        <!--rm失联后重新链接的时间-->
        <property>
            <name>yarn.resourcemanager.connect.retry-interval.ms</name>
            <value>2000</value>
        </property>
    
        <!--开启resourcemanagerHA,默认为false-->
        <property>
            <name>yarn.resourcemanager.ha.enabled</name>
            <value>true</value>
        </property>
    
        <!--配置resourcemanager-->
        <property>
            <name>yarn.resourcemanager.ha.rm-ids</name>
            <value>rm1,rm2</value>
        </property>
    
        <property>
            <name>ha.zookeeper.quorum</name>
            <value>zk1:2181,zk2:2181,zk3:2181</value>
        </property>
    
        <!--开启故障自动切换-->
        <property>
            <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
            <value>true</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.hostname.rm1</name>
            <value>master1</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.hostname.rm2</name>
            <value>master2</value>
        </property>
    
        <!--
         在master1上配置rm1,在master2上配置rm2,
         注意:一般都喜欢把配置好的文件远程复制到其它机器上,但这个在YARN的另一个机器上一定要修改
         -->
        <property>
            <name>yarn.resourcemanager.ha.id</name>
            <value>rm1</value>
            <description>If we want to launch more than one RM in single node,we need this configuration</description>
        </property>
    
        <!--开启自动恢复功能-->
        <property>
            <name>yarn.resourcemanager.recovery.enabled</name>
            <value>true</value>
        </property>
    
        <!--配置与zookeeper的连接地址-->
        <property>
            <name>yarn.resourcemanager.zk-state-store.address</name>
            <value>zk1:2181,zk2:2181,zk3:2181</value>
        </property>
        <property>
            <name>yarn.resourcemanager.store.class</name>
            <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.zk-address</name>
            <value>zk1:2181,zk2:2181,zk3:2181</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.cluster-id</name>
            <value>appcluster-yarn</value>
        </property>
    
        <!--schelduler失联等待连接时间-->
        <property>
            <name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name>
            <value>5000</value>
        </property>
    
        <!--配置rm1-->
        <property>
            <name>yarn.resourcemanager.address.rm1</name>
            <value>master1:8032</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.scheduler.address.rm1</name>
            <value>master1:8030</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.webapp.address.rm1</name>
            <value>master1:8088</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.resource-tracker.address.rm1</name>
            <value>master1:8031</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.admin.address.rm1</name>
            <value>master1:8033</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.ha.admin.address.rm1</name>
            <value>master1:23142</value>
        </property>
        <!--配置rm2-->
        <property>
            <name>yarn.resourcemanager.address.rm2</name>
            <value>master2:8032</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.scheduler.address.rm2</name>
            <value>master2:8030</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.webapp.address.rm2</name>
            <value>master2:8088</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.resource-tracker.address.rm2</name>
            <value>master2:8031</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.admin.address.rm2</name>
            <value>master2:8033</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.ha.admin.address.rm2</name>
            <value>master2:23142</value>
        </property>
    
        <property>
            <name>yarn.nodemanager.aux-services</name>
            <value>mapreduce_shuffle</value>
        </property>
    
        <property>
            <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
            <value>org.apache.hadoop.mapred.ShuffleHandler</value>
        </property>
    
        <property>
            <name>yarn.nodemanager.local-dirs</name>
            <value>/letv/soft/hadoop-2.7.3/yarn/local</value>
        </property>
    
        <property>
            <name>yarn.nodemanager.log-dirs</name>
            <value>/letv/soft/hadoop-2.7.3/yarn/log</value>
        </property>
    
        <property>
            <name>mapreduce.shuffle.port</name>
            <value>23080</value>
        </property>
    
        <!--故障处理类-->
        <property>
            <name>yarn.client.failover-proxy-provider</name>
            <value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value>
        </property>
    
        <property>
            <name>yarn.resourcemanager.ha.automatic-failover.zk-base-path</name>
            <value>/yarn-leader-election</value>
            <description>Optionalsetting.Thedefaultvalueis/yarn-leader-election</description>
        </property>
    </configuration>
    

    2. 启动

    正常格式化,hdfs,yarn正常启动。

    ./sbin/start-dfs.sh 
    
    ./sbin/start-yarn.sh  

    rm2上yarn需要单独启动:

    ./sbin/yarn-daemon.sh start resourcemanager
    

    over  

      

  • 相关阅读:
    NOIP2011 D1T1 铺地毯
    NOIP2013 D1T3 货车运输 倍增LCA OR 并查集按秩合并
    POJ 2513 trie树+并查集判断无向图的欧拉路
    599. Minimum Index Sum of Two Lists
    594. Longest Harmonious Subsequence
    575. Distribute Candies
    554. Brick Wall
    535. Encode and Decode TinyURL(rand and srand)
    525. Contiguous Array
    500. Keyboard Row
  • 原文地址:https://www.cnblogs.com/kisf/p/7460235.html
Copyright © 2011-2022 走看看