Hadoop 2.6.0集群搭建

yum install gcc 

yum install gcc-c++

yum install make

yum install autoconfautomake libtool cmake

yum install ncurses-devel

yum install openssl-devel

groupadd hadoop  添加一个组

useradd hadoop -g hadoop  添加用户

安装protoc(需用root用户)

1 tar -xvf protobuf-2.5.0.tar.bz2 

2 cd protobuf-2.5.0 

3 ./configure --prefix=/opt/protoc/ 

4 make && make install

编译hadoop

mvn clean package -Pdist,native -DskipTests -Dtar 

编译完的hadoop在 /home/hadoop/ocdc/hadoop-2.6.0-src/hadoop-dist/target 路径下

配置hosts文件

10.1.245.244 master

10.1.245.243 slave1

命令行输入 hostname master

免密码登录：

执行命令生成密钥： ssh-keygen -t rsa -P ""

进入文件夹cd  .ssh (进入文件夹后可以执行ls  -a 查看文件)

将生成的公钥id_rsa.pub 内容追加到authorized_keys（执行命令：cat id_rsa.pub >> authorized_keys）

---------------------------(core-site.xml)------------------------------

<configuration>

 <!--指定hdfs的nameservice为ns1-->

<property>

  <name>fs.defaultFS</name>

  <value>hdfs://master</value>

 </property>

 <property>

  <name>io.file.buffer.size</name>

  <value>131072</value>

 </property>

<!--指定hadoop数据存放目录-->

 <property>

  <name>hadoop.tmp.dir</name>

  <value>/home/hadoop/ocdc/hadoop-2.6.0/tmp</value>

  <description>Abasefor other temporary directories.</description>

 </property>

 <property>

  <name>hadoop.proxyuser.spark.hosts</name>

  <value>*</value>

 </property>

<property>

  <name>hadoop.proxyuser.spark.groups</name>

  <value>*</value>

 </property>

</configuration>

<!--指定zookeeper地址-->

 <property>

   <name>ha.zookeeper.quorum</name>

   <value>h4:2181,h5:2181,h6:2181</value>

 </property>

</configuration>

 

-------------------------------(hdfs-site.xml )-----------------------------------

<configuration>

 <property>

  <name>dfs.namenode.secondary.http-address</name>

  <value>master:9001</value>

 </property>

  <property>

   <name>dfs.namenode.name.dir</name>

   <value>/home/hadoop/ocdc/hadoop-2.6.0/name</value>

 </property>

 <property>

  <name>dfs.datanode.data.dir</name>

  <value>/home/hadoop/ocdc/hadoop-2.6.0/data</value>

  </property>

 <property>

  <name>dfs.replication</name>

  <value>3</value>

 </property>

 <property>

  <name>dfs.webhdfs.enabled</name>

  <value>true</value>

 </property>

<property>

   <name>dfs.nameservices</name>

   <value>ns1</value>

</property>

<!-- ns1下面有两个NameNode，分别是nn1，nn2 -->

<property>

   <name>dfs.ha.namenodes.ns1</name>

   <value>nn1,nn2</value>

</property>

</configuration>

----------------------------------------(yarn-site.xml)--------------------------

<configuration>

<!-- Site specific YARN configuration properties -->

<!-- 指定nodemanager启动时加载server的方式为shuffle server -->

<property>

   <name>yarn.nodemanager.aux-services</name>

   <value>mapreduce_shuffle</value>

  </property>

  <property>

   <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

   <value>org.apache.hadoop.mapred.ShuffleHandler</value>

  </property>

  <property>

   <name>yarn.resourcemanager.address</name>

   <value>master:8032</value>

  </property>

  <property>

   <name>yarn.resourcemanager.scheduler.address</name>

   <value>master:8030</value>

  </property>

  <property>

   <name>yarn.resourcemanager.resource-tracker.address</name>

   <value>master:8035</value>

  </property>

  <property>

   <name>yarn.resourcemanager.admin.address</name>

   <value>master:8033</value>

  </property>

  <property>

   <name>yarn.resourcemanager.webapp.address</name>

   <value>master:8088</value>

  </property>

<property>

<name>yarn.nodemanager.resource.memory-mb</name>

<value>16384</value>

</property>

</configuration>

<!-- 指定resourcemanager地址 -->

   <property>

       <name>yarn.resourcemanager.hostname</name>

            <value>h3</value>

   </property>

-----------------------------(mapred-site.xml)-------------------------------

<configuration>

<property>

<name>mapreduce.framework.name</name>

   <value>yarn</value>

 </property>

 <property>

  <name>mapreduce.jobhistory.address</name>

  <value>master:10020</value>

 </property>

 <property>

  <name>mapreduce.jobhistory.webapp.address</name>

  <value>master:19888</value>

 </property>

<property>

    <name>yarn.scheduler.maximum-allocation-mb</name>

    <value>16384</value>

  </property>

</configuration>