查看下载地址：http://archive.cloudera.com/cdh5/cdh/5/
hadoop-2.6.0-cdh5.4.0-src.tar.gz
解压hadoop-2.6.0-cdh5.4.0-src.tar.gz
在etc/hadoop下修改下面文件
修改salves为主机名hostname
hadoop-spark.dragon.org
Configuration
#环境变量
hadoop-env.sh
export JAVA_HOME=/opt/modules/jdk1.7.0_67
#缓存 hadoop-spark.dragon.org 为主机hostname ,如何修改请参考如何修改centos主机名
core-site.xml
<property>
<name>fs.defaultFS</name>
<value>hdfs://hadoop-spark.dragon.org:8020</value>
</property>
#缓存 mkdir data mkdir tmp
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/data02/hadoop-2.6.0-cdh5.4.0/data/tmp</value>
</property>
#副本 默认为3
hdfs-site.xml
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
配置完需要格式化hdfs
bin/hdfs namenode -format
slaves
hadoop-spark.dragon.org
Start HDFS
NameNode Format
bin/hdfs namenode -format
Start NN/DN
sbin/hadoop-daemon.sh start namenode
sbin/hadoop-daemon.sh start datanode
WEB UI
http://hadoop-spark.dragon.org:50070
============================ SetUp Spark=============================
Configuration
spark-env.sh
HADOOP_CONF_DIR=/opt/data02/hadoop-2.6.0-cdh5.4.0/etc/hadoop
JAVA_HOME=/opt/modules/jdk1.7.0_67
SCALA_HOME=/opt/modules/scala-2.10.4
#######################################################
SPARK_MASTER_IP=hadoop-spark.dragon.org
SPARK_MASTER_PORT=7077
SPARK_MASTER_WEBUI_PORT=8080
SPARK_WORKER_CORES=1
SPARK_WORKER_MEMORY=1000m
SPARK_WORKER_PORT=7078
SPARK_WORKER_WEBUI_PORT=8081
SPARK_WORKER_INSTANCES=1
slaves
hadoop-spark.dragon.org
spark-defaults.conf
spark.master spark://hadoop-spark.dragon.org:7077
Start Spark
Start Master
sbin/start-master.sh
Start Slaves
sbin/start-slaves.sh
WEB UI 浏览器查看
http://hadoop-spark.dragon.org:8080
============================ Test Spark=============================
scala> val rdd=sc.textFile("hdfs://hadoop-spark.dragon.org:8020/user/hadoop/data/wc.input")
scala> rdd.cache()
scala> val wordcount=rdd.flatMap(_.split(" ")).map(x=>(x,1)).reduceByKey(_+_)
scala> wordcount.take(10)
scala> val wordsort=wordcount.map(x=>(x._2,x._1)).sortByKey(false).map(x=>(x._2,x._1))
scala> wordsort.take(10)