Centos7下HDFS伪分布式环境+HBASE搭建记录

一、环境准备
IP地址查看
ifconfig
192.168.118.129

修改主机名
hostnamectl set-hostname master
hostname

关闭防火墙
systemctl disable firewalld

配置hosts文件
vi /etc/hosts
192.168.118.129 master

重启网络
systemctl restart network

创建hadoop用户
useradd hadoop
passwd hadoop
密码hadoop7890
su - hadoop
ssh-keygen -t rsa -P ''
ls .ssh/
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

设置SSH配置
su - root
vi /etc/ssh/sshd_config

PubkeyAuthentication yes # 启用公钥私钥配对认证方式
AuthorizedKeysFile .ssh/authorized_keys # 公钥文件路径（默认不需要修改）

systemctl restart sshd

验证ssh
su - hadoop
ssh master
exit
su - root

二、安装jdk
tar -zxvf jdk-8u162-linux-x64.tar.gz
mv /root/res/jdk1.8.0_162/ /usr/local/src/
cd /usr/local/src/

设置环境变量
vi /etc/profile

export JAVA_HOME=/usr/local/src/jdk1.8.0_162
export JRE_HOME=/usr/local/src/jdk1.8.0_162/jre
export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$JAVA_HOME:$PATH

source /etc/profile
echo $JAVA_HOME
java -version

三、安装hadoop
tar -zxvf hadoop-2.7.7.tar.gz
mv hadoop-2.7.7 /usr/local/src/
cd /usr/local/src/
mv hadoop-2.7.7 hadoop

设置环境变量
vi /etc/profile

export HADOOP_HOME=/usr/local/src/hadoop
export PATH=$HADOOP_HOME:$JAVA_HOME/bin:$JRE_HOME/bin:$JAVA_HOME:$PATH

source /etc/profile
echo $HADOOP_HOME

修改hadoop-env.sh
cd /usr/local/src/hadoop/etc/hadoop/
vi hadoop-env.sh

export JAVA_HOME=/usr/local/src/jdk1.8.0_162

vi core-site.xml

<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/src/hadoop/tmp</value>
</property>

mkdir /usr/local/src/hadoop/tmp

vi hdfs-site.xml

<property>
<name>dfs.datanode.ipc.address</name>
<value>master:50020</value>
</property>
<property>
<name>dfs.datanode.http.address</name>
<value>master:50075</value>
</property>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>

vi yarn-site.xml

<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>

cp mapred-site.xml.template mapred-site.xml
vi mapred-site.xml

<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>master:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>master:19888</value>
</property>

vi masters
master
vi slaves
master
注：因为在伪分布式模式下，作为master的namenode与作为slave的datanode是同一台服务器，所以配置文件中的ip是一样的
chown -R hadoop:hadoop /usr/local/src/hadoop/
cp ~/res/fastjson-1.2.41.jar /usr/local/src/hadoop/share/hadoop/common/

四、测试hadoop
su - hadoop
cd /usr/local/src/hadoop/
bin/hdfs namenode -format
sbin/start-all.sh
bin/hdfs dfsadmin -report
su - root
jps

五、hdfs常用命令
ls 命令
bin/hdfs dfs -ls /
df 命令 查看根目录磁盘空间
bin/hdfs dfs -df /
put 命令
bin/hdfs dfs -put NOTICE.txt /
cat 命令
bin/hdfs dfs -cat /NOTICE.txt
cp 命令
bin/hdfs dfs -cp /NOTICE.txt /test.txt
du 命令
bin/hdfs dfs -du /
rm命令
bin/hdfs dfs -rm /NOTICE.txt
mkdir命令
bin/hdfs dfs -mkdir /user
bin/hdfs dfs -mkdir /user/zhang
rmdir命令
bin/hdfs dfs -rmdir /user/zhang

六、安装hbase
tar -zxvf hbase-1.5.0-bin.tar.gz
mv hbase-1.5.0 /usr/local/src/hbase

设置环境变量
vi /etc/profile

export HBASE_HOME=/usr/local/src/hbase
export PATH=$HBASE_HOME/bin:$PATH

source /etc/profile
hbase version

cd /usr/local/src/hbase/conf
vi hbase-site.xml

<property>
<name>hbase.rootdir</name>
<value>hdfs://master:9000/hbase</value>
</property>
<property>
<name>hbase.tmp.dir</name>
<value>/usr/local/src/hbase/tmp</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>master</value>
</property>
<property>
<name>hbase.zookeeper.property.dataDir</name>
<value>/usr/local/src/hbase/zk_data</value>
</property>

mkdir /usr/local/src/hbase/tmp
mkdir /usr/local/src/hbase/zk_data
vi hbase-env.sh

export JAVA_HOME=/usr/local/src/jdk1.8.0_162
export HADOOP_HOME=/usr/local/src/hadoop
export HBASE_HOME=/usr/local/src/hbase
export HBASE_CLASSPATH=/usr/local/src/hadoop/etc/hadoop
export HBASE_MANAGES_ZK=true

cd /usr/local/src/hbase/
chown -R hadoop:hadoop /usr/local/src/hbase/
bin/start-hbase.sh
jps

4144 DataNode
9555 HRegionServer
10723 Jps
4004 NameNode
4327 SecondaryNameNode
10524 HMaster
10413 HQuorumPeer
4622 NodeManager
4495 ResourceManager

kill -9

七、测试hbase
hbase shell
list
create 'student','info'
list

put 'student','1001','info:sex','male'
put 'student','1001','info:name','zhangsan'
put 'student','1002','info:sex','female'
put 'student','1002','info:name','lisi'

scan 'student'
get 'student','1001'
get 'student','1001','info:sex'
count 'student'

put 'student','1001','info:name','zhangsan0'
scan 'student'
alter 'student',{NAME=>'info',VERSIONS=>3}
put 'student','1001','info:name','zhangsan1'
scan 'student'
get 'student','1001',{COLUMN=>'info:name',VERSIONS=>3}
put 'student','1001','info:name','zhangsan2'
scan 'student'
get 'student','1001',{COLUMN=>'info:name',VERSIONS=>3}
put 'student','1001','info:name','zhangsan3'
scan 'student'
get 'student','1001',{COLUMN=>'info:name',VERSIONS=>3}

deleteall 'student','1001'
delete 'student','1002','info:sex'