pof补充

[初始化系统]

sudo systemctl stop firewalld
sudo systemctl disable firewalld
sudo setenforce 0
sudo vi /etc/sysconfig/selinux
SELINUX=disabled

sudo tar -zxf docker-18.06.3-ce.tgz -C /usr/local/
cp /usr/local/docker/* /usr/bin/
mkdir -p /home/docker

[修正docker自启动问题]

添加docker自启动需要文件

sudo vi /usr/lib/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
[Service]
Type=notify
ExecStart=/usr/bin/dockerd --graph /home/docker
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target

sudo groupadd docker

将您的用户添加到该docker组

sudo usermod -aG docker $USER

将登陆用户加入到docker用户组中

sudo gpasswd -a $USER docker

更新用户组

newgrp docker

配置docker以systemctl方式启动

sudo systemctl start docker

配置docker开机启动

sudo systemctl enable docker

---

数据库是否换成maridb
[数据库需确认]

copy sql脚本到数据库容器
[完善docker cp sql脚本到容器]

使用docker cp方式拷贝dataexa-insight-microservice-poc.sql到容器根目录下

docker cp /home/dataexa/insight-deploy/resources/mirrors/mysql/dataexa-insight-microservice-poc.sql mysql:/
docker exec -it mysql bash
mysql -uroot -pDataExa5528280
create database dataexa-insight-microservice-poc default character set utf8 default collate utf8_general_ci;
show databases;
use dataexa-insight-microservice-poc;

source生效 执行完整路径下的sql脚本

source /dataexa-insight-microservice-poc.sql
show tables;

使用docker cp方式拷贝xxl-job-poc.sql到容器根目录下

docker cp /home/dataexa/insight-deploy/resources/xxl/xxl-job-poc.sql mysql:/
source /xxl-job-poc.sql

平滑方式退出docker容器

键盘: curl+ p +q 三个键位一起按出现 read escape sequence 即可平滑退出容器

---

[unzip /home/dataexa/insight-deploy/resources/html/platform.zip -d] /home/dataexa/insight-microservice/workspace/nginx_workspace/html/

需要先安装unzip

unzip [缺unzip离线包]
sudo rpm -ivh /home/dataexa/insight-deploy/resources/tools_package/offline_installer/centos/unzip-6.0-20.el7.x86_64.rpm

查看unzip是否安装成功
which unzip
/usr/bin/unzip

[粘贴复制的时候-v 没有空格]
[修改完nginx.conf]
docker restart nginx

重启前需要curl+p+q 平滑退出容器

重启elasticsearch容器

docker restart elasticsearch

离线安装bzip2

sudo rpm -ivh insight-deploy/resources/tools_package/offline_installer/centos/bzip2-1.0.6-13.el7.x86_64.rpm

检查 bzip2 是否安装

which bzip2
/usr/bin/bzip2

conda使用的python3.5.tar.gz包变成了zip包
[修正命令]
unzip /home/dataexa/insight-deploy/resources/conda/python/python3.5.zip -d /home/dataexa/anaconda3/envs

---

增加创建路径[python监控路径]

mkdir -p /home/dataexa/insight-microservice/workspace/python_service_workspace/container_monitor
mkdir -p /home/dataexa/insight-microservice/workspace/python_service_workspace/dlv5

---

vim 补充 [待修改] vim 不能用

sudo cp /home/dataexa/insight-deploy/resources/tools_package/offline_installer/ubuntu/vim.zip
sudo cd /usr/local
sudo unzip vim.zip
sudo ln -s /usr/local/vim/bin/vim /usr/local/bin/vim

查看vim是否安装

sudo which vim

python [修正]
dlv5 修改[增加nohup python]
nohup python /home/dataexa/insight-microservice/workspace/python_service_workspace/dlv5/insight-v5service/tornado_service.py >> /home/dataexa/insight-microservice/workspace/python_service_workspace/dlv5/insight-v5service/nohup.out 2>&1 &

insight-tagging [创建目录,cp之前没目录]
mkdir -p /home/dataexa/insight-microservice/workspace/python_service_workspace/data_labeling
cd /home/dataexa/insight-microservice/workspace/python_service_workspace/data_labeling/insight-tagging/

启动之前，先把缓存清除

find . -name "pycache" |xargs rm -r

启动命令

nohup python tornado_service.py >/dev/null 2>&1 &

解压前创建spark⼯作⽬录

mkdir -p /home/dataexa/insight-microservice/workspace/spark_workspace

解压前创建hadoop⼯作⽬录

mkdir -p /home/dataexa/insight-microservice/workspace/hadoop_workspace/

解压前创建 yarn-conf⼯作⽬录

mkdir -p /home/dataexa/insight-microservice/workspace/hadoopworkspace/yarn-conf/

Hadoop部署[修正环境变量] 修改jdk和hadoop环境变量都设置到dataexa用户下
vi ~/.bashrc
export JAVA_HOME=/home/dataexa/insight-deploy/resources/jdk/jdk1.8.0_231
export PATH=$JAVA_HOME/bin:$PATH
export HADOOP_HOME=/home/dataexa/insight-deploy/resources/hadoop/hadoop-2.9.0
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

vi etc/hadoop/hadoop-env.sh
export JAVA_HOME=/home/dataexa/insight-deploy/resources/jdk/jdk1.8.0_231
export PATH=$JAVA_HOME/bin:$PATH

如果出现格式化文件系统 hdfs namenode -format 出错

cd /tmp
将标识有hadoop文件名称的hadoop相关文件 备份移动到另一个目录下
重新格式化文件系统

出现20/05/21 07:23:26 INFO ipc.Client: Retrying connect to server: 0.0.0.0/0.0.0.0:8032. Already tried 0 time(s); retry policy is RetryUpT

sbin/start-yarn.sh #启动yarn即可

出现org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory hdfs://localhost:9000/user/dataexa/output already exists

hdfs dfs -rm -r output
再重新执行即可

完整hadoop部署
[修改了jdk和hadoop的环境变量到dataexa用户下]

解压

tar xf /home/dataexa/insight-deploy/resources/jdk/jdk-8u231-linux-x64.tar.gz -C /home/dataexa/insight-deploy/resources/jdk

添加变量环境

vi ~/.bashrc
export JAVA_HOME=/home/dataexa/insight-deploy/resources/jdk/jdk1.8.0_231
export PATH=$JAVA_HOME/bin:$PATH
export HADOOP_HOME=/home/dataexa/insight-deploy/resources/hadoop/hadoop-2.9.0
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

生效dataexa下的用户环境

source ~/.bashrc

进入hadoop环境

cd /home/dataexa/insight-deploy/resources/hadoop/hadoop-2.9.0
vi etc/hadoop/hadoop-env.sh
export JAVA_HOME=/home/dataexa/insight-deploy/resources/jdk/jdk1.8.0_231
export PATH=$JAVA_HOME/bin:$PATH

在Hadoop的安装目录下，创建input目录

mkdir input

拷贝input文件到input目录下

cp etc/hadoop/*.xml input

执行Hadoop job

hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.9.0.jar grep input output 'dfs[a-z.]+'
上面的job是使用hadoop自带的样例，在input中统计含有dfs的字符串。

确认执行结果

cat output/*

添加主机地址和映射

sudo vi /etc/hosts
192.168.1.237 localhost

修改设定文件

vi etc/hadoop/core-site.xml


fs.defaultFS
hdfs://localhost:9000

vi etc/hadoop/hdfs-site.xml


dfs.replication
1

设置本机⽆密码ssh登陆

ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys

测试

ssh dataexa@192.168.1.237

格式化文件系统

hdfs namenode -format

启动名称节点和数据节点后台进程

sbin/start-dfs.sh

确认

jps

访问NameNode的web页面

http://192.168.1.237:50070/

创建HDFS

hdfs dfs -mkdir /user/test

拷贝input文件到HDFS目录下

hdfs dfs -put etc/hadoop /user/test/input

确认，查看

hadoop fs -ls /user/test/input

执行Hadoop job

hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.9.0.jar grep /user/test/input output 'dfs[a-z.]+'

确认执行结果

hdfs dfs -cat output/*

或者从HDFS拷贝到本地查看

bin/hdfs dfs -get output output
cat output/*

停止daemon

sbin/stop-dfs.sh

执行YARN job

修改设定文件
cp etc/hadoop/mapred-site.xml.template etc/hadoop/mapred-site.xml
vi etc/hadoop/mapred-site.xml


mapreduce.framework.name
yarn

vi etc/hadoop/yarn-site.xml


yarn.nodemanager.aux-services
mapreduce_shuffle

启动ResourceManger和NodeManager后台进程

sbin/start-yarn.sh

访问ResourceManger的web页面

http://192.168.1.237:8088/

＃执行hadoop job
hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.９.0.jar grep /user/test/input output 'dfs[a-z.]+'

如果这里报错,需要sbin/stop-yarn.sh再sbin/start-yarn.sh

确认执行结果

hdfs dfs -cat output/*

停止daemon

sbin/stop-yarn.sh

启动jobhistory daemon

sbin/mr-jobhistory-daemon.sh start historyserver

访问Job History Server的web页面 出现数据即可

http://localhost:19888/