CDH6.2安装

一、准备工作：
1、普通用户root权限准备（sudo，此处用aiot用户，sudoer）
groupadd -r test
useradd -d /home/test/ -m -s /bin/bash -c "user" -g test -p 2019 -r test
#另外特别注意：aiot及密码用户在cloudera manager server中会配置进去，作为集群管理【host添加、agent安装&卸载、parcels管理os user】，
所以不能轻易在os修改aiot用户密码，如果需要修改，运维同学需要通知cdh集群管理员同步修改cms保存的密码）
passwd test
root： vim /etc/sudoers
test ALL=(ALL) NOPASSWD: ALL
2.必备工具
sudo yum install yum-utils createrepo -y
sudo yum install ansible -y
sudo yum install httpd -y
sudo yum install systemctl -y
sudo yum install ntp -y
sudo yum install hostnamectl -y
~~~~~~~~~~~~~~~~~~~

3.集群 /etc/hosts设置
sudo vim /etc/hosts
cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.0.3 cdh3
192.168.0.4   cdh4
192.168.0.5  cdh5

#修改主机名
sudo vim /etc/sysconfig/network：
NETWORKING=yes
HOSTNAME=cdh3

reboot

4.关闭防火墙
getenforce
sudo systemctl stop firewalld
sudo systemctl disable firewalld.service
sudo vim /etc/selinux/config 

-------------------------------

SELINUX=disabled

-------------------------------

5 ssh 互信 （所有节点）test用户
$ mkdir .ssh
$ chmod 750 .ssh
$ ssh-keygen -t rsa
##192.168.0.3操作
$cat /home/aiot/.ssh/id_rsa.pub >> /home/aiot/.ssh/authorized_keys
$ chmod 600 ~/.ssh/authorized_keys

所有节点测试互信测试

6.配置NTP服务（所有节点）两种方法二选一

sudo yum install -y ntp
方法一时间同步主节点

----------------------------------------

主节点配置(192.168.0.3 )
sudo vim /etc/ntp.conf
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
server ntp.aliyun.com
从节点配置(192.168.0.4 192.168.0.5)
sudo vim /etc/ntp.conf
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
server 192.168.0.3  cdh3

----------------------------------------------

sudo systemctl enable ntpd
sudo systemctl start ntpd

6.所有节点禁用大透明页 和vm.swappiness设置
sysctl vm.swappiness=10
vim /etc/sysctl.conf

vm.swappiness=10

echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
上面两条加入/etc/rc.local开机自启
vim /etc/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
chmod +x /etc/rc.d/rc.local

7.JDK安装（所有节点）
sudo rpm -qa | grep java # 查询已安装的java
sudo yum remove java* # 卸载
sudo mkdir -p /usr/java
sudo tar -zxf /tmp/installer/jdk-8u181-linux-x64.tar.gz -C /usr/java/
sudo ln -s /usr/java/jdk1.8.0_181 /usr/java/jdk
sudo vim /etc/profile

export JAVA_HOME=/usr/java/jdk
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib:$CLASSPATH

sudo source /etc/profile
#如果不是root用户 需要添加
~/.bashrc
~/.bash_profile

检查环境变量 echo $JAVA_HOME

所有节点以下操作
sudo mkdir -p /usr/share/java
sudo cp /data/mysql-connector-java-5.1.46.jar /usr/share/java/mysql-connector-java.jar

二#mysql5.7.20安装
sudo rpm -qa | grep mariadb
sudo rpm -e mariadb-libs-5.5.60-1.el7_5.x86_64 --nodeps
sudo yum -y install libaio
#官方下载
#sudo wget https://dev.mysql.com/get/Downloads/MySQL-5.7/mysql-5.7.20-1.el7.x86_64.rpm-bundle.tar
#解压
#sudo tar -xvf mysql-5.7.20-1.el7.x86_64.rpm-bundle.tar
#本地rpm安装
sudo rpm -ivh mysql-community-common-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-libs-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-libs-compat-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-client-5.7.20-1.el7.x86_64.rpm
sudo rpm -ivh mysql-community-server-5.7.20-1.el7.x86_64.rpm
#如果指定了mysql用户需要授权给mysql
没有/data/mysqldata/mysql目录
需要
sudo mkdir -p /data/mysqldata/mysql
sudo vim /etc/my.cnf:
--------------------------------
[mysqld]
datadir=/data/mysqldata/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation=READ-COMMITTED
symbolic-links=0
key_buffer_size=32M
max_allowed_packet=32M
thread_stack=256K
thread_cache_size=64
query_cache_limit=8M
query_cache_size=64M
query_cache_type=1
max_connections=550
log_bin=/data/mysqldata/mysql/mysql_binary_log
server_id=1
binlog_format=mixed
read_buffer_size=2M
read_rnd_buffer_size=16M
sort_buffer_size=8M
join_buffer_size=8M
innodb_file_per_table=1
innodb_flush_log_at_trx_commit=2
innodb_log_buffer_size=64M
innodb_buffer_pool_size=4G
innodb_thread_concurrency=8
innodb_flush_method=O_DIRECT
innodb_log_file_size=512M
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
sql_mode=STRICT_ALL_TABLES
-----------------------------
sudo systemctl enable mysqld
#sudo systemctl stop mysqld
sudo systemctl start mysqld
sudo systemctl status mysqld -l
sudo grep 'temporary password' /var/log/mysqld.log
#过滤出初始密码 然后登陆
mysql -u root -p
#修改用户密码用下面这条
ALTER USER USER() IDENTIFIED BY '2019';
#必要的库和用户、权限
grant all privileges on *.* to 'root'@'%' identified by '2019' with grant option;
create database scm DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database nas DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database actmonitor DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database rptmgn DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database navmetas DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database sentry DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

grant all privileges on *.* to 'admin'@'%' identified by '2019' with grant option;

grant all privileges on scm.* to 'scm'@'%' identified by '2019' with grant option;
grant all privileges on hive.* to 'hive'@'%' identified by '2019' with grant option;
grant all privileges on oozie.* to 'oozie'@'%' identified by '2019' with grant option;
grant all privileges on hue.* to 'hue'@'%' identified by '2019' with grant option;
grant all privileges on nas.* to 'nas'@'%' identified by '2019' with grant option;
grant all privileges on actmonitor.* to 'actmonitor'@'%' identified by '2019' with grant option;
grant all privileges on rptmgn.* to 'rptmgn'@'%' identified by '2019' with grant option;
grant all privileges on navmetas.* to 'navmetas'@'%' identified by '2019' with grant option;
grant all privileges on sentry.* to 'sentry'@'%' identified by '2019' with grant option;
flush privileges;

安装CMS 主节点 192.168.0.3
准备: CMS、CDH-Parcel internal REPO搭建 [6.2.0为例]
1）sudo yum install httpd
2）sudo vim /etc/httpd/conf/httpd.conf添加如下：
AddType application/x-gzip .gz .tgz .parcel
3)sudo systemctl start httpd
sudo systemctl enable httpd
#创建本地yum源仓库 实现本地yum安装
4)sudo mkdir -p /var/www/html/cloudera-repos
####5-10步骤没有加速特别慢 自己下的时候 发现是在dump网站 建议拷贝 可以删除里面其它不用的包
###换为下面的单独下载包5）sudo proxychains4 wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/ -P /var/www/html/cloudera-repos
####5 centos选择CDH-6.2.0-1.cdh6.2.0.p0.967373-el7开头的三个下载即可

sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha1 -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha256 -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/parcels/manifest.json -P /var/www/html/cloudera-repos

6）sudo proxychains4 wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/gplextras6/6.2.0/parcels/ -P /var/www/html/cloudera-repos
######6 centos选择GPLEXTRAS-6.2.0-1.gplextras6.2.0.p0.967373-el7开头的三个下载即可
7）sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cm6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
8）sudo wget https://archive.cloudera.com/cm6/6.2.0/allkeys.asc -P /var/www/html/cloudera-repos/cm6/6.2.0/
9）sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/kafka/parcels/4/ -P /var/www/html/cloudera-repos
10）sudo wget --recursive --no-parent --no-host-directories http://archive.cloudera.com/sqoop-connectors/parcels/latest/ -P /var/www/html/cloudera-repos
11)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cdh6
12)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/gplextras6
13)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cm6
14)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/kafka
15)sudo chmod -R ugo+rX /var/www/html/cloudera-repos/sqoop-connectors
16）curl http://localhost/cloudera-repos/
17)sudo vim /etc/yum.repos.d/cloudera-manager6.2.0.repo

[cloudera-manager]
name=Cloudera Manager 6.2.0
baseurl=http://本机IP或者本地hosts解析/cloudera-repos/cm6/6.2.0/redhat7/yum/
gpgkey=http://本机IP或者本地hosts解析/cloudera-repos/cm6/6.2.0/redhat7/yum/RPM-GPG-KEY-cloudera
gpgcheck=1
enabled=1
autorefresh=0
type=rpm-md

17.2)sudo vim /etc/yum.repos.d/cloudera-manager-gplextras6.2.0.repo

#两模块不要一样
[cloudera-manager-gplextras]
name=Cloudera Manager gplextras 6.2.0
baseurl=http://本机IP或者本地hosts解析/cloudera-repos/gplextras6/6.2.0/redhat7/yum/
gpgkey=http://本机IP或者本地hosts解析/cloudera-repos/gplextras6/6.2.0/redhat7/yum/RPM-GPG-KEY-cloudera
gpgcheck=1
enabled=1
autorefresh=0
type=rpm-md

18)sudo vim /etc/httpd/conf/httpd.conf

<Directory "/var/www/cgi-bin">
AllowOverride None
Options FollowSymLinks
Order deny,allow
allow from all
</Directory>

19)sudo systemctl restart httpd
20)sudo yum clean all
21)sudo yum makecache
#执行 yum makecache 报错 缺少/repodata/repomd.xml: [Errno 14] HTTP Error 404 - Not Found
#解决办法
#proxychains4 wget --recursive --no-parent --no-host-directories http://archive.cloudera.com/gplextras6/6.2.0/redhat7/yum/repodata/ -P /var/www/html/cloudera-repos
22)sudo yum update
23)sudo yum -y search cloudera-manager
24)sudo systemctl enable httpd

三）6.2.0 internal repo
#如果要求全部内网下载可以把主节点的repo直接ansible推过去[主节点一定要执行从节点可选吧 查资料上说为避免尽可能的减少出错 三节点都提前安装好但只需要主节点启动server端]

sudo yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
echo $JAVA_HOME

#只需要主节点执行------
sudo JAVA_HOME=/usr/java/jdk /opt/cloudera/cm-agent/bin/certmanager --location /opt/cloudera/CMCA setup --configure-services
sudo -E /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm '2019'