cdh-完整

安装包

CLOUDERA管理安装包

http://archive.cloudera.com/cm5/cm/5/

http://archive.cloudera.com/cm5/cm/5/cloudera-manager-centos7-cm5.13.3_x86_64.tar.gz

rpm包位置
https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.13.3/

CDH版本包

CDH-5.13.3-1.cdh5.13.3.p0.2-el7.parcel

CDH-5.13.3-1.cdh5.13.3.p0.2-el7.parcel.sha1,

manifest.json

http://archive.cloudera.com/cdh5/parcels/5.13.3/

http://archive.cloudera.com/cdh5/parcels/5.13.3/CDH-5.13.3-1.cdh5.13.3.p0.2-el7.parcel

http://archive.cloudera.com/cdh5/parcels/5.13.3/CDH-5.13.3-1.cdh5.13.3.p0.2-el7.parcel.sha1

http://archive.cloudera.com/cdh5/parcels/5.13.3/manifest.json

CDH组件安装包

cdh5.13.3-centos7.tar.gz

https://archive.cloudera.com/cdh5/repo-as-tarball/5.13.3/

https://archive.cloudera.com/cdh5/repo-as-tarball/5.13.3/cdh5.13.3-centos7.tar.gz

Java链接Mysql驱动

mysql-connector-java.jar

离线tarballs 安装

• 需要下载的安装包
  • cloudera-manager-centos7-cm5.13.0_x86_64.tar.gz
  • CDH相关（注意下载相应系统的安装包）
    • CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel
    • CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha1
    • manifest.json（manifest.json需要网页上打开后复制全部内容，在桌面新建txt文件，粘贴复制的内容后修改后缀名为.json）
• • jdk-8u144-linux-x64.tar.gz
  • mysql-5.7.21-1.el7.x86_64.rpm-bundle.tar
  • mysql-connector-java-5.1.47.tar

1，环境

主机名

/etc/hosts

hostnamectl  set-hostname  foo-1.xxx.com 
hostnamectl  set-hostname  foo-2.xxx.com
hostnamectl  set-hostname  foo-2.xxx.com

hostnamectl  set-hostname   cdh-1.xxx.com
hostnamectl  set-hostname   cdh-2.xxx.com

192.168.2.150  cdh-1.xxx.com  cdh-1
192.168.2.117  cdh-2.xxx.com  cdh-2

192.168.1.142   foo-1.xxx.com  foo-1
192.168.1.143   foo-2.xxx.com  foo-2

swappiness

/proc/sys/vm/swappiness 设置为 10
 
echo never > /sys/kernel/mm/transparent_hugepage/defrag

selinux

vi /etc/selinux/config
sed  -i 's/SELINUX=enforcing/SELINUX=disabled/'  /etc/selinux/confi
getenforce 

防火墙

systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld

ssh 免密

ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
在一台机器上自己ssh 自己，成功后将.ssh目录拷贝到其他机器

ntp

主节点
grep  -v   "^#"   /etc/ntp.conf |   grep  -v "^$"
cat >  /etc/ntp.conf <<EOF
driftfile /var/lib/ntp/drift
restrict default nomodify notrap nopeer noquery
restrict 127.0.0.1 
restrict ::1
restrict  192.168.2.0  mask 255.255.255.0 nomodify notrap
server 127.127.1.0
Fudge 127.127.1.0 stratum 1
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor
EOF

客户端
grep  -v   "^#"   /etc/ntp.conf |   grep  -v "^$"
cat >  /etc/ntp.conf <<EOF
driftfile /var/lib/ntp/drift
restrict default nomodify notrap nopeer noquery
####restrict 192.168.1.135 nomodify notrap noquery  这个不知道要不要
restrict 127.0.0.1 
restrict ::1
restrict 192.168.2.0  mask 255.255.255.0 nomodify notrap
server 192.168.2.150 profer
Fudge  192.168.2.150 stratum 10
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor
EOF
开机自动启动
 systemctl enable ntpd.service
 systemctl  start ntpd.service
 手动同步
 ntpdate -u  cdh-01
  ntpq -p 

查看ntp运行状态

    ntpstat

jdk（所有节点）

卸载自带的jdk-open
rpm -qa | grep java
rpm -qa | grep jdk - yum -y remove xxjdk   #删除所有的jdk
yum remove -y *jdk*
yum remove -y *java*

rpm  -ivh jdk-8u202-linux-x64.rpm

数据库安装

卸载原有的mariadb 
rpm -qa | grep mariadb #查看安装信息
rpm --nodeps -e  `rpm -qa | grep mariadb`
安装数据库顺序  最好解压安装
tar -xf mysql-5.7.21-1.el7.x86_64.rpm-bundle.tar    
首先安装（其他节点也需要安装，同时也安装 mysql-community-libs-compat-5.7.21-1.el7.x86_64.rpm ）
yum install net-tools #安装net-tools包
yum install perl #安装perl包


安装顺序
rpm -ivh mysql-community-common-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-server-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-compat-5.7.21-1.el7.x86_64.rpm(compat解决hue数据库连接不上)   比较重要(其他节点都要安装)
rpm -ivh mysql-community-devel-5.7.21-1.el7.x86_64.rpm                                   #安装mysqlclient需要用到

yum  install  mysql-community-common
yum  install  mysql-community-libs
yum  install  mysql-community-client
yum  install  mysql-community-libs-compat
yum  install  mysql-community-server
yum  install  mysql-community-devel


yum remove  mysql-community*
yum  install  -y  perl 
tar  xvf  mysql-5*

rpm -ivh mysql-community-common-*
rpm  -ivh mysql-community-libs-5.*
rpm -ivh rpm -ivh mysql-community-client*
rpm -ivh mysql-community-server-*
rpm -ivh mysql-community-devel-5*

cat /var/log/mysqld.log |grep password





数据库默认配置
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0

key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1

max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M

#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log

#In later versions of MySQL, if you enable the binary log and do not set
#a server_id, MySQL will not start. The server_id must be unique within
#the replicating group.
server_id=1

binlog_format = mixed

read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M

# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit  = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M

[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid

sql_mode=STRICT_ALL_TABLES




启动mysql 
systemctl start mysqld
systemctl enable mysqld
systemctl daemon-reload

查看root初始密码
cat /var/log/mysqld.log |grep password
登录mysql数据库
mysql -uroot –p密码 # 密码为上一步看到的
首先，修改validate_password_policy参数的值


set global validate_password_policy=0;
set global validate_password_length=2;
set global validate_password_special_char_count=0;
set global validate_password_mixed_case_count=0;
set global validate_password_number_count=0;
set password = password('xxxx@123.com');

查看密码策略:
SHOW VARIABLES LIKE 'validate_password%';


授权用户远程连接
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'xxxx@123.com' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON *.* TO 'cdh'@'%' IDENTIFIED BY 'xxxx@123.com' WITH GRANT OPTION;



set global validate_password_policy=0;
set global validate_password_length=2;
set global validate_password_special_char_count=0;
set global validate_password_mixed_case_count=0;
set global validate_password_number_count=0;

grant all privileges on *.* to 'scm'@'localhost' identified by 'scm' with grant option;
grant all privileges on *.* to 'scm'@'127.0.0.1' identified by 'scm' with grant option;
grant all privileges on *.* to 'scm'@'%' identified by 'scm' with grant option;


grant all privileges on *.* to 'root'@'localhost' identified by 'xxxx@123.com' with grant option;
grant all privileges on *.* to 'root'@'127.0.0.1' identified by 'xxxx@123.com'  with grant option;
grant all privileges on *.* to 'root'@'%' identified by 'xxxx@123.com'  with grant option;


grant all privileges on *.* to 'cdh'@'localhost' identified by 'xxxx@123.com' with grant option;
grant all privileges on *.* to 'cdh'@'127.0.0.1' identified by 'xxxx@123.com'  with grant option;
grant all privileges on *.* to 'cdh'@'%' identified by 'xxxx@123.com'  with grant option;


flush privileges;


查看用户权限:  mysql> show grants for test;

创建数据库：

CREATE DATABASE IF NOT EXISTS `dataight` DEFAULT CHARSET utf8 COLLATE utf8_general_ci;



创建需要的数据库：
create database metastore default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database scm default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database amon default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database rmon default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database hue default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database sentry default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database nav default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database navms default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database oozie default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database hive default character set utf8  DEFAULT COLLATE utf8_general_ci;





 
（不用做） 在mysql连接工具下执行（否则可能会报错）
create database metastore default character set utf8;
CREATE USER 'hive'@'%' IDENTIFIED BY 'hive';
GRANT ALL PRIVILEGES ON metastore. * TO 'hive'@'%';
create database hive default character set utf8;
GRANT ALL PRIVILEGES ON hive. * TO 'hive'@'%';
create database cm default character set utf8;
CREATE USER 'cm'@'%' IDENTIFIED BY 'cm';
GRANT ALL PRIVILEGES ON cm.* TO 'cm'@'%';
create database am default character set utf8;
CREATE USER 'am'@'%' IDENTIFIED BY 'am';
GRANT ALL PRIVILEGES ON am. * TO 'am'@'%';
create database rm default character set utf8;
CREATE USER 'rm'@'%' IDENTIFIED BY 'rm';
GRANT ALL PRIVILEGES ON rm. * TO 'rm'@'%';
create database hue default character set utf8;
CREATE USER 'hue'@'%' IDENTIFIED BY 'hue';
GRANT ALL PRIVILEGES ON hue. * TO 'hue'@'%';
create database oozie default character set utf8;
CREATE USER 'oozie'@'%' IDENTIFIED BY 'oozie';
GRANT ALL PRIVILEGES ON oozie. * TO 'oozie'@'%';
flush privileges;

安装CM

• 将下载好的cloudera-manager-centos7-cm5.13.0_x86_64.tar.gz解压到主节点的/opt/,解压出来的文件夹为cloudera与cm-5.13.0

• 为Cloudera Manager建立数据库，此处需要前边下载好的mysql-connector-java-5.1.47-bin.jar

  • cp /root/software/mysql-connector-java-5.1.47/mysql-connector-java-5.1.47-bin.jar /usr/java/
  • cp /root/software/mysql-connector-java-5.1.47/mysql-connector-java-5.1.47-bin.jar /opt/cm-5.13.0/share/cmf/lib/

  主节点执行 执行前确认用户创建无问题(用户创建可能遇到密码策略不符合)

  初始化数据库根据数据库的位置不同而不同：下面为数据库与cm 在统一主机

  /opt/cm-5.13.0/share/cmf/schema/scm_prepare_database.sh mysql -uroot -p123 scm scm scm

配置Agent（统一复制到其他节点，省的重新做了）

vim /opt/cm-5.13.0/etc/cloudera-scm-agent/config.ini
server_host=cdh01

拷贝文件到其他子节点

scp -r /opt/cm-5.13.0/ root@hywx189:/opt/
scp -r /opt/cm-5.13.0/ root@hywx189:/opt/

所有节点上创建用户　 注意家目录的不同/opt/cm-5.13.0/run/cloudera-scm-server/

初始化前将java-mysql 驱动安装到每个节点的给定位置：（见下文测试数据库）

useradd --system --home=/opt/cm-5.13.0/run/cloudera-scm-server --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

useradd --system --home=/opt/cm-5.13.3/run/cloudera-scm-server --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

配置存储目录（主节点貌似）

$ sudo mkdir /var/lib/cloudera-scm-server
$ sudo chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-server

准备parcels，将CDH相关文件拷贝到主节点/opt/cloudera/parcel-repo/*

注：需要将CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha1改成CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha，否则安装时会重新下载，很重要！！

mv CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha1 CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha

启动

##主节点
/opt/cm-5.13.0/etc/init.d/cloudera-scm-server start

#主节点及其他节点
/opt/cm-5.13.0/etc/init.d/cloudera-scm-agent start

启动需要点时间，可以查看数据库scm中的hosts表，如果里面出现节点信息表示server启动完成

启动成功查看方式：

find / -name cloudera-scm-server.log

tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log

grep "Started Jetty server" cloudera-scm-server.log

安装Hadoop集群

• 启动成功后 浏览器访问：

  • http://主节点ip:7180 （用户名、密码：admin）

• 第一个警告直接在所有节点上执行

  • echo 10 > /proc/sys/vm/swappiness

• 第二个警告按照上面的说明，在所有节点上执行这两条命令

  • echo never >/sys/kernel/mm/transparent_hugepage/defrag >> /etc/rc.local
  • echo never > /sys/kernel/mm/transparent_hugepage/enabled>> /etc/rc.local
  • 同时同一命令添加到 /etc/rc.local 等初始化脚本中，以便在系统重启时予以设置。

  测试数据库时

此处hive ，oozie会安装失败，失败信息如下。因为我们使用了MySQL作为hive元数据存储，需要将mysql驱动拷贝到hive和oozie目录下

解决方法（需要在所有节点上执行）

• cp /opt/cm-5.13.0/share/cmf/lib/mysql-connector-java-5.1.47-bin.jar /opt/cloudera/parcels/CDH-5.13.0-1.cdh5.13.0.p0.29/lib/hive/lib/
• cp /opt/cm-5.13.0/share/cmf/lib/mysql-connector-java-5.1.47-bin.jar /var/lib/oozie/

cp /opt/cm-5.7.1/share/cmf/lib/mysql-connector-java-*-bin.jar /opt/cloudera/parcels/CDH-5.7.1-1.cdh5.7.1.p0.11/lib/hive/lib/

cp /opt/cm-5.7.1/share/cmf/lib/mysql-connector-java-*-bin.jar /opt/cloudera/parcels/CDH-5.7.1-1.cdh5.7.1.p0.11/lib/oozie/lib/

cp /opt/cm-5.7.1/share/cmf/lib/mysql-connector-java-*-bin.jar /var/lib/oozie/

初始化  yum安装的scm_prepare_database.sh 位置
 cd    /usr/share/cmf/lib/
 mv   /usr/share/java/mysql-connector-java-5.1.46-bin.jar  .
 
 安装后节点也需要安装
 cd    /usr/share/cmf/lib/
 mv   /usr/share/java/mysql-connector-java-5.1.46-bin.jar  .
 
 
 安装MySQL JDBC驱动程序
从下载的文件中提取JDBC驱动程序JAR文件。例如：
tar zxvf mysql-connector-java-5.1.46.tar.gz
将重命名的JDBC驱动程序复制到 /usr/share/java/。如果目标目录尚不存在，请创建它。例如：
sudo mkdir -p /usr/share/java/ 
cd mysql-connector-java-5.1.46 
sudo cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar  
sudo cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java-5.1.46-bin.jar 

 


cdh 也需要驱动
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib
 
 hive 数据库也需要
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/oozie/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hue/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/var/lib/oozie
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/var/lib/hue

另外如果想添加新的主机，必须要先启动新主机上的agent服务，否则新主机不能被发现，要重新下载cm-agent，很麻烦。安装新的节点以后，记得运行"部署客户端"命令。

yum安装cm

主机名

/etc/hosts

hostnamectl  set-hostname  foo-1.data.com 
hostnamectl  set-hostname  foo-2.data.com
hostnamectl  set-hostname  foo-2.data.com

hostnamectl  set-hostname   cdh-1.data.com
hostnamectl  set-hostname   cdh-2.data.com
cat >> /etc/hosts <<EOF
192.168.2.150  cdh-1.data.com  cdh-1
192.168.2.117  cdh-2.data.com  cdh-2
EOF

192.168.1.142   foo-1.data.com  foo-1
192.168.1.143   foo-2.data.com  foo-2

swappiness

/proc/sys/vm/swappiness 设置为 10
 
echo never > /sys/kernel/mm/transparent_hugepage/defrag

selinux

vi /etc/selinux/config
sed  -i 's/SELINUX=enforcing/SELINUX=disabled/'  /etc/selinux/confi
getenforce 

防火墙

systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld

ssh 免密

ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
在一台机器上自己ssh 自己，成功后将.ssh目录拷贝到其他机器

repo

本地源

cat >   /etc/yum.repos.d/local.repo  <<EOF
[localrepo]
name=localrepo
baseurl=file:///cm-5.13.3/
#gpgkey=http://192.168.2.113/cm-5.13.3/RPM-GPG-KEY-cloudera    
enabled = 1
gpgcheck = 0
EOF

远程源

rm -rf  /etc/yum.repos.d/*
cat >   /etc/yum.repos.d/cloudera-manager.repo  <<EOF
[clouderamanager]
name=Cloudera Manager
baseurl=http://192.168.2.113/cm-5.13.3/
gpgkey=http://192.168.2.113/cm-5.13.3/RPM-GPG-KEY-cloudera    
enabled = 1
gpgcheck = 0
EOF


 yum  clean  all  && yum makecache

ntp

主节点
yum -y install  ntp

grep  -v   "^#"   /etc/ntp.conf |   grep  -v "^$"

cat >  /etc/ntp.conf <<EOF
driftfile /var/lib/ntp/drift
restrict default nomodify notrap nopeer noquery
restrict 127.0.0.1 
restrict ::1
restrict  192.168.2.0  mask 255.255.255.0 nomodify notrap
server 127.127.1.0
Fudge 127.127.1.0 stratum 1
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor
EOF

客户端
grep  -v   "^#"   /etc/ntp.conf |   grep  -v "^$"
cat >  /etc/ntp.conf <<EOF
driftfile /var/lib/ntp/drift
restrict default nomodify notrap nopeer noquery
####restrict 192.168.1.135 nomodify notrap noquery  这个不知道要不要
restrict 127.0.0.1 
restrict ::1
restrict 192.168.2.0  mask 255.255.255.0 nomodify notrap
server 192.168.2.150 profer
Fudge  192.168.2.150 stratum 10
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor
EOF
开机自动启动
 systemctl enable ntpd.service
 systemctl  start ntpd.service
 手动同步
 ntpdate -u  cdh-1
  ntpq -p 

查看ntp运行状态

    ntpstat

jdk（所有节点）

卸载自带的jdk-open
rpm -qa | grep java
rpm -qa | grep jdk - yum -y remove xxjdk   #删除所有的jdk
yum remove -y *jdk*
yum remove -y *java*

rpm  -ivh jdk-8u202-linux-x64.rpm

yum  install   java -y 

数据库安装

卸载原有的mariadb 
rpm -qa | grep mariadb #查看安装信息
rpm --nodeps -e  `rpm -qa | grep mariadb`
安装数据库顺序  最好解压安装
tar -xf mysql-5.7.21-1.el7.x86_64.rpm-bundle.tar    
首先安装（其他节点也需要安装，同时也安装 mysql-community-libs-compat-5.7.21-1.el7.x86_64.rpm ）

yum install net-tools  -y  #安装net-tools包
yum install perl -y        #安装perl包


安装顺序
rpm -ivh mysql-community-common-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-server-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-compat-5.7.21-1.el7.x86_64.rpm(compat解决hue数据库连接不上)   比较重要(其他节点都要安装)
rpm -ivh mysql-community-devel-5.7.21-1.el7.x86_64.rpm                                   #安装mysqlclient需要用到

yum  install  mysql-community-common  -y 
yum  install  mysql-community-libs  -y 
yum  install  mysql-community-client   -y  
yum  install  mysql-community-libs-compat  -y 
yum  install  mysql-community-server   -y 
yum  install  mysql-community-devel  -y 



数据库默认配置
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0

key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1

max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M

#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log

#In later versions of MySQL, if you enable the binary log and do not set
#a server_id, MySQL will not start. The server_id must be unique within
#the replicating group.
server_id=1

binlog_format = mixed

read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M

# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit  = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M

[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid

sql_mode=STRICT_ALL_TABLES




启动mysql 
systemctl start mysqld
systemctl enable mysqld
systemctl daemon-reload

查看root初始密码
cat /var/log/mysqld.log |grep password

# vi /etc/my.cnf 
在[mysqld]的段中加上一句：skip-grant-tables 
例如： 
[mysqld] 
datadir=/var/lib/mysql 
socket=/var/lib/mysql/mysql.sock 
skip-grant-tables 


USE mysql ; 
ALTER USER 'root'@'localhost' IDENTIFIED BY 'xxxx@123.com';
FLUSH PRIVILEGES;


mysql -uroot -pxxxx@123.com

登录mysql数据库
mysql -uroot –p密码 # 密码为上一步看到的
首先，修改validate_password_policy参数的值


set global validate_password_policy=0;
set global validate_password_length=2;
set global validate_password_special_char_count=0;
set global validate_password_mixed_case_count=0;
set global validate_password_number_count=0;
set password = password('xxxx@123.com');

查看密码策略:
SHOW VARIABLES LIKE 'validate_password%';


授权用户远程连接
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'xxxx@123.com' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON *.* TO 'cdh'@'%' IDENTIFIED BY 'xxxx@123.com' WITH GRANT OPTION;



set global validate_password_policy=0;
set global validate_password_length=2;
set global validate_password_special_char_count=0;
set global validate_password_mixed_case_count=0;
set global validate_password_number_count=0;

grant all privileges on *.* to 'scm'@'localhost' identified by 'scm' with grant option;
grant all privileges on *.* to 'scm'@'127.0.0.1' identified by 'scm' with grant option;
grant all privileges on *.* to 'scm'@'%' identified by 'scm' with grant option;


grant all privileges on *.* to 'root'@'localhost' identified by 'xxxx@123.com' with grant option;
grant all privileges on *.* to 'root'@'127.0.0.1' identified by 'xxxx@123.com'  with grant option;
grant all privileges on *.* to 'root'@'%' identified by 'xxxx@123.com'  with grant option;


grant all privileges on *.* to 'cdh'@'localhost' identified by 'xxxx@123.com' with grant option;
grant all privileges on *.* to 'cdh'@'127.0.0.1' identified by 'xxxx@123.com'  with grant option;
grant all privileges on *.* to 'cdh'@'%' identified by 'xxxx@123.com'  with grant option;


flush privileges;





创建需要的数据库：
create database metastore default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database scm default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database amon default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database rmon default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database hue default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database sentry default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database nav default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database navms default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database oozie default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database hive default character set utf8  DEFAULT COLLATE utf8_general_ci;

CM安装

[root@foo-1 yum.repos.d]# cat  cloudera-manager.repo 
[clouderamanager]
name=Cloudera Manager
baseurl=http://192.168.2.113/cm-5.13.3/
gpgkey=http://192.168.2.113/cm-5.13.3/RPM-GPG-KEY-cloudera    
enabled = 1
gpgcheck = 0

install the Cloudera Manager packages. 
在管理节点安装cloudera-manager-daemons cloudera-manager-server
#RHEL, if you have a yum repo configured
sudo yum install cloudera-manager-daemons cloudera-manager-server
sudo yum install cloudera-manager-agent


#On all hosts, run the following command to install the Cloudera Manager agent:
在子节点，安装管理客户端
sudo yum install cloudera-manager-daemons cloudera-manager-agent


*** 在所有节点修改管理代理连接管理服务器的主机名
On all hosts, configure the Cloudera Manager Agent to point to the Cloudera Manager Server by setting the following properties in the /etc/cloudera-scm-agent/config.ini configuration file:
server_host	Name of the host where Cloudera Manager Server is running.
server_port	Port on the host where Cloudera Manager Server is running.


sed -i 's/server_host=localhost/server_host=cdh-1/'  /etc/cloudera-scm-agent/config.ini 

数据库连接程序

初始化  yum安装的scm_prepare_database.sh 位置
 mkdir  /usr/share/java/  &&  cd   /usr/share/java/

curl   -O    http://192.168.2.113/mysql-connector-java/mysql-connector-java-5.1.46.tar.gz


mkdir  /usr/share/java/
cp    mysql-connector-java-5.1.46-bin.jar    mysql-connector-java.jar
chmod  755  mysql-connector-java-5.1.46-bin.jar    mysql-connector-java.jar
 
cp  mysql-connector-java-5.1.46-bin.jar    mysql-connector-java.jar     /usr/share/cmf/lib/
mkdir   /usr/share/java/



初始化数据库：
/usr/share/cmf/schema/scm_prepare_database.sh   mysql cm  -hlocalhost  -uroot   -pxxxx@123.com   --scm-host cdh-1  scm scm  scm


所有节点运行管理代理
管理节点
      systemctl start cloudera-scm-server
      systemctl start cloudera-scm-agent
 
判断是否启动成功
find  / -name   cloudera-scm-server.log
tail  -f  /var/log/cloudera-scm-server/cloudera-scm-server.log
grep "Started Jetty server"    cloudera-scm-server.log


      
子节点
systemctl start cloudera-scm-agent



安装后节点也需要安装
cd    /usr/share/cmf/lib/
mv   /usr/share/java/mysql-connector-java-5.1.46-bin.jar  .
 
cdh 也需要驱动
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib
 
 hive 数据库也需要
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/oozie/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hue/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/var/lib/oozie
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/var/lib/hue

http://192.168.2.150:7180/cmf/clusters/1/express-add-services/index#step=commandDetailsStep

问题

据说5.7版本数据开启GTID会导致cdh初始化数据库失败
在线开启步骤：

1、要求：

（1）必须是5.7.6版本以上的mysql

（2）GTID状态为OFF

2、开启步骤：

（1）：SET GLOBAL ENFORCE_GTID_CONSISTENCY = 'WARN';

（2）：SET GLOBAL ENFORCE_GTID_CONSISTENCY = 'ON';

（3）：SET GLOBAL GTID_MODE = 'OFF_PERMISSIVE';

（4）：SET GLOBAL GTID_MODE = 'ON_PERMISSIVE';

（5）：SET GLOBAL GTID_MODE = 'ON';

3、要永久启用，在my.cnf配置文件中添加参数：

gtid-mode=ON

enforce-gtid-consistency

在线关闭步骤：

1、要求：

（1）必须是5.7.6版本以上的mysql

（2）GTID状态为OFF

2、关闭步骤：

（1）：stop slave;

（2）：SET GLOBAL GTID_MODE = 'ON_PERMISSIVE';

（3）：SET GLOBAL GTID_MODE = 'OFF_PERMISSIVE';

（4）：SET GLOBAL GTID_MODE = 'OFF';

注：

每次开启和关闭时，都是这样一个过程：

打开-->过度模式-->完全打开

停止-->过度模式-->完全关闭

hadoop

离线Hadoop,注意java 的路径

#!/bin/bash

if [ -x "$(command -v /usr/local/hadoop-2.9.0/bin/hadoop)" ]; then
        echo "command hadoop already installed"
else
tar xf /home/xxxxxxx/software/hadoop-2.9.0.tar.gz -C /usr/local/
echo "export PATH=/usr/local/hadoop-2.9.0/bin:$PATH" >> /etc/profile
source /etc/profile

cd /usr/local/hadoop-2.9.0
echo "export JAVA_HOME=/usr/local/jdk"  >>  etc/hadoop/hadoop-env.sh

mkdir input

cp etc/hadoop/*.xml input

hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.9.0.jar grep input output 'dfs[a-z.]+'

cd /usr/local/hadoop-2.9.0
cat > etc/hadoop/core-site.xml << EOF
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>
EOF

cat > etc/hadoop/hdfs-site.xml << EOF
<configuration>
   <property>
       <name>dfs.replication</name>
       <value>1</value>
   </property>
</configuration>
EOF

#设定本机的无密码ssh登陆,静默方式生成
echo '---------------配置ssh免密登录----------------------'
echo "" | ssh-keygen -t rsa -P ""
echo '----------秘钥生成完成，开始生成公钥----------------'
echo '---------------请输入您当前账户的密码----------------------'
ssh-copy-id localhost


hdfs namenode -format

sbin/start-dfs.sh
hdfs dfs -mkdir /user
hdfs dfs -mkdir /user/test
hdfs dfs -put etc/hadoop /user/test/input
hadoop fs -ls /user/test/input
hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.9.0.jar grep /user/test/input output 'dfs[a-z.]+'
hdfs dfs -cat output/*
bin/hdfs dfs -get output output
cat output/*
sbin/stop-dfs.sh
sleep 15
cp etc/hadoop/mapred-site.xml.template etc/hadoop/mapred-site.xml
cat > etc/hadoop/mapred-site.xml  << EOF
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>
EOF

cat > etc/hadoop/yarn-site.xml << EOF
<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
</configuration>
EOF
sbin/start-dfs.sh
sleep 15
sbin/start-yarn.sh
sleep 15
hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.9.0.jar grep /user/test/input output 'dfs[a-z.]+'
hdfs dfs -cat output/*
sbin/stop-yarn.sh
sbin/mr-jobhistory-daemon.sh start historyserver
fi