zoukankan html css js c++ java

Apollo 配置中心学习笔记

前言

Apollo（阿波罗）是携程框架部门研发的分布式配置中心，能够集中化管理应用不同环境、不同集群的配置，配置修改后能够实时推送到应用端，并且具备规范的权限、流程治理等特性，适用于微服务配置管理场景。

服务端基于Spring Boot和Spring Cloud开发，打包后可以直接运行，不需要额外安装Tomcat等应用容器。

Java客户端不依赖任何框架，能够运行于所有Java运行时环境，同时对Spring/Spring Boot环境也有较好的支持。

强烈推荐参考官方文档（中文）：
https://www.apolloconfig.com/#/zh/README

本文着重记录搭建及使用过程，分为以下三种部署方式：

本地快速部署
分布式部署
容器快速部署
基于 k8s 部署

本文所使用到的软件包下载地址：
链接：https://pan.baidu.com/s/1Sa9XLGqIDTyTIwls_VaL4A
提取码：hhhh

本地快速部署

准备工作

软件名	版本
java	1.8.5
MySQL	8.0.25
apache-maven	3.8.1
apollo-quick-start	1.8.0

配置java 环境

# tar xf jdk-8u77-linux-x64.tar.gz  -C /usr/local/
# cat << EOF >> /etc/profile
JAVA_HOME=/usr/local/jdk1.8.0_77
JAVA_BIN=$JAVA_HOME/bin
PATH=$PATH:$JAVA_BIN
CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export JAVA_HOME JAVA_BIN PATH CLASSPATH
EOF
# source  /etc/profile
# ln -vs  /usr/local/jdk1.8.0_77/bin/java /usr/bin/java
# java -version
java version "1.8.0_77"
Java(TM) SE Runtime Environment (build 1.8.0_77-b03)
Java HotSpot(TM) 64-Bit Server VM (build 25.77-b03, mixed mode)

安装并配置 MySQL

# tar xf mysql-server-8.0.25.tar.gz 
# yum localinstall mysql-server-8.0.25/*.rpm -y
# systemctl start mysqld
# egrep pass /var/log/mysqld.log 
2021-05-25T06:00:52.071719Z 6 [Note] [MY-010454] [Server] A temporary password is generated for root@localhost: ba6Q6s;RYbrs  # 这里获取初始密码。
# mysql -uroot -p'ba6Q6s;RYbrs'
 ### 注意密码复杂度，MySQL8.0.x 以上默认要求密码复杂度，否则无法更新密码 ###
mysql> alter user 'root'@'localhost' identified by 'mysql@123.COM';
mysql> flush privileges;
mysql> exit
====== 使用修改后的密码登录 MySQL 获取权限并设置远程客户端连接 ====== 
# mysql -uroot -pmysql@123.COM
mysql> use mysql
mysql> update user set host='%' where user ='root';
### 如果这里出现报错信息，请再次执行一遍 ###
mysql> ALTER USER 'root'@'%' IDENTIFIED WITH mysql_native_password BY 'mysql@123.COM';
mysql> flush privileges;

使用 Navicat Premium 连接 MySQL

安装并配置 Maven

# tar xf apache-maven-3.8.1-bin.tar.gz -C /usr/local/
# cat <<EOF>> /etc/profile.d/maven.sh
export PATH=$PATH:/usr/local/apache-maven-3.8.1/bin/
EOF
# source /etc/profile.d/maven.sh
# mvn -v
Apache Maven 3.8.1 (05c21c65bdfed0f71a2f2ada8b84da59348c4c5d)
Maven home: /usr/local/apache-maven-3.8.1
Java version: 1.8.0_77, vendor: Oracle Corporation, runtime: /usr/local/jdk1.8.0_77/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "3.10.0-1160.el7.x86_64", arch: "amd64", family: "unix"

配置并启动 apollo

# mkdir apollo
# unzip apollo-quick-start-1.8.0.zip -d apollo/

首先导入 sql 数据库信息

# ls
apolloconfigdb.sql  apolloportaldb.sql
# pwd
/root/apollo/sql

=== 登录mysql，准备导入数据
# mysql -uroot -pmysql@123.COM
mysql> source /root/apollo/sql/apolloconfigdb.sql;
mysql> source /root/apollo/sql/apolloportaldb.sql;
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| ApolloConfigDB     |
| ApolloPortalDB     |
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
6 rows in set (0.00 sec)

修改 demo.sh 数据库信息

# vim demo.sh
...
# apollo config db info
apollo_config_db_url="jdbc:mysql://localhost:3306/ApolloConfigDB?characterEncoding=utf8&serverTimezone=Asia/Shanghai"
apollo_config_db_username=root	# mysql 用户名
apollo_config_db_password=mysql@123.COM # mysql 密码

# apollo portal db info
apollo_portal_db_url="jdbc:mysql://localhost:3306/ApolloPortalDB?characterEncoding=utf8&serverTimezone=Asia/Shanghai"
apollo_portal_db_username=root	# mysql 用户名
apollo_portal_db_password=mysql@123.COM	# mysql 密码
...

启动 demo.sh

# ./demo.sh start
==== starting service ====
Service logging file is ./service/apollo-service.log
Application is running as root (UID 0). This is considered insecure.
Started [2143]
Waiting for config service startup...
Config service started. You may visit http://localhost:8080 for service status now!
Waiting for admin service startup
Admin service started
==== starting portal ====
Portal logging file is ./portal/apollo-portal.log
Application is running as root (UID 0). This is considered insecure.
Started [2343]
Waiting for portal startup...
Portal started. You can visit http://localhost:8070 now!

# netstat -ntplu | egrep java
tcp6       0      0 :::8090      :::*   LISTEN      2143/java
tcp6       0      0 :::8070      :::*   LISTEN      2343/java
tcp6       0      0 :::8080      :::*   LISTEN      2143/java

浏览器访问

链接：http://10.0.30.101:8070 用户名密码：apollo/admin

配置 apollo 测试样例

maven 生成 jar 包

# vim /usr/local/apache-maven-3.8.1/conf/settings.xml

### 修改为国内源 ###
...
  <mirrors>
      <mirror>
      <id>alimaven</id>
      <name>aliyun maven</name>
      <url>https://maven.aliyun.com/repository/public/</url>
      <mirrorOf>*</mirrorOf>
      </mirror>
  </mirrors>
...
# tar xf springboot-apollo-demo.tar.gz
# cd springboot-apollo-demo/
# tree -a 
.
├── deploy.yaml
├── Dockerfile
├── pom.xml
├── README.md
└── src
    └── main
        ├── java
        │   └── club
        │       └── mydlq
        │           └── demo
        │               ├── Application.java
        │               └── TestController.java
        └── resources
            └── application.yaml

### 修改配置如下 ###
# cat src/main/resources/application.yaml 
server:
  port: 7070
spring:
  application:
    name: apollo-test

app:
  id: apollo-test	# 应用ID 对应 apollo 项目中的AppId
apollo:
  cacheDir: /opt/data/	# 配置本地配置缓存目录
  meta: http://localhost:8080	# 配置中心地址
  autoUpdateInjectedSpringProperties: true	# 是否开启 Spring 参数自动更新
  bootstrap:
    enabled: true	# 是否开启 Apollo
    eagerLoad:
      enabled: false # 将 Apollo 加载提到初始化日志系统之前

# mvn clean package
# cd target/
# java -jar apollo-demo-0.0.1.jar

通过 apollo 为程序添加变量

发布成功后，通过浏览器访问：http://10.0.30.101:7070/test

程序中apollo 配置解读

[root@apollo ~/springboot-apollo-demo]# tree -a 
.
├── deploy.yaml
├── Dockerfile
├── pom.xml
├── README.md
└── src
    └── main
        ├── java
        │   └── club
        │       └── mydlq
        │           └── demo
        │               ├── Application.java
        │               └── TestController.java
        └── resources
            └── application.yaml	# apollo 主配置文件

7 directories, 7 files

# cat src/main/resources/application.yaml 
server:
  port: 7070
spring:
  application:
    name: apollo-test

app:
  id: apollo-test	# 应用ID 对应 apollo 项目中的AppId
apollo:
  cacheDir: /opt/data/	# 配置本地配置缓存目录
  meta: http://localhost:8080	# 配置中心地址
  autoUpdateInjectedSpringProperties: true	# 是否开启 Spring 参数自动更新
  bootstrap:
    enabled: true	# 是否开启 Apollo
    eagerLoad:
      enabled: false # 将 Apollo 加载提到初始化日志系统之前

测试 apollo

测试目标：测试当apollo中修改参数值后客户端是否能及时刷新。

接下来，通过修改 test 的值来尝试是否热加载。

通过浏览器再次刷新：

测试当apollo执行配置回滚时客户端是否能及时改变

回滚完成后状态将变为未发布状态，通过浏览器访问程序接口：

结论：通过回滚，程序可获取上一个版本参数的值。

测试当不能访问 Apollo 时客户端的变化

直接down掉 config-service

# ./demo.sh stop

关闭之后，通过浏览器刷新访问：

可以发现显示的值并不是定义的默认值，而是 Apollo 配置中心配置的 test 参数的值，查看本地的缓存文件：

# cat /opt/data/apollo-test/config-cache/apollo-test+default+application.properties #Persisted by DefaultConfig#Tue May 25 15:27:50 CST 2021test=123456

结论：当客户端不能连接到 Apollo 配置中心时候，默认使用本地缓存文件中的配置。

现在将缓存也进行清除尝试：

# rm -rf /opt/data/=== 重新启动 jar 程序 ===# java -jar apollo-demo-0.0.1.jar

通过浏览器访问：

结论：删除缓存配置文件后，输出的值为程序定义的默认值。

测试当Apollo 中将参数删除后客户端的变化

进入到 apollo 配置中心，删除之前创建的 test 参数，然后发布。

# ./demo.sh start# java -jar apollo-demo-0.0.1.jar

启动后，访问：

删除参数配置：

删除之后再次刷新浏览器：

结论：删除参数配置后，输出的值为程序定义的默认值。

测试程序切换 apollo 项目实现参数变更

apollo-test 项目参数

apollo-test-1 项目参数

接下来通过，指定不同的启动项来判断 apollo项目的切换，从而获取不同的参数。

默认启动参数为：app.id=apollo-demoapp.meta=http://localhost:8080test 参数为 123456# java -jar apollo-demo-0.0.1.jar

浏览器访问：

添加参数启动：

# java -jar apollo-demo-0.0.1.jar --app.id=apollo-test-1 --apollo.meta=http://localhost:8080

结论：可通过指定不同的apollo 项目来实现多环境的切换。

分布式部署

部署架构

分布式部署需要确定部署环境及部署方式，Apollo目前支持以下环境：

DEV - 开发环境
FAT - 测试环境
UAT - 集成环境
PRO - 生产环境

规划如下：

由于主机有限，本次只做 DEV,PRO 环境
Portal 部署在生产环境，通过它来直接管理多个环境（DEV,PRO）的配置
Config Service 和 Admin Service 在每个环境都单独部署，使用独立数据库
Config Service 和 Admin Service 部署在同一台服务器。

环境

环境说明

apollo-portal是配置的管理端，可以统一管理多套环境的配置（apollo节点）。也可以每套环境独立部署一个apollo-portal，只管理该环境的配置（apollo节点）。本次配置只配置一个 apollo-portal 节点。
每个节点部署一个apollo-configservice和apollo-adminservice
每套环境可以部署多个节点，建议生产环境部署两个以上的节点。开发环境的节点个数可自行决定，可以只部署一个节点。
apollo-portal需要访问ApolloPortalDB数据库。
每套环境需要有一个独立的ApolloConfigDB数据库，同一套环境的apollo-configservice和apollo-configservice访问同一个ApolloConfigDB数据库。

部署案例说明

以下部署将会演示：

在 DEV , PRO 环境中各部署一套 apollo
每套环境部署两个apollo节点，每个节点包含一个apollo-configservice和一个apollo-adminservice。
部署一个apollo-portal，管理多套环境。

软件准备

软件名	版本
java	1.8.5
MySQL	8.0.25
apache-maven	3.8.1
apollo-adminservice	1.8.1
apollo-configservice	1.8.1
apollo-portal	1.8.1

服务器准备

环境	服务器	服务	端口
/	10.0.30.105	apollo-portal	8070
/	10.0.30.106	MySQL数据库ApolloPortalDB	3306
DEV	10.0.30.101	apollo-configservice	8080
	10.0.30.101	apollo-adminservice	8090
	10.0.30.102	apollo-configservice	8080
	10.0.30.102	apollo-adminservice	8090
	10.0.30.103	MySQL数据库ApolloPortalDB	3306
PRO	10.0.30.111	apollo-configservice	8080
	10.0.30.111	apollo-adminservice	8090
	10.0.30.112	apollo-configservice	8080
	10.0.30.112	apollo-adminservice	8090
	10.0.30.113	MySQL数据库ApolloPortalDB	3306

数据库

需要安装并导入数据库的主机如下：

主机	导入库名
10.0.30.106	ApolloPortalDB.sql
10.0.30.103	ApolloConfigDB.sql
10.0.30.113	ApolloConfigDB.sql

安装数据库并导入：

【10.0.30.103、10.0.30.113】操作如下：

tar xf mysql-server-8.0.25.tar.gzcd mysql-server-8.0.25/yum localinstall *.rpm -ysystemctl start mysqldegrep pass /var/log/mysqld.log  # 获取初始化密码mysql -uroot -p'8A-Qkq8)./Ds'	# 通过初始化密码登录mysql> alter user 'root'@'localhost' identified by 'mysql@123.COM';	# 修改密码mysql> flush privileges;mysql> exitmysql -uroot -pmysql@123.COM # 通过修改后的密码登录导入数据库mysql> source /root/apollo/apolloconfigdb.sql;  # 注意导入数据库的sql脚本=== 设置远程连接 ===mysql> use mysql;## 执行报错再次执行一次mysql> alter user 'root'@'%' identified with mysql_native_password by 'mysql@123.COM';mysql> alter user 'root'@'%' identified with mysql_native_password by 'mysql@123.COM';mysql> flush privileges;

调整 ApolloConfigDB.ServerConfig 表数据

如果当前环境只有一个节点时，则可以不用修改。如果有多个节点时，需要修改 eureka.serviceurl 的值为当前环境的 apollo-configservice 的地址和端口，多个地址之间用英文逗号隔开。

DEV 修改如下:http://10.0.30.101:8080/eureka/,http://10.0.30.102:8080/eureka/PRO 修改如下：http://10.0.30.111:8080/eureka/,http://10.0.30.112:8080/eureka/

【10.0.30.106】操作如下：

安装数据库并导入

tar xf mysql-server-8.0.25.tar.gzcd mysql-server-8.0.25/yum localinstall *.rpm -ysystemctl start mysqldegrep pass /var/log/mysqld.log  # 获取初始化密码mysql -uroot -p'8A-Qkq8)./Ds'	# 通过初始化密码登录mysql> alter user 'root'@'localhost' identified by 'mysql@123.COM';	# 修改密码mysql> flush privileges;mysql> exitmysql -uroot -pmysql@123.COM # 通过修改后的密码登录导入数据库mysql> source /root/apollo/apolloportaldb.sql;  # 注意导入数据库的sql脚本=== 设置远程连接 ===mysql> use mysql;## 执行报错再次执行一次mysql> alter user 'root'@'%' identified with mysql_native_password by 'mysql@123.COM';mysql> alter user 'root'@'%' identified with mysql_native_password by 'mysql@123.COM';mysql> flush privileges;

调整 ApolloPortalDB.ServerConfig表的数据

portal默认支持的环境是DEV，若需要支持其它环境，需要修改apollo.portal.envs的值，默认值是DEV，修改为以下值，表示支持DEV、PRO两个环境，注意每个环境值之间用英文逗号隔开：

DEV,PRO

部署 ConfigService 、AdminService

环境	服务	主机IP
DEV	apollo-configService	10.0.30.101
	apollo-configService	10.0.30.102
	apollo-adminService	10.0.30.101
	apollo-adminService	10.0.30.102
PRO	apollo-configService	10.0.30.111
	apollo-configService	10.0.30.112
	apollo-adminService	10.0.30.111
	apollo-adminService	10.0.30.112

部署 java

tar xf jdk-8u77-linux-x64.tar.gz  -C /usr/local/cat << EOF >> /etc/profileJAVA_HOME=/usr/local/jdk1.8.0_77JAVA_BIN=$JAVA_HOME/binPATH=$PATH:$JAVA_BINCLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jarexport JAVA_HOME JAVA_BIN PATH CLASSPATHEOFsource  /etc/profileln -vs  /usr/local/jdk1.8.0_77/bin/java /usr/bin/java

修改配置启动apolloconfig、apolloadmin

yum install unzip -ymkdir -pv apollo/{adminservice,configservice}unzip apollo-configservice-1.8.1-github.zip -d apollo/configservice/unzip apollo-adminservice-1.8.1-github.zip -d apollo/adminservice/=== 修改如下 ===cat apollo/configservice/config/application-github.properties# DataSourcespring.datasource.url = jdbc:mysql://10.0.30.103:3306/ApolloConfigDB?characterEncoding=utf8	# mysql 地址【注意修改为对应集群的地址】spring.datasource.username = root	# 用户名spring.datasource.password = mysql@123.COM	# 密码cat apollo/adminservice/config/application-github.properties# DataSourcespring.datasource.url = jdbc:mysql://10.0.30.103:3306/ApolloConfigDB?characterEncoding=utf8spring.datasource.username = rootspring.datasource.password = mysql@123.COM

注意：这里修改mysql地址请对应集群，比如这里是 DEV 环境对应的数据库地址：10.0.30.103 。如果是PRO 环境，对应的数据库地址：10.0.30.113

启动服务：

# ./apollo/configservice/scripts/startup.sh 
Wed May 26 11:14:30 CST 2021 ==== Starting ==== 
Application is running as root (UID 0). This is considered insecure.
Started [1935]
Waiting for server startup...
Wed May 26 11:14:45 CST 2021 Server started in 15 seconds!
# ./apollo/adminservice/scripts/startup.sh 
Wed May 26 11:14:55 CST 2021 ==== Starting ==== 
Application is running as root (UID 0). This is considered insecure.
Started [2118]
Waiting for server startup..
Wed May 26 11:15:06 CST 2021 Server started in 10 seconds!

### 查看端口
# netstat -ntplu | egrep java
tcp6       0      0 :::8080   :::*   LISTEN 1935/java
tcp6       0      0 :::8090   :::*   LISTEN 2118/java

部署 apollo-portal

只部署了一个apollo-portal，用来管理所有环境的apollo节点。实际部署中，也可以每套环境都部署一个apollo-portal，管理各自环境的apollo-portal，下面也会有提示如何在每个环境单独部署一个apollo-portal。

环境	服务器	服务	端口
/	10.0.30.105	apollo-portal	8070
/	10.0.30.106	MySQL数据库ApolloPortalDB	3306

【10.0.30.105】操作如下：

java 环境

tar xf jdk-8u77-linux-x64.tar.gz  -C /usr/local/cat << EOF >> /etc/profileJAVA_HOME=/usr/local/jdk1.8.0_77JAVA_BIN=$JAVA_HOME/binPATH=$PATH:$JAVA_BINCLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jarexport JAVA_HOME JAVA_BIN PATH CLASSPATHEOFsource  /etc/profileln -vs  /usr/local/jdk1.8.0_77/bin/java /usr/bin/java

配置 apollo-portal

mkdir -pv apollo/portalyum install unzip -yunzip apollo-portal-1.8.1-github.zip -d apollo/portal/vim apollo/portal/config/application-github.properties# DataSourcespring.datasource.url = jdbc:mysql://10.0.30.106:3306/ApolloPortalDB?characterEncoding=utf8	# mysql 地址spring.datasource.username = root	# 用户名spring.datasource.password = mysql@123.COM	# 密码

注意：上面修改为对应数据库的连接地址、用户名、密码

配置meta service信息。

meta service的地址和端口，默认就是apollo-configservice的地址和端口。修改config/apollo-env.properties，填写当前的apollo-portal要管理的环境的apollo-configservice的地址和端口，多个地址之间用英文逗号隔开。修改完后的效果如下：

vim apollo/portal/config/apollo-env.propertiesdev.meta=http://10.0.30.101:8080,http://10.0.30.102:8080pro.meta=http://10.0.30.111:8080,http://10.0.30.112:8080

注意：上面的配置表示当前的apollo-portal要管理dev、pro两套环境的apollo节点，每个环境的节点数是两个。

启动 apollo-portal

# ./apollo/portal/scripts/startup.shWed May 26 14:05:29 CST 2021 ==== Starting ====Application is running as root (UID 0). This is considered insecure.Started [13345]Waiting for server startup..Wed May 26 14:05:39 CST 2021 Server started in 10 seconds!

通过浏览器访问

Docker 方式部署Quick Start

安装docker-ce

curl http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -o /etc/yum.repos.d/docker-ce.repoyum install git docker-ce docker-compose -ymkdir -pv /etc/docker/cat << EOF >> /etc/docker/daemon.json{  "registry-mirrors": [    "https://hub-mirror.c.163.com",    "https://mirror.baidubce.com"  ]}EOFsystemctl start dockerdocker info

下载 docker quick start 文件到本地

git clone https://github.com/ctripcorp/apollo.git

启动 Apollo 配置中心

cd apollo/scripts/docker-quick-start/docker-compose up

报错1：

ERROR: Get https://registry-1.docker.io/v2/: dial tcp: lookup registry-1.docker.io on 223.6.6.6:53: server misbehaving

修改 dns ，不知道为什么阿里的DNS 解析有问题，修改如下：

cat /etc/resolv.conf# Generated by NetworkManagernameserver 223.5.5.5nameserver 223.6.6.6sed -i 's@223.5.5.5@114.114.114.114@g' /etc/resolv.confcat /etc/resolv.conf# Generated by NetworkManagernameserver 114.114.114.114nameserver 223.6.6.6

出现如下信息，表示启动成功。

查看本地端口：

# netstat -ntplu | egrep dockertcp        0  0 0.0.0.0:8080   0.0.0.0:*   LISTEN  3734/docker-proxy   tcp        0  0 0.0.0.0:8090   0.0.0.0:*   LISTEN  3710/docker-proxy   tcp        0  0 0.0.0.0:13306  0.0.0.0:*   LISTEN  3580/docker-proxy   tcp        0  0 0.0.0.0:8070   0.0.0.0:*   LISTEN  3754/docker-proxy   tcp6       0  0 :::8080        :::*        LISTEN  3740/docker-proxy   tcp6       0  0 :::8090        :::*        LISTEN  3719/docker-proxy   tcp6       0  0 :::13306       :::*        LISTEN  3586/docker-proxy   tcp6       0  0 :::8070        :::*        LISTEN  3761/docker-proxy

浏览器访问本机 8070 端口即可。

连接数据库的方法如下：

# 注意：这里必须写 IP 地址，不能为 localhostmysql -uroot -p -h 127.0.0.1 -P 13306 # 密码为空，直接回车

k8s+Apollo

主机规划

（网络环境因素，主机配置双网卡）

IP	主机用途
192.168.1.201/192.168.137.201	k8s-master
192.168.1.202/192.168.137.202	k8s-node1
192.168.1.203/192.168.137.203	k8s-node2
192.168.1.204/192.168.137.204	harbor仓库
192.168.1.205/192.168.137.205	apollo-MySQL

部署k8s

k8s v1.8.6 离线部署：https://www.cnblogs.com/hukey/category/1858851.html

首先，部署 k8s+apollo 首先得有 k8s 环境。按照上面的文档三台主机搭建 k8s 集群，搭建成功如下：

部署Harbor仓库

harbor 部署：https://www.cnblogs.com/hukey/p/14113396.html

部署好之后界面：

部署Apollo到K8S

创建数据库并导入数据

首先，需要为 apollo 创建数据库，关系型数据库最好的方式还是直接运行在物理机上。

安装并配置数据库

apollo-mysql 对应主机：192.168.1.201

[root@apollo-mysql(192.168.1.205) ~]#yum install -y mysql-server mysql
[root@apollo-mysql(192.168.1.205) ~]#systemctl start mysqld
[root@apollo-mysql(192.168.1.205) ~]#egrep -ri password /var/log/mysqld.log 
2021-07-13T03:04:01.331079Z 6 [Note] [MY-010454] [Server] A temporary password is generated for root@localhost: BYLmmLR29q&!

--- 通过初始化密码连接mysql ---
[root@apollo-mysql(192.168.1.205) ~]#mysql -uroot -p'BYLmmLR29q&!'
mysql> alter user user() identified by 'mysql@123.COM';
mysql> flush privileges;

--- 修改完初始密码后，用修改后的密码登录 ---
[root@apollo-mysql(192.168.1.205) ~]#mysql -uroot -pmysql@123.COM   
mysql> create user 'root'@'%' identified by 'mysql@123.COM';  // 创建用户
mysql> grant all privileges on *.* to 'root'@'%';	// 赋予所有权限
mysql> flush privileges;
mysql> exit;

导入apollo 数据

root@apollo-mysql ~# unzip k8s-apollo.zip
root@apollo-mysql ~# cd k8s-apollo/db/config-db-prod/
root@apollo-mysql ~/k/d/config-db-prod# mysql -uroot -p'mysql@123.COM' < apolloconfigdb.sql
root@apollo-mysql ~/k/d/config-db-prod# cd ../portal-db/
root@apollo-mysql ~/k/d/portal-db# mysql -uroot -p'mysql@123.COM' < apolloportaldb.sql

--- 确认导入数据 ---
root@apollo-mysql ~/k/d/config-db-prod# mysql -uroot -p'mysql@123.COM'
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| ApolloPortalDB     |
| ProdApolloConfigDB |
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
6 rows in set (0.00 sec)

生成镜像并推送到 harbor仓库

首先在 harbor 中新建一个仓库

上传镜像到仓库

本步骤在 k8s-master 中操作

root@master ~#unzip k8s-apollo.zip  // 解压上面从百度云盘下载的包
root@master ~# cd k8s-apollo/k8s-apollo-images-v1.8.1/
root@master ~/k/k8s-apollo-images-v1.8.1# ls
k8s-apollo-docker-images-v1.8.1.tar.gz
root@master ~/k/k8s-apollo-images-v1.8.1# tar xf k8s-apollo-docker-images-v1.8.1.tar.gz
root@master ~/k/k8s-apollo-images-v1.8.1# docker load < k8s-apollo-docker-images-v1.8.1.tar

--- 打 tag 到对应的仓库 ---
root@master ~/k/k8s-apollo-images-v1.8.1# docker tag alpine-bash:3.8 192.168.1.204:80/alpine-bash:3.8
root@master ~/k/k8s-apollo-images-v1.8.1# docker tag apollo-config-server:v1.8.1 192.168.1.204:80/apollo/apollo-config-server:v1.8.1
root@master ~/k/k8s-apollo-images-v1.8.1# docker tag apollo-admin-server:v1.8.1 192.168.1.204:80/apollo/apollo-admin-server:v1.8.1
root@master ~/k/k8s-apollo-images-v1.8.1# docker tag apollo-portal-server:v1.8.1 192.168.1.204:80/apollo/apollo-portal-server:v1.8.1

--- 上传到 harbor ---

1. 修改 docker 配置
root@master ~/k/k8s-apollo-images-v1.8.1# vim /etc/docker/daemon.json   // 加入下面这一行
...
"insecure-registries": ["192.168.1.204:80"],
...

root@master ~/k/k8s-apollo-images-v1.8.1# systemctl restart docker  // 修改完成记得 restart docker

2. 登录harbor 用户名密码就是页面登录所用的用户名密码
root@master ~/k/k8s-apollo-images-v1.8.1# docker login 192.168.1.204:80
Username: admin
Password: 
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

3. 上传镜像到仓库
root@master ~# docker push 192.168.1.204:80/apollo/alpine-bash:3.8
root@master ~# docker push 192.168.1.204:80/apollo/apollo-config-server
root@master ~# docker push 192.168.1.204:80/apollo/apollo-admin-server
root@master ~# docker push 192.168.1.204:80/apollo/apollo-portal-server

为每个k8s 节点配置可信仓库

重点：非常容易遗漏的点，这里不做后面创建Pod是无法下载镜像的！！！

# vim /etc/docker/daemon.json
...
"insecure-registries": ["192.168.1.204:80"],
...

# systemctl restart docker

通过 yaml 文件构建Pod及服务

本步骤在 k8s-master 中操作

root@master ~# cd k8s-apollo/kubernetes/apollo-env-prod/
root@master ~/k/k/apollo-env-prod# ll -tsh 
total 24K
8.0K -rwxr-xr-x 1 root root 4.1K Jul 14 10:08 service-apollo-admin-server-prod.yaml*	// apollo-admin-server
8.0K -rwxr-xr-x 1 root root 4.1K Jul 14 10:08 service-apollo-config-server-prod.yaml*	// apollo-config-server
4.0K -rwxr-xr-x 1 root root 1.1K Jul 14 10:08 service-mysql-for-apollo-prod-env.yaml*	// 将apollo外部mysql引入k8s内部调用

注意，执行顺序如下：

service-mysql-for-apollo-prod-env.yaml
service-apollo-config-server-prod.yaml
service-apollo-admin-server-prod.yaml

在执行之前需要为 apollo 创建专属的命名空间：

root@master ~# kubectl create ns sre
namespace/sre created

为k8s 添加 harbor 登录 secrets

在创建 docker-registry时候一定要指定命名空间，切记！

[root@master(192.168.1.201) ~]#kubectl create secret docker-registry registry-harbor --namespace=sre --docker-server=192.168.1.204:80 --docker-username=admin --docker-password=123456 
secret/registry-harbor created

service-mysql-for-apollo-prod-env.yaml 执行

修改后的 service-mysql-for-apollo-dev-env.yaml 如下：

---
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-mysql-for-apollo-prod-env
  labels:
    app: service-mysql-for-apollo-prod-env
spec:
  ports:
    - protocol: TCP
      port: 3306
      targetPort: 3306
  type: ClusterIP
  sessionAffinity: None

---
kind: Endpoints
apiVersion: v1
metadata:
  namespace: sre
  name: service-mysql-for-apollo-prod-env
subsets:
  - addresses:
      - ip: 192.168.1.205	// 这里需要修改为对应的apollo-mysql 主机地址
    ports:
      - protocol: TCP
        port: 3306

执行：

root@master ~/k/k/apollo-env-prod# kubectl apply -f service-mysql-for-apollo-prod-env.yaml
service/service-mysql-for-apollo-prod-env created
endpoints/service-mysql-for-apollo-prod-env created

--- 查看并尝试连接 ---
root@master ~/k/k/apollo-env-prod# kubectl describe svc -n sre
Name:              service-mysql-for-apollo-prod-env
Namespace:         sre
Labels:            app=service-mysql-for-apollo-prod-env
Annotations:       Selector:  <none>
Type:              ClusterIP
IP:                10.111.52.129
Port:              <unset>  3306/TCP
TargetPort:        3306/TCP
Endpoints:         192.168.1.205:3306   // 确认这里是否是mysql主机
Session Affinity:  None
Events:            <none>


root@master ~/k/k/apollo-env-prod# yum install mysql -y
root@master ~/k/k/apollo-env-prod# mysql -uroot -pmysql@123.COM -h 10.111.52.129
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| ProdApolloConfigDB |
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
5 rows in set (0.01 sec)

确认mysql service 没问题。

service-apollo-config-server-prod.yaml 执行

修改后的 service-apollo-config-server-prod.yaml 如下：

kind: ConfigMap
apiVersion: v1
metadata:
  namespace: sre
  name: configmap-apollo-config-server-prod
data:
  application-github.properties: |
    spring.datasource.url = jdbc:mysql://service-mysql-for-apollo-prod-env.sre:3306/ProdApolloConfigDB?characterEncoding=utf8
    spring.datasource.username = root			// 修改为数据库的用户名
    spring.datasource.password = mysql@123.COM	// 修改为数据库的密码
    eureka.service.url = http://statefulset-apollo-config-server-prod-0.service-apollo-meta-server-prod:8080/eureka/,http://statefulset-apollo-config-server-prod-1.service-apollo-meta-server-prod:8080/eureka/,http://statefulset-apollo-config-server-prod-2.service-apollo-meta-server-prod:8080/eureka/

---
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-apollo-meta-server-prod
  labels:
    app: service-apollo-meta-server-prod
spec:
  ports:
    - protocol: TCP
      port: 8080
      targetPort: 8080
  selector:
    app: pod-apollo-config-server-prod
  type: ClusterIP
  clusterIP: None
  sessionAffinity: ClientIP

---
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-apollo-config-server-prod
  labels:
    app: service-apollo-config-server-prod
spec:
  ports:
    - protocol: TCP
      port: 8080
      targetPort: 8080
      nodePort: 30005
  selector:
    app: pod-apollo-config-server-prod
  type: NodePort
  sessionAffinity: ClientIP

---
kind: StatefulSet
apiVersion: apps/v1
metadata:
  namespace: sre
  name: statefulset-apollo-config-server-prod
  labels:
    app: statefulset-apollo-config-server-prod
spec:
  serviceName: service-apollo-meta-server-prod
  replicas: 1
  selector:
    matchLabels:
      app: pod-apollo-config-server-prod
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: pod-apollo-config-server-prod
    spec:
      imagePullSecrets:			// 添加仓库secret
      - name: registry-harbor	// 添加仓库secret
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - pod-apollo-config-server-prod
              topologyKey: kubernetes.io/hostname
      
      volumes:
        - name: volume-configmap-apollo-config-server-prod
          configMap:
            name: configmap-apollo-config-server-prod
            items:
              - key: application-github.properties
                path: application-github.properties
      
      containers:
        - image: 192.168.1.194:80/apollo/apollo-config-server:v1.8.1   // 修改为本地仓库镜像
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          name: container-apollo-config-server-prod
          ports:
            - protocol: TCP
              containerPort: 8080

          volumeMounts:
            - name: volume-configmap-apollo-config-server-prod
              mountPath: /apollo-config-server/config/application-github.properties
              subPath: application-github.properties
          env:
            - name: APOLLO_CONFIG_SERVICE_NAME
              value: "service-apollo-config-server-prod.sre"
          
          readinessProbe:
            tcpSocket:
              port: 8080
            initialDelaySeconds: 10
            periodSeconds: 5
          
          livenessProbe:
            tcpSocket:
              port: 8080
            initialDelaySeconds: 120
            periodSeconds: 10
          
      dnsPolicy: ClusterFirst
      restartPolicy: Always

执行：

root@master ~/k/k/apollo-env-prod# kubectl apply -f service-apollo-config-server-prod.yaml 
configmap/configmap-apollo-config-server-prod created
service/service-apollo-meta-server-prod created
service/service-apollo-config-server-prod created
statefulset.apps/statefulset-apollo-config-server-prod created

等一段时间查看
root@master ~/k/k/apollo-env-prod# kubectl get pod,svc -n sre
NAME                                          READY   STATUS    RESTARTS   AGE
pod/statefulset-apollo-config-server-prod-0   1/1     Running   0          10m

NAME                                        TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
service/service-apollo-config-server-prod   NodePort    10.102.17.193   <none>        8080:30005/TCP   10m
service/service-apollo-meta-server-prod     ClusterIP   None            <none>        8080/TCP         10m
service/service-mysql-for-apollo-prod-env   ClusterIP   10.111.52.129   <none>        3306/TCP         26m

service-apollo-admin-server-prod.yaml 执行

修改后的 service-apollo-admin-server-prod.yaml 如下：

kind: ConfigMap
apiVersion: v1
metadata:
  namespace: sre
  name: configmap-apollo-admin-server-prod
data:
  application-github.properties: |
    spring.datasource.url = jdbc:mysql://service-mysql-for-apollo-prod-env.sre:3306/ProdApolloConfigDB?characterEncoding=utf8
    spring.datasource.username = root				// mysql用户名
    spring.datasource.password = mysql@123.COM		// mysql密码
    eureka.service.url = http://statefulset-apollo-config-server-prod-0.service-apollo-meta-server-prod:8080/eureka/,http://statefulset-apollo-config-server-prod-1.service-apollo-meta-server-prod:8080/eureka/,http://statefulset-apollo-config-server-prod-2.service-apollo-meta-server-prod:8080/eureka/

---
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-apollo-admin-server-prod
  labels:
    app: service-apollo-admin-server-prod
spec:
  ports:
    - protocol: TCP
      port: 8090
      targetPort: 8090
  selector:
    app: pod-apollo-admin-server-prod  
  type: ClusterIP
  sessionAffinity: ClientIP

---
kind: Deployment
apiVersion: apps/v1
metadata:
  namespace: sre
  name: deployment-apollo-admin-server-prod
  labels:
    app: deployment-apollo-admin-server-prod
spec:
  replicas: 1
  selector:
    matchLabels:
      app: pod-apollo-admin-server-prod
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: pod-apollo-admin-server-prod
    spec:
      imagePullSecrets:
      - name: registry-harbor
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - pod-apollo-admin-server-prod
              topologyKey: kubernetes.io/hostname
      
      volumes:
        - name: volume-configmap-apollo-admin-server-prod
          configMap:
            name: configmap-apollo-admin-server-prod
            items:
              - key: application-github.properties
                path: application-github.properties
      
      initContainers:
        - image: 192.168.1.194:80/apollo/alpine-bash:3.8	// 修改镜像地址
          name: check-service-apollo-config-server-prod
          command: ['bash', '-c', "curl --connect-timeout 2 --max-time 5 --retry 50 --retry-delay 1 --retry-max-time 120 service-apollo-config-server-prod.sre:8080"]
      
      containers:
        - image: 192.168.1.194:80/apollo/apollo-admin-server:v1.8.1	// 修改镜像地址
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          name: container-apollo-admin-server-prod
          ports:
            - protocol: TCP
              containerPort: 8090
          
          volumeMounts:
            - name: volume-configmap-apollo-admin-server-prod
              mountPath: /apollo-admin-server/config/application-github.properties
              subPath: application-github.properties
          
          env:
            - name: APOLLO_ADMIN_SERVICE_NAME
              value: "service-apollo-admin-server-prod.sre"
          
          readinessProbe:
            tcpSocket:
              port: 8090
            initialDelaySeconds: 10
            periodSeconds: 5
          
          livenessProbe:
            tcpSocket:
              port: 8090
            initialDelaySeconds: 120
            periodSeconds: 10

      dnsPolicy: ClusterFirst
      restartPolicy: Always

执行：

root@master ~/k/k/apollo-env-prod# kubectl apply -f service-apollo-admin-server-prod.yaml 
configmap/configmap-apollo-admin-server-prod created
service/service-apollo-admin-server-prod created
deployment.apps/deployment-apollo-admin-server-prod created

过2分钟后查看：
root@master ~/k/k/apollo-env-prod# kubectl get pod,svc -n sre
NAME                                                       READY   STATUS    RESTARTS   AGE
pod/deployment-apollo-admin-server-prod-64c74bdcb9-2d74n   0/1     Running   0          19s
pod/statefulset-apollo-config-server-prod-0                1/1     Running   0          13m

NAME                                        TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
service/service-apollo-admin-server-prod    ClusterIP   10.100.33.188   <none>        8090/TCP         20s
service/service-apollo-config-server-prod   NodePort    10.102.17.193   <none>        8080:30005/TCP   13m
service/service-apollo-meta-server-prod     ClusterIP   None            <none>        8080/TCP         13m
service/service-mysql-for-apollo-prod-env   ClusterIP   10.111.52.129   <none>        3306/TCP         30m

最后，再来构建 service-apollo-portal-server.yaml

root@master ~/k/k/apollo-env-prod# cd ../
root@master ~/k/kubernetes# pwd
/root/k8s-apollo/kubernetes
root@master ~/k/kubernetes# ls service-apollo-portal-server.yaml 
service-apollo-portal-server.yaml*

service-apollo-portal-server.yaml 执行

修改后的 service-apollo-portal-server.yaml 如下：

# 为外部 mysql 服务设置 service
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-mysql-for-portal-server
  labels:
    app: service-mysql-for-portal-server
spec:
  ports:
    - protocol: TCP
      port: 3306
      targetPort: 3306
  type: ClusterIP
  sessionAffinity: None
---
kind: Endpoints
apiVersion: v1
metadata:
  namespace: sre
  name: service-mysql-for-portal-server
subsets:
  - addresses:
      # 更改为你的 mysql addresses, 例如 1.1.1.1
      - ip: 192.168.1.205		// 修改为k8s集群外部访问mysql的地址
    ports:
      - protocol: TCP
        port: 3306

---
# configmap for apollo-portal-server
kind: ConfigMap
apiVersion: v1
metadata:
  namespace: sre
  name: configmap-apollo-portal-server
data:
  application-github.properties: |
    spring.datasource.url = jdbc:mysql://service-mysql-for-portal-server.sre:3306/ApolloPortalDB?characterEncoding=utf8
    # mysql username
    spring.datasource.username = root	// mysql user
    # mysql password
    spring.datasource.password = mysql@123.COM // mysql password
  apollo-env.properties: |
    dev.meta=http://service-apollo-config-server-dev.sre:8080
    fat.meta=http://service-apollo-config-server-test-alpha.sre:8080
    uat.meta=http://service-apollo-config-server-test-beta.sre:8080
    pro.meta=http://service-apollo-config-server-prod.sre:8080

---
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-apollo-portal-server
  labels:
    app: service-apollo-portal-server
spec:
  ports:
    - protocol: TCP
      port: 8070
      targetPort: 8070
      nodePort: 30001
  selector:
    app: pod-apollo-portal-server
  type: NodePort
  # portal session 保持
  sessionAffinity: ClientIP

---
kind: Deployment
apiVersion: apps/v1
metadata:
  namespace: sre
  name: deployment-apollo-portal-server
  labels:
    app: deployment-apollo-portal-server
spec:
  # 3 个实例
  replicas: 3
  selector:
    matchLabels:
      app: pod-apollo-portal-server
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: pod-apollo-portal-server
    spec:
      imagePullSecrets:			// 添加 harbor secret
      - name: registry-harbor	// 添加 harbor secret
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - pod-apollo-portal-server
              topologyKey: kubernetes.io/hostname
      
      volumes:
        - name: volume-configmap-apollo-portal-server
          configMap:
            name: configmap-apollo-portal-server
            items:
              - key: application-github.properties
                path: application-github.properties
              - key: apollo-env.properties
                path: apollo-env.properties
      
      initContainers:
        # 确保 admin-service 正常提供服务
        - image: 192.168.1.204:80/apollo/alpine-bash:3.8	// 修改镜像仓库
          name: check-service-apollo-admin-server-prod
          command: ['bash', '-c', "curl --connect-timeout 2 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 service-apollo-admin-server-prod.sre:8090"]    
      
      containers:
        - image: 192.168.1.204:80/apollo/apollo-portal-server:v1.8.1   // 修改镜像仓库
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          name: container-apollo-portal-server
          ports:
            - protocol: TCP
              containerPort: 8070
          
          volumeMounts:
            - name: volume-configmap-apollo-portal-server
              mountPath: /apollo-portal-server/config/application-github.properties
              subPath: application-github.properties
            - name: volume-configmap-apollo-portal-server
              mountPath: /apollo-portal-server/config/apollo-env.properties
              subPath: apollo-env.properties
          
          env:
            - name: APOLLO_PORTAL_SERVICE_NAME
              value: "service-apollo-portal-server.sre"
          
          readinessProbe:
            tcpSocket:
              port: 8070
            initialDelaySeconds: 10
            periodSeconds: 5
          
          livenessProbe:
            tcpSocket:
              port: 8070
            # 120s 内, server 未启动则重启 container
            initialDelaySeconds: 120
            periodSeconds: 15
          
      dnsPolicy: ClusterFirst
      restartPolicy: Always

执行：

root@master ~/k/kubernetes# kubectl apply -f service-apollo-portal-server.yaml 
service/service-mysql-for-portal-server created
endpoints/service-mysql-for-portal-server created
configmap/configmap-apollo-portal-server created
service/service-apollo-portal-server created
deployment.apps/deployment-apollo-portal-server created

查看：
root@master ~/k/kubernetes# kubectl get pod,svc -n sre
NAME                                                   READY   STATUS    RESTARTS   AGE
deployment-apollo-admin-server-prod-64c74bdcb9-2d74n   1/1     Running   0          26m
deployment-apollo-portal-server-f89ff96db-4bccn        1/1     Running   3          11m
deployment-apollo-portal-server-f89ff96db-hdnh8        1/1     Running   3          11m
deployment-apollo-portal-server-f89ff96db-xg2rs        1/1     Running   3          11m
statefulset-apollo-config-server-prod-0                1/1     Running   0          39m


NAME                                TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
service-apollo-admin-server-prod    ClusterIP   10.100.33.188   <none>        8090/TCP         26m
service-apollo-config-server-prod   NodePort    10.102.17.193   <none>        8080:30005/TCP   40m
service-apollo-meta-server-prod     ClusterIP   None            <none>        8080/TCP         40m
service-apollo-portal-server        NodePort    10.111.67.76    <none>        8070:30001/TCP   11m
service-mysql-for-apollo-prod-env   ClusterIP   10.111.52.129   <none>        3306/TCP         56m
service-mysql-for-portal-server     ClusterIP   10.107.60.175   <none>        3306/TCP         11m

通过浏览器访问：

http://192.168.10.201:30001/

--- EOF ---

查看全文

相关阅读:
微信开放平台扫码登录回调后 state 参数丢失遗漏的解决方案
 WPF 图片源为空时展示默认图片
 使用批处理(.bat)文件一键编译 .NET CORE 网站并发布至 IIS
10倍程序员的思考模型
 以TiDB热点问题来谈Region的调度流程
 Can't perform a React state update on an unmounted component. This is a no-op, but it indicates a memory leak in your application. To fix, cancel的解决办法(react Hooks)
react+tsx+antd表格点击行触发单选复选事件（类型xx上不存在属性xx的解决办法）
does not exist on type 'JSX.IntrinsicElements'.does not exist on type 'JSX.IntrinsicElements'.的解决办法
 antd-react自定义显示表格操作按钮
 react+tsx+antd关于表单二次弹框

原文地址：https://www.cnblogs.com/hukey/p/14822521.html