zoukankan html css js c++ java

Apollo 配置中心学习笔记

前言

Apollo（阿波罗）是携程框架部门研发的分布式配置中心，能够集中化管理应用不同环境、不同集群的配置，配置修改后能够实时推送到应用端，并且具备规范的权限、流程治理等特性，适用于微服务配置管理场景。

服务端基于Spring Boot和Spring Cloud开发，打包后可以直接运行，不需要额外安装Tomcat等应用容器。

Java客户端不依赖任何框架，能够运行于所有Java运行时环境，同时对Spring/Spring Boot环境也有较好的支持。

强烈推荐参考官方文档（中文）：
https://www.apolloconfig.com/#/zh/README

本文着重记录搭建及使用过程，分为以下三种部署方式：

本地快速部署
分布式部署
容器快速部署
基于 k8s 部署

本文所使用到的软件包下载地址：
链接：https://pan.baidu.com/s/1Sa9XLGqIDTyTIwls_VaL4A
提取码：hhhh

本地快速部署

准备工作

软件名	版本
java	1.8.5
MySQL	8.0.25
apache-maven	3.8.1
apollo-quick-start	1.8.0

配置java 环境

# tar xf jdk-8u77-linux-x64.tar.gz  -C /usr/local/
# cat << EOF >> /etc/profile
JAVA_HOME=/usr/local/jdk1.8.0_77
JAVA_BIN=$JAVA_HOME/bin
PATH=$PATH:$JAVA_BIN
CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export JAVA_HOME JAVA_BIN PATH CLASSPATH
EOF
# source  /etc/profile
# ln -vs  /usr/local/jdk1.8.0_77/bin/java /usr/bin/java
# java -version
java version "1.8.0_77"
Java(TM) SE Runtime Environment (build 1.8.0_77-b03)
Java HotSpot(TM) 64-Bit Server VM (build 25.77-b03, mixed mode)

安装并配置 MySQL

# tar xf mysql-server-8.0.25.tar.gz 
# yum localinstall mysql-server-8.0.25/*.rpm -y
# systemctl start mysqld
# egrep pass /var/log/mysqld.log 
2021-05-25T06:00:52.071719Z 6 [Note] [MY-010454] [Server] A temporary password is generated for root@localhost: ba6Q6s;RYbrs  # 这里获取初始密码。
# mysql -uroot -p'ba6Q6s;RYbrs'
 ### 注意密码复杂度，MySQL8.0.x 以上默认要求密码复杂度，否则无法更新密码 ###
mysql> alter user 'root'@'localhost' identified by 'mysql@123.COM';
mysql> flush privileges;
mysql> exit
====== 使用修改后的密码登录 MySQL 获取权限并设置远程客户端连接 ====== 
# mysql -uroot -pmysql@123.COM
mysql> use mysql
mysql> update user set host='%' where user ='root';
### 如果这里出现报错信息，请再次执行一遍 ###
mysql> ALTER USER 'root'@'%' IDENTIFIED WITH mysql_native_password BY 'mysql@123.COM';
mysql> flush privileges;

使用 Navicat Premium 连接 MySQL

安装并配置 Maven

# tar xf apache-maven-3.8.1-bin.tar.gz -C /usr/local/
# cat <<EOF>> /etc/profile.d/maven.sh
export PATH=$PATH:/usr/local/apache-maven-3.8.1/bin/
EOF
# source /etc/profile.d/maven.sh
# mvn -v
Apache Maven 3.8.1 (05c21c65bdfed0f71a2f2ada8b84da59348c4c5d)
Maven home: /usr/local/apache-maven-3.8.1
Java version: 1.8.0_77, vendor: Oracle Corporation, runtime: /usr/local/jdk1.8.0_77/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "3.10.0-1160.el7.x86_64", arch: "amd64", family: "unix"

配置并启动 apollo

# mkdir apollo
# unzip apollo-quick-start-1.8.0.zip -d apollo/

首先导入 sql 数据库信息

# ls
apolloconfigdb.sql  apolloportaldb.sql
# pwd
/root/apollo/sql

=== 登录mysql，准备导入数据
# mysql -uroot -pmysql@123.COM
mysql> source /root/apollo/sql/apolloconfigdb.sql;
mysql> source /root/apollo/sql/apolloportaldb.sql;
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| ApolloConfigDB     |
| ApolloPortalDB     |
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
6 rows in set (0.00 sec)

修改 demo.sh 数据库信息

# vim demo.sh
...
# apollo config db info
apollo_config_db_url="jdbc:mysql://localhost:3306/ApolloConfigDB?characterEncoding=utf8&serverTimezone=Asia/Shanghai"
apollo_config_db_username=root	# mysql 用户名
apollo_config_db_password=mysql@123.COM # mysql 密码

# apollo portal db info
apollo_portal_db_url="jdbc:mysql://localhost:3306/ApolloPortalDB?characterEncoding=utf8&serverTimezone=Asia/Shanghai"
apollo_portal_db_username=root	# mysql 用户名
apollo_portal_db_password=mysql@123.COM	# mysql 密码
...

启动 demo.sh

# ./demo.sh start
==== starting service ====
Service logging file is ./service/apollo-service.log
Application is running as root (UID 0). This is considered insecure.
Started [2143]
Waiting for config service startup...
Config service started. You may visit http://localhost:8080 for service status now!
Waiting for admin service startup
Admin service started
==== starting portal ====
Portal logging file is ./portal/apollo-portal.log
Application is running as root (UID 0). This is considered insecure.
Started [2343]
Waiting for portal startup...
Portal started. You can visit http://localhost:8070 now!

# netstat -ntplu | egrep java
tcp6       0      0 :::8090      :::*   LISTEN      2143/java
tcp6       0      0 :::8070      :::*   LISTEN      2343/java
tcp6       0      0 :::8080      :::*   LISTEN      2143/java

浏览器访问

链接：http://10.0.30.101:8070 用户名密码：apollo/admin

配置 apollo 测试样例

maven 生成 jar 包

# vim /usr/local/apache-maven-3.8.1/conf/settings.xml

### 修改为国内源 ###
...
  <mirrors>
      <mirror>
      <id>alimaven</id>
      <name>aliyun maven</name>
      <url>https://maven.aliyun.com/repository/public/</url>
      <mirrorOf>*</mirrorOf>
      </mirror>
  </mirrors>
...
# tar xf springboot-apollo-demo.tar.gz
# cd springboot-apollo-demo/
# tree -a 
.
├── deploy.yaml
├── Dockerfile
├── pom.xml
├── README.md
└── src
    └── main
        ├── java
        │   └── club
        │       └── mydlq
        │           └── demo
        │               ├── Application.java
        │               └── TestController.java
        └── resources
            └── application.yaml

### 修改配置如下 ###
# cat src/main/resources/application.yaml 
server:
  port: 7070
spring:
  application:
    name: apollo-test

app:
  id: apollo-test	# 应用ID 对应 apollo 项目中的AppId
apollo:
  cacheDir: /opt/data/	# 配置本地配置缓存目录
  meta: http://localhost:8080	# 配置中心地址
  autoUpdateInjectedSpringProperties: true	# 是否开启 Spring 参数自动更新
  bootstrap:
    enabled: true	# 是否开启 Apollo
    eagerLoad:
      enabled: false # 将 Apollo 加载提到初始化日志系统之前

# mvn clean package
# cd target/
# java -jar apollo-demo-0.0.1.jar

通过 apollo 为程序添加变量

发布成功后，通过浏览器访问：http://10.0.30.101:7070/test

程序中apollo 配置解读

[root@apollo ~/springboot-apollo-demo]# tree -a 
.
├── deploy.yaml
├── Dockerfile
├── pom.xml
├── README.md
└── src
    └── main
        ├── java
        │   └── club
        │       └── mydlq
        │           └── demo
        │               ├── Application.java
        │               └── TestController.java
        └── resources
            └── application.yaml	# apollo 主配置文件

7 directories, 7 files

# cat src/main/resources/application.yaml 
server:
  port: 7070
spring:
  application:
    name: apollo-test

app:
  id: apollo-test	# 应用ID 对应 apollo 项目中的AppId
apollo:
  cacheDir: /opt/data/	# 配置本地配置缓存目录
  meta: http://localhost:8080	# 配置中心地址
  autoUpdateInjectedSpringProperties: true	# 是否开启 Spring 参数自动更新
  bootstrap:
    enabled: true	# 是否开启 Apollo
    eagerLoad:
      enabled: false # 将 Apollo 加载提到初始化日志系统之前

测试 apollo

测试目标：测试当apollo中修改参数值后客户端是否能及时刷新。

接下来，通过修改 test 的值来尝试是否热加载。

通过浏览器再次刷新：

测试当apollo执行配置回滚时客户端是否能及时改变

回滚完成后状态将变为未发布状态，通过浏览器访问程序接口：

结论：通过回滚，程序可获取上一个版本参数的值。

测试当不能访问 Apollo 时客户端的变化

直接down掉 config-service

# ./demo.sh stop

关闭之后，通过浏览器刷新访问：

可以发现显示的值并不是定义的默认值，而是 Apollo 配置中心配置的 test 参数的值，查看本地的缓存文件：

# cat /opt/data/apollo-test/config-cache/apollo-test+default+application.properties #Persisted by DefaultConfig#Tue May 25 15:27:50 CST 2021test=123456

结论：当客户端不能连接到 Apollo 配置中心时候，默认使用本地缓存文件中的配置。

现在将缓存也进行清除尝试：

# rm -rf /opt/data/=== 重新启动 jar 程序 ===# java -jar apollo-demo-0.0.1.jar

通过浏览器访问：

结论：删除缓存配置文件后，输出的值为程序定义的默认值。

测试当Apollo 中将参数删除后客户端的变化

进入到 apollo 配置中心，删除之前创建的 test 参数，然后发布。

# ./demo.sh start# java -jar apollo-demo-0.0.1.jar

启动后，访问：

删除参数配置：

删除之后再次刷新浏览器：

结论：删除参数配置后，输出的值为程序定义的默认值。

测试程序切换 apollo 项目实现参数变更

apollo-test 项目参数

apollo-test-1 项目参数

接下来通过，指定不同的启动项来判断 apollo项目的切换，从而获取不同的参数。

默认启动参数为：app.id=apollo-demoapp.meta=http://localhost:8080test 参数为 123456# java -jar apollo-demo-0.0.1.jar

浏览器访问：

添加参数启动：

# java -jar apollo-demo-0.0.1.jar --app.id=apollo-test-1 --apollo.meta=http://localhost:8080

结论：可通过指定不同的apollo 项目来实现多环境的切换。

分布式部署

部署架构

分布式部署需要确定部署环境及部署方式，Apollo目前支持以下环境：

DEV - 开发环境
FAT - 测试环境
UAT - 集成环境
PRO - 生产环境

规划如下：

由于主机有限，本次只做 DEV,PRO 环境
Portal 部署在生产环境，通过它来直接管理多个环境（DEV,PRO）的配置
Config Service 和 Admin Service 在每个环境都单独部署，使用独立数据库
Config Service 和 Admin Service 部署在同一台服务器。

环境

环境说明

apollo-portal是配置的管理端，可以统一管理多套环境的配置（apollo节点）。也可以每套环境独立部署一个apollo-portal，只管理该环境的配置（apollo节点）。本次配置只配置一个 apollo-portal 节点。
每个节点部署一个apollo-configservice和apollo-adminservice
每套环境可以部署多个节点，建议生产环境部署两个以上的节点。开发环境的节点个数可自行决定，可以只部署一个节点。
apollo-portal需要访问ApolloPortalDB数据库。
每套环境需要有一个独立的ApolloConfigDB数据库，同一套环境的apollo-configservice和apollo-configservice访问同一个ApolloConfigDB数据库。

部署案例说明

以下部署将会演示：

在 DEV , PRO 环境中各部署一套 apollo
每套环境部署两个apollo节点，每个节点包含一个apollo-configservice和一个apollo-adminservice。
部署一个apollo-portal，管理多套环境。

软件准备

软件名	版本
java	1.8.5
MySQL	8.0.25
apache-maven	3.8.1
apollo-adminservice	1.8.1
apollo-configservice	1.8.1
apollo-portal	1.8.1

服务器准备

环境	服务器	服务	端口
/	10.0.30.105	apollo-portal	8070
/	10.0.30.106	MySQL数据库ApolloPortalDB	3306
DEV	10.0.30.101	apollo-configservice	8080
	10.0.30.101	apollo-adminservice	8090
	10.0.30.102	apollo-configservice	8080
	10.0.30.102	apollo-adminservice	8090
	10.0.30.103	MySQL数据库ApolloPortalDB	3306
PRO	10.0.30.111	apollo-configservice	8080
	10.0.30.111	apollo-adminservice	8090
	10.0.30.112	apollo-configservice	8080
	10.0.30.112	apollo-adminservice	8090
	10.0.30.113	MySQL数据库ApolloPortalDB	3306

数据库

需要安装并导入数据库的主机如下：

主机	导入库名
10.0.30.106	ApolloPortalDB.sql
10.0.30.103	ApolloConfigDB.sql
10.0.30.113	ApolloConfigDB.sql

安装数据库并导入：

【10.0.30.103、10.0.30.113】操作如下：

tar xf mysql-server-8.0.25.tar.gzcd mysql-server-8.0.25/yum localinstall *.rpm -ysystemctl start mysqldegrep pass /var/log/mysqld.log  # 获取初始化密码mysql -uroot -p'8A-Qkq8)./Ds'	# 通过初始化密码登录mysql> alter user 'root'@'localhost' identified by 'mysql@123.COM';	# 修改密码mysql> flush privileges;mysql> exitmysql -uroot -pmysql@123.COM # 通过修改后的密码登录导入数据库mysql> source /root/apollo/apolloconfigdb.sql;  # 注意导入数据库的sql脚本=== 设置远程连接 ===mysql> use mysql;## 执行报错再次执行一次mysql> alter user 'root'@'%' identified with mysql_native_password by 'mysql@123.COM';mysql> alter user 'root'@'%' identified with mysql_native_password by 'mysql@123.COM';mysql> flush privileges;

调整 ApolloConfigDB.ServerConfig 表数据

如果当前环境只有一个节点时，则可以不用修改。如果有多个节点时，需要修改 eureka.serviceurl 的值为当前环境的 apollo-configservice 的地址和端口，多个地址之间用英文逗号隔开。

DEV 修改如下:http://10.0.30.101:8080/eureka/,http://10.0.30.102:8080/eureka/PRO 修改如下：http://10.0.30.111:8080/eureka/,http://10.0.30.112:8080/eureka/

【10.0.30.106】操作如下：

安装数据库并导入

tar xf mysql-server-8.0.25.tar.gzcd mysql-server-8.0.25/yum localinstall *.rpm -ysystemctl start mysqldegrep pass /var/log/mysqld.log  # 获取初始化密码mysql -uroot -p'8A-Qkq8)./Ds'	# 通过初始化密码登录mysql> alter user 'root'@'localhost' identified by 'mysql@123.COM';	# 修改密码mysql> flush privileges;mysql> exitmysql -uroot -pmysql@123.COM # 通过修改后的密码登录导入数据库mysql> source /root/apollo/apolloportaldb.sql;  # 注意导入数据库的sql脚本=== 设置远程连接 ===mysql> use mysql;## 执行报错再次执行一次mysql> alter user 'root'@'%' identified with mysql_native_password by 'mysql@123.COM';mysql> alter user 'root'@'%' identified with mysql_native_password by 'mysql@123.COM';mysql> flush privileges;

调整 ApolloPortalDB.ServerConfig表的数据

portal默认支持的环境是DEV，若需要支持其它环境，需要修改apollo.portal.envs的值，默认值是DEV，修改为以下值，表示支持DEV、PRO两个环境，注意每个环境值之间用英文逗号隔开：

DEV,PRO

部署 ConfigService 、AdminService

环境	服务	主机IP
DEV	apollo-configService	10.0.30.101
	apollo-configService	10.0.30.102
	apollo-adminService	10.0.30.101
	apollo-adminService	10.0.30.102
PRO	apollo-configService	10.0.30.111
	apollo-configService	10.0.30.112
	apollo-adminService	10.0.30.111
	apollo-adminService	10.0.30.112

部署 java

tar xf jdk-8u77-linux-x64.tar.gz  -C /usr/local/cat << EOF >> /etc/profileJAVA_HOME=/usr/local/jdk1.8.0_77JAVA_BIN=$JAVA_HOME/binPATH=$PATH:$JAVA_BINCLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jarexport JAVA_HOME JAVA_BIN PATH CLASSPATHEOFsource  /etc/profileln -vs  /usr/local/jdk1.8.0_77/bin/java /usr/bin/java

修改配置启动apolloconfig、apolloadmin

yum install unzip -ymkdir -pv apollo/{adminservice,configservice}unzip apollo-configservice-1.8.1-github.zip -d apollo/configservice/unzip apollo-adminservice-1.8.1-github.zip -d apollo/adminservice/=== 修改如下 ===cat apollo/configservice/config/application-github.properties# DataSourcespring.datasource.url = jdbc:mysql://10.0.30.103:3306/ApolloConfigDB?characterEncoding=utf8	# mysql 地址【注意修改为对应集群的地址】spring.datasource.username = root	# 用户名spring.datasource.password = mysql@123.COM	# 密码cat apollo/adminservice/config/application-github.properties# DataSourcespring.datasource.url = jdbc:mysql://10.0.30.103:3306/ApolloConfigDB?characterEncoding=utf8spring.datasource.username = rootspring.datasource.password = mysql@123.COM

注意：这里修改mysql地址请对应集群，比如这里是 DEV 环境对应的数据库地址：10.0.30.103 。如果是PRO 环境，对应的数据库地址：10.0.30.113

启动服务：

# ./apollo/configservice/scripts/startup.sh 
Wed May 26 11:14:30 CST 2021 ==== Starting ==== 
Application is running as root (UID 0). This is considered insecure.
Started [1935]
Waiting for server startup...
Wed May 26 11:14:45 CST 2021 Server started in 15 seconds!
# ./apollo/adminservice/scripts/startup.sh 
Wed May 26 11:14:55 CST 2021 ==== Starting ==== 
Application is running as root (UID 0). This is considered insecure.
Started [2118]
Waiting for server startup..
Wed May 26 11:15:06 CST 2021 Server started in 10 seconds!

### 查看端口
# netstat -ntplu | egrep java
tcp6       0      0 :::8080   :::*   LISTEN 1935/java
tcp6       0      0 :::8090   :::*   LISTEN 2118/java

部署 apollo-portal

只部署了一个apollo-portal，用来管理所有环境的apollo节点。实际部署中，也可以每套环境都部署一个apollo-portal，管理各自环境的apollo-portal，下面也会有提示如何在每个环境单独部署一个apollo-portal。

环境	服务器	服务	端口
/	10.0.30.105	apollo-portal	8070
/	10.0.30.106	MySQL数据库ApolloPortalDB	3306

【10.0.30.105】操作如下：

java 环境

tar xf jdk-8u77-linux-x64.tar.gz  -C /usr/local/cat << EOF >> /etc/profileJAVA_HOME=/usr/local/jdk1.8.0_77JAVA_BIN=$JAVA_HOME/binPATH=$PATH:$JAVA_BINCLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jarexport JAVA_HOME JAVA_BIN PATH CLASSPATHEOFsource  /etc/profileln -vs  /usr/local/jdk1.8.0_77/bin/java /usr/bin/java

配置 apollo-portal

mkdir -pv apollo/portalyum install unzip -yunzip apollo-portal-1.8.1-github.zip -d apollo/portal/vim apollo/portal/config/application-github.properties# DataSourcespring.datasource.url = jdbc:mysql://10.0.30.106:3306/ApolloPortalDB?characterEncoding=utf8	# mysql 地址spring.datasource.username = root	# 用户名spring.datasource.password = mysql@123.COM	# 密码

注意：上面修改为对应数据库的连接地址、用户名、密码

配置meta service信息。

meta service的地址和端口，默认就是apollo-configservice的地址和端口。修改config/apollo-env.properties，填写当前的apollo-portal要管理的环境的apollo-configservice的地址和端口，多个地址之间用英文逗号隔开。修改完后的效果如下：

vim apollo/portal/config/apollo-env.propertiesdev.meta=http://10.0.30.101:8080,http://10.0.30.102:8080pro.meta=http://10.0.30.111:8080,http://10.0.30.112:8080

注意：上面的配置表示当前的apollo-portal要管理dev、pro两套环境的apollo节点，每个环境的节点数是两个。

启动 apollo-portal

# ./apollo/portal/scripts/startup.shWed May 26 14:05:29 CST 2021 ==== Starting ====Application is running as root (UID 0). This is considered insecure.Started [13345]Waiting for server startup..Wed May 26 14:05:39 CST 2021 Server started in 10 seconds!

通过浏览器访问

Docker 方式部署Quick Start

安装docker-ce

curl http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -o /etc/yum.repos.d/docker-ce.repoyum install git docker-ce docker-compose -ymkdir -pv /etc/docker/cat << EOF >> /etc/docker/daemon.json{  "registry-mirrors": [    "https://hub-mirror.c.163.com",    "https://mirror.baidubce.com"  ]}EOFsystemctl start dockerdocker info

下载 docker quick start 文件到本地

git clone https://github.com/ctripcorp/apollo.git

启动 Apollo 配置中心

cd apollo/scripts/docker-quick-start/docker-compose up

报错1：

ERROR: Get https://registry-1.docker.io/v2/: dial tcp: lookup registry-1.docker.io on 223.6.6.6:53: server misbehaving

修改 dns ，不知道为什么阿里的DNS 解析有问题，修改如下：

cat /etc/resolv.conf# Generated by NetworkManagernameserver 223.5.5.5nameserver 223.6.6.6sed -i 's@223.5.5.5@114.114.114.114@g' /etc/resolv.confcat /etc/resolv.conf# Generated by NetworkManagernameserver 114.114.114.114nameserver 223.6.6.6

出现如下信息，表示启动成功。

查看本地端口：

# netstat -ntplu | egrep dockertcp        0  0 0.0.0.0:8080   0.0.0.0:*   LISTEN  3734/docker-proxy   tcp        0  0 0.0.0.0:8090   0.0.0.0:*   LISTEN  3710/docker-proxy   tcp        0  0 0.0.0.0:13306  0.0.0.0:*   LISTEN  3580/docker-proxy   tcp        0  0 0.0.0.0:8070   0.0.0.0:*   LISTEN  3754/docker-proxy   tcp6       0  0 :::8080        :::*        LISTEN  3740/docker-proxy   tcp6       0  0 :::8090        :::*        LISTEN  3719/docker-proxy   tcp6       0  0 :::13306       :::*        LISTEN  3586/docker-proxy   tcp6       0  0 :::8070        :::*        LISTEN  3761/docker-proxy

浏览器访问本机 8070 端口即可。

连接数据库的方法如下：

# 注意：这里必须写 IP 地址，不能为 localhostmysql -uroot -p -h 127.0.0.1 -P 13306 # 密码为空，直接回车

k8s+Apollo

主机规划

（网络环境因素，主机配置双网卡）

IP	主机用途
192.168.1.201/192.168.137.201	k8s-master
192.168.1.202/192.168.137.202	k8s-node1
192.168.1.203/192.168.137.203	k8s-node2
192.168.1.204/192.168.137.204	harbor仓库
192.168.1.205/192.168.137.205	apollo-MySQL

部署k8s

k8s v1.8.6 离线部署：https://www.cnblogs.com/hukey/category/1858851.html

首先，部署 k8s+apollo 首先得有 k8s 环境。按照上面的文档三台主机搭建 k8s 集群，搭建成功如下：

部署Harbor仓库

harbor 部署：https://www.cnblogs.com/hukey/p/14113396.html

部署好之后界面：

部署Apollo到K8S

创建数据库并导入数据

首先，需要为 apollo 创建数据库，关系型数据库最好的方式还是直接运行在物理机上。

安装并配置数据库

apollo-mysql 对应主机：192.168.1.201

[root@apollo-mysql(192.168.1.205) ~]#yum install -y mysql-server mysql
[root@apollo-mysql(192.168.1.205) ~]#systemctl start mysqld
[root@apollo-mysql(192.168.1.205) ~]#egrep -ri password /var/log/mysqld.log 
2021-07-13T03:04:01.331079Z 6 [Note] [MY-010454] [Server] A temporary password is generated for root@localhost: BYLmmLR29q&!

--- 通过初始化密码连接mysql ---
[root@apollo-mysql(192.168.1.205) ~]#mysql -uroot -p'BYLmmLR29q&!'
mysql> alter user user() identified by 'mysql@123.COM';
mysql> flush privileges;

--- 修改完初始密码后，用修改后的密码登录 ---
[root@apollo-mysql(192.168.1.205) ~]#mysql -uroot -pmysql@123.COM   
mysql> create user 'root'@'%' identified by 'mysql@123.COM';  // 创建用户
mysql> grant all privileges on *.* to 'root'@'%';	// 赋予所有权限
mysql> flush privileges;
mysql> exit;

导入apollo 数据

root@apollo-mysql ~# unzip k8s-apollo.zip
root@apollo-mysql ~# cd k8s-apollo/db/config-db-prod/
root@apollo-mysql ~/k/d/config-db-prod# mysql -uroot -p'mysql@123.COM' < apolloconfigdb.sql
root@apollo-mysql ~/k/d/config-db-prod# cd ../portal-db/
root@apollo-mysql ~/k/d/portal-db# mysql -uroot -p'mysql@123.COM' < apolloportaldb.sql

--- 确认导入数据 ---
root@apollo-mysql ~/k/d/config-db-prod# mysql -uroot -p'mysql@123.COM'
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| ApolloPortalDB     |
| ProdApolloConfigDB |
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
6 rows in set (0.00 sec)

生成镜像并推送到 harbor仓库

首先在 harbor 中新建一个仓库

上传镜像到仓库

本步骤在 k8s-master 中操作

root@master ~#unzip k8s-apollo.zip  // 解压上面从百度云盘下载的包
root@master ~# cd k8s-apollo/k8s-apollo-images-v1.8.1/
root@master ~/k/k8s-apollo-images-v1.8.1# ls
k8s-apollo-docker-images-v1.8.1.tar.gz
root@master ~/k/k8s-apollo-images-v1.8.1# tar xf k8s-apollo-docker-images-v1.8.1.tar.gz
root@master ~/k/k8s-apollo-images-v1.8.1# docker load < k8s-apollo-docker-images-v1.8.1.tar

--- 打 tag 到对应的仓库 ---
root@master ~/k/k8s-apollo-images-v1.8.1# docker tag alpine-bash:3.8 192.168.1.204:80/alpine-bash:3.8
root@master ~/k/k8s-apollo-images-v1.8.1# docker tag apollo-config-server:v1.8.1 192.168.1.204:80/apollo/apollo-config-server:v1.8.1
root@master ~/k/k8s-apollo-images-v1.8.1# docker tag apollo-admin-server:v1.8.1 192.168.1.204:80/apollo/apollo-admin-server:v1.8.1
root@master ~/k/k8s-apollo-images-v1.8.1# docker tag apollo-portal-server:v1.8.1 192.168.1.204:80/apollo/apollo-portal-server:v1.8.1

--- 上传到 harbor ---

1. 修改 docker 配置
root@master ~/k/k8s-apollo-images-v1.8.1# vim /etc/docker/daemon.json   // 加入下面这一行
...
"insecure-registries": ["192.168.1.204:80"],
...

root@master ~/k/k8s-apollo-images-v1.8.1# systemctl restart docker  // 修改完成记得 restart docker

2. 登录harbor 用户名密码就是页面登录所用的用户名密码
root@master ~/k/k8s-apollo-images-v1.8.1# docker login 192.168.1.204:80
Username: admin
Password: 
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

3. 上传镜像到仓库
root@master ~# docker push 192.168.1.204:80/apollo/alpine-bash:3.8
root@master ~# docker push 192.168.1.204:80/apollo/apollo-config-server
root@master ~# docker push 192.168.1.204:80/apollo/apollo-admin-server
root@master ~# docker push 192.168.1.204:80/apollo/apollo-portal-server

为每个k8s 节点配置可信仓库

重点：非常容易遗漏的点，这里不做后面创建Pod是无法下载镜像的！！！

# vim /etc/docker/daemon.json
...
"insecure-registries": ["192.168.1.204:80"],
...

# systemctl restart docker

通过 yaml 文件构建Pod及服务

本步骤在 k8s-master 中操作

root@master ~# cd k8s-apollo/kubernetes/apollo-env-prod/
root@master ~/k/k/apollo-env-prod# ll -tsh 
total 24K
8.0K -rwxr-xr-x 1 root root 4.1K Jul 14 10:08 service-apollo-admin-server-prod.yaml*	// apollo-admin-server
8.0K -rwxr-xr-x 1 root root 4.1K Jul 14 10:08 service-apollo-config-server-prod.yaml*	// apollo-config-server
4.0K -rwxr-xr-x 1 root root 1.1K Jul 14 10:08 service-mysql-for-apollo-prod-env.yaml*	// 将apollo外部mysql引入k8s内部调用

注意，执行顺序如下：

service-mysql-for-apollo-prod-env.yaml
service-apollo-config-server-prod.yaml
service-apollo-admin-server-prod.yaml

在执行之前需要为 apollo 创建专属的命名空间：

root@master ~# kubectl create ns sre
namespace/sre created

为k8s 添加 harbor 登录 secrets

在创建 docker-registry时候一定要指定命名空间，切记！

[root@master(192.168.1.201) ~]#kubectl create secret docker-registry registry-harbor --namespace=sre --docker-server=192.168.1.204:80 --docker-username=admin --docker-password=123456 
secret/registry-harbor created

service-mysql-for-apollo-prod-env.yaml 执行

修改后的 service-mysql-for-apollo-dev-env.yaml 如下：

---
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-mysql-for-apollo-prod-env
  labels:
    app: service-mysql-for-apollo-prod-env
spec:
  ports:
    - protocol: TCP
      port: 3306
      targetPort: 3306
  type: ClusterIP
  sessionAffinity: None

---
kind: Endpoints
apiVersion: v1
metadata:
  namespace: sre
  name: service-mysql-for-apollo-prod-env
subsets:
  - addresses:
      - ip: 192.168.1.205	// 这里需要修改为对应的apollo-mysql 主机地址
    ports:
      - protocol: TCP
        port: 3306

执行：

root@master ~/k/k/apollo-env-prod# kubectl apply -f service-mysql-for-apollo-prod-env.yaml
service/service-mysql-for-apollo-prod-env created
endpoints/service-mysql-for-apollo-prod-env created

--- 查看并尝试连接 ---
root@master ~/k/k/apollo-env-prod# kubectl describe svc -n sre
Name:              service-mysql-for-apollo-prod-env
Namespace:         sre
Labels:            app=service-mysql-for-apollo-prod-env
Annotations:       Selector:  <none>
Type:              ClusterIP
IP:                10.111.52.129
Port:              <unset>  3306/TCP
TargetPort:        3306/TCP
Endpoints:         192.168.1.205:3306   // 确认这里是否是mysql主机
Session Affinity:  None
Events:            <none>


root@master ~/k/k/apollo-env-prod# yum install mysql -y
root@master ~/k/k/apollo-env-prod# mysql -uroot -pmysql@123.COM -h 10.111.52.129
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| ProdApolloConfigDB |
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
5 rows in set (0.01 sec)

确认mysql service 没问题。

service-apollo-config-server-prod.yaml 执行

修改后的 service-apollo-config-server-prod.yaml 如下：

kind: ConfigMap
apiVersion: v1
metadata:
  namespace: sre
  name: configmap-apollo-config-server-prod
data:
  application-github.properties: |
    spring.datasource.url = jdbc:mysql://service-mysql-for-apollo-prod-env.sre:3306/ProdApolloConfigDB?characterEncoding=utf8
    spring.datasource.username = root			// 修改为数据库的用户名
    spring.datasource.password = mysql@123.COM	// 修改为数据库的密码
    eureka.service.url = http://statefulset-apollo-config-server-prod-0.service-apollo-meta-server-prod:8080/eureka/,http://statefulset-apollo-config-server-prod-1.service-apollo-meta-server-prod:8080/eureka/,http://statefulset-apollo-config-server-prod-2.service-apollo-meta-server-prod:8080/eureka/

---
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-apollo-meta-server-prod
  labels:
    app: service-apollo-meta-server-prod
spec:
  ports:
    - protocol: TCP
      port: 8080
      targetPort: 8080
  selector:
    app: pod-apollo-config-server-prod
  type: ClusterIP
  clusterIP: None
  sessionAffinity: ClientIP

---
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-apollo-config-server-prod
  labels:
    app: service-apollo-config-server-prod
spec:
  ports:
    - protocol: TCP
      port: 8080
      targetPort: 8080
      nodePort: 30005
  selector:
    app: pod-apollo-config-server-prod
  type: NodePort
  sessionAffinity: ClientIP

---
kind: StatefulSet
apiVersion: apps/v1
metadata:
  namespace: sre
  name: statefulset-apollo-config-server-prod
  labels:
    app: statefulset-apollo-config-server-prod
spec:
  serviceName: service-apollo-meta-server-prod
  replicas: 1
  selector:
    matchLabels:
      app: pod-apollo-config-server-prod
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: pod-apollo-config-server-prod
    spec:
      imagePullSecrets:			// 添加仓库secret
      - name: registry-harbor	// 添加仓库secret
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - pod-apollo-config-server-prod
              topologyKey: kubernetes.io/hostname
      
      volumes:
        - name: volume-configmap-apollo-config-server-prod
          configMap:
            name: configmap-apollo-config-server-prod
            items:
              - key: application-github.properties
                path: application-github.properties
      
      containers:
        - image: 192.168.1.194:80/apollo/apollo-config-server:v1.8.1   // 修改为本地仓库镜像
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          name: container-apollo-config-server-prod
          ports:
            - protocol: TCP
              containerPort: 8080

          volumeMounts:
            - name: volume-configmap-apollo-config-server-prod
              mountPath: /apollo-config-server/config/application-github.properties
              subPath: application-github.properties
          env:
            - name: APOLLO_CONFIG_SERVICE_NAME
              value: "service-apollo-config-server-prod.sre"
          
          readinessProbe:
            tcpSocket:
              port: 8080
            initialDelaySeconds: 10
            periodSeconds: 5
          
          livenessProbe:
            tcpSocket:
              port: 8080
            initialDelaySeconds: 120
            periodSeconds: 10
          
      dnsPolicy: ClusterFirst
      restartPolicy: Always

执行：

root@master ~/k/k/apollo-env-prod# kubectl apply -f service-apollo-config-server-prod.yaml 
configmap/configmap-apollo-config-server-prod created
service/service-apollo-meta-server-prod created
service/service-apollo-config-server-prod created
statefulset.apps/statefulset-apollo-config-server-prod created

等一段时间查看
root@master ~/k/k/apollo-env-prod# kubectl get pod,svc -n sre
NAME                                          READY   STATUS    RESTARTS   AGE
pod/statefulset-apollo-config-server-prod-0   1/1     Running   0          10m

NAME                                        TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
service/service-apollo-config-server-prod   NodePort    10.102.17.193   <none>        8080:30005/TCP   10m
service/service-apollo-meta-server-prod     ClusterIP   None            <none>        8080/TCP         10m
service/service-mysql-for-apollo-prod-env   ClusterIP   10.111.52.129   <none>        3306/TCP         26m

service-apollo-admin-server-prod.yaml 执行

修改后的 service-apollo-admin-server-prod.yaml 如下：

kind: ConfigMap
apiVersion: v1
metadata:
  namespace: sre
  name: configmap-apollo-admin-server-prod
data:
  application-github.properties: |
    spring.datasource.url = jdbc:mysql://service-mysql-for-apollo-prod-env.sre:3306/ProdApolloConfigDB?characterEncoding=utf8
    spring.datasource.username = root				// mysql用户名
    spring.datasource.password = mysql@123.COM		// mysql密码
    eureka.service.url = http://statefulset-apollo-config-server-prod-0.service-apollo-meta-server-prod:8080/eureka/,http://statefulset-apollo-config-server-prod-1.service-apollo-meta-server-prod:8080/eureka/,http://statefulset-apollo-config-server-prod-2.service-apollo-meta-server-prod:8080/eureka/

---
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-apollo-admin-server-prod
  labels:
    app: service-apollo-admin-server-prod
spec:
  ports:
    - protocol: TCP
      port: 8090
      targetPort: 8090
  selector:
    app: pod-apollo-admin-server-prod  
  type: ClusterIP
  sessionAffinity: ClientIP

---
kind: Deployment
apiVersion: apps/v1
metadata:
  namespace: sre
  name: deployment-apollo-admin-server-prod
  labels:
    app: deployment-apollo-admin-server-prod
spec:
  replicas: 1
  selector:
    matchLabels:
      app: pod-apollo-admin-server-prod
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: pod-apollo-admin-server-prod
    spec:
      imagePullSecrets:
      - name: registry-harbor
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - pod-apollo-admin-server-prod
              topologyKey: kubernetes.io/hostname
      
      volumes:
        - name: volume-configmap-apollo-admin-server-prod
          configMap:
            name: configmap-apollo-admin-server-prod
            items:
              - key: application-github.properties
                path: application-github.properties
      
      initContainers:
        - image: 192.168.1.194:80/apollo/alpine-bash:3.8	// 修改镜像地址
          name: check-service-apollo-config-server-prod
          command: ['bash', '-c', "curl --connect-timeout 2 --max-time 5 --retry 50 --retry-delay 1 --retry-max-time 120 service-apollo-config-server-prod.sre:8080"]
      
      containers:
        - image: 192.168.1.194:80/apollo/apollo-admin-server:v1.8.1	// 修改镜像地址
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          name: container-apollo-admin-server-prod
          ports:
            - protocol: TCP
              containerPort: 8090
          
          volumeMounts:
            - name: volume-configmap-apollo-admin-server-prod
              mountPath: /apollo-admin-server/config/application-github.properties
              subPath: application-github.properties
          
          env:
            - name: APOLLO_ADMIN_SERVICE_NAME
              value: "service-apollo-admin-server-prod.sre"
          
          readinessProbe:
            tcpSocket:
              port: 8090
            initialDelaySeconds: 10
            periodSeconds: 5
          
          livenessProbe:
            tcpSocket:
              port: 8090
            initialDelaySeconds: 120
            periodSeconds: 10

      dnsPolicy: ClusterFirst
      restartPolicy: Always

执行：

root@master ~/k/k/apollo-env-prod# kubectl apply -f service-apollo-admin-server-prod.yaml 
configmap/configmap-apollo-admin-server-prod created
service/service-apollo-admin-server-prod created
deployment.apps/deployment-apollo-admin-server-prod created

过2分钟后查看：
root@master ~/k/k/apollo-env-prod# kubectl get pod,svc -n sre
NAME                                                       READY   STATUS    RESTARTS   AGE
pod/deployment-apollo-admin-server-prod-64c74bdcb9-2d74n   0/1     Running   0          19s
pod/statefulset-apollo-config-server-prod-0                1/1     Running   0          13m

NAME                                        TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
service/service-apollo-admin-server-prod    ClusterIP   10.100.33.188   <none>        8090/TCP         20s
service/service-apollo-config-server-prod   NodePort    10.102.17.193   <none>        8080:30005/TCP   13m
service/service-apollo-meta-server-prod     ClusterIP   None            <none>        8080/TCP         13m
service/service-mysql-for-apollo-prod-env   ClusterIP   10.111.52.129   <none>        3306/TCP         30m

最后，再来构建 service-apollo-portal-server.yaml

root@master ~/k/k/apollo-env-prod# cd ../
root@master ~/k/kubernetes# pwd
/root/k8s-apollo/kubernetes
root@master ~/k/kubernetes# ls service-apollo-portal-server.yaml 
service-apollo-portal-server.yaml*

service-apollo-portal-server.yaml 执行

修改后的 service-apollo-portal-server.yaml 如下：

# 为外部 mysql 服务设置 service
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-mysql-for-portal-server
  labels:
    app: service-mysql-for-portal-server
spec:
  ports:
    - protocol: TCP
      port: 3306
      targetPort: 3306
  type: ClusterIP
  sessionAffinity: None
---
kind: Endpoints
apiVersion: v1
metadata:
  namespace: sre
  name: service-mysql-for-portal-server
subsets:
  - addresses:
      # 更改为你的 mysql addresses, 例如 1.1.1.1
      - ip: 192.168.1.205		// 修改为k8s集群外部访问mysql的地址
    ports:
      - protocol: TCP
        port: 3306

---
# configmap for apollo-portal-server
kind: ConfigMap
apiVersion: v1
metadata:
  namespace: sre
  name: configmap-apollo-portal-server
data:
  application-github.properties: |
    spring.datasource.url = jdbc:mysql://service-mysql-for-portal-server.sre:3306/ApolloPortalDB?characterEncoding=utf8
    # mysql username
    spring.datasource.username = root	// mysql user
    # mysql password
    spring.datasource.password = mysql@123.COM // mysql password
  apollo-env.properties: |
    dev.meta=http://service-apollo-config-server-dev.sre:8080
    fat.meta=http://service-apollo-config-server-test-alpha.sre:8080
    uat.meta=http://service-apollo-config-server-test-beta.sre:8080
    pro.meta=http://service-apollo-config-server-prod.sre:8080

---
kind: Service
apiVersion: v1
metadata:
  namespace: sre
  name: service-apollo-portal-server
  labels:
    app: service-apollo-portal-server
spec:
  ports:
    - protocol: TCP
      port: 8070
      targetPort: 8070
      nodePort: 30001
  selector:
    app: pod-apollo-portal-server
  type: NodePort
  # portal session 保持
  sessionAffinity: ClientIP

---
kind: Deployment
apiVersion: apps/v1
metadata:
  namespace: sre
  name: deployment-apollo-portal-server
  labels:
    app: deployment-apollo-portal-server
spec:
  # 3 个实例
  replicas: 3
  selector:
    matchLabels:
      app: pod-apollo-portal-server
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: pod-apollo-portal-server
    spec:
      imagePullSecrets:			// 添加 harbor secret
      - name: registry-harbor	// 添加 harbor secret
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - pod-apollo-portal-server
              topologyKey: kubernetes.io/hostname
      
      volumes:
        - name: volume-configmap-apollo-portal-server
          configMap:
            name: configmap-apollo-portal-server
            items:
              - key: application-github.properties
                path: application-github.properties
              - key: apollo-env.properties
                path: apollo-env.properties
      
      initContainers:
        # 确保 admin-service 正常提供服务
        - image: 192.168.1.204:80/apollo/alpine-bash:3.8	// 修改镜像仓库
          name: check-service-apollo-admin-server-prod
          command: ['bash', '-c', "curl --connect-timeout 2 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 service-apollo-admin-server-prod.sre:8090"]    
      
      containers:
        - image: 192.168.1.204:80/apollo/apollo-portal-server:v1.8.1   // 修改镜像仓库
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          name: container-apollo-portal-server
          ports:
            - protocol: TCP
              containerPort: 8070
          
          volumeMounts:
            - name: volume-configmap-apollo-portal-server
              mountPath: /apollo-portal-server/config/application-github.properties
              subPath: application-github.properties
            - name: volume-configmap-apollo-portal-server
              mountPath: /apollo-portal-server/config/apollo-env.properties
              subPath: apollo-env.properties
          
          env:
            - name: APOLLO_PORTAL_SERVICE_NAME
              value: "service-apollo-portal-server.sre"
          
          readinessProbe:
            tcpSocket:
              port: 8070
            initialDelaySeconds: 10
            periodSeconds: 5
          
          livenessProbe:
            tcpSocket:
              port: 8070
            # 120s 内, server 未启动则重启 container
            initialDelaySeconds: 120
            periodSeconds: 15
          
      dnsPolicy: ClusterFirst
      restartPolicy: Always

执行：

root@master ~/k/kubernetes# kubectl apply -f service-apollo-portal-server.yaml 
service/service-mysql-for-portal-server created
endpoints/service-mysql-for-portal-server created
configmap/configmap-apollo-portal-server created
service/service-apollo-portal-server created
deployment.apps/deployment-apollo-portal-server created

查看：
root@master ~/k/kubernetes# kubectl get pod,svc -n sre
NAME                                                   READY   STATUS    RESTARTS   AGE
deployment-apollo-admin-server-prod-64c74bdcb9-2d74n   1/1     Running   0          26m
deployment-apollo-portal-server-f89ff96db-4bccn        1/1     Running   3          11m
deployment-apollo-portal-server-f89ff96db-hdnh8        1/1     Running   3          11m
deployment-apollo-portal-server-f89ff96db-xg2rs        1/1     Running   3          11m
statefulset-apollo-config-server-prod-0                1/1     Running   0          39m


NAME                                TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
service-apollo-admin-server-prod    ClusterIP   10.100.33.188   <none>        8090/TCP         26m
service-apollo-config-server-prod   NodePort    10.102.17.193   <none>        8080:30005/TCP   40m
service-apollo-meta-server-prod     ClusterIP   None            <none>        8080/TCP         40m
service-apollo-portal-server        NodePort    10.111.67.76    <none>        8070:30001/TCP   11m
service-mysql-for-apollo-prod-env   ClusterIP   10.111.52.129   <none>        3306/TCP         56m
service-mysql-for-portal-server     ClusterIP   10.107.60.175   <none>        3306/TCP         11m

通过浏览器访问：

http://192.168.10.201:30001/

--- EOF ---

查看全文

相关阅读:
SharePoint 2013 商务智能报表发布
 sharepoint designer web 服务器似乎没有安装microsoft sharepoint foundation
SharePoint 2013 Designer系列之数据视图
 SharePoint 2013 Designer系列之数据视图筛选
 SharePoint 2013 Designer系列之自定义列表表单
 SharePoint 2013 入门教程之创建及修改母版页
 SharePoint 2013 入门教程之创建页面布局及页面
 SharePoint 2010 级联下拉列表 (Cascading DropDownList)
使用SharePoint Designer定制开发专家库系统实例！
PL/SQL Developer 建立远程连接数据库的配置和安装包+汉化包+注册机

原文地址：https://www.cnblogs.com/hukey/p/14822521.html