k8s+jenkins：部署SpringCloud微服务

接着上一篇：https://www.cnblogs.com/wwjj4811/p/14590698.html

拉取代码，构建镜像

配置nfs

vi /etc/exports

/opt/nfs/jenkins *(rw,no_root_squash)
/opt/nfs/maven   *(rw,no_root_squash)

systemctl restart nfs
mkdir -p /opt/nfs/maven
chown -R jenkins:jenkins /opt/nfs/maven
chmod -R 777 /opt/nfs/maven
#这里有点坑，卡了我很长时间，需要给docker.sock777权限，不然jenkins用户无法执行docker命令
#并且这里需要把每一台k8s机器的docker.sock赋权
cd /run/
chmod 777 docker.sock

配置流水线项目

创建harbor凭证

新建一个流水线项目，配置参数化构建

添加文本参数：

配置流水线脚本：

def git_address = "http://192.168.1.50:82/root/tensquare_back.git"
def git_auth = "070a1a0f-6f41-4b47-8b4e-9621087df6fd"
//构建版本的名称
def tag = "latest"
//Harbor私服地址
def harbor_url = "192.168.1.52:85"
//Harbor的项目名称
def harbor_project_name = "tensquare"
//Harbor的凭证
def harbor_auth = "067c32d4-92d1-4d73-bd8b-853dbb5186ee"
//pod模板
podTemplate(label: 'jenkins-slave', cloud: 'kubernetes', containers: [
	containerTemplate(
		name: 'jnlp',
		image: "192.168.1.52:85/library/jenkins-slave-maven:lasted"
	),
	containerTemplate(
		name: 'docker',
		image: "docker:stable",
		ttyEnabled: true,
		command: 'cat'
	),
],
volumes: [
	hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
	nfsVolume(mountPath: '/usr/local/apache-maven/repo', serverAddress: '192.168.1.36' , serverPath: '/opt/nfs/maven'),
],
)
{
node("jenkins-slave"){
// 第一步
	stage('拉取代码'){
		checkout([$class: 'GitSCM', branches: [[name: '${branch}']],
		userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]]])
	}
	// 第二步
	stage('代码编译'){
		//编译并安装公共工程
		sh "mvn -f tensquare_common clean install"
	}
// 第三步
stage('构建镜像，部署项目'){
	//把选择的项目信息转为数组
	def selectedProjects = "${project_name}".split(',')
	for(int i=0;i<selectedProjects.size();i++){
		//取出每个项目的名称和端口
		def currentProject = selectedProjects[i];
		//项目名称
		def currentProjectName = currentProject.split('@')[0]
		//项目启动端口
		def currentProjectPort = currentProject.split('@')[1]
		//定义镜像名称注意：在构建过程会发现无法创建仓库目录，是因为NFS共享目录权限不足，需更改权限
		//还有Docker命令执行权限问题
		//需要手动上传父工程依赖到NFS的Maven共享仓库目录中
		//微服务部署到K8S
		def imageName = "${currentProjectName}:${tag}"
		//编译，构建本地镜像
		sh "mvn -f ${currentProjectName} clean package dockerfile:build"
		container('docker') {
			//给镜像打标签
			sh "docker tag ${imageName} ${harbor_url}/${harbor_project_name}/${imageName}"
			//登录Harbor，并上传镜像
			withCredentials([usernamePassword(credentialsId: "${harbor_auth}", passwordVariable: 'password', usernameVariable: 'username')]){
				//登录
				sh "docker login -u ${username} -p ${password} ${harbor_url}"
				//上传镜像
				sh "docker push ${harbor_url}/${harbor_project_name}/${imageName}"
			}
			//删除本地镜像
			sh "docker rmi -f ${imageName}"
			sh "docker rmi -f ${harbor_url}/${harbor_project_name}/${imageName}"
		}
	}
}
}
}

保存，然后我们测试注册中心的构建：

等待一段时间，第一次部署，会安装许多maven依赖。

安装Kubernetes Continuous Deploy插件

安装完成后，重启jenkins。

获取k8s的token

cd ~/.kube/
cat config

创建k8s凭证：把上面config文件的内容复制到content中

修改流水线脚本

def git_address = "http://192.168.1.50:82/root/tensquare_back.git"
def git_auth = "070a1a0f-6f41-4b47-8b4e-9621087df6fd"
//构建版本的名称
def tag = "latest"
//Harbor私服地址
def harbor_url = "192.168.1.52:85"
//Harbor的项目名称
def harbor_project_name = "tensquare"
//Harbor的凭证
def harbor_auth = "067c32d4-92d1-4d73-bd8b-853dbb5186ee"
def k8s_auth = "f8de51c5-6d18-4f5b-8e8a-b645f45a750e"
//定义k8s-harbor的凭证
def secret_name = "registry-auth-secret"
//pod模板
podTemplate(label: 'jenkins-slave', cloud: 'kubernetes', containers: [
	containerTemplate(
		name: 'jnlp',
		image: "192.168.1.52:85/library/jenkins-slave-maven:lasted"
	),
	containerTemplate(
		name: 'docker',
		image: "docker:stable",
		ttyEnabled: true,
		command: 'cat'
	),
],
volumes: [
	hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
	nfsVolume(mountPath: '/usr/local/apache-maven/repo', serverAddress: '192.168.1.36' , serverPath: '/opt/nfs/maven'),
],
)
{
node("jenkins-slave"){
    // 第一步
	stage('拉取代码'){
		checkout([$class: 'GitSCM', branches: [[name: '${branch}']],userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]]])
	}
	// 第二步
	stage('代码编译'){
		//编译并安装公共工程
		sh "mvn -f tensquare_common clean install"
	}
// 第三步
stage('构建镜像，部署项目'){
	//把选择的项目信息转为数组
	def selectedProjects = "${project_name}".split(',')
	for(int i=0;i<selectedProjects.size();i++){
		//取出每个项目的名称和端口
		def currentProject = selectedProjects[i];
		//项目名称
		def currentProjectName = currentProject.split('@')[0]
		//项目启动端口
		def currentProjectPort = currentProject.split('@')[1]
		def imageName = "${currentProjectName}:${tag}"
		//编译，构建本地镜像
		sh "mvn -f ${currentProjectName} clean package dockerfile:build"
		container('docker') {
			//给镜像打标签
			sh "docker tag ${imageName} ${harbor_url}/${harbor_project_name}/${imageName}"
			//登录Harbor，并上传镜像
			withCredentials([usernamePassword(credentialsId: "${harbor_auth}", passwordVariable: 'password', usernameVariable: 'username')]){
				//登录
				sh "docker login -u ${username} -p ${password} ${harbor_url}"
				//上传镜像
				sh "docker push ${harbor_url}/${harbor_project_name}/${imageName}"
			}
			//删除本地镜像
			sh "docker rmi -f ${imageName}"
			sh "docker rmi -f ${harbor_url}/${harbor_project_name}/${imageName}"
		}
		def deploy_image_name = "${harbor_url}/${harbor_project_name}/${imageName}"
        //部署到K8S
        sh """
            sed -i 's#$IMAGE_NAME#${deploy_image_name}#' ${currentProjectName}/deploy.yml
            sed -i 's#$SECRET_NAME#${secret_name}#' ${currentProjectName}/deploy.yml
        """
        kubernetesDeploy configs: "${currentProjectName}/deploy.yml", kubeconfigId: "${k8s_auth}"

	}
}
}
}

配置deploy.yml

放到eureka项目的根路径下面：

---
apiVersion: v1
kind: Service
metadata:
  name: eureka
  labels:
    app: eureka
spec:
  type: NodePort
  ports:
    - port: 10086
      name: eureka
      targetPort: 10086
  selector:
    app: eureka
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: eureka
spec:
  serviceName: "eureka"
  replicas: 2
  selector:
    matchLabels:
      app: eureka
  template:
    metadata:
      labels:
        app: eureka
    spec:
      imagePullSecrets:
        - name: $SECRET_NAME
      containers:
        - name: eureka
          image: $IMAGE_NAME
          ports:
            - containerPort: 10086
          env:
            - name: MY_POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: EUREKA_SERVER
              value: "http://eureka-0.eureka:10086/eureka/,http://eureka-1.eureka:10086/eureka/"
            - name: EUREKA_INSTANCE_HOSTNAME
              value: ${MY_POD_NAME}.eureka
  podManagementPolicy: "Parallel"

修改application.yml

server:
  port: ${PORT:10086}
spring:
  application:
    name: eureka

eureka:
  server:
    # 续期时间，即扫描失效服务的间隔时间（缺省为60*1000ms）
    eviction-interval-timer-in-ms: 5000
    enable-self-preservation: false
    use-read-only-response-cache: false
  client:
    # eureka client间隔多久去拉取服务注册信息 默认30s
    registry-fetch-interval-seconds: 5
    serviceUrl:
      defaultZone: ${EUREKA_SERVER:http://127.0.0.1:${server.port}/eureka/}
  instance:
    # 心跳间隔时间，即发送一次心跳之后，多久在发起下一次（缺省为30s）
    lease-renewal-interval-in-seconds: 5
    # 在收到一次心跳之后，等待下一次心跳的空档时间，大于心跳间隔即可，即服务续约到期时间（缺省为90s）
    lease-expiration-duration-in-seconds: 10
    instance-id: ${EUREKA_INSTANCE_HOSTNAME:${spring.application.name}}:${server.port}@${random.long(1000000,9999999)}
    hostname: ${EUREKA_INSTANCE_HOSTNAME:${spring.application.name}}

生成docker凭证

k8s master执行:

docker login -u admin -p Harbor12345 192.168.1.52:85
#生成证书
kubectl create secret docker-registry registry-auth-secret --docker-server=192.168.1.52:85 --docker-username=admin --docker-password=Harbor12345 --docker-email=wj@qq.com
#查看密钥
kubectl get secret 

测试部署

访问集群暴露的30876端口

可以看到，两个eureka实例分散的部署到node1和node2节点了

测试部署服务网关zuul

修改zuul网关的eureka连接配置：

网关的deploy.yml文件,放到网关服务的根目录下

---
apiVersion: v1
kind: Service
metadata:
  name: zuul
  labels:
    app: zuul
spec:
  type: NodePort
  ports:
    - port: 10020
      name: zuul
      targetPort: 10020
  selector:
    app: zuul
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zuul
spec:
  serviceName: "zuul"
  replicas: 2
  selector:
    matchLabels:
      app: zuul
  template:
    metadata:
      labels:
        app: zuul
    spec:
      imagePullSecrets:
        - name: $SECRET_NAME
      containers:
        - name: zuul
          image: $IMAGE_NAME
          ports:
            - containerPort: 10020
  podManagementPolicy: "Parallel"

测试部署网关：

网关服务注册到eureka：