实战项目部署应用到kubernetes流程(jenkins+docker+k8s)

 说明

通过jenkins构建java应用程序发布到k8s集群中

本文已一个大数据的java项目来演示构建部署过程

支持发布和回滚

支持一套模板应用不同项目

k8s基础准备

创建项目名称空间

[root@k8s-master ~]# kubectl create ns architect
namespace/architect created

[root@k8s-master ~]# kubectl label namespace architect istio-injection=enabled
namespace/architect labeled
[root@k8s-master ~]# kubectl get namespace -L istio-injection
NAME           STATUS   AGE     ISTIO-INJECTION
architect      Active   87s     enabled
default        Active   39d     enabled
istio-system   Active   7d17h   
kube-public    Active   39d     
kube-system    Active   39d     
[root@k8s-master ~]# 

　　

创建全局配置文件

＃创建大数据配置文件
[root@k8s-master ~]# ls bigdata-counter
core-site.xml  dmp_operator1.keytab  hbase-site.xml  hdfs-site.xml  krb5.conf
[root@k8s-master ~]#
[root@k8s-master ~]# kubectl create configmap bigdata-config --from-file=/root/bigdata-counter/ -n architect 
configmap/bigdata-config created
[root@k8s-master ~]# 
#全局环境变量
[root@k8s-master ~]# cat yaml/global-config.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: global-config
  namespace: architect
data:
  CONFIG_SERVER_URL: http://config.ms.xx.com
  CONFIG_SERVER_USER: myconfig
  ILOG_BOOTSTRAP_SERVERS: bootstrap.servers=192.168.1.122:9092
  CONFIG_SERVER_PWDS: mc123456

[root@k8s-master ~]# 
[root@k8s-master ~]# kubectl apply -f yaml/global-config.yaml 
configmap/global-config created
[root@k8s-master ~]# kubectl get configmap -n architect
NAME DATA AGE
bigdata-config 4 2m33s
global-config 4 3s
[root@k8s-master ~]#

创建habror认证信息

[root@k8s-master ~]# kubectl create secret docker-registry registry-secret --docker-server=dev-hub.xx.net --docker-username=admin --docker-password=Harbor12345 --docker-email=admin@dev-hub.xx.net -n architect
secret/registry-secret created
[root@k8s-master ~]# kubectl get secret -n architect
NAME                  TYPE                                  DATA   AGE
default-token-5wsxl   kubernetes.io/service-account-token   3      2m24s
istio.default         istio.io/key-and-cert                 3      2m24s
registry-secret       kubernetes.io/dockerconfigjson        1      10s

　

项目deployment文件

可以做成模本，通过变量替换形式．后面有讲到　

[root@k8s-master ~]# cat bigdata-interface-deploy.yaml 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    app: bigdata-interface
    version: default
  name: bigdata-interface
  namespace: architect
spec:
  replicas: 1
  selector:
    matchLabels:
      app: bigdata-interface
      version: default
  template:
    metadata:
      labels:
        app: bigdata-interface
        version: default
    spec:
      containers:
      - env:
        - name: izone
        - name: MY_SERVICE_NAME
          value: bigdata-interface
        - name: ilogEnvs
          value: izone,buildNumber
        - name: ILOG_BOOTSTRAP_SERVERS
          valueFrom:
            configMapKeyRef:
              key: ILOG_BOOTSTRAP_SERVERS
              name: global-config
        - name: CONFIG_SERVER_URL
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_URL
              name: global-config
        - name: CONFIG_SERVER_USER
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_USER
              name: global-config
        - name: CONFIG_SERVER_PWDS
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_PWDS
              name: global-config
        - name: CONFIG_SERVER_PWDS
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_PWDS
              name: global-config
        - name: JAR_BOOT_ARG
          value: --spring.cloud.config.fail-fast=true
        - name: JDK_HEAP_OPTS
          value: -Xms256m -Xmx256m
        image: dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b63
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 2
          httpGet:
            path: /actuator/info
            port: 80
            scheme: HTTP
          initialDelaySeconds: 300
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 30
        name: bigdata-interface
        ports:
        - containerPort: 80
          name: http
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /actuator/info
            port: 80
            scheme: HTTP
          initialDelaySeconds: 60
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 30
        volumeMounts:
        - mountPath: /opt/jdk
          name: jdk-path
        - mountPath: /srv/applogs
          name: applogs
      imagePullSecrets:
      - name: registry-secret
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      volumes:
      - hostPath:
          path: /usr/local/jdk/default
          type: ""
        name: jdk-path
      - hostPath:
          path: /data/applogs
          type: ""
        name: applogs
[root@k8s-master ~]# 

jenkins pipeline 模板文件

node {
    
   
     //项目名称
    String APPNAME='bigdata-interface'
     //名称空间
    String NAMESPACE='architect'
    String HarborUrl='dev-hub.jiatuiyun.net'
    String HarborUser='admin'
    String HarborPasswd='Harbor12345'
    String KubernetHost = 'root@192.168.100.101'
    


      properties([
         parameters([
         string(name:'GIT_BRANCH_TAG', defaultValue: 'master',description:'', trim: false) 
    ])])
           

   
    stage('git clone') {
        checkout([$class: 'GitSCM', branches: [[name: params.GIT_BRANCH_TAG]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bc', url: 'git@git.jiatuiyun.net:bigdata-center/bigdata-interface.git']]])
    }
       
    stage('maven build') {
        
         sh 'mvn clean package -U'  
         
    }
      
    stage('docker build && push habror'){
             sh "cd ${WORKSPACE}"
             restapi = '`ls -d *-restapi||echo .`'
           
             sh "ls -l ${restapi}/target/"
             sh "echo ${restapi}"
             sh "curl -o ${restapi}/target/Dockerfile http://192.168.1.55:8089/files/Dockerfile-java.v1"
             sh "docker build -t ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${restapi}/target"
             sh "docker tag ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}"
             sh "docker login ${HarborUrl} -u ${HarborUser} -p ${HarborPasswd}"
             sh "docker push ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}"
         
        }
    stage('deploy'){
             echo "${params.GIT_BRANCH_TAG}"
             echo "${BUILD_NUMBER}"
             //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect
             updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE}  ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record"
             sh "ssh -p 22 ${KubernetHost} ${updateImage}"
             //test
    } 
  
 }

　

进一步改造pipeline　

node {
      
    String HarborUrl='dev-hub.jiatuiyun.net'
    String HarborUser='admin'
    String HarborPasswd='Harbor12345'
    String KubernetHost = 'root@192.168.100.101'
     
 
 
      properties([
         parameters([
            string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false),
            string(defaultValue: 'bigdata-interface', description: '项目名', name: 'XBUILD_DEPLOY', trim: false),
            string(defaultValue: 'git@git.jiatuiyun.net:bigdata-center/bigdata-interface.git', description: 'git仓库地址', name: 'XBUILD_REPO', trim: false),
            string(defaultValue: 'master', description: '版本名称', name: 'XBUILD_BRANCHE', trim: false)
    ])])
            
 
    
    stage('git clone') {
        checkout([$class: 'GitSCM', branches: [[name: params.XBUILD_BRANCHE]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bc', url: params.XBUILD_REPO]]])
    }
        
    stage('maven build') {
         
         sh 'mvn clean package -U' 
          
    }
       
    stage('docker build && push habror'){
             sh "cd ${WORKSPACE}"
             restapi = '`ls -d *-restapi||echo .`'
            
             sh "ls -l ${restapi}/target/"
             sh "echo ${restapi}"
             sh "curl -o ${restapi}/target/Dockerfile http://192.168.1.55:8089/files/Dockerfile-java.v1"
             NAMESPACE = params.XBUILD_TEAM
             APPNAME = params.XBUILD_DEPLOY
            
             sh "docker build -t ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${restapi}/target"
             sh "docker tag ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}"
             sh "docker login ${HarborUrl} -u ${HarborUser} -p ${HarborPasswd}"
             sh "docker push ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}"
          
        }
    stage('deploy'){
             NAMESPACE = params.XBUILD_TEAM
             APPNAME = params.XBUILD_DEPLOY
             echo "${params.GIT_BRANCH_TAG}"
             echo "${BUILD_NUMBER}"
             //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect
             //updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE}  ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record"
             //sh "ssh -p 22 ${KubernetHost} ${updateImage}"
             //test
    } 
   
 }

　

jenkins上创建自由风格项目

 效果如下 支持参数化构建。但每项目单独JOB

查看镜像仓库已经推送正常

 项目回滚

node {
    
      properties([
         parameters([
        string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false),
        string(defaultValue: 'bigdata-interface', description: '项目名', name: 'XBUILD_DEPLOY', trim: false),
        string(defaultValue: '1', description: '镜像标签', name: 'XBUILD_NUMBER', trim: false)     

    ])])   

    stage('deploy'){
             
             echo "${BUILD_NUMBER}"
              NAMESPACE = params.XBUILD_TEAM
             APPNAME = params.XBUILD_DEPLOY
             BUILD_NUMBER = params.XBUILD_NUMBER
             echo "----${params.XBUILD_NUMBER}---${params.XBUILD_DEPLOY} ----${params.XBUILD_TEAM} "
             //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect
             //updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE}  ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record"
             //sh "ssh -p 22 ${KubernetHost} ${updateImage}"
             //test
    }
  
 }

　　

jenkins效果如下：输入namespace和deployment 名称和上次构建版本

查看k8s集群

可以灵活的在已知版本回滚操作

[root@k8s-master ~]# kubectl get deploy -n architect -o wide
NAME                READY   UP-TO-DATE   AVAILABLE   AGE     CONTAINERS          IMAGES                                                           SELECTOR
bigdata-interface   0/1     1            0           3h12m   bigdata-interface   dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b1   app=bigdata-interface,version=default
[root@k8s-master ~]# kubectl get pods -n architect
NAME                                 READY   STATUS    RESTARTS   AGE
bigdata-interface-6bf7b5646d-sf9j5   1/2     Running   0          24s
[root@k8s-master ~]# 

　

通过以上实践，可以将deployment做成模板

deployment模板文件　

-bash-4.2$ cat k8s-deployment.tpl 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    app: {APPNAME}
    version: default
  name: {APPNAME}
  namespace: {NAMESPACE}
spec:
  replicas: 1
  selector:
    matchLabels:
      app: {APPNAME}
      version: default
  template:
    metadata:
      labels:
        app: {APPNAME}
        version: default
    spec:
      containers:
      - env:
        - name: izone
        - name: MY_SERVICE_NAME
          value: {APPNAME}
        - name: ilogEnvs
          value: izone,buildNumber
        - name: ILOG_BOOTSTRAP_SERVERS
          valueFrom:
            configMapKeyRef:
              key: ILOG_BOOTSTRAP_SERVERS
              name: global-config
        - name: CONFIG_SERVER_URL
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_URL
              name: global-config
        - name: CONFIG_SERVER_USER
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_USER
              name: global-config
        - name: CONFIG_SERVER_PWDS
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_PWDS
              name: global-config
        - name: CONFIG_SERVER_PWDS
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_PWDS
              name: global-config
        - name: JAR_BOOT_ARG
          value: --spring.cloud.config.fail-fast=true
        - name: JDK_HEAP_OPTS
          value: -Xms256m -Xmx256m
        image: {HarborUrl}/{NAMESPACE}/{APPNAME}:{NAMESPACE}_b{BUILD_NUMBER}
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 2
          httpGet:
            path: /actuator/info
            port: 80
            scheme: HTTP
          initialDelaySeconds: 300
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 30
        name: {APPNAME}
        ports:
        - containerPort: 80
          name: http
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /actuator/info
            port: 80
            scheme: HTTP
          initialDelaySeconds: 60
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 30
        volumeMounts:
        - mountPath: /opt/jdk
          name: jdk-path
        - mountPath: /srv/applogs
          name: applogs
      imagePullSecrets:
      - name: registry-secret
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      volumes:
      - hostPath:
          path: /usr/local/jdk/default
          type: ""
        name: jdk-path
      - hostPath:
          path: /data/applogs
          type: ""
        name: applogs
-bash-4.2$ 

　

　　

通过模板动态生成项目的deploy文件．进行升级回滚．

比如下面的：

node {
    String HarborUrl='dev-hub.jiatuiyun.net'
    String HarborUser='admin'
    String HarborPasswd='Harbor12345'
    String KubernetHost = 'root@192.168.100.101'
      properties([
         parameters([
        string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false),
        string(defaultValue: 'bigdata-interface', description: '项目名', name: 'XBUILD_DEPLOY', trim: false),
        string(defaultValue: '1', description: '镜像标签', name: 'XBUILD_NUMBER', trim: false)     

    ])])   

    stage('deploy'){
             
             echo "${BUILD_NUMBER}"
              NAMESPACE = params.XBUILD_TEAM
             APPNAME = params.XBUILD_DEPLOY
             BUILD_NUMBER = params.XBUILD_NUMBER
             echo "----${params.XBUILD_NUMBER}---${params.XBUILD_DEPLOY} ----${params.XBUILD_TEAM} "
             //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect
             updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE}  ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record"
             command="sed -e 's#{APPNAME}#${APPNAME}#g;s#{NAMESPACE}#${NAMESPACE}#g;s#{HarborUrl}#${HarborUrl}#g;s#{BUILD_NUMBER}#${BUILD_NUMBER}#g' /data/jenkins_slave/k8s-deployment.tpl > /data/jenkins_slave/${APPNAME}-deploy.yaml"
             sh "${command}"
             sh "scp -P 22 /data/jenkins_slave/${APPNAME}-deploy.yaml ${KubernetHost}:/root/ "
             //sh "ssh -p 22 ${KubernetHost} ${updateImage}"
             sh "ssh -p 22 ${KubernetHost} kubectl apply -f /root/${APPNAME}-deploy.yaml"
             //test
    }
  
 }

pipeline参数化构建

 示例：

pipeline {
      agent any

 parameters {
  gitParameter branch: '', branchFilter: '.*', defaultValue: '', description: '', name: 'XBUILD_BRANCHE', quickFilterEnabled: false, selectedValue: 'NONE', sortMode: 'NONE', tagFilter: '*', type: 'PT_BRANCH_TAG'
  string defaultValue: 'git@git.xx.net:company/xx-xx.git', description: '', name: 'XBUILD_REPO', trim: false
}


 
 stages {
        stage('Example') {
            steps {
          checkout([$class: 'GitSCM', branches: [[name: params.XBUILD_BRANCHE]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bb', url: params.XBUILD_REPO]]])
            }
        }
    }


 }

　　构建效果如下：