1.构建镜像
下载基础镜像，这里使用openvz的包，下载地址为：https://wiki.openvz.org/Download/template/precreated，下载centos7的镜像
下载镜像后导入到本地
docker import centos-7-x86_64.tar.gz openvz-centos7
从官方下载最新版的Jenkins的war包和适配的jdk，这里使用jdk8。构建Jenkins的dockerfile文件dockerfile-jenkins
[root@dataserver jenkins]#cat dockerfile-jenkins FROM openvz-centos7 ADD jdk-8u241-linux-x64.tar.gz /home ADD jenkins2.222.war /home/jenkins.war ENV JAVA_HOME=/home/jdk1.8.0_241 ENV PATH=$PATH:/home/jdk1.8.0_241/bin ENV JENKINS_HOME=/var/jenkins_home WORKDIR /home CMD java -jar jenkins.war --httpPort=8080
构建后推送到本地仓库
docker build -t 192.168.31.9:5000/jenkins-self:2.222 -f dockerfile-jenkins . docker push 192.168.31.9:5000/jenkins-self:2.222
构建Jenkins-agent的dockerfile-agent
[root@dataserver jenkins]# cat dockerfile-agent FROM openvz-centos7 ADD jdk-8u241-linux-x64.tar.gz /home ADD agent.jar /home ENV JAVA_HOME=/home/jdk1.8.0_241 ENV PATH=$PATH:/home/jdk1.8.0_241/bin RUN yum -y install docker kubernetes-client kubernetes* WORKDIR /home CMD exec /home/jdk1.8.0_241/bin/java -Dorg.jenkinsci.remoting.engine.JnlpProtocol3.disabled=true -cp /home/agent.jar hudson.remoting.jnlp.Main -headless -url ${JENKINS_URL} -workDir ${JENKINS_AGENT_WORKDIR} ${JENKINS_SECRET} ${JENKINS_AGENT_NAME}
构建后推送到本地仓库
docker build -t 192.168.31.9:5000/jenkins-agent-self:v2.222.11 -f dockerfile-agent . docker push 192.168.31.9:5000/jenkins-agent-self:v2.222.11
agent.jar可以从Jenkins中下载。manager Jenkins --> 节点管理 --> 新建节点 --> 输入test，选择固定节点 --> 远程工作目录输入 /home --> 启动方式选择通过web web启动代理后点击保存。
这里这个节点是不在线的，点击后提示如下：
点击下面界面显示的agent.jar后就可以下载对应的agent.jar包
2.部署Jenkins到k8s中
namespace.yaml
[root@env11 cicd]# cat namespace.yaml apiVersion: v1 kind: Namespace metadata: name: kube-ops
pvc.yaml这里使用nfs服务提供外置存储
[root@env11 cicd]# cat pvc.yaml apiVersion: v1 kind: PersistentVolume metadata: name: opspv spec: capacity: storage: 20Gi accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Delete nfs: server: 192.168.31.9 path: /data/nfsData/jenkins --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: opspvc namespace: kube-ops spec: accessModes: - ReadWriteMany resources: requests: storage: 20Gi
rbac.yaml
[root@env11 cicd]# cat rbac.yaml apiVersion: v1 kind: ServiceAccount metadata: name: jenkins2 namespace: kube-ops --- kind: Role apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: jenkins2 namespace: kube-ops rules: - apiGroups: ["extensions", "apps"] resources: ["deployments"] verbs: ["create", "delete", "get", "list", "watch", "patch", "update"] - apiGroups: [""] resources: ["services"] verbs: ["create", "delete", "get", "list", "watch", "patch", "update"] - apiGroups: [""] resources: ["pods"] verbs: ["create","delete","get","list","patch","update","watch"] - apiGroups: [""] resources: ["pods/exec"] verbs: ["create","delete","get","list","patch","update","watch"] - apiGroups: [""] resources: ["pods/log"] verbs: ["get","list","watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding metadata: name: jenkins2 namespace: kube-ops roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: jenkins2 subjects: - kind: ServiceAccount name: jenkins2 namespace: kube-ops
jenkins2.yaml
[root@env11 cicd]# cat jenkins2.yaml --- apiVersion: extensions/v1beta1 kind: Deployment metadata: name: jenkins2 namespace: kube-ops spec: template: metadata: labels: app: jenkins2 spec: terminationGracePeriodSeconds: 10 serviceAccount: jenkins2 containers: - name: jenkins image: 192.168.31.9:5000/jenkins-self:v2.222 imagePullPolicy: IfNotPresent command: ["/bin/sh","-c","java -jar jenkins.war --httpPort=8080"] ports: - containerPort: 8080 name: web protocol: TCP - containerPort: 50000 name: agent protocol: TCP resources: limits: cpu: 2000m memory: 2Gi requests: cpu: 500m memory: 512Mi #livenessProbe: # httpGet: # path: /login # port: 8080 # initialDelaySeconds: 60 # timeoutSeconds: 5 # failureThreshold: 12 #readinessProbe: # httpGet: # path: /login # port: 8080 # initialDelaySeconds: 60 # timeoutSeconds: 5 # failureThreshold: 12 volumeMounts: - name: jenkinshome subPath: jenkinsenv mountPath: /var/jenkins_home env: - name: LIMITS_MEMORY valueFrom: resourceFieldRef: resource: limits.memory divisor: 1Mi - name: JAVA_OPTS value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85 -Duser.timezone=Asia/Shanghai securityContext: fsGroup: 1000 volumes: - name: jenkinshome persistentVolumeClaim: claimName: opspvc --- apiVersion: v1 kind: Service metadata: name: jenkins2 namespace: kube-ops labels: app: jenkins2 spec: selector: app: jenkins2 type: NodePort ports: - name: web port: 8080 targetPort: web nodePort: 30002 - name: agent port: 50000 targetPort: agent
部署完成后，就可以通过集群IP:3002后登陆Jenkins
3.需要安装插件：
Jenkins需要安装插件Kubernetes plugin插件才可以动态调用k8s的api接口完成pod的创建
4.配置Jenkins调用k8s配置
配置Jenkins代理为固定端口50000，因为service定义是对外暴露的端口是50000。
点击Jenkins的节点管理
点击节点管理
点击configure clouds，添加一个远端服务
配置Kubernetes地址为http://kubernetes.default:443，命名空间为kube-ops，和Jenkins是同一个命名空间，然后点击连接测试。因为Jenkins的pod在提交是配置了rbac授权，所以可以访问k8s。
配置Jenkins地址为http://jenkins2.kube-ops.svc.cluster.local:8080，如果上面的service配置的名字为jenkins，这里就配置为jenkins而不是jenkins2。
配置pod模版，名字配置为jenkins-slave-001，命名空间为kube-ops，标签列表为haimaxy-jnlp，这个标签列表很重要，后面需要用到。容器列表配置jnlp，镜像就是自己构建的192.168.31.9:5000/jenkins-agent-self:v2.222.11，工作目录写/home/jenkins
下面的运行的命令和参数都不要写，否则会覆盖掉镜像中定义的启动命令。
这里把/var/run/docker.sock和/home/jenkins/.kube挂载到容器中，这样可以使用docker和kubectl命令。注意kubectl必须在每个节点都可以执行。
5.测试
添加一个项目来测试。
这里选择节点标签就是前面的定义的haimaxy-jnlp。
写入shell命令。
添加完成后开始构建
构建过程，就是在k8s中创建了一个pod来运行。
查看构建日志
此时在k8s中查看命名空间kube-ops下有自动创建的pod。
执行完成后查看日志，发现同样获取了命名空间下的pod信息。
