kubernetes容器集群自签TLS证书

集群部署

1、环境规划
2、安装docker
3、自签TLS证书
4、部署Flannel网络
5、部署Etcd集群
6、创建Node节点kubeconfig文件
7、获取K8S二进制包
8、运行Master组件
9、运行Node组件
10、查询集群状态
11、启动一个测试实例
12、部署Web UI（Dashboard）

集群部署环境规划

软件	版本
Linux操作系统	CentOS7.2_x64
kubernetes	1.9
docker	18.09.7
etcd	3.0

注意：linux关闭selinux。

[root@master ~]# sed -i s#SELINUX=enforcing#SELINUX=disabled#g /etc/selinux/config`
[root@master ~]# getenforce
Enforcing
[root@master ~]# setenforce 0
[root@master ~]# getenforce
Permissive

角色	IP	组件
master	192.168.238.130	kube-apiserver、kube-controller-manager、kube-scheduler、etcd
node01	192.168.238.129	kubelet、kube-proxy、docker、flannel、etcd
node02	192.168.238.128	kubelet、kube-proxy、docker、flannel、etcd

集群部署安装docker

安装docker依赖包

[root@master ~]# yum install -y yum-utils device-mapper-persistent-data lvm2

安装docker

[root@master ~]# yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
[root@master ~]# ls /etc/yum.repos.d/docker-ce.repo 
/etc/yum.repos.d/docker-ce.repo
[root@master ~]# yum install -y docker-ce
配置国内镜像
[root@master ~]# cat /etc/docker/daemon.json 
{
    "registry-mirrors":["https://registry.docker-cn.com"]
}
设置docker开机自启动
[root@master ~]# systemctl enable docker
启动docker
[root@master ~]# systemctl start docker
查看docker信息
[root@master ~]# docker info

集群部署自签TLS证书

组件	使用的证书
etcd	ca.pem、server.pem、server-key.pem
kube-apiserver	ca.pem、server.pem、server-key.pem
kubelet	ca.pem、ca-key.pem
kube-proxy	ca.pem、kube-proxy.pem、kube-proxy-key.pem
kubectl	ca.pem、admin.pem、admin-key.pem

安装证书生产工具cfssl

[root@master ~]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
[root@master ~]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
[root@master ~]# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
[root@master ~]# chmod +x cfssljson_linux-amd64 cfssl-certinfo_linux-amd64 cfssl_linux-amd64
[root@master ~]# mv cfssljson_linux-amd64.1 /usr/local/bin/cfssljson
[root@master ~]# mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
[root@master ~]# mv cfssl_linux-amd64 /usr/local/bin/cfssl              
[root@master ~]# ls /usr/local/bin/cfssl*
/usr/local/bin/cfssl  /usr/local/bin/cfssl-certinfo  /usr/local/bin/cfssljson
[root@master ssl]# cfssl --help
Usage:
Available commands:
        serve
        gencert
        ocspdump
        ocspserve
        certinfo
        ocspsign
        info
        sign
        gencrl
        selfsign
        print-defaults
        bundle
        version
        genkey
        ocsprefresh
        scan
        revoke
Top-level flags:
  -allow_verification_with_non_compliant_keys
        Allow a SignatureVerifier to use keys which are technically non-compliant with RFC6962.
  -loglevel int
        Log level (0 = DEBUG, 5 = FATAL) (default 1)

生成证书

创建保存证书目录
[root@master ~]# mkdir ssl
[root@master ~]# cd ssl
生成证书模板文件
[root@master ssl]# cfssl print-defaults config >config.json
[root@master ssl]# ls
config.json
[root@master ssl]# cat config.json
{
    "signing": {
        "default": {
            "expiry": "168h"
        },
        "profiles": {
            "www": {
                "expiry": "8760h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth"
                ]
            },
            "client": {
                "expiry": "8760h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            }
        }
    }
}
[root@master ssl]# cfssl print-defaults csr >csr.json
[root@master ssl]# cat csr.json
{
    "CN": "example.net",
    "hosts": [
        "example.net",
        "www.example.net"
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "US",
            "L": "CA",
            "ST": "San Francisco"
        }
    ]
}
[root@master ssl]# cat > ca-config.json <<EOF
> {
>     "signing":{
>         "default":{
>             "expiry":"87600h"
>         },
>         "profiles":{
>             "kubernetes":{
>             "expiry":"87600h",
>                 "usages":[
>                     "signing",
>                     "key encipherment",
>                     "server auth",
>                     "client auth"
>                 ]
>             }
>         }
>     }
> }
> EOF
[root@master ssl]# cat ca-config.json 
{
    "signing":{
        "default":{
            "expiry":"87600h"
        },
        "profiles":{
            "kubernetes":{
            "expiry":"87600h",
                "usages":[
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}

[root@master ssl]# cat > ca-csr.json <<EOF
> {
>     "CN":"kubernetes",
>     "key":{
>         "algo":"rsa",
>         "size":2048
>     },
>     "name":[
>         {
>             "C":"CN",
>             "L":"Wuhan",
>             "ST":"Wuhan",
>             "O":"k8s",
>             "OU":"System"
>         }
>     ]
> 
> }
> EOF
[root@master ssl]# cat ca-csr.json
{
    "CN":"kubernetes",
    "key":{
        "algo":"rsa",
        "size":2048
    },
    "name":[
        {
            "C":"CN",
            "L":"Wuhan",
            "ST":"Wuhan",
            "O":"k8s",
            "OU":"System"
        }
    ]

}
[root@master ssl]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2019/06/30 11:51:14 [INFO] generating a new CA key and certificate from CSR
2019/06/30 11:51:14 [INFO] generate received request
2019/06/30 11:51:14 [INFO] received CSR
2019/06/30 11:51:14 [INFO] generating key: rsa-2048
2019/06/30 11:51:14 [INFO] encoded CSR
2019/06/30 11:51:14 [INFO] signed certificate with serial number 357684144253379560050468419609693070989434498568
生成证书ca-key.pem、ca.pem
[root@master ssl]# ls ca*
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem
[root@master ssl]# cat > server-csr.json <<EOF
> {
>     "CN":"kubernetes",
>     "hosts":[
>         "127.0.0.1",
>         "192.168.238.130",
>         "192.168.238.129",
>         "192.168.238.128",
>         "kubernetes.default",
>         "kubernetes.default.svc",
>         "kubernetes.default.svc.cluster",
>         "kubernetes.default.svc.cluster.local"
>     ],
>     "key":{
>         "algo":"rsa",
>         "size":2048
>     },
>     "names":[
>         {
>             "C":"CN",
>             "L":"Wuhan",
>             "ST":"Wuhan",
>             "O":"k8s",
>             "OU":"System"
>          }
>     ]
> }
> EOF
[root@master ssl]# cat server-csr.json
{
    "CN":"kubernetes",
    "hosts":[
        "127.0.0.1",
        "192.168.238.130",
        "192.168.238.129",
        "192.168.238.128",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.svc.cluster",
        "kubernetes.default.svc.cluster.local"
    ],
    "key":{
        "algo":"rsa",
        "size":2048
    },
    "names":[
        {
            "C":"CN",
            "L":"Wuhan",
            "ST":"Wuhan",
            "O":"k8s",
            "OU":"System"
         }
    ]
}
[root@master ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
2019/06/30 12:26:45 [INFO] generate received request
2019/06/30 12:26:45 [INFO] received CSR
2019/06/30 12:26:45 [INFO] generating key: rsa-2048
2019/06/30 12:26:45 [INFO] encoded CSR
2019/06/30 12:26:45 [INFO] signed certificate with serial number 349804933480633404809478762244384990113466024768
2019/06/30 12:26:45 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@master ssl]# ls server*
server.csr  server-csr.json  server-key.pem  server.pem
[root@master ssl]# cat > admin-csr.json <<EOF
> {
>     "CN":"admin",
>     "hosts":[],
>     "key":{
>         "algo":"rsa",
>         "size":2048
>     },
>     "names":[
>         {
>             "C":"CN",
>             "L":"Wuhan",
>             "ST":"Wuhan",
>             "O":"system:masters",
>             "OU":"System"
>         }
>     ]
> 
> }
> EOF
[root@master ssl]# cat admin-csr.json
{
    "CN":"admin",
    "hosts":[],
    "key":{
        "algo":"rsa",
        "size":2048
    },
    "names":[
        {
            "C":"CN",
            "L":"Wuhan",
            "ST":"Wuhan",
            "O":"system:masters",
            "OU":"System"
        }
    ]

}
[root@master ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
2019/06/30 12:34:53 [INFO] generate received request
2019/06/30 12:34:53 [INFO] received CSR
2019/06/30 12:34:53 [INFO] generating key: rsa-2048
2019/06/30 12:34:53 [INFO] encoded CSR
2019/06/30 12:34:53 [INFO] signed certificate with serial number 7605307211369238746660755012651019629332863527
2019/06/30 12:34:53 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@master ssl]# ls admin*
admin.csr  admin-csr.json  admin-key.pem  admin.pem
[root@master ssl]# cat > kube-proxy-csr.json <<EOF
> {
>     "CN":"system:kube-proxy",
>     "hosts":[],
>     "key":{
>         "algo":"rsa",
>         "size":2048
>     },
>     "names":[
>         {
>             "C":"CN",
>             "L":"Wuhan",
>             "ST":"Wuhan",
>             "O":"k8s",
>             "OU":"System"
>         }
> 
>     ]
> }
> EOF
[root@master ssl]# cat kube-proxy-csr.json
{
    "CN":"system:kube-proxy",
    "hosts":[],
    "key":{
        "algo":"rsa",
        "size":2048
    },
    "names":[
        {
            "C":"CN",
            "L":"Wuhan",
            "ST":"Wuhan",
            "O":"k8s",
            "OU":"System"
        }

    ]
}
[root@master ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
2019/06/30 12:42:07 [INFO] generate received request
2019/06/30 12:42:07 [INFO] received CSR
2019/06/30 12:42:07 [INFO] generating key: rsa-2048
2019/06/30 12:42:07 [INFO] encoded CSR
2019/06/30 12:42:07 [INFO] signed certificate with serial number 469894574335691035633190543464468828048263055138
2019/06/30 12:42:07 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@master ssl]# ls kube-proxy*
kube-proxy.csr  kube-proxy-csr.json  kube-proxy-key.pem  kube-proxy.pem
[root@master ssl]# ls *pem
admin-key.pem  ca-key.pem  kube-proxy-key.pem  server-key.pem
admin.pem      ca.pem      kube-proxy.pem      server.pem