2.etcd集群的安装（cfssl版）

etcd的安装注意两点 1.systemd的配置文件   2. 证书

1. 解决 systemd的问题，想安装指定版本的etcd可以通过 yum方式安装 etcd 可以获得 systemc 和 etcd.conf 的模板。

[root@master01 etcd-v3.3.13-linux-amd64]# rpm -ql etcd
/etc/etcd
/etc/etcd/etcd.conf
/usr/bin/etcd
/usr/bin/etcdctl
/usr/lib/systemd/system/etcd.service
/usr/share/doc/etcd-3.3.11
/usr/share/doc/etcd-3.3.11/CHANGELOG.md
........
.......
.....
/usr/share/man/man1/etcdctl3.1.gz
/var/lib/etcd
------------------------------------------------------------------------------------
[root@master01 etcd-v3.3.13-linux-amd64]# !cat 
cat /usr/lib/systemd/system/etcd.service  
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
User=etcd
# set GOMAXPROCS to number of processors
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name="${ETCD_NAME}" --data-dir="${ETCD_DATA_DIR}" --listen-client-urls="${ETCD_LISTEN_CLIENT_URLS}""
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
[root@master01 etcd-v3.3.13-linux-amd64]# cat /etc/etcd/etcd.conf 
#[Member]
#ETCD_CORS=""
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
#ETCD_LISTEN_PEER_URLS="http://localhost:2380"
ETCD_LISTEN_CLIENT_URLS="http://localhost:2379"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
ETCD_NAME="default"
#ETCD_SNAPSHOT_COUNT="100000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
#ETCD_QUOTA_BACKEND_BYTES="0"
#ETCD_MAX_REQUEST_BYTES="1572864"
#ETCD_GRPC_KEEPALIVE_MIN_TIME="5s"
#ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s"
#ETCD_GRPC_KEEPALIVE_TIMEOUT="20s"
#
#[Clustering]
#ETCD_INITIAL_ADVERTISE_PEER_URLS="http://localhost:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://localhost:2379"
#ETCD_DISCOVERY=""
#ETCD_DISCOVERY_FALLBACK="proxy"
#ETCD_DISCOVERY_PROXY=""
#ETCD_DISCOVERY_SRV=""
#ETCD_INITIAL_CLUSTER="default=http://localhost:2380"
#ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
#ETCD_INITIAL_CLUSTER_STATE="new"
#ETCD_STRICT_RECONFIG_CHECK="true"
#ETCD_ENABLE_V2="true"
#
#[Proxy]
#ETCD_PROXY="off"
#ETCD_PROXY_FAILURE_WAIT="5000"
#ETCD_PROXY_REFRESH_INTERVAL="30000"
#ETCD_PROXY_DIAL_TIMEOUT="1000"
#ETCD_PROXY_WRITE_TIMEOUT="5000"
#ETCD_PROXY_READ_TIMEOUT="0"
#
#[Security]
#ETCD_CERT_FILE=""
#ETCD_KEY_FILE=""
#ETCD_CLIENT_CERT_AUTH="false"
#ETCD_TRUSTED_CA_FILE=""
#ETCD_AUTO_TLS="false"
#ETCD_PEER_CERT_FILE=""
#ETCD_PEER_KEY_FILE=""
#ETCD_PEER_CLIENT_CERT_AUTH="false"
#ETCD_PEER_TRUSTED_CA_FILE=""
#ETCD_PEER_AUTO_TLS="false"
#
#[Logging]
#ETCD_DEBUG="false"
#ETCD_LOG_PACKAGE_LEVELS=""
#ETCD_LOG_OUTPUT="default"
#
#[Unsafe]
#ETCD_FORCE_NEW_CLUSTER="false"
#
#[Version]
#ETCD_VERSION="false"
#ETCD_AUTO_COMPACTION_RETENTION="0"
#
#[Profiling]
#ETCD_ENABLE_PPROF="false"
#ETCD_METRICS="basic"
#
#[Auth]
#ETCD_AUTH_TOKEN="simple"

2. 解决证书问题

curl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64  -o cfssl_linux-amd64
curl https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64  -o cfssljson_linux-amd64
mv  cfssl_linux_amd64 /bin/cfssl
mv cfssljson_linux_amd64 /bin/cfssljson
-------------------------------------------------------------------------
生成根证书文件
[root@master01 ~]# cat ca-config.json ca-csr.json 
{
    "signing": {
        "default": {
            "expiry": "175200h"
        },
        "profiles": {
            "kubernetes": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
            "client auth"
                ]
            },
            "etcd": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}
{
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "cnpc",
            "OU": "RF"
        }
    ]
}
-----------------------------------------------------------------------
[root@master01 ~]# ./cfssl_linux-amd64 gencert --initca ca-csr.json | cfssljson --bare ca
2019/05/14 04:46:17 [INFO] generating a new CA key and certificate from CSR
2019/05/14 04:46:17 [INFO] generate received request
2019/05/14 04:46:17 [INFO] received CSR
2019/05/14 04:46:17 [INFO] generating key: rsa-2048
2019/05/14 04:46:17 [INFO] encoded CSR
2019/05/14 04:46:17 [INFO] signed certificate with serial number 542129512178184951176207072980197727070484397354
[root@master01 ~]# ls ca* -l
-rw-r--r-- 1 root root  640 May 14 03:52 ca-config.json
-rw-r--r-- 1 root root  968 May 14 04:46 ca.csr
-rw-r--r-- 1 root root  237 May 14 04:45 ca-csr.json
-rw------- 1 root root 1679 May 14 04:46 ca-key.pem
-rw-r--r-- 1 root root 1294 May 14 04:46 ca.pem
------------------------------------------------------------------------
生产集群验证证书
[root@master01 ~]# cat etcd-csr.json 
{
  "CN": "etcd-server",
  "hosts": [
    "localhost",
    "0.0.0.0",
    "127.0.0.1",
    "192.168.141.135",
    "192.168.141.136",
    "192.168.141.137"
  ],
  "key": {
    "algo": "rsa",
    "size": 4096
  },
  "names": [
    {
        "C": "CN",
        "L": "Beijing",
        "ST": "Beijing",
        "O": "cnpc",
        "OU": "RF"}
  ]
}

[root@master01 ~]# ./cfssl_linux-amd64 gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=etcd etcd-csr.json | cfssljson -bare etcd
2019/05/14 04:55:19 [INFO] generate received request
2019/05/14 04:55:19 [INFO] received CSR
2019/05/14 04:55:19 [INFO] generating key: rsa-4096
2019/05/14 04:55:23 [INFO] encoded CSR
2019/05/14 04:55:23 [INFO] signed certificate with serial number 246077356353194423743124239784275664122753186375
2019/05/14 04:55:23 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

-------------------------------------------------------------------------
etcd客户端访问证书 可有可不用。

[root@master01 ~]# cat etcd-client-csr.json 
{
  "CN": "etcd-client",
  "hosts": [
    ""
  ],
  "key": {
    "algo": "rsa",
    "size": 4096
  },
  "names": [
    {
        "C": "CN",
        "L": "Beijing",
        "ST": "Beijing",
        "O": "cnpc",
        "OU": "RF"}
  ]
}

[root@master01 ~]# ./cfssl_linux-amd64 gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=etcd etcd-client-csr.json | cfssljson -bare etcd-client
2019/05/14 04:58:46 [INFO] generate received request
2019/05/14 04:58:46 [INFO] received CSR
2019/05/14 04:58:46 [INFO] generating key: rsa-4096
2019/05/14 04:58:48 [INFO] encoded CSR
2019/05/14 04:58:48 [INFO] signed certificate with serial number 627937418614823301041449342112313001983243456545
2019/05/14 04:58:48 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@master01 ~]# ls etcd-client* -l
-rw-r--r-- 1 root root 1732 May 14 04:58 etcd-client.csr
-rw-r--r-- 1 root root  230 May 14 03:56 etcd-client-csr.json
-rw------- 1 root root 3247 May 14 04:58 etcd-client-key.pem
-rw-r--r-- 1 root root 1724 May 14 04:58 etcd-client.pem
------------------------------------

3.修改配置文件进行etcd启动

　

[root@master01 ~]# cat /etc/etcd/etcd.conf 
# [member]
ETCD_NAME=etcd1
ETCD_DATA_DIR="/var/lib/etcd/etcd1.etcd"
ETCD_WAL_DIR="/var/lib/etcd/wal"
ETCD_SNAPSHOT_COUNT="100"
ETCD_HEARTBEAT_INTERVAL="100"
ETCD_ELECTION_TIMEOUT="1000"
ETCD_LISTEN_PEER_URLS="https://192.168.141.136:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.141.136:2379,http://127.0.0.1:2379"
ETCD_MAX_SNAPSHOTS="5"
ETCD_MAX_WALS="5"
#ETCD_CORS=""
# [cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.141.136:2380"
# if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
ETCD_INITIAL_CLUSTER="etcd0=https://192.168.141.135:2380,etcd1=https://192.168.141.136:2380,etcd2=https://192.168.141.137:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.141.136:2379"
 
#ETCD_DISCOVERY="" 
#ETCD_DISCOVERY_SRV="" 
#ETCD_DISCOVERY_FALLBACK="proxy" 
#ETCD_DISCOVERY_PROXY="" 
#ETCD_STRICT_RECONFIG_CHECK="false" 
#ETCD_AUTO_COMPACTION_RETENTION="0" 
# [proxy] 
#ETCD_PROXY="off"/ 
#ETCD_PROXY_FAILURE_WAIT="5000" 
#ETCD_PROXY_REFRESH_INTERVAL="30000" 
#ETCD_PROXY_DIAL_TIMEOUT="1000" 
#ETCD_PROXY_WRITE_TIMEOUT="5000" 
#ETCD_PROXY_READ_TIMEOUT="0" 
# [security] 
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem" 
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" 
ETCD_CLIENT_CERT_AUTH="true" 
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem" 
ETCD_AUTO_TLS="true" 
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem" 
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" 
ETCD_PEER_CLIENT_CERT_AUTH="true" 
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem" 
ETCD_PEER_AUTO_TLS="true" 
# [logging] 
#ETCD_DEBUG="false" 
# examples for -log-package-levels etcdserver=WARNING,security=DEBUG 
#ETCD_LOG_PACKAGE_LEVELS=""　　

记得修改不通的node上的配置文件，保证统一合理。

4.启动查看状态

[root@master01 ~]# etcdctl --cacert=/etc/etcd/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints=https://192.168.141.135:2379,https://192.168.141.136:2379,https://192.168.141.137:2379 endpoint status
https://192.168.141.135:2379, b306da3cfa564bfe, 3.3.11, 20 kB, false, 2, 8
https://192.168.141.136:2379, e43238dbe44b3543, 3.3.11, 20 kB, true, 2, 8
https://192.168.141.137:2379, 95d8aab064c5a521, 3.3.11, 20 kB, false, 2, 8
[root@master01 ~]# etcdctl --cacert=/etc/etcd/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints=https://192.168.141.135:2379,https://192.168.141.136:2379,https://192.168.141.137:2379 endpoint health
https://192.168.141.137:2379 is healthy: successfully committed proposal: took = 6.097643ms
https://192.168.141.136:2379 is healthy: successfully committed proposal: took = 6.426622ms
https://192.168.141.135:2379 is healthy: successfully committed proposal: took = 2.639766ms