Centos7 k8s v1.5.2二进制部署安装-k8s组件

一、安装kube-apiserver服务

1、安装supervisor

yum install -y supervisor

2、创建supervisor的启动文件

vi /etc/supervisord.d/etcd-server.ini
#内容如下
[program:etcd-server-112-23]
command=/opt/etcd/etcd-server-startup.sh
numprocs=1
directory=/opt/etcd
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=etcd
redirect_stderr=true
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=4
stdout_capture_maxbytes=1MB
stdout_events_enabled=false

启动服务
systemctl restart supervisord

supervisorctl update
supervisorctl status
tail -fn 200 /data/logs/etcd-server/etcd.stdout.log
查看监听端口
netstat -luntp | grep etcd

检查集群状态
cd /opt/etcd
./etcdctl cluster-health
./etcdctl member list

3、安装kube

wget https://storage.googleapis.com/kubernetes-release/release/v1.15.2/kubernetes-server-linux-amd64.tar.gz
tar -zxvf kubernetes-v1.15.2-server-linux-amd64.tar.gz 
mv kubernetes kubernetes-v1.15.2
ln -s /opt/kubernetes-v1.15.2 /opt/kubernetes

rm -rf /opt/kubernetes/server/bin/*.tar
rm -rf  /opt/kubernetes/server/bin/*_tag

4、签发证书

vim client-csr.json
{
  "CN": "k8s-node",
  "hosts":[
  ],
  "key": {
      "algo": "rsa",
      "size": 2048
  },
  "names": [
      {
          "C": "CN",
          "L": "Beijing",
          "ST": "Beijing",
          "O": "od",
          "OU": "ops"
      }
  ]
}
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json | cfssl-json -bare client
这个证书目前专属于 apiserver加了一个 *.kubernetes.master 域名以便内部私有 DNS 解析使用(可删除)；至于很多人问过 kubernetes 这几个能不能删掉，答案是不可以的；因为当集群创建好后，default namespace 
下会创建一个叫 kubenretes 的 svc，有一些组件会直接连接这个 svc 来跟 api 通讯的，证书如果不包含可能会出现无法连接的情况；其他几个 kubernetes 开头的域名作用相同hosts包含的是授权范围，不在此范围的的节点
或者服务使用此证书就会报证书不匹配错误。

5、签发apiserver证书

vim apiserver-csr.json 
{
  "CN": "k8s-apiserver",
  "hosts": [
    "127.0.0.1",
    "192.168.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local",
    "192.168.112.10",
    "192.168.112.22",
    "192.168.112.23",
    "192.168.112.25"
  ],
  "key": {
      "algo": "rsa",
      "size": 2048
  },
  "names": [
      {
          "C": "CN",
          "L": "Beijing",
          "ST": "Beijing",
          "O": "od",
          "OU": "ops"
      }
  ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server apiserver-csr.json | cfssl-json -bare apiserver

5、拷贝三套证书

cd /opt/kubernetes/server/bin/cert
scp operations:/opt/certs/ca.pem ./
scp operations:/opt/certs/ca-key.pem ./
scp operations:/opt/certs/client.pem ./
scp operations:/opt/certs/client-key.pem ./
scp operations:/opt/certs/apiserver.pem ./
scp operations:/opt/certs/apiserver-key.pem ./

mkdir ../conf && cd ../conf

 6、日志审计配置

vim  /opt/kubernetes/server/bin/conf/audit.yaml

apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
- "RequestReceived"
rules:
# Log pod changes at RequestResponse level
- level: RequestResponse
  resources:
  - group: ""
    # Resource "pods" doesn't match requests to any subresource of pods,
    # which is consistent with the RBAC policy.
    resources: ["pods"]
# Log "pods/log", "pods/status" at Metadata level
- level: Metadata
  resources:
  - group: ""
    resources: ["pods/log", "pods/status"]

# Don't log requests to a configmap called "controller-leader"
- level: None
  resources:
  - group: ""
    resources: ["configmaps"]
    resourceNames: ["controller-leader"]

# Don't log watch requests by the "system:kube-proxy" on endpoints or services
- level: None
  users: ["system:kube-proxy"]
  verbs: ["watch"]
  resources:
  - group: "" # core API group
    resources: ["endpoints", "services"]

# Don't log authenticated requests to certain non-resource URL paths.
- level: None
  userGroups: ["system:authenticated"]
  nonResourceURLs:
  - "/api*" # Wildcard matching.
  - "/version"

# Log the request body of configmap changes in kube-system.
- level: Request
  resources:
  - group: "" # core API group
    resources: ["configmaps"]
  # This rule only applies to resources in the "kube-system" namespace.
  # The empty string "" can be used to select non-namespaced resources.
  namespaces: ["kube-system"]

# Log configmap and secret changes in all other namespaces at the Metadata level.
- level: Metadata
  resources:
  - group: "" # core API group
    resources: ["secrets", "configmaps"]

# Log all other resources in core and extensions at the Request level.
- level: Request
  resources:
  - group: "" # core API group
  - group: "extensions" # Version of group should NOT be included.

# A catch-all rule to log all other requests at the Metadata level.
- level: Metadata
  # Long-running requests like watches that fall under this rule will not
  # generate an audit event in RequestReceived.
  omitStages:
    - "RequestReceived"

  7、配置apiserver启动文件

vim /opt/kubernetes/server/bin/kube-apiserver.sh
#!/bin/bash
./kube-apiserver 
  --apiserver-count 2 
  --audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log 
  --audit-policy-file ./conf/audit.yaml 
  --authorization-mode RBAC 
  --client-ca-file ./cert/ca.pem 
  --requestheader-client-ca-file ./cert/ca.pem 
  --enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota 
  --etcd-cafile ./cert/ca.pem 
  --etcd-certfile ./cert/client.pem 
  --etcd-keyfile ./cert/client-key.pem 
  --etcd-servers https://192.168.112.21:2379,https://192.168.112.22:2379,https://192.168.112.23:2379 
  --service-account-key-file ./cert/ca-key.pem 
  --service-cluster-ip-range 192.168.0.0/16 
  --service-node-port-range 3000-29999 
  --target-ram-mb=1024 
  --kubelet-client-certificate ./cert/client.pem 
  --kubelet-client-key ./cert/client-key.pem 
  --log-dir  /data/logs/kubernetes/kube-apiserver 
  --tls-cert-file ./cert/apiserver.pem 
  --tls-private-key-file ./cert/apiserver-key.pem 
  --v 2

chmod +x /opt/kubernetes/server/bin/kube-apiserver.sh

 8、创建ini配置文件，program根据主机ip更改

vim  /etc/supervisord.d/kube-apiserver.ini
[program:kube-apiserver-112-21]
command=/opt/kubernetes/server/bin/kube-apiserver.sh            ; the program (relative uses PATH, can take args)
numprocs=1                                                      ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin                            ; directory to cwd to before exec (def no cwd)
autostart=true                                                  ; start at supervisord start (default: true)
autorestart=true                                                ; retstart at unexpected quit (default: true)
startsecs=30                                                    ; number of secs prog must stay running (def. 1)
startretries=3                                                  ; max # of serial start failures (default 3)
exitcodes=0,2                                                   ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                 ; signal used to kill process (default TERM)
stopwaitsecs=10                                                 ; max num secs to wait b4 SIGKILL (default 10)
user=root                                                       ; setuid to this UNIX account to run the program
redirect_stderr=true                                            ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stdout.log        ; stderr log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                    ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                        ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                     ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                     ; emit events on stdout writes (default false)

mkdir -p /data/logs/kubernetes/kube-apiserver

 9、启动

supervisorctl update
supervisorctl status
netstat -nutlp|grep kube-api

11、在前两台安装nginx

yum install nginx -y
vim /etc/nginx/nginx.conf  
stream {
    upstream kube-apiserver {
        server 192.168.112.22:6443     max_fails=3 fail_timeout=30s;
        server 192.168.112.23:6443     max_fails=3 fail_timeout=30s;
    }
    server {
        listen 7443;
        proxy_connect_timeout 2s;
        proxy_timeout 900s;
        proxy_pass kube-apiserver;
    }
}
nginx -t
systemctl restart nginx
systemctl enable nginx

12、安装keepalived

yum install keepalived -y

编写监听端口得执行脚本

vim /etc/keepalived/check_port.sh
#!/bin/bash
#keepalived 监控端口脚本
#使用方法：
#在keepalived的配置文件中
#vrrp_script check_port {#创建一个vrrp_script脚本,检查配置
# script "/etc/keepalived/check_port.sh 6379" #配置监听的端口
# interval 2 #检查脚本的频率,单位（秒）
#}
CHK_PORT=$1
if [ -n "$CHK_PORT" ];then
        PORT_PROCESS=`ss -lnt|grep $CHK_PORT|wc -l`
        if [ $PORT_PROCESS -eq 0 ];then
                echo "Port $CHK_PORT Is Not Used,End."
                exit 1
        fi
else
        echo "Check Port Cant Be Empty!"
fi

chmod +x /etc/keepalived/check_port.sh

13、配置keepalived文件

主：

vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id 192.168.112.20

}

vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 7443"
    interval 2
    weight -20
}

vrrp_instance VI_1 {
    state MASTER
    interface ens192
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 192.168.112.20
    nopreempt   #非抢占式 ，当主节点挂了以后，从节点vip飘到从上，主节点恢复以后，不主动飘回主，需要手动重启keepalived

    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        192.168.112.10
    }
}
systemctl start keepalived
systemctl enable keepalived

从：

! Configuration File for keepalived
global_defs {
    router_id 192.168.112.21
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 7443"
    interval 2
    weight -20
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 251
    mcast_src_ip 192.168.112.21
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
        chk_nginx
    }
    virtual_ipaddress {
        192.168.112.10
    }
}

 检查VIP：