k8s搭建redis集群

redis集群在k8s

• 安装nfs

yum -y install nfs-utils
yum -y install rpcbind

• 新增/etc/exports

/usr/local/k8s/redis/pv1 172.16.215.0/24(rw,sync,no_root_squash)
/usr/local/k8s/redis/pv2 172.16.215.0/24(rw,sync,no_root_squash)
/usr/local/k8s/redis/pv3 172.16.215.0/24(rw,sync,no_root_squash)
/usr/local/k8s/redis/pv4 172.16.215.0/24(rw,sync,no_root_squash)
/usr/local/k8s/redis/pv5 172.16.215.0/24(rw,sync,no_root_squash)
/usr/local/k8s/redis/pv6 172.16.215.0/24(rw,sync,no_root_squash)

• 创建相应目录

mkdir -p /usr/local/k8s/redis/pv{1..6}

• 启动nfs和rpcbind服务

systemctl restart rpcbind
systemctl restart nfs
systemctl enable nfs

• 客户端安装

yum -y install nfs-utils

• 客户端查看存储共享

[root@k8s-node01 ~]# showmount -e 172.16.215.139
Export list for 172.16.215.139:
/usr/local/k8s/redis/pv6 172.16.215.0/24
/usr/local/k8s/redis/pv5 172.16.215.0/24
/usr/local/k8s/redis/pv4 172.16.215.0/24
/usr/local/k8s/redis/pv3 172.16.215.0/24
/usr/local/k8s/redis/pv2 172.16.215.0/24
/usr/local/k8s/redis/pv1 172.16.215.0/24

• master创建pv

apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv1
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.16.215.139
    path: "/usr/local/k8s/redis/pv1"

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-vp2
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.16.215.139
    path: "/usr/local/k8s/redis/pv2"

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv3
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.16.215.139
    path: "/usr/local/k8s/redis/pv3"

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv4
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.16.215.139
    path: "/usr/local/k8s/redis/pv4"

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv5
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.16.215.139
    path: "/usr/local/k8s/redis/pv5"

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv6
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.16.215.139
    path: "/usr/local/k8s/redis/pv6"

• master执行创建

[root@k8s-master ~]# kubectl create -f redis-pv.yaml 
persistentvolume/nfs-pv1 created
persistentvolume/nfs-vp2 created
persistentvolume/nfs-pv3 created
persistentvolume/nfs-pv4 created
persistentvolume/nfs-pv5 created
persistentvolume/nfs-pv6 created

• 创建configmap
  • 这里，我们可以直接将Redis的配置文件转化为Configmap，这是一种更方便的配置读取方式。配置文件redis.conf如下

appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379

• 创建名为redis-conf的Configmap：

kubectl create configmap redis-conf --from-file=redis.conf

• 查看创建的configmap:

[root@k8s-master redis]# kubectl describe cm redis-conf
Name:         redis-conf
Namespace:    default
Labels:       <none>
Annotations:  <none>

Data
====
redis.conf:
----
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379

Events:  <none>

• 创建Headless service
  Headless service是StatefulSet实现稳定网络标识的基础，我们需要提前创建。准备文件headless-service.yml如下：

apiVersion: v1
kind: Service
metadata:
  name: redis-service
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    port: 6379
  clusterIP: None
  selector:
    app: redis
    appCluster: redis-cluster

• 创建

[root@k8s-master redis]# kubectl create -f headless-service.yaml 

• 查看

[root@k8s-master redis]# kubectl get svc
NAME            TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE
kubernetes      ClusterIP   10.96.0.1      <none>        443/TCP        4d22h
redis-service   ClusterIP   None           <none>        6379/TCP       23s
web             NodePort    10.99.137.71   <none>        80:30003/TCP   2d19h

• 创建Redis 集群节点
  • 创建好Headless service后，就可以利用StatefulSet创建Redis 集群节点，这也是本文的核心内容。我们先创建redis.yml文件：

apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: redis-app
spec:
  serviceName: "redis-service"
  selector:
    matchLabels:
      app: redis
  replicas: 6
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      terminationGracePeriodSeconds: 20
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - redis
              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: redis
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
        resources:
          requests:
            cpu: "100m"
            memory: "100Mi"
        ports:
            - name: redis
              containerPort: 6379
              protocol: "TCP"
            - name: cluster
              containerPort: 16379
              protocol: "TCP"
        volumeMounts:
          - name: "redis-conf"
            mountPath: "/etc/redis"
          - name: "redis-data"
            mountPath: "/var/lib/redis"
      volumes:
      - name: "redis-conf"
        configMap:
          name: "redis-conf"
          items:
            - key: "redis.conf"
              path: "redis.conf"
  volumeClaimTemplates:
  - metadata:
      name: redis-data
    spec:
      accessModes: [ "ReadWriteMany" ]
      resources:
        requests:
          storage: 200M

• 创建

kubectl apply -f redis.yaml

• 查看

kubectl get pods -o wide

• 创建乌班图容器

kubectl run -it ubuntu --image=ubuntu --restart=Never /bin/bash
# 进入乌班图
kubectl exec -it ubuntu -- /bash/sh

• 使用阿里云乌班图

cat > /etc/apt/sources.list << EOF
deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
 
deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
EOF

• 成功后，原项目要求执行如下命令安装基本的软件环境

apt-get update
apt-get install -y vim wget python2.7 python-pip redis-tools dnsutils

• 初始化集群

pip install redis-trib==0.5.1

• 然后，创建只有Master节点的集群：

redis-trib.py create 
  `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 
  `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 
  `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379

• 其次，为每个Master添加Slave

redis-trib.py replicate 
  --master-addr `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 
  --slave-addr `dig +short redis-app-3.redis-service.default.svc.cluster.local`:6379

redis-trib.py replicate 
  --master-addr `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 
  --slave-addr `dig +short redis-app-4.redis-service.default.svc.cluster.local`:6379

redis-trib.py replicate 
  --master-addr `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379 
  --slave-addr `dig +short redis-app-5.redis-service.default.svc.cluster.local`:6379

• 至此，我们的Redis集群就真正创建完毕了，连到任意一个Redis Pod中检验一下：

[root@k8s-master redis]# kubectl exec -it redis-app-2 -- /bin/bash

root@redis-app-2:/data# /usr/local/bin/redis-cli -c

127.0.0.1:6379> cluster nodes
aa924ed36ce31be94daa0dab5fedd57abc1ce7fe 10.244.85.238:6379@16379 slave d6784ef6a90177cadba2f5ac054b154a8036f863 0 1619702340000 4 connected
a95195aed4250491f5840aa26f8c83b2911c5e60 10.244.85.224:6379@16379 master - 0 1619702339000 1 connected 0-5461
69861b643fdeb6b94ccd1888170fd893bfffc1d7 10.244.85.232:6379@16379 master - 0 1619702339000 2 connected 10923-16383
d6784ef6a90177cadba2f5ac054b154a8036f863 10.244.85.235:6379@16379 myself,master - 0 1619702338000 4 connected 5462-10922
05ef6287b585a9a3427a72c6ec938ecc5516cd57 10.244.85.236:6379@16379 slave a95195aed4250491f5840aa26f8c83b2911c5e60 0 1619702340879 1 connected
daa70741f10c9361069ca1012ada5569b851d28a 10.244.85.237:6379@16379 slave 69861b643fdeb6b94ccd1888170fd893bfffc1d7 0 1619702340000 2 connected

127.0.0.1:6379> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:4
cluster_my_epoch:4
cluster_stats_messages_ping_sent:255
cluster_stats_messages_pong_sent:266
cluster_stats_messages_meet_sent:2
cluster_stats_messages_sent:523
cluster_stats_messages_ping_received:266
cluster_stats_messages_pong_received:257
cluster_stats_messages_received:523

• 另外，还可以在NFS上查看Redis挂载的数据：

[root@k8s-master redis]# ll /usr/local/k8s/redis/pv3
总用量 12
-rw-r--r-- 1 root root  92 4月  29 22:05 appendonly.aof
-rw-r--r-- 1 root root 175 4月  29 22:05 dump.rdb
-rw-r--r-- 1 root root 805 4月  29 22:05 nodes.conf

• 创建用于访问Service
  前面我们创建了用于实现StatefulSet的Headless Service，但该Service没有Cluster Ip，因此不能用于外界访问。所以，我们还需要创建一个Service，专用于为Redis集群提供访问和负载均衡：

apiVersion: v1
kind: Service
metadata:
  name: redis-access-service
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    protocol: "TCP"
    port: 6379
    targetPort: 6379
  selector:
    app: redis
    appCluster: redis-cluster

• 如上，该Service名称为 redis-access-service，在K8S集群中暴露6379端口，并且会对labels name为app: redis或appCluster: redis-cluster的pod进行负载均衡。

[root@k8s-master redis]# kubectl get svc redis-access-service -o wide
NAME                   TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE   SELECTOR
redis-access-service   ClusterIP   10.108.198.163   <none>        6379/TCP   12s   app=redis,appCluster=redis-cluster

• 进入一个pod测试

kubectl exec -it redis-app-2 -- /bin/bash
# 进入redis
/usr/local/bin/redis-cli -h 10.108.198.163
10.108.198.163:6379> keys *
1) "name"

• 参考：
  https://blog.csdn.net/zhutongcloud/article/details/90768390