在Openshift搭建Redis Cluster实践

集群模式: 三主三从

pv.yaml

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: cluster-redis-pv-0
  labels:
    pvname: cluster-redis-pv-0
spec:
  capacity:
    storage: 100M
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  nfs:
    server: 192.168.64.108
    path: /data/nfs/cluster/redis/0
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: cluster-redis-pv-1
  labels:
    pvname: cluster-redis-pv-1
spec:
  capacity:
    storage: 100M
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  nfs:
    server: 192.168.64.108
    path: /data/nfs/cluster/redis/1
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: cluster-redis-pv-2
  labels:
    pvname: cluster-redis-pv-2
spec:
  capacity:
    storage: 100M
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  nfs:
    server: 192.168.64.108
    path: /data/nfs/cluster/redis/2
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: cluster-redis-pv-3
  labels:
    pvname: cluster-redis-pv-3
spec:
  capacity:
    storage: 100M
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  nfs:
    server: 192.168.64.108
    path: /data/nfs/cluster/redis/3
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: cluster-redis-pv-4
  labels:
    pvname: cluster-redis-pv-4
spec:
  capacity:
    storage: 100M
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  nfs:
    server: 192.168.64.108
    path: /data/nfs/cluster/redis/4
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: cluster-redis-pv-5
  labels:
    pvname: cluster-redis-pv-5
spec:
  capacity:
    storage: 100M
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  nfs:
    server: 192.168.64.108
    path: /data/nfs/cluster/redis/5

redis-conf.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: cluster-redis-conf
data:
  redis.conf: |+
    cluster-enabled yes
    cluster-require-full-coverage no
    cluster-node-timeout 15 # 这里为了测试故意改小了1000倍
    cluster-config-file /data/nodes.conf
    cluster-migration-barrier 1

    appendonly yes
    protected-mode no
    requirepass passwd
    masterauth passwd
    maxmemory 1024M
    maxmemory-policy noeviction # 内存占满时不删数据直接报错

  update-node.sh: |
    #!/bin/sh
    REDIS_NODES="/data/nodes.conf"
    sed -i -e "/myself/ s/[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}/${POD_IP}/" ${REDIS_NODES}
    exec "$@"

redis-sts.yaml

---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-cluster-0
spec:
  serviceName: redis-cluster-0
  replicas: 2
  selector:
    matchLabels:
      app: redis-cluster-0
  template:
    metadata:
      labels:
        app: redis-cluster-0
        cluster: redis
    spec:
      containers:
      - name: redis
        image: redis:5.0.1-alpine
        ports:
        - containerPort: 6379
          name: client
        - containerPort: 16379
          name: gossip
        command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"]
        env:
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        volumeMounts:
        - name: conf
          mountPath: /conf
          readOnly: false
        - name: data
          mountPath: /data
          readOnly: false
      volumes:
      - name: conf
        configMap:
          name: cluster-redis-conf
          defaultMode: 0755
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 10M
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-cluster-1
spec:
  serviceName: redis-cluster-1
  replicas: 2
  selector:
    matchLabels:
      app: redis-cluster-1
  template:
    metadata:
      labels:
        cluster: redis
        app: redis-cluster-1
    spec:
      containers:
        - name: redis
          image: redis:5.0.1-alpine
          ports:
            - containerPort: 6379
              name: client
            - containerPort: 16379
              name: gossip
          command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"]
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
          volumeMounts:
            - name: conf
              mountPath: /conf
              readOnly: false
            - name: data
              mountPath: /data
              readOnly: false
      volumes:
        - name: conf
          configMap:
            name: cluster-redis-conf
            defaultMode: 0755
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: [ "ReadWriteOnce" ]
        resources:
          requests:
            storage: 10M
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-cluster-2
spec:
  serviceName: redis-cluster-2
  replicas: 2
  selector:
    matchLabels:
      app: redis-cluster-2
  template:
    metadata:
      labels:
        cluster: redis
        app: redis-cluster-2
    spec:
      containers:
        - name: redis
          image: redis:5.0.1-alpine
          ports:
            - containerPort: 6379
              name: client
            - containerPort: 16379
              name: gossip
          command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"]
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
          volumeMounts:
            - name: conf
              mountPath: /conf
              readOnly: false
            - name: data
              mountPath: /data
              readOnly: false
      volumes:
        - name: conf
          configMap:
            name: cluster-redis-conf
            defaultMode: 0755
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: [ "ReadWriteOnce" ]
        resources:
          requests:
            storage: 10M

redis-svc.yaml

---
apiVersion: v1
kind: Service
metadata:
  name: redis-cluster-0
spec:
  type: ClusterIP
  ports:
  - port: 6379
    targetPort: 6379
    name: client
  - port: 16379
    targetPort: 16379
    name: gossip
  selector:
    app: redis-cluster-0
---
apiVersion: v1
kind: Service
metadata:
  name: redis-cluster-1
spec:
  type: ClusterIP
  ports:
    - port: 6379
      targetPort: 6379
      name: client
    - port: 16379
      targetPort: 16379
      name: gossip
  selector:
    app: redis-cluster-1
---
apiVersion: v1
kind: Service
metadata:
  name: redis-cluster-2
spec:
  type: ClusterIP
  ports:
    - port: 6379
      targetPort: 6379
      name: client
    - port: 16379
      targetPort: 16379
      name: gossip
  selector:
    app: redis-cluster-2

build.sh

#!/usr/bin/env bash

oc new-project cluster
oc project cluster

oc create -f pv.yaml -n cluster
oc create -f redis-conf.yaml -n cluster

oc create -f redis-sts.yaml -n cluster
oc create -f redis-svc.yaml -n cluster

oc adm policy add-role-to-user admin dev  -n cluster

sleep 10

oc exec -it redis-cluster-0-0 -- redis-cli --cluster create --cluster-replicas 1 $(oc get pods -l cluster=redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ')

delete.sh

#!/usr/bin/env bash

oc project cluster



oc delete -f pv.yaml -n cluster
oc delete -f redis-conf.yaml -n cluster

oc delete -f redis-sts.yaml -n cluster
oc delete -f redis-svc.yaml -n cluster

oc delete project cluster

建立集群

sh build.sh