k8s-rabbitmq集群(三磁盘/一磁盘两内存)镜像模式 安装延迟插件

转载表明出处

断断续续费时两周。。。一步一个坑,修改下NFS_IP 直接apply -f  就可以了

----------------------------------------------------------------------------------------------------

补充下，经过多次修改测试 集群中内存模式的pod有BUG。改为内存模式后

貌似集群通讯配置也会放到内存中，stop_app && start_app后进程会退出，

导致pod一直重启，报错为找不到node通讯节点。还是老实的用三硬盘集群了

----------------------------------------------------------------------------------------------------

NFS存储

------存储RBAC

apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: rabbitmq
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner
namespace: rabbitmq
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create","watcg","get", "update", "patch", "list"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["create","watcg","get", "update", "patch", "list", "delete"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
namespace: rabbitmq
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: rabbitmq
roleRef:
kind: ClusterRole
name: nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io

存储sts

apiVersion: storage.k8s.io/v1beta1
kind: StorageClass
metadata:
name: managed-nfs-storage-rabbitmq
provisioner: kubernetes.io/rabbitmq

存储PV

kind: Deployment
apiVersion: apps/v1
metadata:
name: nfs-client-provisioner-rabbitmq
namespace: rabbitmq
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: nfs-client-provisioner
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
#serviceAccount: nfs-provisioner-rabbitmq
serviceAccount: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
image: registry.cn-hangzhou.aliyuncs.com/open-ali/nfs-client-provisioner
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: kubernetes.io/rabbitmq
- name: NFS_SERVER
value: NFS_IP
- name: NFS_PATH
value: /data1/rabbitmq
volumes:
- name: nfs-client-root
nfs:
server: NFS_IP
path: /data1/rabbitmq
nodeSelector:
role: "app"

statefulset_rabbitmq集群

service

kind: Service
apiVersion: v1
metadata:
labels:
app: rabbitmq-cluster
name: rabbitmq-cluster
namespace: rabbitmq
spec:
clusterIP: None
ports:
- name: rmqport
port: 5672
targetPort: 5672
selector:
app: rabbitmq-cluster
type: ClusterIP

---
kind: Service
apiVersion: v1
metadata:
labels:
app: rabbitmq-cluster
name: rabbitmq-cluster-manage
namespace: rabbitmq
spec:
ports:
- name: http
port: 15672
protocol: TCP
targetPort: 15672
selector:
app: rabbitmq-cluster
type: NodePort

rabbimq_configmap

kind: ConfigMap
apiVersion: v1
metadata:
name: rabbitmq-cluster-config
namespace: rabbitmq
labels:
addonmanager.kubernetes.io/mode: Reconcile
data:
enabled_plugins: |
[rabbitmq_management,rabbitmq_peer_discovery_k8s,rabbitmq_delayed_message_exchange].
rabbitmq.conf: |
default_user = admin
default_pass = RABBITMQ_PASSWORD
## Cluster formation. See https://www.rabbitmq.com/cluster-formation.html to learn more.
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s
cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
## Should RabbitMQ node name be computed from the pod's hostname or IP address?
## IP addresses are not stable, so using [stable] hostnames is recommended when possible.
## Set to "hostname" to use pod hostnames.
## When this value is changed, so should the variable used to set the RABBITMQ_NODENAME
## environment variable.
cluster_formation.k8s.address_type = hostname
## How often should node cleanup checks run?
cluster_formation.node_cleanup.interval = 30
## Set to false if automatic removal of unknown/absent nodes
## is desired. This can be dangerous, see
## * https://www.rabbitmq.com/cluster-formation.html#node-health-checks-and-cleanup
## * https://groups.google.com/forum/#!msg/rabbitmq-users/wuOfzEywHXo/k8z_HWIkBgAJ
cluster_formation.node_cleanup.only_log_warning = true
cluster_partition_handling = autoheal
## See https://www.rabbitmq.com/ha.html#master-migration-data-locality
queue_master_locator=min-masters
## See https://www.rabbitmq.com/access-control.html#loopback-users
loopback_users.guest = false
cluster_formation.randomized_startup_delay_range.min = 0
cluster_formation.randomized_startup_delay_range.max = 2
# default is rabbitmq-cluster's namespace
# hostname_suffix
cluster_formation.k8s.hostname_suffix = .rabbitmq-cluster.rabbitmq.svc.cluster.local
# memory
vm_memory_high_watermark.absolute = 1GB
# disk
disk_free_limit.absolute = 2GB

rabbitmq集群

kind: StatefulSet
apiVersion: apps/v1
metadata:
labels:
app: rabbitmq-cluster
name: rabbitmq-cluster
namespace: rabbitmq
spec:
replicas: 3
selector:
matchLabels:
app: rabbitmq-cluster
serviceName: rabbitmq-cluster
template:
metadata:
labels:
app: rabbitmq-cluster
spec:
containers:
- args:
- "-c"
- cp -v /etc/rabbitmq/rabbitmq.conf ${RABBITMQ_CONFIG_FILE}; exec docker-entrypoint.sh
rabbitmq-server
command:
- sh
env:
- name: TZ
value: 'Asia/Shanghai'
- name: RABBITMQ_ERLANG_COOKIE
value: 'SWvCP0Hrqv43NG7GybHC95ntCJKoW8UyNFWnBEWG8TY='
- name: K8S_SERVICE_NAME
value: rabbitmq-cluster
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: RABBITMQ_USE_LONGNAME
value: "true"
- name: RABBITMQ_NODENAME
value: rabbit@$(POD_NAME).$(K8S_SERVICE_NAME).$(POD_NAMESPACE).svc.cluster.local
- name: RABBITMQ_CONFIG_FILE
value: /var/lib/rabbitmq/rabbitmq.conf
#image: ccr.ccs.tencentyun.com/duia/rabbitmq:v2
image: rabbitmq:3.8.3-management
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- rabbitmq-diagnostics
- status
# See https://www.rabbitmq.com/monitoring.html for monitoring frequency recommendations.
initialDelaySeconds: 60
periodSeconds: 60
timeoutSeconds: 15
name: rabbitmq
ports:
- containerPort: 15672
name: http
protocol: TCP
- containerPort: 5672
name: amqp
protocol: TCP
readinessProbe:
exec:
command:
- rabbitmq-diagnostics
- status
initialDelaySeconds: 20
periodSeconds: 60
timeoutSeconds: 10
volumeMounts:
#- mountPath: /etc/rabbitmq/
# name: enable-plugins
- mountPath: /etc/rabbitmq
name: config-volume
readOnly: false
- mountPath: /var/lib/rabbitmq
name: rabbitmq-storage
readOnly: false
- name: timezone
mountPath: /etc/localtime
readOnly: true
lifecycle:
postStart:
exec:
command:
- /bin/sh
- -c
- sleep 61 && if [ $POD_NAME = rabbitmq-cluster-1 -o $POD_NAME = rabbitmq-cluster-2 ];then rabbitmqctl stop_app; rabbitmqctl change_cluster_node_type ram >> /home/rabbitmq.log ; rabbitmqctl start_app;rabbitmqctl cluster_status >> /home/rabbitmq.log;fi && apt-get update > /home/apt.log && apt-get -y install wget sudo && wget -P /plugins/ https://github.com/rabbitmq/rabbitmq-delayed-message-exchange/releases/download/v3.8.0/rabbitmq_delayed_message_exchange-3.8.0.ez && chown -R rabbitmq. /plugins/rabbitmq_delayed_message_exchange-3.8.0.ez
serviceAccountName: nfs-client-provisioner
terminationGracePeriodSeconds: 30
volumes:
# - name: enable-plugins
# nfs:
# path: /data1/rabbitmq/enabled_plugins
# server: 10.0.0.53
- name: config-volume
configMap:
items:
- key: rabbitmq.conf
path: rabbitmq.conf
- key: enabled_plugins
path: enabled_plugins
name: rabbitmq-cluster-config
- name: timezone
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
- name: rabbitmq-storage
persistentVolumeClaim:
claimName: rabbitmq

volumeClaimTemplates:
- metadata:
name: rabbitmq-storage
spec:
accessModes:
- ReadWriteMany
storageClassName: "managed-nfs-storage-rabbitmq"
resources:
requests:
storage: 5Gi

三磁盘集群

kind: StatefulSet
apiVersion: apps/v1
metadata:
labels:
app: rabbitmq-cluster
name: rabbitmq-cluster
namespace: rabbitmq
spec:
replicas: 3
selector:
matchLabels:
app: rabbitmq-cluster
serviceName: rabbitmq-cluster
template:
metadata:
labels:
app: rabbitmq-cluster
spec:
containers:
- args:
- -c
- cp -v /etc/rabbitmq/rabbitmq.conf ${RABBITMQ_CONFIG_FILE}; exec docker-entrypoint.sh
rabbitmq-server
command:
- sh
env:
- name: TZ
value: 'Asia/Shanghai'
- name: RABBITMQ_ERLANG_COOKIE
value: 'SWvCP0Hrqv43NG7GybHC95ntCJKoW8UyNFWnBEWG8TY='
- name: K8S_SERVICE_NAME
value: rabbitmq-cluster
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: RABBITMQ_USE_LONGNAME
value: "true"
- name: RABBITMQ_NODENAME
value: rabbit@$(POD_NAME).$(K8S_SERVICE_NAME).$(POD_NAMESPACE).svc.cluster.local
- name: RABBITMQ_CONFIG_FILE
value: /var/lib/rabbitmq/rabbitmq.conf
#image: ccr.ccs.tencentyun.com/duia/rabbitmq:v2
image: rabbitmq:3.8.3-management
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- rabbitmq-diagnostics
- status
# See https://www.rabbitmq.com/monitoring.html for monitoring frequency recommendations.
initialDelaySeconds: 60
periodSeconds: 60
timeoutSeconds: 15
name: rabbitmq
ports:
- containerPort: 15672
name: http
protocol: TCP
- containerPort: 5672
name: amqp
protocol: TCP
readinessProbe:
exec:
command:
- rabbitmq-diagnostics
- status
initialDelaySeconds: 20
periodSeconds: 60
timeoutSeconds: 10
volumeMounts:
#- mountPath: /etc/rabbitmq/
# name: enable-plugins
- mountPath: /etc/rabbitmq
name: config-volume
readOnly: false
- mountPath: /var/lib/rabbitmq
name: rabbitmq-storage
readOnly: false
- name: timezone
mountPath: /etc/localtime
readOnly: true
serviceAccountName: nfs-client-provisioner
terminationGracePeriodSeconds: 30
volumes:
# - name: enable-plugins
# nfs:
# path: /data1/rabbitmq/enabled_plugins
# server: 10.0.0.53
- name: config-volume
configMap:
items:
- key: rabbitmq.conf
path: rabbitmq.conf
- key: enabled_plugins
path: enabled_plugins
name: rabbitmq-cluster-config
- name: timezone
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
- name: rabbitmq-storage
persistentVolumeClaim:
claimName: rabbitmq
volumeClaimTemplates:
- metadata:
name: rabbitmq-storage
spec:
accessModes:
- ReadWriteMany
storageClassName: "managed-nfs-storage-rabbitmq"
resources:
requests:
storage: 5Gi