k8s1.19.8配置glusterfs动态存储

我测试了1.20.2版本配置glusterfs结果使用不了。记录时间2021-2-19

一、环境介绍

2台服务器都为存储服务器，都有一块20G的数据盘

注意:数据盘不要格式化操作

[root@master ~]# fdisk -l   /dev/vdb 

Disk /dev/vdb: 21.5 GB, 21474836480 bytes, 41943040 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes

[root@master ~]# kubectl get node 
NAME     STATUS   ROLES    AGE     VERSION
master   Ready    master   5m28s   v1.19.8
node1    Ready    <none>   4m52s   v1.19.8

[root@master ~]# cat /etc/hosts
::1    localhost    localhost.localdomain    localhost6    localhost6.localdomain6
127.0.0.1    localhost    localhost.localdomain    localhost4    localhost4.localdomain4

172.16.208.210 master
172.16.208.211 node1

二、2台服务器都安装glusterfs和heketi

注意:GlusterFS只需要安装并启动即可，不必组建受信存储池

#所有存储服务器下载安装glusterfs
yum install centos-release-gluster -y
yum install glusterfs-server -y

#启动
systemctl  start glusterd 

#安装heketi
yum install -y heketi heketi-client

三、配置heketi.json(2台服务器都修改)

[root@master ~]# cat  /etc/heketi/heketi.json 
{
  "_port_comment": "Heketi Server Port Number",
  "port": "8080", 

  "_use_auth": "Enable JWT authorization. Please enable for deployment",
  "use_auth": true, 

  "_jwt": "Private keys for access",
  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "key": "admin@key"  
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "key": "user@key"  
    }
  },

  "_glusterfs_comment": "GlusterFS Configuration",
  "glusterfs": {
    "_executor_comment": [
      "Execute plugin. Possible choices: mock, ssh",
      "mock: This setting is used for testing and development.",
      "      It will not send commands to any node.",
      "ssh:  This setting will notify Heketi to ssh to the nodes.",
      "      It will need the values in sshexec to be configured.",
      "kubernetes: Communicate with GlusterFS containers over",
      "            Kubernetes exec api."
    ],
    "executor": "ssh", 

    "_sshexec_comment": "SSH username and private key file information",
    "sshexec": {
      "keyfile": "/etc/heketi/heketi_key", 
      "user": "root", 
      "port": "22", 
      "fstab": "/etc/fstab" 
    },

    "_kubeexec_comment": "Kubernetes configuration",
    "kubeexec": {
      "host" :"https://kubernetes.host:8443",
      "cert" : "/path/to/crt.file",
      "insecure": false,
      "user": "kubernetes username",
      "password": "password for kubernetes user",
      "namespace": "OpenShift project or Kubernetes namespace",
      "fstab": "Optional: Specify fstab file on node.  Default is /etc/fstab"
    },

    "_db_comment": "Database file name",
    "db": "/var/lib/heketi/heketi.db",

    "_loglevel_comment": [
      "Set log level. Choices are:",
      "  none, critical, error, warning, info, debug",
      "Default is warning"
    ],
    "loglevel" : "warning" 
  }
}

三、设置heketi免密访问GlusterFS与公钥分发

#设置heketi免密访问GlusterFS
[root@master heketi]# ssh-keygen -t rsa -q -f /etc/heketi/heketi_key -N ""
[root@master heketi]# chown heketi:heketi /etc/heketi/heketi_key

#分发公钥
[root@master heketi]# ssh-copy-id -i /etc/heketi/heketi_key.pub root@master
[root@master heketi]# ssh-copy-id -i /etc/heketi/heketi_key.pub root@node1
#将秘钥充master服务器复制到node1服务器上
[root@master heketi]# rsync -avz /etc/heketi/heketi_key root@node1:/etc/heketi/

四、启动heketi

systemctl enable heketi
systemctl restart heketi
systemctl status heketi

验证

[root@master ~]# curl http://localhost:8080/hello
Hello from Heketi

五、设置GlusterFS集群

1、创建topology.json文件

[root@master ~]# cat /etc/heketi/topology.json 
{
    "clusters": [
        {
            "nodes": [
                {
                    "node": {
                        "hostnames": {
                            "manage": [
                                "172.16.208.210"
                            ],
                            "storage": [
                                "172.16.208.210"
                            ]
                        },
                        "zone": 1
                    },
                    "devices": [
                        "/dev/vdb"
                    ]
                },
                {
                    "node": {
                        "hostnames": {
                            "manage": [
                                "172.16.208.211"
                            ],
                            "storage": [
                                "172.16.208.211"
                            ]
                        },
                        "zone": 2
                    },
                    "devices": [
                        "/dev/vdb"
                    ]
                }
            ]
        }
    ]
}

2、通过topology.json组建GlusterFS集群

[root@master ~]# heketi-cli --server http://localhost:8080 --user admin --secret admin@key topology load --json=/etc/heketi/topology.json
Creating cluster ... ID: 2865ef5ac77aae777bbfaf3f27e456ef
    Allowing file volumes on cluster.
    Allowing block volumes on cluster.
    Creating node 172.16.208.210 ... ID: 474894862effef22952e7c0d4542605b
        Adding device /dev/vdb ... OK
    Creating node 172.16.208.211 ... ID: 156c6b793ef761f68b317d0cfe8e7ec1
        Adding device /dev/vdb ... OK

3、查看命令（后面添加的，就不贴粘贴内容了）

[root@master ~]# heketi-cli --server http://localhost:8080 --user admin --secret admin@key topology info

六、配置StorageClass

1、定义并导入secret

[root@master ~]# cat  gluster-heketi-secret.yaml 
apiVersion: v1
kind: Secret
metadata:
  name: heketi-secret
  namespace: default
data:
  # base64 encoded password. E.g.: echo -n "mypassword" | base64
  key: YWRtaW5Aa2V5
type: kubernetes.io/glusterfs

[root@master ~]# kubectl  apply -f gluster-heketi-secret.yaml 
secret/heketi-secret created

2、定义StorageClass并导入

注意:allowVolumeExpansion可以根据情况是否设置为true(默认为false),allowVolumeExpansion为flase时不能动态扩容（例如不能直接修改pvc大小，当allowVolumeExpansion为true时可以修改）

文档：https://kubernetes.io/zh/docs/concepts/storage/storage-classes/

[root@master ~]# cat  gluster-heketi-storageclass.yaml 
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: gluster-heketi-storageclass
provisioner: kubernetes.io/glusterfs
reclaimPolicy: Delete
parameters:
  resturl: "http://172.16.208.210:8080"
  restauthenabled: "true"
  restuser: "admin"
  secretNamespace: "default"
  secretName: "heketi-secret"
  volumetype: "replicate:2"

[root@master ~]# kubectl apply -f  gluster-heketi-storageclass.yaml 
storageclass.storage.k8s.io/gluster-heketi-storageclass created

3、查看

[root@master ~]# kubectl get secret  heketi-secret
NAME            TYPE                      DATA   AGE
heketi-secret   kubernetes.io/glusterfs   1      20m
[root@master ~]# kubectl get sc 
NAME                          PROVISIONER               RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
gluster-heketi-storageclass   kubernetes.io/glusterfs   Delete          Immediate           false                  19m

七、测试

1、创建的测试pvc

[root@master ~]# cat test-pvc.yaml 
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-claim
  annotations:
    volume.beta.kubernetes.io/storage-class: "gluster-heketi-storageclass"
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi

2、导入并查看是否自动创建了pv

[root@master ~]# kubectl apply -f test-pvc.yaml 
persistentvolumeclaim/test-claim unchanged
[root@master ~]# kubectl get pvc 
NAME         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS                  AGE
test-claim   Bound    pvc-34c1b3e5-76ec-4c5e-98b6-f8d5bca6c2eb   1Gi        RWX            gluster-heketi-storageclass   21m
[root@master ~]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                STORAGECLASS                  REASON   AGE
pvc-34c1b3e5-76ec-4c5e-98b6-f8d5bca6c2eb   1Gi        RWX            Delete           Bound    default/test-claim   gluster-heketi-storageclass            21m