CEPH之块存储

一、官方文档

https://docs.ceph.com/en/latest/
http://docs.ceph.org.cn/rbd/rbd/

二、块存储

​ 块存储简称（RADOS Block Device）,是一种有序的字节序块，也是Ceph三大存储类型中最为常用的存储方式，Ceph的块存储时基于RADOS的，因此它也借助RADOS的快照，复制和一致性等特性提供了快照，克隆和备份等操作。Ceph的块设备值一种精简置备模式，可以拓展块存储的大小且存储的数据以条带化的方式存储到Ceph集群中的多个OSD中。

2.1、创建pool

官方文档：http://docs.ceph.org.cn/rados/operations/pools/

# 1、 查看pool命令
[root@node1 ceph-deploy]# ceph osd lspools

# 2、首先得创建一个pool（名字为ceph-demo pg数量64  pgp数量64 副本数【replicated】默认是3）
[root@node1 ceph-deploy]# ceph osd pool create ceph-demo 64 64 
pool 'ceph-demo' created

# 3、查看pool
[root@node1 ceph-deploy]# ceph osd lspools
1 ceph-demo

# 4、查看pg、pgp、副本的数量
[root@node1 ceph-deploy]# ceph osd pool get ceph-demo pg_num
pg_num: 64
[root@node1 ceph-deploy]# ceph osd pool get ceph-demo pgp_num
pgp_num: 64
[root@node1 ceph-deploy]# ceph osd pool get ceph-demo size
size: 3

# 5、查看调度算法
[root@node1 ceph-deploy]# ceph osd pool get ceph-demo crush_rule
crush_rule: replicated_rule

# 6、调整就用set
[root@node1 ceph-deploy]# ceph osd pool set ceph-demo pg_num  128 
set pool 1 pg_num to 128
[root@node1 ceph-deploy]# ceph osd pool set ceph-demo pgp_num  128
set pool 1 pgp_num to 128

# 7、查看调整后的pg、pgp
[root@node1 ceph-deploy]# ceph osd pool get ceph-demo pg_num  
pg_num: 128
[root@node1 ceph-deploy]# ceph osd pool get ceph-demo pgp_num  
pgp_num: 128

2.2、创建块存储文件

官方文档：http://docs.ceph.org.cn/rbd/rados-rbd-cmds/

# 1、创建方式（2种方式都可）  -p 指定pool名称、--image 指定image（块名字）名字
[root@node1 ~]# rbd create -p ceph-demo --image rbd-demo.img --size 1G
[root@node1 ~]# rbd create ceph-demo/rbd-demo1.img --size 1G 

# 2、查看列表
[root@node1 ~]# rbd -p ceph-demo ls
rbd-demo.img
rbd-demo1.img

# 3、查看某个块的信息（可以看到一个块被分成了256个objects）
[root@node1 ~]# rbd info ceph-demo/rbd-demo.img
rbd image 'rbd-demo.img':
        size 1 GiB in 256 objects
        order 22 (4 MiB objects)
        snapshot_count: 0
        id: 14a48cb7303ce
        block_name_prefix: rbd_data.14a48cb7303ce
        format: 2
        features: layering, exclusive-lock, object-map, fast-diff, deep-flatten  # 等会把这几个features都去掉
        op_features: 
        flags: 
        create_timestamp: Sun Jan 24 14:13:37 2021
        access_timestamp: Sun Jan 24 14:13:37 2021
        modify_timestamp: Sun Jan 24 14:13:37 2021

# 4、删除块
[root@node1 ~]#  rbd rm ceph-demo/rbd-demo1.img 
Removing image: 100% complete...done.

2.3、使用块存储文件

# 1、查看当前pool有几个块文件
[root@node1 ~]# rbd list  ceph-demo
rbd-demo.img

# 2、直接使用会报错，因为内核级别的一些东西不支持
[root@node1 ~]# rbd map ceph-demo/rbd-demo.img
rbd: sysfs write failed
RBD image feature set mismatch. You can disable features unsupported by the kernel with "rbd feature disable ceph-demo/rbd-demo.img object-map fast-diff deep-flatten".
In some cases useful info is found in syslog - try "dmesg | tail".
rbd: map failed: (6) No such device or address

# 3、disable 模块
[root@node1 ~]# rbd feature disable ceph-demo/rbd-demo.img deep-flatten
[root@node1 ~]# rbd feature disable ceph-demo/rbd-demo.img fast-diff
[root@node1 ~]# rbd feature disable ceph-demo/rbd-demo.img object-map
[root@node1 ~]# rbd feature disable ceph-demo/rbd-demo.img 

# 4、查看是否成功禁用
[root@node1 ~]# rbd info ceph-demo/rbd-demo.img
rbd image 'rbd-demo.img':
        size 1 GiB in 256 objects
        order 22 (4 MiB objects)
        snapshot_count: 0
        id: 14a48cb7303ce
        block_name_prefix: rbd_data.14a48cb7303ce
        format: 2
        features: layering   # 看到这里是layering状态就可以测试挂载了
        op_features: 
        flags: 
        create_timestamp: Sun Jan 24 14:13:37 2021
        access_timestamp: Sun Jan 24 14:13:37 2021
        modify_timestamp: Sun Jan 24 14:13:37 2021
        
# 5、再次使用刚刚创建的块存储文件
[root@node1 ~]# rbd map ceph-demo/rbd-demo.img
/dev/rbd0

# 6、查看device
[root@node1 ~]# rbd device  list
id pool      namespace image        snap device    
0  ceph-demo           rbd-demo.img -    /dev/rbd0 （这就相当于我们本地的一块磁盘一样，可以进行分区格式化操作）

# 7、fdisk查看可以查看到相应信息
[root@node1 ~]# fdisk -l | grep rbd0
Disk /dev/rbd0: 1073 MB, 1073741824 bytes, 2097152 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes

# 8、比如格式化
[root@node1 ~]# mkfs.ext4 /dev/rbd0

# 9、然后挂载
[root@node1 ~]# mkdir /mnt/rbd-demo
[root@node1 ~]# mount /dev/rbd0 /mnt/rbd-demo

# 10、df查看
[root@node1 ~]# df -h
Filesystem               Size  Used Avail Use% Mounted on
/dev/rbd0                976M  2.6M  907M   1% /mnt/rbd-demo

2.4、块存储扩容

# 1、就拿之前创建的盘来操作
[root@node1 ~]# rbd -p ceph-demo ls
rbd-demo.img

# 2、查看它的信息
[root@node1 ~]# rbd -p ceph-demo info --image rbd-demo.img
rbd image 'rbd-demo.img':
        size 1 GiB in 256 objects   （目前是一个g）
        order 22 (4 MiB objects)
        snapshot_count: 0
        id: 14a48cb7303ce
        block_name_prefix: rbd_data.14a48cb7303ce
        format: 2
        features: layering
        op_features: 
        flags: 
        create_timestamp: Sun Jan 24 14:13:37 2021
        access_timestamp: Sun Jan 24 14:13:37 2021
        modify_timestamp: Sun Jan 24 14:13:37 2021

# 3、扩容（缩容也可，但是不建议）
[root@node1 ~]# rbd resize ceph-demo/rbd-demo.img --size 2G
Resizing image: 100% complete...done.

# 4、扩容后查看
[root@node1 ~]# rbd -p ceph-demo info --image rbd-demo.img
rbd image 'rbd-demo.img':
        size 2 GiB in 512 objects
        order 22 (4 MiB objects)
        snapshot_count: 0
        id: 14a48cb7303ce
        block_name_prefix: rbd_data.14a48cb7303ce
        format: 2
        features: layering
        op_features: 
        flags: 
        create_timestamp: Sun Jan 24 14:13:37 2021
        access_timestamp: Sun Jan 24 14:13:37 2021
        modify_timestamp: Sun Jan 24 14:13:37 2021

# 5、此时的磁盘大小是扩上去了，可是文件系统挂载的是不会自动扩的
[root@node1 ~]# fdisk -l | grep rbd0
Disk /dev/rbd0: 2147 MB, 2147483648 bytes, 4194304 sectors
[root@node1 ~]# df -h
Filesystem               Size  Used Avail Use% Mounted on
/dev/rbd0                976M  2.6M  907M   1% /mnt/rbd-demo  # 此处还是1个G

# 6、扩容文件系统（注意不建议对这种磁盘进行分区，云上的也是一样，建议多买几块）
[root@node1 ~]# blkid 
/dev/sr0: UUID="2018-11-25-23-54-16-00" LABEL="CentOS 7 x86_64" TYPE="iso9660" PTTYPE="dos" 
/dev/sdb: UUID="k4g1pw-rOvV-NG7w-ajnZ-qipH-kXwq-h0jY0o" TYPE="LVM2_member" 
/dev/sda1: UUID="ccb430ea-66c9-4c91-a4b4-ba870ca15943" TYPE="xfs" 
/dev/sda2: UUID="NegVJw-3XZn-BJeZ-NfKW-VROQ-roSa-LcKgoy" TYPE="LVM2_member" 
/dev/mapper/centos-root: UUID="59c5d6b6-e34d-4149-b28a-8a3b9c32536d" TYPE="xfs" 
/dev/sdc: UUID="h65OWm-ELDd-R5pO-HaQ0-Ejjs-cUWn-8Ejcpq" TYPE="LVM2_member" 
/dev/mapper/centos-swap: UUID="25f54a98-e472-4438-9641-eac952a46e3e" TYPE="swap" 
/dev/rbd0: UUID="911aadb8-bbf4-48d2-a62d-d86886af79dc" TYPE="ext4" 
# 扩它
[root@node1 ~]# resize2fs /dev/rbd0

2.5、RBD数据写入流程

2.6、解决告警排查

# 1、发现问题
[root@node1 ~]# ceph -s
  cluster:
    id:     081dc49f-2525-4aaa-a56d-89d641cef302
    health: HEALTH_WARN
            application not enabled on 1 pool(s)
 
  services:
    mon: 3 daemons, quorum node1,node2,node3 (age 3h)
    mgr: node2(active, since 3h), standbys: node3, node1
    osd: 3 osds: 3 up (since 3h), 3 in (since 13h)
 
  data:
    pools:   1 pools, 128 pgs
    objects: 22 objects, 38 MiB
    usage:   3.1 GiB used, 57 GiB / 60 GiB avail
    pgs:     128 active+clean

# 2、通过他提供的命令查看
[root@node1 ~]# ceph health detail
HEALTH_WARN application not enabled on 1 pool(s)
POOL_APP_NOT_ENABLED application not enabled on 1 pool(s)
    application not enabled on pool 'ceph-demo'   # 看这句
    use 'ceph osd pool application enable <pool-name> <app-name>', where <app-name> is 'cephfs', 'rbd', 'rgw', or freeform for custom applications.       # 这句是提示你怎么操作（就是把这个资源类型进行分类就行）
    
# 3、解决命令
[root@node1 ~]# ceph osd pool application enable ceph-demo rbd
enabled application 'rbd' on pool 'ceph-demo'

# 4、再次查看
[root@node1 ~]# ceph -s
  cluster:
    id:     081dc49f-2525-4aaa-a56d-89d641cef302
    health: HEALTH_OK