(原)Ubuntu系统中基于Docker单机搭建Ceph集群学习笔记

# 要用root用户创建，或有sudo权限
# 注：docker镜像源(镜像加速)：https://registry.docker-cn.com
# 1、修改docker镜像源
cat > /etc/docker/deamon.json << EOF
{
"registry-mirrors": [
"https://registry.docker-cn.com"
]
}
EOF

# 需要用到的镜像：
docker pull ceph/daemon
ceph/mon、ceph/osd…等一个一个镜像拉取是比较旧的方法
# 重启docker
systemctl restart docker
# 2、创建ceph专用网络
docker network create --driver bridge --subnet 172.20.0.0/16 ceph-network
docker network inspect ceph-network

# 3、删除旧的ceph相关容器
docker rm -f $(docker ps -a | grep ceph | awk '{print $1}')
# 4、清理旧的ceph相关目录文件，
rm -rf /www/ceph /var/lib/ceph /www/osd
# 5、创建相关目录及修改权限，用于挂载volume
mkdir -p /www/ceph /var/lib/ceph/osd /www/osd /var/log/ceph
chown -R 64045:64045 /var/lib/ceph/osd
chown -R 64045:64045 /www/osd/

# 6、创建monitor节点
docker run -itd --name monnode --network ceph-network --ip 172.20.0.10 -e NON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph ceph/mon

例子：
docker run -itd --name monnode --network ceph-network --ip 172.20.0.10 --restart always -v /etc/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /var/log/ceph/:/var/log/ceph/ -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_IP=172.20.0.10 -e MON_NAME=monnode -e CEPH_PUBLIC_NETWORK=172.20.0.0/16 ceph/daemon mon

实际：
docker run -itd --name monnode --network ceph-network --ip 172.20.0.10 -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /var/log/ceph/:/var/log/ceph/ -e CEPH_PUBLIC_NETWORK=172.20.0.0/16 ceph/daemon mon

–ip：设置的是容器的IP
ceph/daemon mon：中的mon是参数，不可随意更改

# 7、在monitor节点上标识3个osd节点
docker exec monnode ceph osd create
docker exec monnode ceph osd create
docker exec monnode ceph osd create
注意：实际操作过程中这一步没有执行。

# 8、创建osd节点
docker run -itd --name osdnode0 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /www/osd0:/var/lib/ceph/osd/ceph-0 ceph/osd

docker run -itd --name osdnode1 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /www/osd1:/var/lib/ceph/osd/ceph-1 ceph/osd

docker run -itd --name osdnode2 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /www/osd2:/var/lib/ceph/osd/ceph-2 ceph/osd

新：
创建OSD密钥文件：
docker exec monnode ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring

修改目录权限：
chown 167:167 -R /www/ceph
chmod 777 -R /www/ceph

执行部署osd操作：
docker run -itd --privileged=true --name osdnode0 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /var/lib/ceph/osd/0:/var/lib/ceph/osd -e OSD_TYPE=directory -v /etc/localtime:/etc/localtime:ro ceph/daemon osd

docker run -itd --privileged=true --name osdnode1 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /var/lib/ceph/osd/1:/var/lib/ceph/osd -e OSD_TYPE=directory -v /etc/localtime:/etc/localtime:ro ceph/daemon osd

docker run -itd --privileged=true --name osdnode2 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /var/lib/ceph/osd/2:/var/lib/ceph/osd -e OSD_TYPE=directory -v /etc/localtime:/etc/localtime:ro ceph/daemon osd

查看日志：
root@ybelzjlpa9xmngcb-0315223:~/ceph_cluster# docker logs osdnode0
2021-09-03 14:58:55 /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2021-09-03 14:58:55 /opt/ceph-container/bin/entrypoint.sh: Creating osd
2021-09-03T14:58:55.246+0800 7f2386b2c700 -1 auth: unable to find a keyring on /var/lib/ceph/bootstrap-osd/ceph.keyring: (2) No such file or directory
2021-09-03T14:58:55.246+0800 7f2386b2c700 -1 AuthRegistry(0x7f238005b458) no keyring found at /var/lib/ceph/bootstrap-osd/ceph.keyring, disabling cephx
2021-09-03T14:58:55.246+0800 7f2387d8e700 -1 auth: unable to find a keyring on /var/lib/ceph/bootstrap-osd/ceph.keyring: (2) No such file or directory
2021-09-03T14:58:55.246+0800 7f2387d8e700 -1 AuthRegistry(0x7f238005ef00) no keyring found at /var/lib/ceph/bootstrap-osd/ceph.keyring, disabling cephx
2021-09-03T14:58:55.246+0800 7f2387d8e700 -1 auth: unable to find a keyring on /var/lib/ceph/bootstrap-osd/ceph.keyring: (2) No such file or directory
2021-09-03T14:58:55.246+0800 7f2387d8e700 -1 AuthRegistry(0x7f2387d8cea0) no keyring found at /var/lib/ceph/bootstrap-osd/ceph.keyring, disabling cephx
2021-09-03T14:58:55.246+0800 7f2385329700 -1 monclient(hunting): handle_auth_bad_method server allowed_methods [2] but i only support [1]
2021-09-03T14:58:55.250+0800 7f2387d8e700 -1 monclient: authenticate NOTE: no keyring found; disabled cephx authentication
[errno 95] error connecting to the cluster
root@ybelzjlpa9xmngcb-0315223:~/ceph_cluster#

如果出现这种情况的话，检查是否执行了上面的有关目录权限的问题。

# 9、创建mgr节点
docker run -itd --privileged=true --name mgrnode --network ceph-network -e CLUSTER=ceph --pid=container:monnode -v /www/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ ceph/daemon mgr

需要注意的参数：–pid

# 10、启动rgw服务
在mon节点生成rgw密钥信息
docker exec monnode ceph auth get client.bootstrap-rgw -o /var/lib/ceph/bootstrap-rgw/ceph.keyring

docker run -itd --privileged=true --name rgwnode --network ceph-network -e CLUSTER=ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /www/ceph:/etc/ceph -v /etc/localtime:/etc/localtime:ro -e RGW_NAME=rgw0 ceph/daemon rgw

# 10、启动mds服务

docker run -d
--net=host
--name=mds
--privileged=true
-v /etc/localtime:/etc/localtime
-v /usr/local/ceph/etc:/etc/ceph
-v /usr/local/ceph/lib:/var/lib/ceph
-v /usr/local/ceph/logs:/var/log/ceph
-e CEPHFS_CREATE=0
-e CEPHFS_METADATA_POOL_PG=512
-e CEPHFS_DATA_POOL_PG=512
ceph/daemon mds

说明下：
CEPHFS_CREATE 是为METADATA服务生成文件系统， 0表示不自动创建文件系统（默认值）， 1表示自动创建。
CEPHFS_DATA_POOL_PG是数据池的数量，默认为8。
CEPHFS_METADATA_POOL_PG是元数据池的数量，默认为8。

实际执行：
docker run -d --name=mds0 --privileged=true --network ceph-network -v /etc/localtime:/etc/localtime -v /www/ceph:/etc/ceph -v /var/lib/ceph:/var/lib/ceph -v /var/log/ceph:/var/log/ceph -e CEPHFS_CREATE=0 -e CEPHFS_METADATA_POOL_PG=16 -e CEPHFS_DATA_POOL_PG=16 ceph/daemon mds

# 11、创建FS文件系统
创建Data Pool:
docker exec osdnode0 ceph osd pool create cephfs_data 128 128

创建Metadata Pool:
docker exec osdnode0 ceph osd pool create cephfs_metadata 64 64
注意： 如果受mon_max_pg_per_osd限制， 不能设为128，可以调小点， 改为64。
当执行上面64的时候出错：
Error ERANGE: pg_num 64 size 3 would mean 888 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
改为：
docker exec osdnode0 ceph osd pool create cephfs_metadata 32 32

创建CephFS:
docker exec osdnode0 ceph fs new cephfs cephfs_metadata cephfs_data
上面的数据池与元数据池关联， 创建cephfs的文件系统。

查看FS信息:
docker exec osdnode0 ceph fs ls
输出：
name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]

# 12、查看ceph集群状态
docker exec monnode ceph -s

输出：
root@ybelzjlpa9xmngcb-0315223:~/ceph_cluster# docker exec monnode ceph -s
cluster:
id: 53e5a82c-f2ab-4e54-a9fb-d14f936cea62
health: HEALTH_WARN
mon is allowing insecure global_id reclaim
Degraded data redundancy: 211/633 objects degraded (33.333%), 53 pgs degraded, 288 pgs undersized
OSD count 2 < osd_pool_default_size 3
too many PGs per OSD (288 > max 250)

services:
mon: 1 daemons, quorum monnode (age 55m)
mgr: 13bb5037212c(active, since 39m)
mds: 1/1 daemons up
osd: 2 osds: 2 up (since 48m), 2 in (since 48m)
rgw: 1 daemon active (1 hosts, 1 zones)

data:
volumes: 1/1 healthy
pools: 6 pools, 288 pgs
objects: 211 objects, 7.2 KiB
usage: 24 MiB used, 200 GiB / 200 GiB avail
pgs: 211/633 objects degraded (33.333%)
235 active+undersized
53 active+undersized+degraded

progress:
Global Recovery Event (34m)
[............................]

# 13、增加monitor节点，组件成机器
docker run -itd --name monnode_1 --network ceph-network --ip 172.20.0.11 -e NON_NAME=monnode_1 -e MON_IP=172.20.0.11 -v /www/ceph:/etc/ceph ceph/mon
docker run -itd --name monnode_2 --network ceph-network --ip 172.20.0.12 -e NON_NAME=monnode_2 -e MON_IP=172.20.0.12 -v /www/ceph:/etc/ceph ceph/mon

# 14、开始使用ceph-fs文件系统
挂载文件系统，支持两种方式：
a,使用内核驱动挂载
这里我选择在osdnode0的容器内执行：
mkdir -p /mnt/cephfs
mount -t ceph monnode:6789:/ /mnt/mycephfs

要挂载启用了 cephx 认证的 Ceph 文件系统，你必须指定用户名、密钥。

ceph-authtool --print-key /etc/ceph/ceph.client.admin.keyring

mount -t ceph 172.20.0.10:6789:/ /mnt/mycephfs -o name=admin,secret=AQBfOTdhwfU7DBAA+Q2gOumgEPvcbGO1YqC1BQ==

但是这种方式一直报错：(目前还没找到解决方案)
modprobe: FATAL: Module ceph not found in directory /lib/modules/4.4.0-186-generic
failed to load ceph kernel module (1)
mount error 110 = Connection timed out

要卸载 Ceph 文件系统，可以用 unmount 命令，例如：

umount /mnt/mycephfs

b,使用ceph-fuse方式挂载(执行成功)
ceph-fuse -c /etc/ceph/ceph.conf /mnt/mycephfs

卸载:
fusermount -u /mnt/mycephfs