007 Ceph手动部署单节点

前面已经介绍了Ceph的自动部署，本次介绍一下关于手动部署Ceph节点操作

一、环境准备

一台虚拟机部署单节点Ceph集群

IP：172.25.250.14     

内核： Red Hat Enterprise Linux Server release 7.4 (Maipo)

磁盘：/dev/vab,/dev/vac,/dev/vad

二、部署mon

2.1 安装Ceph软件包

[root@cepe5 ~]#  yum install -y ceph-common ceph-mon ceph-mgr ceph-mds ceph-osd ceph-radosgw

2.2 创建mon节点准备

登录到ceph5查看ceph目录是否生成

[root@ceph5 ceph]# ll
-rw-r--r-- 1 root root 92 Nov 23  2017 rbdmap
[root@ceph6 ~]# ll /etc/ceph/
-rw-r--r-- 1 root root 92 Nov 23  2017 rbdmap

执行uuidgen命令，得到一个唯一的标识，作为ceph集群的ID

[root@ceph5 ceph]# uuidgen
82bf91ae-6d5e-4c09-9257-3d9e5992e6ef

 生成ceph配置文件

[root@ceph5 ceph]# vim /etc/ceph/backup.conf

fsid = 51dda18c-7545-4edb-8ba9-27330ead81a7
mon_initial_members = ceph5
mon_host = 172.25.250.14
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public_network = 172.25.250.0/24
cluster_network = 172.25.250.0/24
[mgr]
mgr modules = dashboard
[client.rgw.ceph5]
host = ceph5
keyring = /etc/ceph/backup.client.rgw.ceph5.keyring
log_file=/var/log/ceph/$cluster.$name.log
rgw_frontends = civetweb port=80
rgw_print_continue = false
rgw_dns_name = lab.example.com

设置集群名为backup

 [root@ceph5 ceph]# vim /etc/sysconfig/ceph 

 

2.3 为监控节点创建管理密钥

创建monitor使用的key

[root@ceph5 ceph]# ceph-authtool --create-keyring /tmp/backup.mon.keyring --gen-key -n mon. --cap mon 'allow *' 

 

为ceph amin用户创建管理集群的密钥并赋予访问权限

[root@ceph5 ceph]# ceph-authtool --create-keyring /etc/ceph/backup.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mgr 'allow *' --cap mds 'allow *'

添加client.admin的秘钥到/tmp/backup.mon.keyring文件中

[root@ceph5 ceph]# ceph-authtool /tmp/backup.mon.keyring --import-keyring /etc/ceph/backup.client.admin.keyring

2.4 创建monitormap

[root@ceph5 ~]# monmaptool --create --add ceph5 172.25.250.14 --fsid 51dda18c-7545-4edb-8ba9-27330ead81a7  /tmp/monmap

[root@ceph5 ceph]# file /tmp/monmaptool

[root@ceph5 ceph]# monmaptool --print  /tmp/monmaptool

创建monitor使用的目录

[root@ceph5 ~]# mkdir -p /var/lib/ceph/mon/backup-ceph5

设置文件相关的权限

[root@ceph5 ~]# chown ceph.ceph -R /var/lib/ceph /etc/ceph /tmp/backup.mon.keyring /tmp/monmap

2.5 初始化monitor

[root@ceph5 ~]# sudo -u ceph ceph-mon --cluster backup --mkfs -i ceph5 --monmap /tmp/monmap --keyring /tmp/backup.mon.keyring

2.6 启动monitor：

[root@ceph5 ~]# systemctl start ceph-mon@ceph5

[root@ceph5 ~]# systemctl enable ceph-mon@ceph5

Created symlink from /etc/systemd/system/ceph-mon.target.wants/ceph-mon@ceph5.service to /usr/lib/systemd/system/ceph-mon@.service.

[root@ceph5 ~]# systemctl status ceph-mon@ceph5

ceph-mon@ceph5.service - Ceph cluster monitor daemon
   Loaded: loaded (/usr/lib/systemd/system/ceph-mon@.service; enabled; vendor preset: disabled)
   Active: active (running) since Mon 2019-03-18 15:46:45 CST; 2min 21s ago
 Main PID: 1527 (ceph-mon)
   CGroup: /system.slice/system-cephx2dmon.slice/ceph-mon@ceph5.service
           └─1527 /usr/bin/ceph-mon -f --cluster backup --id ceph5 --setuser ceph --setgroup ceph

Mar 18 15:46:45 ceph5 systemd[1]: Started Ceph cluster monitor daemon.
Mar 18 15:46:45 ceph5 systemd[1]: Starting Ceph cluster monitor daemon...

2.7 获取monitor

[root@ceph5 ~]# ceph mon getmap -o monmap --cluster backup

[root@ceph5 ~]# monmaptool --print monmap 

[root@ceph5 ~]# ceph auth get mon. --cluster backup

exported keyring for mon.
[mon.]
    key = AQDLTI9cCRjlLRAASL2nSyFnRX9uHSxbGXTycQ==
    caps mon = "allow *"

三、配置MGR

3.1 创建秘钥

[root@ceph5 ~]# mkdir /var/lib/ceph/mgr/backup-ceph5

[root@ceph5 ~]# chown ceph.ceph -R /var/lib/ceph

[root@ceph5 ~]# ceph-authtool --create-keyring /etc/ceph/backup.mgr.ceph5.keyring --gen-key -n mgr.ceph5 --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *'

[root@ceph5 ~]# ceph auth import -i /etc/ceph/backup.mgr.ceph5.keyring --cluster backup 

[root@ceph5 ~]# ceph auth get-or-create mgr.ceph5 -o /var/lib/ceph/mgr/backup-ceph5/keyring --cluster backup

3.2 启动服务

[root@ceph5 ~]# systemctl start ceph-mgr@ceph5

[root@ceph5 ~]# systemctl enable ceph-mgr@ceph5

Created symlink from /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@ceph5.service to /usr/lib/systemd/system/ceph-mgr@.service.

 3.3 查看集群状态

 [root@ceph5 ~]# ceph -s --cluster backup

cluster:
id: 51dda18c-7545-4edb-8ba9-27330ead81a7
health: HEALTH_OK
services:
mon: 1 daemons, quorum ceph5
mgr: ceph5(active, starting)
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 bytes
usage: 0 kB used, 0 kB / 0 kB avail
pgs:

3.5 启用MGR的dashboard

[root@ceph5 ~]# ceph mgr dump  --conf /etc/ceph/backup.conf

{
    "epoch": 2,
    "active_gid": 4100,
    "active_name": "ceph5",
    "active_addr": "-",
    "available": false,
    "standbys": [],
    "modules": [
        "restful",
        "status"
    ],
    "available_modules": [
        "dashboard",
        "prometheus",
        "restful",
        "status",
        "zabbix"
    ]
}

[root@ceph5 ~]# ceph mgr module enable dashboard --conf /etc/ceph/backup.conf

[root@ceph5 ~]# ceph mgr module ls --conf /etc/ceph/backup.conf

[
    "dashboard",
    "restful",
    "status"
]

 [root@ceph5 ~]# ceph config-key put mgr/dashboard/server_addr 172.25.250.14 --conf /etc/ceph/backup.conf 

[root@ceph5 ~]# ceph config-key put mgr/dashboard/server_port 7000 --conf /etc/ceph/backup.conf 

[root@ceph5 ~]# ceph config-key dump --conf /etc/ceph/backup.conf 

{
    "mgr/dashboard/server_addr": "172.25.250.14",
    "mgr/dashboard/server_port": "7000"
}

做一个别名

[root@ceph5 ~]# alias ceph='ceph --cluster backup'

[root@ceph5 ~]# alias rbd='rbd --cluster backup' 

四、配置OSD

4.1 添加osd.0

[root@ceph5 ~]# uuidgen

创建日志盘

[root@ceph5 ~]# sgdisk --new=1:0:+5G --change-name=1:'ceph journal' --partition-guid=1:aa25b6e9-384a-4012-b597-f4af96da3d5e --typecode=1:aa25b6e9-384a-4012-b597-f4af96da3d5e --mbrtogpt -- /dev/vdb

[root@ceph5 ~]# uuidgen

创建数据盘

[root@ceph5 ~]# sgdisk --new=2:0:0 --change-name=2:'ceph data' --partition-guid=2:5aaec435-b7fd-4a50-859f-3d26ed09f185 --typecode=2:5aaec435-b7fd-4a50-859f-3d26ed09f185 --mbrtogpt -- /dev/vdb

格式化

[root@ceph5 ~]# mkfs.xfs -f -i size=2048 /dev/vdb1

[root@ceph5 ~]# mkfs.xfs -f -i size=2048 /dev/vdb2

挂载数据盘

[root@ceph5 ~]# mkdir /var/lib/ceph/osd/backup-{0,1,2}

[root@ceph5 ~]# mount -o noatime,largeio,inode64,swalloc /dev/vdb2 /var/lib/ceph/osd/backup-0

[root@ceph5 ~]# uuid_blk=`blkid /dev/vdb2|awk '{print $2}'`

[root@ceph5 ~]# echo $uuid_blk

[root@ceph5 ~]# echo "$uuid_blk /var/lib/ceph/osd/backup-0 xfs defaults,noatime,largeio,inode64,swalloc 0 0" >> /etc/fstab

[root@ceph5 ~]# cat /etc/fstab

#
# /etc/fstab
# Created by anaconda on Tue Jul 11 05:34:11 2017
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
UUID=716e713d-4e91-4186-81fd-c6cfa1b0974d / xfs defaults 0 0
UUID="2fc06d6c-39b1-47f8-b9ab-cf034599cbd1" /var/lib/ceph/osd/backup-0 xfs defaults,noatime,largeio,inode64,swalloc 0 0

创建Ceph osd秘钥

[root@ceph5 ~]# ceph-authtool --create-keyring /etc/ceph/backup.osd.0.keyring --gen-key -n osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *'

[root@ceph5 ~]# ceph auth import -i /etc/ceph/backup.osd.0.keyring --cluster backup

[root@ceph5 ~]# ceph auth get-or-create osd.0 -o /var/lib/ceph/osd/backup-0/keyring --cluster backup

初始化osd数据目录

[root@ceph5 ~]# ceph-osd -i 0 --mkfs --cluster backup

2019-03-18 16:35:46.788193 7f030624ad00 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway
2019-03-18 16:35:46.886027 7f030624ad00 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway
2019-03-18 16:35:46.886514 7f030624ad00 -1 read_settings error reading settings: (2) No such file or directory
2019-03-18 16:35:46.992516 7f030624ad00 -1 created object store /var/lib/ceph/osd/backup-0 for osd.0 fsid 51dda18c-7545-4edb-8ba9-27330ead81a7

挂载日志盘

[root@ceph5 ~]# cd /var/lib/ceph/osd/backup-0

[root@ceph5 backup-0]# rm -f journal

[root@ceph5 backup-0]# partuuid_0=`blkid /dev/vdb1|awk -F "[""]" '{print $8}'`

[root@ceph5 backup-0]# echo $partuuid_0

[root@ceph5 backup-0]# ln -s /dev/disk/by-partuuid/$partuuid_0 ./journal

[root@ceph5 backup-0]# ll

[root@ceph5 backup-0]# chown ceph.ceph -R /var/lib/ceph

[root@ceph5 backup-0]# ceph-osd --mkjournal -i 0 --cluster backup

[root@ceph5 backup-0]# chown ceph.ceph /dev/disk/by-partuuid/$partuuid_0

将osd节点机加入crushmap

[root@ceph5 backup-0]# ceph osd crush add-bucket ceph5 host --cluster backup

将osd节点机移动到默认的root default下面

[root@ceph5 backup-0]# ceph osd crush move ceph5 root=default --cluster backup

将osd.0添加到ceph5节点下

[root@ceph5 backup-0]# ceph osd crush add osd.0 0.01500 root=default host=ceph5 --cluster backup

启动OSD

[root@ceph5 backup-0]# systemctl start ceph-osd@0

[root@ceph5 backup-0]# systemctl enable ceph-osd@0

Created symlink from /etc/systemd/system/ceph-osd.target.wants/ceph-osd@0.service to /usr/lib/systemd/system/ceph-osd@.service.

 4.2 添加osd.1

[root@ceph5 backup-0]# ceph osd create
[root@ceph5 backup-0]# sgdisk --new=1:0:+5G --change-name=1:'ceph journal' --partition-guid=1:3af9982e-8324-4666-bf3a-771f0e981f00 --typecode=1:3af9982e-8324-4666-bf3a-771f0e981f00  --mbrtogpt -- /dev/vdc
[root@ceph5 backup-0]# sgdisk --new=2:0:0 --change-name=2:'ceph data' --partition-guid=2:3f1d1456-2ebe-4702-96dc-e8ba975a772c --typecode=2:3f1d1456-2ebe-4702-96dc-e8ba975a772c --mbrtogpt -- /dev/vdc
[root@ceph5 backup-0]# mkfs.xfs -f -i size=2048 /dev/vdc1
[root@ceph5 backup-0]# mkfs.xfs -f -i size=2048 /dev/vdc2
[root@ceph5 backup-0]# mount -o noatime,largeio,inode64,swalloc /dev/vdc2 /var/lib/ceph/osd/backup-1
[root@ceph5 backup-0]# blkid /dev/vdc2
/dev/vdc2: UUID="e0a2ccd2-549d-474f-8d36-fd1e3f20a4f8" TYPE="xfs" PARTLABEL="ceph data" PARTUUID="3f1d1456-2ebe-4702-96dc-e8ba975a772c" 
[root@ceph5 backup-0]# uuid_blk_1=`blkid /dev/vdc2|awk '{print $2}'`
[root@ceph5 backup-0]# echo "$uuid_blk_1 /var/lib/ceph/osd/backup-1 xfs defaults,noatime,largeio,inode64,swalloc 0 0" >> /etc/fstab
[root@ceph5 backup-0]# ceph-osd -i 1 --mkfs --mkkey --cluster backup 
2019-03-18 17:01:46.680469 7f4156f0ed00 -1 asok(0x5609ed1701c0) AdminSocketConfigObs::init: failed: AdminSocket::bind_and_listen: failed to bind the UNIX domain socket to '/var/run/ceph/backup-osd.1.asok': (17) File exists
[root@ceph5 backup-0]# cd /var/lib/ceph/osd/backup-1
[root@ceph5 backup-1]# rm -f journal
[root@ceph5 backup-1]# partuuid_1=`blkid /dev/vdc1|awk -F "[""]" '{print $8}'`
[root@ceph5 backup-1]# ln -s /dev/disk/by-partuuid/$partuuid_1 ./journal
[root@ceph5 backup-1]# chown ceph.ceph -R /var/lib/ceph
[root@ceph5 backup-1]# ceph-osd --mkjournal -i 1 --cluster backup 
[root@ceph5 backup-1]# chown ceph.ceph   /dev/disk/by-partuuid/$partuuid_1
[root@ceph5 backup-1]# ceph auth add osd.1 osd 'allow *' mon 'allow profile osd' mgr 'allow profile osd' -i /var/lib/ceph/osd/backup-1/keyring --cluster backup
[root@ceph5 backup-1]# ceph osd crush add osd.1 0.01500 root=default host=ceph5 --cluster backup
[root@ceph5 backup-1]# systemctl start ceph-osd@1
[root@ceph5 backup-1]# systemctl enable ceph-osd@1

 4.3 添加osd.2

[root@ceph5 ~]# ceph osd create
[root@ceph5 backup-1]# sgdisk --new=1:0:+5G --change-name=1:'ceph journal' --partition-guid=1:4e7b40e7-f303-4362-8b57-e024d4987892 --typecode=1:4e7b40e7-f303-4362-8b57-e024d4987892  --mbrtogpt -- /dev/vdd
Creating new GPT entries.
chown ceph.ceph -R /var/lib/ceph
ceph-osd --mkjournal -i 2 --cluster backup 
chown ceph.ceph   /dev/disk/by-partuuid/$partuuid_2
ceph auth add osd.2 osd 'allow *' mon 'allow profile osd' mgr 'allow profile osd' -i /var/lib/ceph/osd/backup-2/keyring --cluster backup
ceph osd crush add osd.2 0.01500 root=default host=ceph5 --cluster backup
systemctl start ceph-osd@2
systemctl enable ceph-osd@2The operation has completed successfully.
[root@ceph5 backup-1]# sgdisk --new=2:0:0 --change-name=2:'ceph data' --partition-guid=2:c82ce8a4-a762-4ba9-b987-21ef51df2eb2 --typecode=2:c82ce8a4-a762-4ba9-b987-21ef51df2eb2 --mbrtogpt -- /dev/vdd
The operation has completed successfully.
[root@ceph5 backup-1]# mkfs.xfs -f -i size=2048 /dev/vdd1
[root@ceph5 backup-1]# mkfs.xfs -f -i size=2048 /dev/vdd2
[root@ceph5 backup-1]# mount -o noatime,largeio,inode64,swalloc /dev/vdd2 /var/lib/ceph/osd/backup-2
[root@ceph5 backup-1]# blkid /dev/vdd2
/dev/vdd2: UUID="8458389b-c453-4c1f-b918-5ebde3a3b7b9" TYPE="xfs" PARTLABEL="ceph data" PARTUUID="c82ce8a4-a762-4ba9-b987-21ef51df2eb2" 
[root@ceph5 backup-1]# uuid_blk_2=`blkid /dev/vdd2|awk '{print $2}'`
[root@ceph5 backup-1]# echo "$uuid_blk_2 /var/lib/ceph/osd/backup-2 xfs defaults,noatime,largeio,inode64,swalloc 0 0">> /etc/fstab
[root@ceph5 backup-1]# ceph-osd -i 2 --mkfs --mkkey --cluster backup 
[root@ceph5 backup-1]# cd /var/lib/ceph/osd/backup-2
[root@ceph5 backup-2]# rm -f journal
[root@ceph5 backup-2]# partuuid_2=`blkid /dev/vdd1|awk -F "[""]" '{print $8}'`
[root@ceph5 backup-2]# ln -s /dev/disk/by-partuuid/$partuuid_2  ./journal
[root@ceph5 backup-2]# chown ceph.ceph -R /var/lib/ceph
[root@ceph5 backup-2]# ceph-osd --mkjournal -i 2 --cluster backup 
2019-03-18 17:06:43.123486 7efe77573d00 -1 journal read_header error decoding journal header
[root@ceph5 backup-2]# chown ceph.ceph   /dev/disk/by-partuuid/$partuuid_2
[root@ceph5 backup-2]# ceph auth add osd.2 osd 'allow *' mon 'allow profile osd' mgr 'allow profile osd' -i /var/lib/ceph/osd/backup-2/keyring --cluster backup
added key for osd.2
[root@ceph5 backup-2]# ceph osd crush add osd.2 0.01500 root=default host=ceph5 --cluster backup
add item id 2 name 'osd.2' weight 0.015 at location {host=ceph5,root=default} to crush map
[root@ceph5 backup-2]# systemctl start ceph-osd@2
[root@ceph5 backup-2]# systemctl enable ceph-osd@2

五、验证测试

5.1 健康测试

[root@ceph5 backup-2]# ceph -s

cluster:
    id:     51dda18c-7545-4edb-8ba9-27330ead81a7
    health: HEALTH_OK
  services:
    mon: 1 daemons, quorum ceph5
    mgr: ceph5(active)
    osd: 3 osds: 3 up, 3 in
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 bytes
    usage:   5441 MB used, 40605 MB / 46046 MB avail
    pgs:     

由于ceph默认crushmap策略是基于host的故障域，在三副本的情况下，由于只有一个host，这时创建存储池后，ceph状态会变为HEALTH_WARN，而且一直无法重平衡PG，故障表现为：

* 写入数据会hang住

* 如果安装了radosgw，会发现civetweb端口一直无法正常监听且radosgw进程反复重启

5.2 尝试创建一个池的结果

[root@ceph5 backup-2]#  ceph osd pool create rbd 64 64 --cluster backup
pool 'rbd' created
[root@ceph5 backup-2]# ceph -s
  cluster:
    id:     51dda18c-7545-4edb-8ba9-27330ead81a7
    health: HEALTH_WARN
            too few PGs per OSD (21 < min 30)
  services:
    mon: 1 daemons, quorum ceph5
    mgr: ceph5(active)
    osd: 3 osds: 3 up, 3 in
  data:
    pools:   1 pools, 64 pgs
    objects: 0 objects, 0 bytes
    usage:   5442 MB used, 40604 MB / 46046 MB avail
    pgs:     100.000% pgs not active
             64 creating+peering

5.3  修改为osd级别

[root@ceph5 ~]# cd /etc/ceph/
[root@ceph5 ceph]# ceph osd getcrushmap -o /etc/ceph/crushmap
9
[root@ceph5 ceph]# crushtool -d /etc/ceph/crushmap -o /etc/ceph/crushmap.txt
[root@ceph5 ceph]# sed -i 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' /etc/ceph/crushmap.txt
[root@ceph5 ceph]# grep 'step chooseleaf' /etc/ceph/crushmap.txt
    step chooseleaf firstn 0 type osd
[root@ceph5 ceph]# crushtool -c /etc/ceph/crushmap.txt -o /etc/ceph/crushmap-new
[root@ceph5 ceph]# ceph osd setcrushmap -i /etc/ceph/crushmap-new
10

5.4 再次检测变成健康状态

[root@ceph5 ceph]# ceph -s
  cluster:
    id:     51dda18c-7545-4edb-8ba9-27330ead81a7
    health: HEALTH_OK
  services:
    mon: 1 daemons, quorum ceph5
    mgr: ceph5(active)
    osd: 3 osds: 3 up, 3 in 
  data:
    pools:   1 pools, 64 pgs
    objects: 0 objects, 0 bytes
    usage:   5443 MB used, 40603 MB / 46046 MB avail
    pgs:     64 active+clean
[root@ceph5 ceph]# ceph osd pool application enable rbd rbd
enabled application 'rbd' on pool 'rbd'
[root@ceph5 ceph]# rbd create --size 1G test
[root@ceph5 ceph]# rbd ls
test

六、配置radosgw

[root@ceph5 backup-2]#  ceph auth get-or-create client.rgw.ceph5 mon 'allow rwx' osd 'allow rwx' -o ceph.client.rgw.ceph5.keyring --backup cluster --conf  /etc/ceph/backup.conf

添加配置文件

启动服务

[root@ceph5 ceph]#  systemctl start ceph-radosgw@rgw.ceph5

[root@ceph5 ceph]#  systemctl enbale ceph-radosgw@rgw.ceph5

所有命令

#可以把这些命令直接写一个bash文件执行安装，也可以进行修改，做一个shell脚本一键安装
yum install -y ceph-common ceph-mon ceph-mgr ceph-mds ceph-osd ceph-radosgw
echo 'CLUSTER = backup'>>/etc/sysconfig/ceph
uuid=`uuidgen`
echo "fsid = $uuid
mon_initial_members = ceph5
mon_host = 172.25.250.14

auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

public_network = 172.25.250.0/24
cluster_network = 172.25.250.0/24

[mgr]
mgr modules = dashboard 

[client.rgw.ceph5]
host = ceph5
keyring = /etc/ceph/backup.client.rgw.ceph5.keyring
log_file=/var/log/ceph/$cluster.$name.log
rgw_frontends = civetweb port=80
rgw_print_continue = false
rgw_dns_name = lab.example.com

" >>/etc/ceph/backup.conf
ceph-authtool --create-keyring /tmp/backup.mon.keyring --gen-key -n mon. --cap mon 'allow *'
ceph-authtool --create-keyring /etc/ceph/backup.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mgr 'allow *' --cap mds 'allow *'
ceph-authtool /tmp/backup.mon.keyring --import-keyring /etc/ceph/backup.client.admin.keyring
monmaptool --create --add ceph5 172.25.250.14 --fsid $uuid  /tmp/monmap
monmaptool --print /tmp/monmap
mkdir -p /var/lib/ceph/mon/backup-ceph5
chown ceph.ceph -R /var/lib/ceph /etc/ceph /tmp/backup.mon.keyring /tmp/monmap
sudo -u ceph ceph-mon --cluster backup --mkfs -i ceph5 --monmap /tmp/monmap --keyring /tmp/backup.mon.keyring
systemctl start ceph-mon@ceph5
systemctl enable ceph-mon@ceph5
ceph mon getmap -o monmap --cluster backup 
monmaptool --print monmap 
ceph auth get mon. --cluster backup
mkdir /var/lib/ceph/mgr/backup-ceph5
chown ceph.ceph -R /var/lib/ceph
ceph-authtool --create-keyring /etc/ceph/backup.mgr.ceph5.keyring --gen-key -n mgr.ceph5 --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *'
ceph auth import -i /etc/ceph/backup.mgr.ceph5.keyring --cluster backup 
ceph auth get-or-create mgr.ceph5 -o /var/lib/ceph/mgr/backup-ceph5/keyring  --cluster backup 
systemctl start ceph-mgr@ceph5
systemctl enable ceph-mgr@ceph5
ceph -s --cluster backup
ceph mgr dump  --conf /etc/ceph/backup.conf 
ceph mgr module enable dashboard  --conf /etc/ceph/backup.conf 
ceph mgr module ls   --conf /etc/ceph/backup.conf 
ceph config-key put mgr/dashboard/server_addr 172.25.250.14 --conf /etc/ceph/backup.conf 
ceph config-key put mgr/dashboard/server_port 7000 --conf /etc/ceph/backup.conf 
ceph config-key dump --conf /etc/ceph/backup.conf 
alias ceph='ceph --cluster backup'
alias rbd='rbd --cluster backup'
ceph osd create 
sgdisk --new=1:0:+5G --change-name=1:'ceph journal' --partition-guid=1:baf21303-1f6a-4345-965e-4b75a12ec3c6 --typecode=1:baf21303-1f6a-4345-965e-4b75a12ec3c6  --mbrtogpt -- /dev/vdb
sgdisk --new=2:0:0 --change-name=2:'ceph data' --partition-guid=2:04aad6e0-0232-4578-8b52-a289f3777f07 --typecode=2:04aad6e0-0232-4578-8b52-a289f3777f07 --mbrtogpt -- /dev/vdb
mkfs.xfs -f -i size=2048 /dev/vdb1
mkfs.xfs -f -i size=2048 /dev/vdb2
mkdir /var/lib/ceph/osd/backup-{0,1,2}
mount -o noatime,largeio,inode64,swalloc /dev/vdb2 /var/lib/ceph/osd/backup-0
uuid_blk=`blkid /dev/vdb2|awk '{print $2}'`
echo "$uuid_blk /var/lib/ceph/osd/backup-0 xfs defaults,noatime,largeio,inode64,swalloc 0 0" >> /etc/fstab
ceph-authtool --create-keyring /etc/ceph/backup.osd.0.keyring --gen-key -n osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *'
ceph auth import -i /etc/ceph/backup.osd.0.keyring --cluster backup 
ceph auth get-or-create osd.0 -o /var/lib/ceph/osd/backup-0/keyring --cluster backup
ceph-osd -i 0 --mkfs --cluster backup 
cd /var/lib/ceph/osd/backup-0
rm -f journal
partuuid_0=`blkid /dev/vdb1|awk -F "[""]" '{print $8}'`
ln -s /dev/disk/by-partuuid/$partuuid_0 ./journal
chown ceph.ceph -R /var/lib/ceph
ceph-osd --mkjournal -i 0 --cluster backup 
chown ceph.ceph   /dev/disk/by-partuuid/$partuuid_0
ceph osd crush add-bucket ceph5 host --cluster backup
ceph osd crush move ceph5 root=default --cluster backup
ceph osd crush add osd.0 0.01500 root=default host=ceph5 --cluster backup
systemctl start ceph-osd@0
systemctl enable ceph-osd@0

ceph osd create
sgdisk --new=1:0:+5G --change-name=1:'ceph journal' --partition-guid=1:3af9982e-8324-4666-bf3a-771f0e981f00 --typecode=1:3af9982e-8324-4666-bf3a-771f0e981f00  --mbrtogpt -- /dev/vdc
sgdisk --new=2:0:0 --change-name=2:'ceph data' --partition-guid=2:3f1d1456-2ebe-4702-96dc-e8ba975a772c --typecode=2:3f1d1456-2ebe-4702-96dc-e8ba975a772c --mbrtogpt -- /dev/vdc
mkfs.xfs -f -i size=2048 /dev/vdc1
mkfs.xfs -f -i size=2048 /dev/vdc2
mount -o noatime,largeio,inode64,swalloc /dev/vdc2 /var/lib/ceph/osd/backup-1
blkid /dev/vdc2
uuid_blk_1=`blkid /dev/vdc2|awk '{print $2}'`
echo "$uuid_blk_1 /var/lib/ceph/osd/backup-1 xfs defaults,noatime,largeio,inode64,swalloc 0 0" >> /etc/fstab
ceph-osd -i 1 --mkfs --mkkey --cluster backup 
cd /var/lib/ceph/osd/backup-1
rm -f journal
partuuid_1=`blkid /dev/vdc1|awk -F "[""]" '{print $8}'`
ln -s /dev/disk/by-partuuid/$partuuid_1 ./journal
chown ceph.ceph -R /var/lib/ceph
ceph-osd --mkjournal -i 1 --cluster backup 
chown ceph.ceph   /dev/disk/by-partuuid/$partuuid_1
ceph auth add osd.1 osd 'allow *' mon 'allow profile osd' mgr 'allow profile osd' -i /var/lib/ceph/osd/backup-1/keyring --cluster backup
ceph osd crush add osd.1 0.01500 root=default host=ceph5 --cluster backup
systemctl start ceph-osd@1
systemctl enable ceph-osd@1

ceph osd create 
sgdisk --new=1:0:+5G --change-name=1:'ceph journal' --partition-guid=1:4e7b40e7-f303-4362-8b57-e024d4987892 --typecode=1:4e7b40e7-f303-4362-8b57-e024d4987892  --mbrtogpt -- /dev/vdd
sgdisk --new=2:0:0 --change-name=2:'ceph data' --partition-guid=2:c82ce8a4-a762-4ba9-b987-21ef51df2eb2 --typecode=2:c82ce8a4-a762-4ba9-b987-21ef51df2eb2 --mbrtogpt -- /dev/vdd
mkfs.xfs -f -i size=2048 /dev/vdd1
mkfs.xfs -f -i size=2048 /dev/vdd2
mount -o noatime,largeio,inode64,swalloc /dev/vdd2 /var/lib/ceph/osd/backup-2
blkid /dev/vdd2
uuid_blk_2=`blkid /dev/vdd2|awk '{print $2}'`
echo "$uuid_blk_2 /var/lib/ceph/osd/backup-2 xfs defaults,noatime,largeio,inode64,swalloc 0 0">> /etc/fstab
ceph-osd -i 2 --mkfs --mkkey --cluster backup 
cd /var/lib/ceph/osd/backup-2
rm -f journal
partuuid_2=`blkid /dev/vdd1|awk -F "[""]" '{print $8}'`
ln -s /dev/disk/by-partuuid/$partuuid_2  ./journal
chown ceph.ceph -R /var/lib/ceph
ceph-osd --mkjournal -i 2 --cluster backup 
chown ceph.ceph   /dev/disk/by-partuuid/$partuuid_2
ceph auth add osd.2 osd 'allow *' mon 'allow profile osd' mgr 'allow profile osd' -i /var/lib/ceph/osd/backup-2/keyring --cluster backup
ceph osd crush add osd.2 0.01500 root=default host=ceph5 --cluster backup
systemctl start ceph-osd@2
systemctl enable ceph-osd@2

ceph osd getcrushmap -o /etc/ceph/crushmap
crushtool -d /etc/ceph/crushmap -o /etc/ceph/crushmap.txt
sed -i 's/step chooseleaf firstn 0 type host/step chooseleaf firstn 0 type osd/' /etc/ceph/crushmap.txt
grep 'step chooseleaf' /etc/ceph/crushmap.txt
crushtool -c /etc/ceph/crushmap.txt -o /etc/ceph/crushmap-new
ceph osd setcrushmap -i /etc/ceph/crushmap-new

命令

---

博主声明：本文的内容来源主要来自誉天教育晏威老师，由本人实验完成操作验证，需要的博友请联系誉天教育（http://www.yutianedu.com/），获得官方同意或者晏老师（https://www.cnblogs.com/breezey/）本人同意即可转载，谢谢！