ceph最佳实践

基础环境：Centos7.2

192.168.200.126 ceph2
192.168.200.127 ceph3
192.168.200.129 ceph4

关闭防火墙和selinux

# setenforce 0

# sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/confi

# systemctl stop firewalld

# systemctl disable firewalld

ceph Yum源：
[root@ceph2 ~]# cat /etc/yum.repos.d/ceph.repo
[Ceph-mimic]
name=Ceph x86_64 packages
baseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/x86_64/
enabled=1
gpgcheck=0


ceph-deploy Yum源
[root@ceph2 ~]# cat /etc/yum.repos.d/ceph-deploy.repo
[ceph-deploy]
name=ceph-deploy
baseurl=https://download.ceph.com/rpm-mimic/el7/noarch/
enabled=1
gpgcheck=0

所有节点安装ntp后进行时间同步
# yum install -y ntp
# ntpdate pool.ntp.org


集群免密钥配置
[root@ceph2 ~]# ssh-keygen
[root@ceph2 ~]# ssh-copy-id ceph2
[root@ceph2 ~]# ssh-copy-id ceph3
[root@ceph2 ~]# ssh-copy-id ceph4

同步配置

[root@ceph2 ~]# scp /etc/hosts ceph3:/etc/hosts
[root@ceph2 ~]# scp /etc/hosts ceph4:/etc/hosts
[root@ceph2 ~]# scp /etc/yum.repos.d/ceph-deploy.repo ceph3:/etc/yum.repos.d/
[root@ceph2 ~]# scp /etc/yum.repos.d/ceph-deploy.repo ceph4:/etc/yum.repos.d/

部署ceph
[root@ceph2 ~]# mkdir /etc/ceph
[root@ceph2 ~]# yum install -y ceph-deploy python-pip
[root@ceph2 ceph]# ceph-deploy new ceph2 ceph3 ceph4
[root@ceph2 ceph]# ls
ceph.conf ceph-deploy-ceph.log ceph.mon.keyring
[root@ceph2 ceph]# vi ceph.conf
[global]
fsid = d5dec480-a9df-4833-b740-de3a0ae4c755
mon_initial_members = ceph2, ceph3, ceph4
mon_host = 192.168.200.126,192.168.200.127,192.168.200.129
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public network = 192.168.200.0/24
cluster network = 192.168.200.0/24


所有节点安装ceph组件：
yum install -y ceph

ceph2节点 初始monitor 并收集密钥

[root@ceph2 ceph]# ceph-deploy mon create-initial

分发密钥给其他节点

[root@ceph2 ceph]# ceph-deploy admin ceph{1..3}

配置OSD
[root@ceph2 ceph]# ceph-deploy osd create --data /dev/sdb ceph2
[root@ceph2 ceph]# ceph-deploy osd create --data /dev/sdb ceph3
[root@ceph2 ceph]# ceph-deploy osd create --data /dev/sdb ceph4

[root@ceph2 ceph]# ceph -s
cluster:
id: d5dec480-a9df-4833-b740-de3a0ae4c755
health: HEALTH_WARN
no active mgr

services:
mon: 3 daemons, quorum ceph2,ceph3,ceph4
mgr: no daemons active
osd: 3 osds: 3 up, 3 in

data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:



如果出现以下警告，说明是集群内主机未同步:同步时间即可
health: HEALTH_WARN
clock skew detected on mon.ceph3
[root@ceph3 ~]# ntpdate pool.ntp.org
[root@ceph2 ceph]# systemctl restart ceph-mon.target


[root@ceph2 ceph]# ceph -s
cluster:
id: d5dec480-a9df-4833-b740-de3a0ae4c755
health: HEALTH_OK

services:
mon: 3 daemons, quorum ceph2,ceph3,ceph4
mgr: ceph2(active), standbys: ceph4, ceph3
osd: 3 osds: 3 up, 3 in

data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 3.0 GiB used, 57 GiB / 60 GiB avail
pgs:

开启dashboard，web管理
[root@ceph2 ceph]# vi /etc/ceph/ceph.conf
# 添加
[mgr]
mgr_modules = dashboard
[root@ceph2 ceph]# ceph mgr module enable dashboard
[root@ceph2 ceph]# ceph-deploy mgr create ceph2

生成并安装一个 自签名证书
[root@ceph2 ceph]# ceph dashboard create-self-signed-cert
生成密钥，生成两个文件dashboard.crt dashboard.key
[root@ceph2 ceph]#openssl req -new -nodes -x509 -subj "/O=IT/CN=ceph-mgr-dashboard" -days 3650 -keyout dashboard.key -out dashboard.crt -extensions v3_ca
配置服务地址、端口，默认的端口是8443，这里改为7000
[root@ceph2 ceph]# ceph config set mgr mgr/dashboard/server_addr 192.168.200.126
[root@ceph2 ceph]# ceph config set mgr mgr/dashboard/server_port 7000
[root@ceph2 ceph]# ceph dashboard set-login-credentials admin admin
[root@ceph2 ceph]# systemctl restart ceph-mgr@ceph2.service
[root@ceph2 ceph]# ceph mgr services
{
"dashboard": "https://192.168.200.126:7000/"
}

同步集群ceph配置文件
[root@ceph2 ceph]# ceph-deploy --overwrite-conf config push ceph3
[root@ceph2 ceph]# ceph-deploy --overwrite-conf config push ceph4



https://192.168.200.126:7000/#/login

块存储的使用

[root@ceph4 ceph]# ceph osd pool create rbd 128
[root@ceph4 ceph]# ceph osd pool get rbd pg_num
pg_num: 128
[root@ceph4 ceph]# ceph auth add client.rbd mon 'allow r' osd 'allow rwx pool=rbd'
[root@ceph4 ceph]# ceph auth export client.rbd -o ceph.client.rbd.keyring

[root@ceph4 ceph]# rbd create rbd1 --size 1024 --name client.rbd
[root@ceph4 ceph]# rbd ls -p rbd --name client.rbd
rbd1
[root@ceph4 ceph]# rbd --image rbd1 info --name client.rbd
rbd image 'rbd1':
size 1 GiB in 256 objects
order 22 (4 MiB objects)
id: 85d36b8b4567
block_name_prefix: rbd_data.85d36b8b4567
format: 2
features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
op_features:
flags:
create_timestamp: Sun Nov 17 04:33:17 2019

place group(pg) 为存储对象数量 一个磁盘为1个OSD，本次是三个sdb所以小于5个为128

[root@ceph4 ceph]# rbd feature disable rbd1 exclusive-lock object-map deep-flatten fast-diff --name client.rbd
[root@ceph4 ceph]# rbd map --image rbd1 --name client.rbd
/dev/rbd0
[root@ceph4 ceph]# rbd showmapped --name client.rbd
id pool image snap device
0 rbd rbd1 - /dev/rbd0
[root@ceph4 ceph]# mkfs.xfs /dev/rbd0
meta-data=/dev/rbd0 isize=256 agcount=8, agsize=32752 blks
= sectsz=512 attr=2, projid32bit=1
= crc=0 finobt=0
data = bsize=4096 blocks=262016, imaxpct=25
= sunit=16 swidth=16 blks
naming =version 2 bsize=4096 ascii-ci=0 ftype=0
log =internal log bsize=4096 blocks=768, version=2
= sectsz=512 sunit=16 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
[root@ceph4 ceph]# mount /dev/rbd0 /mnt/
[root@ceph4 ceph]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 467M 0 467M 0% /dev
tmpfs 479M 0 479M 0% /dev/shm
tmpfs 479M 13M 466M 3% /run
tmpfs 479M 0 479M 0% /sys/fs/cgroup
/dev/mapper/centos-root 50G 1.9G 49G 4% /
/dev/mapper/centos-home 28G 33M 28G 1% /home
/dev/sda1 497M 139M 359M 28% /boot
tmpfs 479M 24K 479M 1% /var/lib/ceph/osd/ceph-2
tmpfs 96M 0 96M 0% /run/user/0
/dev/rbd0 1021M 33M 989M 4% /mnt

删除存储池
[root@ceph4 ceph]# umount /dev/rbd0

[root@ceph4 ceph]# rbd unmap /dev/rbd/rbd/rbd1
[root@ceph4 ceph]# ceph osd pool delete rbd rbd --yes-i-really-really-mean-it
pool 'rbd' removed