如何在Kubernetes里使用cephfs作为存储

1 创建一个 ceph pool 创建存储池#

1.1 ceph 集群请看这里：[[RookCeph安装]]#

1
ceph osd pool create cephfs_data 128
2
ceph osd pool set-quota cephfs_data max_bytes $((20 * 1024 * 1024 * 1024)) #20G的存储池
3
cephfs pool init cephfs_data

1.2 查看集群状态#

1
ceph -s
2
  cluster:
3
    id:     xxxxxxxxx-xxxx-11ec-bfb9-67a0f564e0d6
4
    health: HEALTH_OK
5

6
  services:
7
    mon: 3 daemons, quorum ceph01,ceph02,ceph03 (age 7d)
8
    mgr: ceph01.pjvndt(active, since 7d), standbys: ceph02.injlkl, ceph03.sulrio
9
    mds: 1/1 daemons up, 2 standby
10
    osd: 8 osds: 8 up (since 7d), 8 in (since 7d)
11
    rgw: 3 daemons active (3 hosts, 1 zones)
12

13
  data:
14
    volumes: 1/1 healthy
15
    pools:   10 pools, 265 pgs
16
    objects: 269 objects, 8.3 KiB
17
    usage:   20 GiB used, 400 GiB / 420 GiB avail
18
    pgs:     265 active+clean

1.3 查看用户 key#

1
ceph auth get client.admin
2
[client.admin]
3
        key = AQBTyCpiHHhbARAAfHqI0X9iMd3rnzJQHaMLkw==
4
        caps mds = "allow *"
5
        caps mgr = "allow *"
6
        caps mon = "allow *"
7
        caps osd = "allow *"
8
exported keyring for client.admin

或者自己创建存储池、用户以及用户 key

1
ceph osd pool create kubernetes
2
cephfs pool init kubernetes
3
ceph auth get-or-create client.kubernetes mon 'profile cephfs' osd 'profile cephfs pool=kubernetes' mgr 'profile cephfs pool=kubernetes'
4
[client.kubernetes]
5
        key = AQD9o0Fd6hEWSDAAAt7fMaSZXduT3NWEqylNpmg==

2 k8s ceph 配置#

2.1 生成 ceph-csi 的 kubernetes configmap#

1
ceph mon dump
2
epoch 3
3
fsid xxxxxxxxx-xxxx-11ec-bfb9-67a0f564e0d6
4
last_changed 2022-03-11T03:57:44.615026+0000
5
created 2022-03-11T03:56:03.739853+0000
6
min_mon_release 16 (pacific)
7
election_strategy: 1
8
0: [v2:10.1.1.1:3300/0,v1:10.1.1.1:6789/0] mon.ceph01
9
1: [v2:10.1.1.2:3300/0,v1:10.1.1.2:6789/0] mon.ceph02
10
2: [v2:10.1.1.3:3300/0,v1:10.1.1.3:6789/0] mon.ceph03
11
dumped monmap epoch 3

2.2 用以上的的信息生成 configmap#

下载此目录所有文件可以参考官网的,需要自己改镜像

1
cat csi-config-map.yaml
2
---
3
apiVersion: v1
4
kind: ConfigMap
5
metadata:
6
  name: "ceph-csi-config"
7
  namespace: cephfs
8
data:
9
  config.json: |-
10
    [
11
      {
12
        "clusterID": "xxxxxxxxx-xxxx-11ec-bfb9-67a0f564e0d6",
13
        "monitors": [
14
          "10.1.1.1:6789",
15
          "10.1.1.2:6789",
16
          "10.1.1.3:6789"
17
        ]
18
      }
19
    ]

2.3 修改 kubectl 路径(可选)#

这里我的 kubelet 存储路径在/data/kubelet/
如果这里不对的话会报在不到驱动

1
grep plugins_registry *
2
csi-cephfsplugin.yaml:            - "--kubelet-registration-path=/var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock"
3
csi-cephfsplugin.yaml:              mountPath: /var/lib/kubelet/pods
4
csi-cephfsplugin.yaml:              mountPath: /var/lib/kubelet/plugins
5
csi-cephfsplugin.yaml:            path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/
6
csi-cephfsplugin.yaml:            path: /var/lib/kubelet/plugins_registry/
7
csi-cephfsplugin.yaml:            path: /var/lib/kubelet/pods
8
csi-cephfsplugin.yaml:            path: /var/lib/kubelet/plugins
9
csi-cephfsplugin.yaml:            path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/mountinfo
10
csi-nodeplugin-psp.yaml:    - pathPrefix: '/var/lib/kubelet/pods'
11
csi-nodeplugin-psp.yaml:    - pathPrefix: '/var/lib/kubelet/plugins/cephfs.csi.ceph.com'
12
csi-nodeplugin-psp.yaml:    - pathPrefix: '/var/lib/kubelet/plugins_registry'
13
csi-nodeplugin-psp.yaml:    - pathPrefix: '/var/lib/kubelet/plugins'

sed -i "s/var\/lib/data/g" *

1
grep kubelet *
2
csi-cephfsplugin.yaml:            - "--kubelet-registration-path=/data/kubelet/plugins/cephfs.csi.ceph.com/csi.sock"
3
csi-cephfsplugin.yaml:              mountPath: /data/kubelet/pods
4
csi-cephfsplugin.yaml:              mountPath: /data/kubelet/plugins
5
csi-cephfsplugin.yaml:            path: /data/kubelet/plugins/cephfs.csi.ceph.com/
6
csi-cephfsplugin.yaml:            path: /data/kubelet/plugins_registry/
7
csi-cephfsplugin.yaml:            path: /data/kubelet/pods
8
csi-cephfsplugin.yaml:            path: /data/kubelet/plugins
9
csi-cephfsplugin.yaml:            path: /data/kubelet/plugins/cephfs.csi.ceph.com/mountinfo
10
csi-nodeplugin-psp.yaml:    - pathPrefix: '/data/kubelet/pods'
11
csi-nodeplugin-psp.yaml:    - pathPrefix: '/data/kubelet/plugins/cephfs.csi.ceph.com'
12
csi-nodeplugin-psp.yaml:    - pathPrefix: '/data/kubelet/plugins_registry'
13
csi-nodeplugin-psp.yaml:    - pathPrefix: '/data/kubelet/plugins'

2.4 生成 ceph-csi cephx 的 secret#

1
cat <<EOF > csi-cephfs-secret.yaml
2
apiVersion: v1
3
kind: Secret
4
metadata:
5
  name: csi-cephfs-secret
6
  namespace: cephfs
7
stringData:
8
  userID: admin
9
  userKey: xxxxx3rnzJQHaMLkw==
10

11
  adminID: admin
12
  adminKey: xxxxx0X9iMd3rnzJQHaMLkw==
13
EOF

2.5 部署 csi#

kubectl apply -f . -n cephfs

3 使用 ceph 块儿设备#

3.1 查看 fsName#

1
ceph fs ls
2
name: testfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]

3.2 创建 storageclass#

1
apiVersion: storage.k8s.io/v1
2
kind: StorageClass
3
metadata:
4
   name: csi-cephfs-sc
5
provisioner: cephfs.csi.ceph.com
6
parameters:
7
   clusterID: 2a8e37c8-a0ef-11ec-bfb9-67a0f564e0d6
8
   pool: cephfs_data
9
   fsName: testfs
10
   csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret
11
   csi.storage.k8s.io/provisioner-secret-namespace: cephfs
12
   csi.storage.k8s.io/controller-expand-secret-name: csi-cephfs-secret
13
   csi.storage.k8s.io/controller-expand-secret-namespace: cephfs
14
   csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret
15
   csi.storage.k8s.io/node-stage-secret-namespace: cephfs
16
reclaimPolicy: Delete
17
mountOptions:
18
   - discard

clusterID 对应之前的步骤中的 fsid
secret-name 对应之前的 Secret 名字
fsName 对应 ceph fs ls 查到的名字
imageFeatures，这个是用来确定创建的 image 的特征的
allowVolumeExpansion: true 是否开启在线扩容

kubectl apply -f sc.yaml -n cephfs

3.3 查看 storageclass#

1
get StorageClass -n cephfs
2
NAME            PROVISIONER           RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
3
csi-cephfs-sc   cephfs.csi.ceph.com   Delete          Immediate           false                  8m48s

3.4 创建 PVC#

1
apiVersion: v1
2
kind: PersistentVolumeClaim
3
metadata:
4
  name: csi-cephfs-pvc
5
  namespace: cephfs
6
spec:
7
  accessModes:
8
    - ReadWriteOnce
9
  resources:
10
    requests:
11
      storage: 1Gi
12
  storageClassName: csi-cephfs-sc

kubectl aaply -f pvc.yaml

3.5 查看 pvc#

1
kubectl get pvc -n cephfs
2
NAME             STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS    AGE
3
csi-cephfs-pvc   Bound    pvc-42da0f11-baba-4b73-a1ea-bbaee1f5fad4   1Gi        RWO            csi-cephfs-sc   2s

3.6 测试挂载 pod 写入#

1
cat <<EOF > csi-cephfs-secret.yaml
2
apiVersion: v1
3
kind: Pod
4
metadata:
5
  name: csi-cephfs-demo-pod
6
  namespace: cephfs
7
spec:
8
  containers:
9
    - name: web-server
10
      image: nginx
11
      volumeMounts:
12
        - name: mypvc
13
          mountPath: /var/lib/www/html
14
  volumes:
15
    - name: mypvc
16
      persistentVolumeClaim:
17
        claimName: csi-cephfs-pvc
18
        readOnly: false
19
EOF

kubectl apply -f pod.yaml -n cephfs

1
kubectl apply -f pod.yaml -n rbd
2
kubectl exec -it rbd-pod -n rbd -- sh
3
echo "hello" > /mnt/1
4
cat /mnt/1

4 静态创建#

4.1 配置 secret#

在 pvc 所在的 namespaces 创建

1
apiVersion: v1
2
kind: Secret
3
metadata:
4
  name: ceph-secret
5
data:
6
  key: QVFCVHlDcGlISGhiQVJxxxxxxxx # 上面获取的token要base64加密,很重要

4.2 创建 pv#

1
apiVersion: v1
2
kind: PersistentVolume
3
metadata:
4
  name: cai-pv
5
spec:
6
  accessModes:
7
  - ReadWriteOnce
8
  capacity:
9
    storage: 1Gi
10
  cephfs:
11
    monitors:
12
    - 10.1.1.1:6789   #ceph的monitors
13
    - 10.1.1.2:6789
14
    path: /volumes/pvc-cephfs #默认为 / ，不会自动创建目录,可通过在cephfs增加subvolume或subvolumegroup,并获取其path来创建新的目录
15
    user: admin           #ceph账号，需要后端存储提供
16
    secretRef:
17
      name: ceph-secret   #下面的cephfs-seret.yml
18
    #secretFile: "/etc/ceph/cephfs-seret.yml"  #如果不用secretRef，也可直接指定secret文件
19
  persistentVolumeReclaimPolicy: Retain
20
  volumeMode: Filesystem

4.3 pvc 挂载#

1
apiVersion: v1
2
kind: PersistentVolumeClaim
3
metadata:
4
  name: cai-pvc
5
spec:
6
  volumeName: cai-pv
7
  accessModes: ["ReadWriteOnce"] # 与上面一样
8
  resources:
9
    requests:
10
      storage: 1Gi

4.4 查看状态#

1
kubectl get pvc
2
NAME        STATUS   VOLUME    CAPACITY   ACCESS MODES   STORAGECLASS   AGE
3
cai-pvc     Bound    cai-pv       1Gi        RWO                        63m

5 排错#

5.1 查看日志#

kubectl logs -f deployment.apps/csi-cephfsplugin-provisioner -n cephfs --all-containers=true --max-log-requests=7

5.2 查看 pvc 状态#

kubectl describe pvc csi-cephfs-pvc01 -n cephfs

5.3 测试挂载#

mount -t ceph 10.1.1.1:6789,10.1.1.2:6789,10.1.1.3:6789:/volumes/pvc-cephfs -o name=admin,secret=xxxxxx9iMd3rnzJQHaMLkw==