Kubernetes 存储实现原理及简单示例

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

# PV
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-nfs
spec:
	# 存储类名称
	storageClassName: manual
	# 存储资源
  capacity:
    storage: 10Gi
	# 访问模式
  accessModes:
    - ReadWriteMany
	# 数据源
  nfs:
    path: /opt/nfs-volume
    server: 172.26.204.144
---
# PVC
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: pvc-nfs
spec:
	storageClassName: manual
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
---
# 用户使用
apiVersion: apps/v1
kind: Deployment
metadata:
  name: test-nfs
  labels:
    app: test
spec:
  replicas: 1
  selector:
    matchLabels:
      app: test
  template:
    metadata:
      labels:
        app: test
    spec:
      containers:
      - name: busybox
        image: busybox
				# 使用卷
        volumeMounts:
        - name: data
          mountPath: /data
			# 声明卷
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: pvc-nfs

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# SC示例
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs
# 指定存储服务方（Provisioner，预配器），基于该字段判断使用的存储插件，内置插件以kubernetes.io/为前缀
provisioner: nfs.csi.k8s.io
parameters:
  server: 10.0.0.11
  share: /data
# 默认回收策略
reclaimPolicy: Delete
# 定义如何为PVC完成预配和绑定
volumeBindingMode: Immediate
# 当前类动态创建的PV资源的默认挂载选项
mountOptions:
  - nfsvers=4.1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
  # 和CSI驱动的全名一致
  name: mycsidriver.example.com
spec:
  # 提示K8S，此驱动需要Attach操作（因为驱动实现了ControllerPublishVolume方法），
  # 并且需要在Attach之后等待操作完成，然后再进行后续的Mount操作
  # 默认值true
  attachRequired: true
  # 提示K8S，在挂载阶段，此驱动需要Pod的信息（名称、Pod的UID等）
  # Pod信息会在NodePublishVolume调用中作为volume_context传递：
  #   "csi.storage.k8s.io/pod.name": pod.Name
  #   "csi.storage.k8s.io/pod.namespace": pod.Namespace
  #   "csi.storage.k8s.io/pod.uid": string(pod.UID)
  #   "csi.storage.k8s.io/serviceAccount.name": pod.Spec.ServiceAccountName
  podInfoOnMount: true
  # 1.16中添加，到1.18为止beta
  # 此驱动支持的Volume Mode
  volumeLifecycleModes:
  - Persistent    # 默认，常规PV/PVC机制
  - Ephemeral     # 内联临时存储（inline ephemeral volumes）

1

kubectl get csidrivers.storage.k8s.io

1
2
3
4
5
6
7
8
9

apiVersion: storage.k8s.io/v1
kind: CSINode
metadata:
  name: node1
spec:
  drivers:
  - name: mycsidriver.example.com
    nodeID: storageNodeID1
    topologyKeys: ['mycsidriver.example.com/regions', "mycsidriver.example.com/zones"]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

// 让调用者（K8S组件、CSI Sidecar容器）能识别驱动，知晓它具有哪些可选特性
service Identity {
	// 获取插件名称和版本
  rpc GetPluginInfo(GetPluginInfoRequest)
    returns (GetPluginInfoResponse) {}
 // 获取插件功能
  rpc GetPluginCapabilities(GetPluginCapabilitiesRequest)
    returns (GetPluginCapabilitiesResponse) {}
	// 检查插件是否运行
  rpc Probe (ProbeRequest)
    returns (ProbeResponse) {}
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

service Controller {
	// 必须实现
  rpc CreateVolume (CreateVolumeRequest)
    returns (CreateVolumeResponse) {}

  // 必须实现
  rpc DeleteVolume (DeleteVolumeRequest)
    returns (DeleteVolumeResponse) {}
 
  rpc ControllerPublishVolume (ControllerPublishVolumeRequest)
    returns (ControllerPublishVolumeResponse) {}
 
  rpc ControllerUnpublishVolume (ControllerUnpublishVolumeRequest)
    returns (ControllerUnpublishVolumeResponse) {}
  
  // 必须实现
  rpc ValidateVolumeCapabilities (ValidateVolumeCapabilitiesRequest)
    returns (ValidateVolumeCapabilitiesResponse) {}
 
  rpc ListVolumes (ListVolumesRequest)
    returns (ListVolumesResponse) {}
 
  rpc GetCapacity (GetCapacityRequest)
    returns (GetCapacityResponse) {}

  // 必须实现
  rpc ControllerGetCapabilities (ControllerGetCapabilitiesRequest)
    returns (ControllerGetCapabilitiesResponse) {}
 
	// nfs-csi已实现
  rpc CreateSnapshot (CreateSnapshotRequest)
    returns (CreateSnapshotResponse) {}

	// nfs-csi已实现
  rpc DeleteSnapshot (DeleteSnapshotRequest)
    returns (DeleteSnapshotResponse) {}
 
  rpc ListSnapshots (ListSnapshotsRequest)
    returns (ListSnapshotsResponse) {}
 
  rpc ControllerExpandVolume (ControllerExpandVolumeRequest)
    returns (ControllerExpandVolumeResponse) {}
 
  rpc ControllerGetVolume (ControllerGetVolumeRequest)
    returns (ControllerGetVolumeResponse) {
        option (alpha_method) = true;
    }

  rpc ControllerModifyVolume (ControllerModifyVolumeRequest)
	  returns (ControllerModifyVolumeResponse) {
	      option (alpha_method) = true;
	  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

service Node {
  rpc NodeStageVolume (NodeStageVolumeRequest)
    returns (NodeStageVolumeResponse) {}
 
  rpc NodeUnstageVolume (NodeUnstageVolumeRequest)
    returns (NodeUnstageVolumeResponse) {}
 
	// 将卷挂载到节点上
  rpc NodePublishVolume (NodePublishVolumeRequest)
    returns (NodePublishVolumeResponse) {}

	// 卸载卷
  rpc NodeUnpublishVolume (NodeUnpublishVolumeRequest)
    returns (NodeUnpublishVolumeResponse) {}

  // 获取卷状态
  rpc NodeGetVolumeStats (NodeGetVolumeStatsRequest)
    returns (NodeGetVolumeStatsResponse) {}
 
  rpc NodeExpandVolume(NodeExpandVolumeRequest)
    returns (NodeExpandVolumeResponse) {}

  // 必须实现
  rpc NodeGetCapabilities (NodeGetCapabilitiesRequest)
    returns (NodeGetCapabilitiesResponse) {}
	 
	// 必须实现，返回运行插件的节点的信息
  rpc NodeGetInfo (NodeGetInfoRequest)
    returns (NodeGetInfoResponse) {}
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
  labels:
    app: nginx
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.14.2
        ports:
        - containerPort: 80
        volumeMounts:
        - name: data
          mountPath: /usr/share/nginx/html
      volumes:
      - name: data
        nfs:
          path: /opt/nfs-deployment
          server: 10.1.33.53

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

# PV
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-nfs
spec:
	storageClassName: manual
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteMany 
  nfs:
    path: /opt/nfs-deployment
    server: 172.26.204.144
---
# PVC
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: pvc-nfs
spec:
	storageClassName: manual
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
---
# 使用
apiVersion: apps/v1
kind: Deployment
metadata:
  name: busybox
  labels:
    app: busybox
spec:
  replicas: 1
  selector:
    matchLabels:
      app: busybox
  template:
    metadata:
      labels:
        app: busybox
    spec:
      containers:
      - name: busybox
        image: busybox
        command: ['sh', '-c', 'echo "Hello, Kubernetes!" && sleep 3600']
        volumeMounts:
        - name: data
          mountPath: /data
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: pvc-nfs

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

# 静态绑定
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-nfs-csi
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  mountOptions:
    - hard
    - nfsvers=4.1
  csi:
    driver: nfs.csi.k8s.io
    readOnly: false
    volumeHandle: unique-volumeid  # #确保它是集群中的唯一 ID
    volumeAttributes:
      server: 172.26.204.144
      share: /opt/nfs-deployment
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: pvc-nfs-csi-static
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
  volumeName: pv-nfs-csi
  storageClassName: ""

# 动态绑定
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs-csi
provisioner: nfs.csi.k8s.io
parameters:
  server: 172.26.204.144
  share: /opt/nfs-deployment
reclaimPolicy: Delete
volumeBindingMode: Immediate
mountOptions:
  - hard
  - nfsvers=4.1
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: pvc-nfs-csi-dynamic
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
  storageClassName: nfs-csi