k8s存储之挂载阿里云oss对象存储

Posted 2021-06-27 爱上口袋的天空

1.准备工作

1）k8s集群，我是通过kubeadm搭建的2节点集群

2）准备好阿里云的OSS账号，并创建bucket

2.在每台机器上安装阿里云ossfs软件。这个软件必须要安装，因为pv/pvc如果想要用阿里云的oss的话，这是必须的软件。安装步骤参见阿里云OSS官方文档，我的机器是二节点centos7,下边我就写一下centos7该怎么安装ossfs。

1)下载安装包

      wget http://gosspublic.alicdn.com/ossfs/ossfs_1.80.6_centos7.0_x86_64.rpm

2)安装

    yum install ossfs_1.80.6_centos7.0_x86_64.rpm

3.yaml文件准备

1）rbac.yaml

# This YAML file contains all RBAC objects that are necessary to run external
# CSI provisioner.
#
# In production, each CSI driver deployment has to be customized:
# - to avoid conflicts, use non-default namespace and different names
#   for non-namespaced entities like the ClusterRole
# - decide whether the deployment replicates the external CSI
#   provisioner, in which case leadership election must be enabled;
#   this influences the RBAC setup, see below
 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin
  # replace with the same namespace name with plugin
  namespace: kube-system
 
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: alicloud-csi-plugin
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "update", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["csinodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "watch", "list", "delete", "update", "create"]
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["get", "watch", "list", "delete", "update", "create"]
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["csi.storage.k8s.io"]
    resources: ["csinodeinfos"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["volumeattachments"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotcontents"]
    verbs: ["create", "get", "list", "watch", "update", "delete"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshots"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["apiextensions.k8s.io"]
    resources: ["customresourcedefinitions"]
    verbs: ["create", "list", "watch", "delete"]
 
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: alicloud-csi-plugin
subjects:
  - kind: ServiceAccount
    name: admin
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: alicloud-csi-plugin
  apiGroup: rbac.authorization.k8s.io

2）oss-plugin.yaml

apiVersion: storage.k8s.io/v1beta1
kind: CSIDriver
metadata:
  name: ossplugin.csi.alibabacloud.com
spec:
  attachRequired: false
---
# This YAML defines all API objects to create RBAC roles for csi node plugin.
kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: csi-ossplugin
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: csi-ossplugin
  template:
    metadata:
      labels:
        app: csi-ossplugin
    spec:
      tolerations:
      - operator: Exists
      priorityClassName: system-node-critical
      serviceAccount: admin
      hostNetwork: true
      hostPID: true
      containers:
      - name: driver-registrar
        image: registry.cn-hangzhou.aliyuncs.com/acs/csi-node-driver-registrar:v1.1.0
        imagePullPolicy: Always
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "rm -rf /registration/ossplugin.csi.alibabacloud.com /registration/ossplugin.csi.alibabacloud.com-reg.sock"]
        args:
        - "--v=5"
        - "--csi-address=/var/lib/kubelet/plugins/ossplugin.csi.alibabacloud.com/csi.sock"
        - "--kubelet-registration-path=/var/lib/kubelet/plugins/ossplugin.csi.alibabacloud.com/csi.sock"
        env:
        - name: KUBE_NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        volumeMounts:
        - name: kubelet-dir
          mountPath: /var/lib/kubelet/
        - name: registration-dir
          mountPath: /registration
 
      - name: csi-ossplugin
        securityContext:
          privileged: true
          capabilities:
            add: ["SYS_ADMIN"]
          allowPrivilegeEscalation: true
        image: registry.cn-hangzhou.aliyuncs.com/acs/csi-plugin:v1.14.8.32-c77e277b-aliyun
        imagePullPolicy: "Always"
        args:
        - "--endpoint=$(CSI_ENDPOINT)"
        - "--v=5"
        - "--driver=ossplugin.csi.alibabacloud.com"
        - "--nodeid=$(KUBE_NODE_NAME)"
        env:
        - name: CSI_ENDPOINT
          value: unix://var/lib/kubelet/plugins/ossplugin.csi.alibabacloud.com/csi.sock
        - name: KUBE_NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        volumeMounts:
        - name: kubelet-dir
          mountPath: /var/lib/kubelet/
          mountPropagation: "Bidirectional"
        - name: etc
          mountPath: /host/etc
        - mountPath: /var/log/
          name: host-log
        - mountPath: /host/usr/
          name: flexvolumedir
      volumes:
      - name: kubelet-dir
        hostPath:
          path: /var/lib/kubelet/
          type: Directory
      - name: registration-dir
        hostPath:
          path: /var/lib/kubelet/plugins_registry
          type: DirectoryOrCreate
      - name: etc
        hostPath:
          path: /etc
      - name: flexvolumedir
        hostPath:
          path: /usr/
      - name: host-log
        hostPath:
          path: /var/log/
  updateStrategy:
    type: RollingUpdate

3）my-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  namespace: kgf-dev
  name: oss-pv
  labels:
    alicloud-pvname: oss-pv
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  csi:
    driver: ossplugin.csi.alibabacloud.com
    # set volumeHandle same value pv name
    volumeHandle: oss-pv
    volumeAttributes:
      bucket: "kgf"
      url: "oss-cn-shanghai.aliyuncs.com"
      otherOpts: "-o max_stat_cache_size=0 -o allow_other"
      akId: "xxxxxxx"
      akSecret: "xxxxxxxxx"
      path: "/"

说明：

      bucket：目前只支持挂载Bucket，不支持挂载Bucket下面的子目录或文件。
      url：OSS endpoint，挂载OSS的接入域名，挂载节点和bucket相同region时，可使用内网地址。
      akId：用户的access id值。
      akSecret：用户的access secret值。
      otherOpts：挂载OSS时支持定制化参数输入，格式为：-o *** -o ***。

4）my-pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  namespace: kgf-dev
  name: oss-pvc
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi
  selector:
    matchLabels:
      alicloud-pvname: oss-pv

5)my-deploy.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kgf-gateway
  namespace: kgf-dev
  labels:
    app: kgf-gateway
spec:
  replicas: 1
  selector:
    matchLabels: { app: kgf-gateway }
  strategy: 
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate  
  template:
    metadata:
      labels: { app: kgf-gateway }
    spec:
      containers:
        - name: kgf-gateway
          env:
            - name: kgf-gateway
              valueFrom:
                fieldRef: { fieldPath: metadata.name }
          image: hub.harbor.com/library/kgf-gateway:v1
          imagePullPolicy: IfNotPresent
          resources: 
            limits:
              cpu: 1
              memory: 512Mi
            requests:
              cpu: 0.4
              memory: 256Mi
          volumeMounts:
            - name: application-dev-yaml
              mountPath: /config
            - name: oss-pvc
              mountPath: /kgf/kgf-gateway  
      volumes:
        - name: oss-pvc
          persistentVolumeClaim:
            claimName: oss-pvc 
        - name: application-dev-yaml
          configMap:
            name: application-dev-yaml
            items: 
              - key: application.yaml
                path: application.yaml              

4.部署

#创建rbac权限
$ kubectl create -f ./rbac.yaml 
serviceaccount/admin created
clusterrole.rbac.authorization.k8s.io/alicloud-csi-plugin created
clusterrolebinding.rbac.authorization.k8s.io/alicloud-csi-plugin created
 
#创建oss-plugin
$ kubectl create -f ./oss-plugin.yaml
 
#检查创建情况
$ kubectl get pod -n kube-system | grep csi-oss
kube-system             csi-ossplugin-9jdhw                                  2/2     Running             0          55m
kube-system             csi-ossplugin-f7n5f                                  2/2     Running             0          55m
kube-system             csi-ossplugin-vgkcp                                  2/2     Running             0          55m
 
#查验CSIDriver安装情况
$ kubectl get CSIDriver
NAME                             CREATED AT
ossplugin.csi.alibabacloud.com   2020-06-23T14:48:18Z
 
#创建pv
$ kubectl create -f ./my-pv.yaml
 
#创建pvc
$ kubectl create -f ./my-pvc.yaml
 
#检验一下阿里云oss是否可以成功挂载到k8s集群中做pv使用
$ kubectl create -f ./my-deploy.yaml

5.验证

1)kubectl get pods -n kgf-dev