Kubernetes持久化存储

kubectl explain 资源类型.属性.属性... #查看指定资源的特定帮助信息
[root@k8s-master ~]# kubectl -s http://10.0.0.110:10000 explain pod.spec.volumes
RESOURCE: volumes <[]Object>
......
   emptyDir <Object>
     EmptyDir represents a temporary directory that shares a pod's lifetime.
     More info: http://kubernetes.io/docs/user-guide/volumes#emptydir
......
   glusterfs    <Object>
     Glusterfs represents a Glusterfs mount on the host that shares a pod's
     lifetime. More info:
     http://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md
......
   nfs  <Object>
     NFS represents an NFS mount on the host that shares a pod's lifetime More
     info: http://kubernetes.io/docs/user-guide/volumes#nfs
......
   hostPath <Object>
     HostPath represents a pre-existing file or directory on the host machine
     that is directly exposed to the container. This is generally used for system
     agents or other privileged things that are allowed to see the host machine.
     Most containers will NOT need this. More info:
     http://kubernetes.io/docs/user-guide/volumes#hostpath

   name <string> -required-
     Volume's name. Must be a DNS_LABEL and unique within the pod. More info:
     http://kubernetes.io/docs/user-guide/identifiers#names

[root@k8s-master ~]# kubectl -s http://10.0.0.110:10000 explain pod.spec.containers
RESOURCE: containers <[]Object>
......
[root@k8s-master ~]# kubectl -s http://10.0.0.110:10000 explain pod.spec.containers.volumeMounts
RESOURCE: volumeMounts <[]Object>

DESCRIPTION:
     Pod volumes to mount into the container's filesystem. Cannot be updated.

    VolumeMount describes a mounting of a Volume within a container.

FIELDS:
   mountPath    <string> -required-
     Path within the container at which the volume should be mounted.  Must not
     contain ':'.

   name <string> -required-
     This must match the Name of a Volume.

   readOnly <boolean>
     Mounted read-only if true, read-write otherwise (false or unspecified).
     Defaults to false.

   subPath  <string>
     Path within the volume from which the container's volume should be mounted.
     Defaults to "" (volume's root).
apiVersion: 版本
kind: 资源类型
metadata:

name: 名称

spec:

replicas: n
template:
metadata:
labels:
app: 名称
spec:
volumes:
- name: 名称 #主机持久化卷名称
持久化类型: 参数
containers:
- name: 名称
image: 镜像仓库:端口/镜像名称:版本
volumeMounts:
- name: 名称 #容器持久化卷名称(需与主机持久化卷名称一致)
mountPath: 持久化卷路径 #容器中需要持久化的目录。

一、 emptyDir类型

volumes:
- name: 名称

emptyDir: {}

emptyDir类型的持久化目录会随着Pod的删除而被删除

emptyDir 类型会将指定的容器目录持久化目录至宿主机的 /var/lib/kubelet/pods/pod的ID/volumes/kubernetes.io~empty~dir/持久化卷名称/ 目录下;
注: 更换Pod节点或者修改Pod名称会导致 emptyDir 类型的持久化无效
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  namespace: tomcat
  name: mysql
spec:
  replicas: 1
  minReadySeconds: 30
  template:
    metadata:
      labels:
        app: mysql
    spec:
      volumes:
      - name: mysql
        emptyDir: {}
      containers:
      - name: mysql
        image: 10.0.0.140:5000/mysql:5.7
        volumeMounts:
        - name: mysql
          mountPath: /var/lib/mysql
        ports:
        - containerPort: 3306
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: '123456'
[root@k8s-master ~]# cd k8s_yaml/tomcat+mysql/
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# cat k8s_deploy_mysql.yaml 
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 delete -f k8s_deploy_mysql.yaml 
deployment "mysql" deleted
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 create -f k8s_deploy_mysql.yaml 
deployment "mysql" created
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 exec mysql-2698284748-8x9d4 -it /bin/bash -n tomcat
root@mysql-2698284748-8x9d4:/# mysql -uroot -p123456
......
mysql> use HPE_APP;
mysql> desc T_USERS;
+-----------+--------------+------+-----+---------+----------------+
| Field     | Type         | Null | Key | Default | Extra          |
+-----------+--------------+------+-----+---------+----------------+
| ID        | int(11)      | NO   | PRI | NULL    | auto_increment |
| USER_NAME | varchar(100) | YES  |     | NULL    |                |
| LEVEL     | varchar(20)  | YES  |     | NULL    |                |
+-----------+--------------+------+-----+---------+----------------+
3 rows in set (0.00 sec)
mysql> insert into T_USERS(ID,USER_NAME,LEVEL) values('8','Young','99');
Query OK, 1 row affected (0.00 sec)

mysql> insert into T_USERS(ID,USER_NAME,LEVEL) values('9','Aspen','101');
Query OK, 1 row affected (0.00 sec)

mysql> select * from T_USERS;
+----+-----------+-------+
| ID | USER_NAME | LEVEL |
+----+-----------+-------+
|  1 | me        | 100   |
|  2 | our team  | 100   |
|  3 | HPE       | 100   |
|  4 | teacher   | 100   |
|  5 | docker    | 100   |
|  6 | google    | 100   |
|  8 | Young     | 99    |
|  9 | Aspen     | 101   |
+----+-----------+-------+
8 rows in set (0.00 sec)
[root@k8s-node01 ~]# docker ps | grep mysql:5.7
d83d10c120d0        10.0.0.140:5000/mysql:5.7                           "docker-entrypoint..."   8 minutes ago       Up 8 minutes                            k8s_mysql.51f141ef_mysql-2698284748-5mq60_tomcat_98b04c16-e519-11eb-8479-000c29870b5c_d1622f6d
[root@k8s-node01 ~]# ls /var/lib/kubelet/pods/ |grep 98b04c16-e519-11eb-8479-000c29870b5c
98b04c16-e519-11eb-8479-000c29870b5c
[root@k8s-node01 ~]# tree /var/lib/kubelet/pods/98b04c16-e519-11eb-8479-000c29870b5c/volumes/kubernetes.io~empty-dir/mysql/ -L 1
/var/lib/kubelet/pods/98b04c16-e519-11eb-8479-000c29870b5c/volumes/kubernetes.io~empty-dir/mysql/
├── auto.cnf
├── HPE_APP
├── ib_buffer_pool
├── ibdata1
├── ib_logfile0
├── ib_logfile1
├── ibtmp1
├── mysql
├── performance_schema
└── sys

4 directories, 6 files
[root@k8s-node02 ~]# docker rm -f  `docker ps -a -q`
.....
4cf64f6855a1
da8f3bef7d5c
8f4c4b763a60
6126c05a9b76
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 get pod -n tomcat -o wide
NAME                     READY     STATUS    RESTARTS   AGE       IP             NODE
mysql-2698284748-8x9d4   1/1       Running   0          10m       192.168.24.7   10.0.0.120
tomcat-627834460-2khwr   1/1       Running   1          1d        192.168.24.4   10.0.0.120
tomcat-627834460-czx1p   1/1       Running   2          1d        192.168.50.2   10.0.0.130
tomcat-627834460-klrbb   1/1       Running   0          1d        192.168.50.6   10.0.0.130
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 exec mysql-2698284748-8x9d4 -it /bin/bash -n tomcat
root@mysql-2698284748-8x9d4:/# mysql -uroot -p123456
......
mysql> use HPE_APP;
mysql> select  * from T_USERS;
+----+-----------+-------+
| ID | USER_NAME | LEVEL |
+----+-----------+-------+
|  1 | me        | 100   |
|  2 | our team  | 100   |
|  3 | HPE       | 100   |
|  4 | teacher   | 100   |
|  5 | docker    | 100   |
|  6 | google    | 100   |
|  8 | Young     | 99    |
|  9 | Aspen     | 101   |
+----+-----------+-------+
8 rows in set (0.00 sec)

emptyDir持久化失效演示

[root@k8s-master ~]# kubectl -s http://10.0.0.110:10000 delete pod --all -n tomcat
pod "mysql-2698284748-8x9d4" deleted
pod "tomcat-627834460-2khwr" deleted
pod "tomcat-627834460-czx1p" deleted
pod "tomcat-627834460-klrbb" deleted
[root@k8s-master ~]# kubectl -s http://10.0.0.110:10000 get pod -n tomcat -o wide
NAME                     READY     STATUS    RESTARTS   AGE       IP             NODE
mysql-2698284748-5mq60   1/1       Running   0          56s       192.168.24.9   10.0.0.120
tomcat-627834460-j0bvb   1/1       Running   0          56s       192.168.50.7   10.0.0.130
tomcat-627834460-jb3q5   1/1       Running   0          56s       192.168.50.8   10.0.0.130
tomcat-627834460-nfh46   1/1       Running   0          56s       192.168.24.8   10.0.0.120
[root@k8s-master ~]# kubectl -s http://10.0.0.110:10000 exec mysql-2698284748-5mq60  -it /bin/bash -n tomcat
root@mysql-2698284748-5mq60:/# mysql -uroot -p123456
.....
mysql> use HPE_APP;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
mysql> select * from T_USERS;
+----+-----------+-------+
| ID | USER_NAME | LEVEL |
+----+-----------+-------+
|  1 | me        | 100   |
|  2 | our team  | 100   |
|  3 | HPE       | 100   |
|  4 | teacher   | 100   |
|  5 | docker    | 100   |
|  6 | google    | 100   |
+----+-----------+-------+
6 rows in set (0.00 sec)

二、 HostPath类型

volumes:
- name: 名称

hostPath:
path: 路径 #指定宿主机持久化路径

若指定的持久化宿主机目录不存在,docker会自动创建指定目录

更换Pod节点会导致 hostPath 类型的持久化无效
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  namespace: tomcat
  name: mysql
spec:
  replicas: 1
  minReadySeconds: 30
  template:
    metadata:
      labels:
        app: mysql
    spec:
      volumes:
      - name: mysql
        hostPath:
          path: /data/tomcat_db
      containers:
      - name: mysql
        image: 10.0.0.140:5000/mysql:5.7
        volumeMounts:
        - name: mysql
          mountPath: /var/lib/mysql
        ports:
        - containerPort: 3306
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: '123456'
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# cat k8s_deploy_mysql.yaml
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 delete -f k8s_deploy_mysql.yaml 
deployment "mysql" deleted
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 create -f k8s_deploy_mysql.yaml 
deployment "mysql" created
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 get pod -o wide -n tomcat
NAME                     READY     STATUS    RESTARTS   AGE       IP             NODE
mysql-2177208214-2g0w5   1/1       Running   0          20s       192.168.24.4   10.0.0.120
tomcat-627834460-j0bvb   1/1       Running   0          2h        192.168.50.7   10.0.0.130
tomcat-627834460-jb3q5   1/1       Running   0          2h        192.168.50.8   10.0.0.130
tomcat-627834460-nfh46   1/1       Running   0          2h        192.168.24.8   10.0.0.120
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# ssh root@10.0.0.120 ls /data/tomcat_db
root@10.0.0.120's password: 
auto.cnf
ib_buffer_pool
ibdata1
ib_logfile0
ib_logfile1
ibtmp1
mysql
performance_schema
sys
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 exec -it mysql-2177208214-2g0w5 /bin/bash -n tomcat
root@mysql-2177208214-2g0w5:/# mysql -uroot -p123456
......
mysql> use HPE_APP;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
mysql> insert into T_USERS(ID,USER_NAME,LEVEL)
    -> values('8','Young','101');
Query OK, 1 row affected (0.01 sec)

mysql> insert into T_USERS(ID,USER_NAME,LEVEL) values('9','Aspen','95');
Query OK, 1 row affected (0.00 sec)

mysql> select * from T_USERS;
+----+-----------+-------+
| ID | USER_NAME | LEVEL |
+----+-----------+-------+
|  1 | me        | 100   |
|  2 | our team  | 100   |
|  3 | HPE       | 100   |
|  4 | teacher   | 100   |
|  5 | docker    | 100   |
|  6 | google    | 100   |
|  8 | Young     | 101   |
|  9 | Aspen     | 95    |
+----+-----------+-------+
8 rows in set (0.00 sec)
mysql> ^DBye
root@mysql-2177208214-2g0w5:/# exit
# 删除POD
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 delete pod --all -n tomcat
pod "mysql-2177208214-2g0w5" deleted
pod "tomcat-627834460-j0bvb" deleted
pod "tomcat-627834460-jb3q5" deleted
pod "tomcat-627834460-nfh46" deleted
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 get pod -o wide -n tomcat
NAME                     READY     STATUS    RESTARTS   AGE       IP             NODE
mysql-2177208214-1fqpb   1/1       Running   0          17s       192.168.24.9   10.0.0.120
tomcat-627834460-1pzvh   1/1       Running   0          17s       192.168.24.7   10.0.0.120
tomcat-627834460-61lq0   1/1       Running   0          17s       192.168.50.3   10.0.0.130
tomcat-627834460-zqvv0   1/1       Running   0          17s       192.168.50.4   10.0.0.130
root@mysql-2177208214-1fqpb:/# mysql -uroot -p123456 -e "use HPE_APP;select * from T_USERS;"
mysql: [Warning] Using a password on the command line interface can be insecure.
+----+-----------+-------+
| ID | USER_NAME | LEVEL |
+----+-----------+-------+
|  1 | me        | 100   |
|  2 | our team  | 100   |
|  3 | HPE       | 100   |
|  4 | teacher   | 100   |
|  5 | docker    | 100   |
|  6 | google    | 100   |
|  8 | Young     | 101   |
|  9 | Aspen     | 95    |
+----+-----------+-------+

三、 NFS类型

1. 工作原理

2. 设置NFS类型持久化

step 1 准备NFS环境

详细步骤请参照NFS章节

# NFS_Server端
[root@k8s-registry ~]# yum install -y nfs-utils
......
Complete!
[root@k8s-registry ~]# mkdir -p /data/tomcat_db
[root@k8s-registry ~]# tail /etc/exports
/data 10.0.0.0/24(rw,sync,no_root_squash,no_all_squash)
[root@k8s-registry ~]# systemctl enable rpcbind
[root@k8s-registry ~]# systemctl start rpcbind
[root@k8s-registry ~]# systemctl enable nfs
Created symlink from /etc/systemd/system/multi-user.target.wants/nfs-server.service to /usr/lib/systemd/system/nfs-server.service.
[root@k8s-registry ~]# systemctl start nfs
# Node 1
[root@k8s-node01 ~]# yum install -y nfs-utils
......
Complete!
[root@k8s-node01 ~]# showmount -e 10.0.0.140
Export list for 10.0.0.140:
/data 10.0.0.0/24
# Node 2
[root@k8s-node02 ~]# yum install -y nfs-utils
......
Complete!
[root@k8s-node02 ~]# showmount -e 10.0.0.140
Export list for 10.0.0.140:
/data 10.0.0.0/24

nfs

volumes:
- name: 名称

nfs:
path: 路径 #指定挂载路径
server: IP #指定NFS服务端IP
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  namespace: tomcat
  name: mysql
spec:
  replicas: 1
  minReadySeconds: 30
  template:
    metadata:
      labels:
        app: mysql
    spec:
      volumes:
      - name: mysql
        nfs:
          path: /data/tomcat_db
          server: 10.0.0.140
      containers:
      - name: mysql
        image: 10.0.0.140:5000/mysql:5.7
        volumeMounts:
        - name: mysql
          mountPath: /var/lib/mysql
        ports:
        - containerPort: 3306
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: '123456'
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# cat k8s_deploy_mysql.yaml
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 delete -f k8s_deploy_mysql.yaml 
deployment "mysql" deleted
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 create -f k8s_deploy_mysql.yaml 
deployment "mysql" created
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 get pod -o wide -n tomcat
NAME                     READY     STATUS    RESTARTS   AGE       IP             NODE
mysql-2940313263-7c3k3   1/1       Running   0          17s       192.168.24.4   10.0.0.120
tomcat-627834460-1pzvh   1/1       Running   0          47m       192.168.24.7   10.0.0.120
tomcat-627834460-61lq0   1/1       Running   0          47m       192.168.50.3   10.0.0.130
tomcat-627834460-zqvv0   1/1       Running   0          47m       192.168.50.4   10.0.0.130
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# ssh root@10.0.0.120 "df -Th | grep -i nfs "
root@10.0.0.120's password: 
10.0.0.140:/data/tomcat_db nfs4       99G  8.2G   91G   9% /var/lib/kubelet/pods/5ceb30d6-e534-11eb-8479-000c29870b5c/volumes/kubernetes.io~nfs/mysql
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# ssh root@10.0.0.120 
root@10.0.0.120's password: 
Last login: Thu Jul 15 09:43:12 2021 from 10.0.0.2
[root@k8s-node01 ~]# ls /var/lib/kubelet/pods/5ceb30d6-e534-11eb-8479-000c29870b5c/volumes/kubernetes.io~nfs/mysql
auto.cnf  ib_buffer_pool  ibdata1  ib_logfile0  ib_logfile1  ibtmp1  mysql  performance_schema  sys
[root@k8s-node01 ~]# logout
Connection to 10.0.0.120 closed.
root@mysql-2940313263-7c3k3:/# mysql -uroot -p123456 
......
mysql> use HPE_APP;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
mysql>  insert into T_USERS(ID,USER_NAME,LEVEL) values('8','Young','8080');
Query OK, 1 row affected (0.00 sec)

mysql>  insert into T_USERS(ID,USER_NAME,LEVEL) values('9','Aspen','9090');
Query OK, 1 row affected (0.00 sec)

mysql> select * from T_USERS;
+----+-----------+-------+
| ID | USER_NAME | LEVEL |
+----+-----------+-------+
|  1 | me        | 100   |
|  2 | our team  | 100   |
|  3 | HPE       | 100   |
|  4 | teacher   | 100   |
|  5 | docker    | 100   |
|  6 | google    | 100   |
|  8 | Young     | 8080  |
|  9 | Aspen     | 9090  |
+----+-----------+-------+
8 rows in set (0.00 sec)
mysql> ^DBye
root@mysql-2940313263-7c3k3:/# exit
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 delete -f k8s_deploy_mysql.yaml 
deployment "mysql" deleted
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 create -f k8s_deploy_mysql.yaml 
deployment "mysql" created
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# grep nodeName k8s_deploy_mysql.yaml
      nodeName: 10.0.0.130
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 get pod -o wide -n tomcat
NAME                     READY     STATUS    RESTARTS   AGE       IP             NODE
mysql-2663358606-tmtpf   1/1       Running   0          49s       192.168.50.7   10.0.0.130
tomcat-627834460-1pzvh   1/1       Running   0          1h        192.168.24.7   10.0.0.120
tomcat-627834460-61lq0   1/1       Running   0          1h        192.168.50.3   10.0.0.130
tomcat-627834460-zqvv0   1/1       Running   0          1h        192.168.50.4   10.0.0.130
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 -n tomcat exec -it mysql-2663358606-tmtpf  /bin/bash
root@mysql-2663358606-tmtpf:/# mysql -uroot -p123456 -e "use HPE_APP; select * From T_USERS;"
mysql: [Warning] Using a password on the command line interface can be insecure.
+----+-----------+-------+
| ID | USER_NAME | LEVEL |
+----+-----------+-------+
|  1 | me        | 100   |
|  2 | our team  | 100   |
|  3 | HPE       | 100   |
|  4 | teacher   | 100   |
|  5 | docker    | 100   |
|  6 | google    | 100   |
|  8 | Young     | 8080  |
|  9 | Aspen     | 9090  |
+----+-----------+-------+
root@mysql-2663358606-tmtpf:/# exit
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# ssh root@10.0.0.120 "df -Th | grep nfs"
root@10.0.0.120's password: 
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# ssh root@10.0.0.130 "df -Th | grep nfs"
root@10.0.0.130's password: 
10.0.0.140:/data/tomcat_db nfs4       99G  8.2G   91G   9% /var/lib/kubelet/pods/9dee9626-e538-11eb-8479-000c29870b5c/volumes/kubernetes.io~nfs/mysql
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# ssh root@10.0.0.130 
root@10.0.0.130's password: 
[root@k8s-node02 ~]# ls /var/lib/kubelet/pods/9dee9626-e538-11eb-8479-000c29870b5c/volumes/kubernetes.io~nfs/mysql
auto.cnf  ib_buffer_pool  ib_logfile0  ibtmp1  performance_schema
HPE_APP   ibdata1         ib_logfile1  mysql   sys
[root@k8s-node02 ~]# logout
Connection to 10.0.0.130 closed.

四、 GlusterFS类型

step 0 准备GFS环境

详细步骤请参照GlusterFS章节

[root@k8s-node01 ~]# gluster pool list 
UUID                    Hostname    State
9b3ef84e-ddd2-4b8f-8cca-4009e8e7c34d    10.0.0.130  Connected 
78fb00a8-8a8d-41ad-91dc-cf7827a4b608    10.0.0.140  Connected 
c1b8a1d1-b44e-4918-a7fb-53711f45ff00    localhost   Connected 
[root@k8s-node01 ~]# gluster volume info Young

Volume Name: Young
Type: Distributed-Replicate
Volume ID: d8947a21-54a7-4ab3-b807-9f7beb1daf07
Status: Started
Snapshot Count: 0
Number of Bricks: 3 x 3 = 9
Transport-type: tcp
Bricks:
Brick1: 10.0.0.120:/data/volume1
Brick2: 10.0.0.120:/data/volume2
Brick3: 10.0.0.120:/data/volume3
Brick4: 10.0.0.130:/data/volume1
Brick5: 10.0.0.130:/data/volume2
Brick6: 10.0.0.130:/data/volume3
Brick7: 10.0.0.140:/data/volume1
Brick8: 10.0.0.140:/data/volume2
Brick9: 10.0.0.140:/data/volume3
Options Reconfigured:
transport.address-family: inet
nfs.disable: on
performance.client-io-threads: off

step 1 创建Endpoint资源

apiVersion: v1
kind: Endpoints
metadata:

name: 名称
namespace: 名称

subnets:
- addresses:

- ip: IP地址1
- ip: IP地址2
- ip: IP地址3
...
ports:
- port: 端口
protocol: 协议
apiVersion: v1
kind: Endpoints
metadata:
  name: glusterfs
  namespace: tomcat
subsets:
- addresses:
  - ip: 10.0.0.120
  - ip: 10.0.0.130
  - ip: 10.0.0.140
  ports:
  - port: 49152
    protocol: TCP
[root@k8s-master ~]# mkdir -p k8s_yaml/gluster && cd k8s_yaml/gluster
[root@k8s-master ~/k8s_yaml/gluster]# cat  k8s_endpoints_glusterfs.yml
[root@k8s-master ~/k8s_yaml/gluster]# kubectl -s http://10.0.0.110:10000 create -f  k8s_endpoints_glusterfs.yml
endpoints "glusterfs" created
[root@k8s-master ~/k8s_yaml/gluster]# kubectl -s http://10.0.0.110:10000 describe endpoints glusterfs -n tomcat
Name:       glusterfs
Namespace:  tomcat
Labels:     <none>
Subsets:
  Addresses:        10.0.0.120,10.0.0.130,10.0.0.140
  NotReadyAddresses:    <none>
  Ports:
    Name    Port    Protocol
    ----    ----    --------
    <unset>   49152   TCP

No events.

step 2 创建Service资源

要求Service资源与Endpoint资源,名称一致;此时创建的Service资源不能指定标签选择器
两者名称一致,会自动关联;
apiVersion: v1
kind: Service
metadata:

name: 名称

spec:

type: ClusterIP
ports:
- port: 端口
protocol: 协议
targetPort: 端口
apiVersion: v1
kind: Service
metadata:
  name: glusterfs
  namespace: tomcat
spec:
  ports:
  - port: 49152
    protocol: TCP
    targetPort: 49152
  type: ClusterIP
[root@k8s-master ~/k8s_yaml/gluster]# cat k8s_svc_glusterfs.yml 
[root@k8s-master ~/k8s_yaml/gluster]# kubectl -s http://10.0.0.110:10000 create -f k8s_svc_glusterfs.yml service "glusterfs" created
[root@k8s-master ~/k8s_yaml/gluster]# kubectl -s http://10.0.0.110:10000 describe svc glusterfs -n tomcatName:          glusterfs
Namespace:      tomcat
Labels:         <none>
Selector:       <none>
Type:           ClusterIP
IP:         10.100.196.251
Port:           <unset>   49152/TCP
Endpoints:      10.0.0.120:49152,10.0.0.130:49152,10.0.0.140:49152
Session Affinity:   None
No events.

step 3 准备GFS相关参数

volumes:
- name: 名称

glusterfs:
path: GFS卷名称 #指定挂载卷
endpoints: 资源名称 #指定endpoint资源名称
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  namespace: tomcat
  name: mysql
spec:
  replicas: 1
  minReadySeconds: 30
  template:
    metadata:
      labels:
        app: mysql
    spec:
      volumes:
      - name: mysql
        glusterfs:
          path: Young
          endpoints: glusterfs
      nodeName: 10.0.0.130
      containers:
      - name: mysql
        image: 10.0.0.140:5000/mysql:5.7
        volumeMounts:
        - name: mysql
          mountPath: /var/lib/mysql
        ports:
        - containerPort: 3306
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: '123456'
[root@k8s-master ~/k8s_yaml/gluster]# cd ../tomcat+mysql/
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# cat k8s_deploy_mysql.yaml 
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 delete -f k8s_deploy_mysql.yaml 
deployment "mysql" deleted
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 create -f k8s_deploy_mysql.yaml 
deployment "mysql" created

step 4 测试

[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 exec -it -n tomcat mysql-330277304-n8t96 /bin/bash
root@mysql-330277304-n8t96:/# mysql -uroot -p123456 -e "use HPE_APP; insert into T_USERS(ID,USER_NAME,LEVEL) values('8','Young','8080');"
mysql: [Warning] Using a password on the command line interface can be insecure.
root@mysql-330277304-n8t96:/# mysql -uroot -p123456 -e "use HPE_APP; insert into T_USERS(ID,USER_NAME,LEVEL) values('9','Aspen','9090');"
mysql: [Warning] Using a password on the command line interface can be insecure.
root@mysql-330277304-n8t96:/# mysql -uroot -p123456 -e "use HPE_APP; select * from T_USERS"             
mysql: [Warning] Using a password on the command line interface can be insecure.
+----+-----------+-------+
| ID | USER_NAME | LEVEL |
+----+-----------+-------+
|  1 | me        | 100   |
|  2 | our team  | 100   |
|  3 | HPE       | 100   |
|  4 | teacher   | 100   |
|  5 | docker    | 100   |
|  6 | google    | 100   |
|  8 | Young     | 8080  |
|  9 | Aspen     | 9090  |
+----+-----------+-------+
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 delete pod  -n tomcat mysql-330277304-n8t96
pod "mysql-330277304-n8t96" deleted
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 get pod -n tomcat
NAME                     READY     STATUS    RESTARTS   AGE
mysql-330277304-wwv40    1/1       Running   1          5s
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 exec -it -n tomcat mysql-330277304-wwv40 /bin/bash
root@mysql-330277304-wwv40:/# mysql -uroot -p123456 -e "use HPE_APP; select * from T_USERS"
mysql: [Warning] Using a password on the command line interface can be insecure.
+----+-----------+-------+
| ID | USER_NAME | LEVEL |
+----+-----------+-------+
|  1 | me        | 100   |
|  2 | our team  | 100   |
|  3 | HPE       | 100   |
|  4 | teacher   | 100   |
|  5 | docker    | 100   |
|  6 | google    | 100   |
|  8 | Young     | 8080  |
|  9 | Aspen     | 9090  |
+----+-----------+-------+
root@mysql-330277304-wwv40:/# exit
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 get pod -n tomcat -o wide
NAME                     READY     STATUS    RESTARTS   AGE       IP             NODE
mysql-330277304-wwv40    1/1       Running   1          1m        192.168.23.8   10.0.0.130
tomcat-627834460-1pzvh   1/1       Running   2          3d        192.168.12.2   10.0.0.120
tomcat-627834460-61lq0   1/1       Running   3          3d        192.168.23.5   10.0.0.130
tomcat-627834460-zqvv0   1/1       Running   3          3d        192.168.23.2   10.0.0.130
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# ssh root@10.0.0.130 df -Th | grep -i gluster
root@10.0.0.130's password: 
10.0.0.120:Young fuse.glusterfs   99G  6.8G   93G   7% /var/lib/kubelet/pods/a9c5ca2e-e84b-11eb-86e8-000c29870b5c/volumes/kubernetes.io~glusterfs/mysql
[root@k8s-node02 ~]# mount 10.0.0.120:/Young /mnt -t glusterfs
[root@k8s-node02 ~]# ls /mnt/
auto.cnf  ib_buffer_pool  ib_logfile0  ibtmp1  performance_schema
HPE_APP   ibdata1         ib_logfile1  mysql   sys
[root@k8s-node02 ~]# ls /var/lib/kubelet/pods/a9c5ca2e-e84b-11eb-86e8-000c29870b5c/volumes/kubernetes.io~glusterfs/mysql/
auto.cnf  ib_buffer_pool  ib_logfile0  ibtmp1  performance_schema
HPE_APP   ibdata1         ib_logfile1  mysql   sys
[root@k8s-node02 ~]# ls /data/volume1/
HPE_APP  ibdata1  ib_logfile0  mysql  performance_schema  sys
[root@k8s-node02 ~]# ls /data/volume1/
HPE_APP  ibdata1  ib_logfile0  mysql  performance_schema  sys
[root@k8s-node02 ~]# ssh root@10.0.0.120 'ls /data/volume1;'
root@10.0.0.120's password: 
HPE_APP
ib_buffer_pool
ib_logfile1
mysql
performance_schema
sys
[root@k8s-node02 ~]# ssh root@10.0.0.140 'ls /data/volume1;'
root@10.0.0.140's password: 
auto.cnf
HPE_APP
ib_buffer_pool
ib_logfile0
ibtmp1
mysql
performance_schema
sys

五、 pv和pvc类型

PV 全称 Persistent Volumes #持久化卷[全局资源: 在任一NameSpace下都能看到该资源]
PVC 全称 Persistent Volume Claims #持久化卷分配[局部资源]

1. 工作原理

  • PV不提供真实的存储,真实的存储有外部服务提供;
  • PV类似于资源池,可以创建多个类似于客户端的资源用于连接外部存储系统,用于提供存储;
  • PV需要声明其名字、对接存储的类型以及所能提供的存储能力
  • PVC需要声明所需的存储空间
  • PV和PVC必须一一对应,一旦某个PV被分配给某一个PVC,那么这个PV就不能再分配给其他PVC了

2. 创建PV

PV的存储能力参数仅是一个标签,不代表实际存储容量
apiVersion: v1
kind: PersistentVolume
metadata:

name: 名称
labels:
type: 名称

spec:

capacity:
storge: nGi #声明存储能力(存储容量)
accessModes: #定义访问模式
- ReadWriteMany #允许多个Pod同时读写
persistentVolumeReclaimPolicy: Recycle #PVC允许回收
后端存储对接参数...
apiVersion: v1
kind: PersistentVolume
metadata: 
  name: mysql
  labels:
    type: gfs
spec:
  capacity: 
    storage: 50Gi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Recycle
  glusterfs:
    path: Young
    endpoints: glusterfs
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# mkdir ../pv && cd ../pv
[root@k8s-master ~/k8s_yaml/pv]# cat k8s_pv_mysql.yml 
[root@k8s-master ~/k8s_yaml/pv]# kubectl -s http://10.0.0.110:10000 create -f k8s_pv_mysql.yml 
persistentvolume "mysql" created
[root@k8s-master ~/k8s_yaml/pv]# kubectl -s http://10.0.0.110:10000 get pv
NAME      CAPACITY   ACCESSMODES   RECLAIMPOLICY   STATUS      CLAIM     REASON    AGE
mysql     50Gi       RWX           Recycle         Available                       9m

3. 创建PVC

系统会自动将满足PVC要求的最小的PV资源分配给PVC,以节省资源。
apiVersion: v1
kind: PersistentVolumeClaim
metadata:

name: 名称
namespace: 名称

spec:

accessModes: #定义访问模式
- ReadWriteMany #允许多个Pod同时读写
resource:
requests:
storage: nGi
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: tomcat-db
  namespace: tomcat
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 20Gi
  selector:
    matchLabels:
      type: gfs
[root@k8s-master ~/k8s_yaml/pv]# cat k8s_pvc_mysql.yml 
[root@k8s-master ~/k8s_yaml/pv]# kubectl -s http://10.0.0.110:10000 create -f k8s_pvc_mysql.yml 
persistentvolumeclaim "tomcat-db" created
[root@k8s-master ~/k8s_yaml/pv]# kubectl -s http://10.0.0.110:10000 get pvc
No resources found.
[root@k8s-master ~/k8s_yaml/pv]# kubectl -s http://10.0.0.110:10000 get pvc -n tomcat
NAME        STATUS    VOLUME    CAPACITY   ACCESSMODES   AGE
tomcat-db   Bound     mysql     50Gi       RWX           2m
[root@k8s-master ~/k8s_yaml/pv]# kubectl -s http://10.0.0.110:10000 get pv
NAME      CAPACITY   ACCESSMODES   RECLAIMPOLICY   STATUS    CLAIM              REASON    AGE
mysql     50Gi       RWX           Recycle         Bound     tomcat/tomcat-db             17m

4. PVC相关参数

volumes:
- name: 名称

persistenrVolumeClaim:
claimName: 名称 #指定pvc名称
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  namespace: tomcat
  name: mysql
spec:
  replicas: 1
  minReadySeconds: 30
  template:
    metadata:
      labels:
        app: mysql
    spec:
      volumes:
      - name: mysql
        persistentVolumeClaim:
          claimName: tomcat-db 
      nodeName: 10.0.0.130
      containers:
      - name: mysql
        image: 10.0.0.140:5000/mysql:5.7
        volumeMounts:
        - name: mysql
          mountPath: /var/lib/mysql
        ports:
        - containerPort: 3306
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: '123456'
[root@k8s-master ~/k8s_yaml/pv]# cd ../tomcat+mysql/
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 delete -f k8s_deploy_mysql.yaml 
deployment "mysql" deleted
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# cat k8s_deploy_mysql.yaml 
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 create -f k8s_deploy_mysql.yaml 
deployment "mysql" created
[root@k8s-master ~/k8s_yaml/tomcat+mysql]# kubectl -s http://10.0.0.110:10000 exec -it -n tomcat mysql-3924662437-b9p1l /bin/bash
root@mysql-3924662437-b9p1l:/# mysql -uroot -p123456 -e "use HPE_APP; select * from T_USERS"
mysql: [Warning] Using a password on the command line interface can be insecure.
+----+-----------+-------+
| ID | USER_NAME | LEVEL |
+----+-----------+-------+
|  1 | me        | 100   |
|  2 | our team  | 100   |
|  3 | HPE       | 100   |
|  4 | teacher   | 100   |
|  5 | docker    | 100   |
|  6 | google    | 100   |
|  8 | Young     | 8080  |
|  9 | Aspen     | 9090  |
+----+-----------+-------+
root@mysql-3924662437-b9p1l:/# exit