在5个worker的k8s集群上部署minio开源版分布式集群，并不建议在master节点上部署minio节点。
minio集群对磁盘性能要求非常高，官方最推荐的方式是使用直接让minio各个节点使用裸盘，且推荐文件系统为xfs
在k8s的5个worker节点上都选出一块相同的硬盘作为minio数据盘，
要先配置好集群的ingress，minio的api和控制台需要依赖ingress对外暴露。
这是我的集群状态
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
| root@k8s-master1:~# kubectl get node -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-master1 Ready control-plane 24d v1.36.1 10.10.0.21 <none> Debian GNU/Linux 13 (trixie) 6.12.100+deb13-cloud-amd64 (amd64) containerd://1.7.24
k8s-master2 Ready control-plane 24d v1.36.1 10.10.0.22 <none> Debian GNU/Linux 13 (trixie) 6.12.100+deb13-cloud-amd64 (amd64) containerd://1.7.24
k8s-master3 Ready control-plane 24d v1.36.1 10.10.0.23 <none> Debian GNU/Linux 13 (trixie) 6.12.100+deb13-cloud-amd64 (amd64) containerd://1.7.24
k8s-node1 Ready <none> 24d v1.36.1 10.10.0.11 <none> Debian GNU/Linux 13 (trixie) 6.12.101+deb13-cloud-amd64 (amd64) containerd://1.7.24
k8s-node2 Ready <none> 24d v1.36.1 10.10.0.12 <none> Debian GNU/Linux 13 (trixie) 6.12.101+deb13-cloud-amd64 (amd64) containerd://1.7.24
k8s-node3 Ready <none> 24d v1.36.1 10.10.0.13 <none> Debian GNU/Linux 13 (trixie) 6.12.101+deb13-cloud-amd64 (amd64) containerd://1.7.24
k8s-node4 Ready <none> 24d v1.36.1 10.10.0.14 <none> Debian GNU/Linux 13 (trixie) 6.12.101+deb13-cloud-amd64 (amd64) containerd://1.7.24
k8s-node5 Ready <none> 12d v1.36.1 10.10.0.15 <none> Debian GNU/Linux 13 (trixie) 6.12.101+deb13-cloud-amd64 (amd64) containerd://2.2.6
root@k8s-master1:~#
root@k8s-master1:~# kubectl get svc -A
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 12d
ingress-nginx ingress-nginx-controller LoadBalancer 10.105.28.234 10.10.0.200 80:30810/TCP,443:30628/TCP 10d
ingress-nginx ingress-nginx-controller-admission ClusterIP 10.107.151.18 <none> 443/TCP 10d
kube-system kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 105d
metallb-system webhook-service ClusterIP 10.99.192.179 <none> 443/TCP 12d
root@k8s-master1:~#
|
初始化minio数据盘
在5个worker节点上给minio数据库盘建立xfs文件系统，这里要要注意每台worker节点的硬盘设备文件路径都有可能不一样，
挂载到一个目录下
1
2
3
| mkdir -p /mnt/disks/sdc
mount /dev/sdc /mnt/disks/sdc
echo "/dev/sdc /mnt/disks/sdc xfs defaults 0 0" | sudo tee -a /etc/fstab
|
声明配置文件
minio-storage.yaml
1
2
3
4
5
6
7
8
9
10
11
| apiVersion: v1
kind: Namespace
metadata:
name: minio
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: local-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer # 延迟绑定模式
|
minio-pvs.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
| apiVersion: v1
kind: List
items:
- apiVersion: v1
kind: PersistentVolume
metadata:
name: minio-pv-1
spec:
capacity:
storage: 20Gi # 容量
volumeMode: Filesystem
accessModes: ["ReadWriteOnce"]
persistentVolumeReclaimPolicy: Retain
storageClassName: local-storage
local:
path: /mnt/disks/sdc # 节点1挂载路径
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: ["k8s-node1"] # 节点1名字
- apiVersion: v1
kind: PersistentVolume
metadata:
name: minio-pv-2
spec:
capacity:
storage: 20Gi # 容量
volumeMode: Filesystem
accessModes: ["ReadWriteOnce"]
persistentVolumeReclaimPolicy: Retain
storageClassName: local-storage
local:
path: /mnt/disks/sdc # 节点2挂载路径
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: ["k8s-node2"] # 节点2名字
- apiVersion: v1
kind: PersistentVolume
metadata:
name: minio-pv-3
spec:
capacity:
storage: 20Gi # 容量
volumeMode: Filesystem
accessModes: ["ReadWriteOnce"]
persistentVolumeReclaimPolicy: Retain
storageClassName: local-storage
local:
path: /mnt/disks/sdc # 节点3挂载路径
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: ["k8s-node3"] # 节点3名字
- apiVersion: v1
kind: PersistentVolume
metadata:
name: minio-pv-4
spec:
capacity:
storage: 20Gi
volumeMode: Filesystem
accessModes: ["ReadWriteOnce"]
persistentVolumeReclaimPolicy: Retain
storageClassName: local-storage
local:
path: /mnt/disks/sdc
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: ["k8s-node4"]
- apiVersion: v1
kind: PersistentVolume
metadata:
name: minio-pv-5
spec:
capacity:
storage: 20Gi
volumeMode: Filesystem
accessModes: ["ReadWriteOnce"]
persistentVolumeReclaimPolicy: Retain
storageClassName: local-storage
local:
path: /mnt/disks/sdc
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: ["k8s-node5"]
|
minio-app.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
| # Headless Service，用于 Pod 间建立分布式通信
apiVersion: v1
kind: Service
metadata:
name: minio-headless
namespace: minio
spec:
clusterIP: None
selector:
app: minio
ports:
- name: api
port: 9000
- name: console
port: 9001
---
# Client Service，提供给集群内部访问
apiVersion: v1
kind: Service
metadata:
name: minio-service
namespace: minio
spec:
type: ClusterIP
selector:
app: minio
ports:
- name: api
port: 9000
targetPort: 9000
- name: console
port: 9001
targetPort: 9001
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: minio
namespace: minio
spec:
serviceName: minio-headless
replicas: 5
selector:
matchLabels:
app: minio
template:
metadata:
labels:
app: minio
spec:
# 保证每个 Worker 节点最多部署 1 个 Pod
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values: ["minio"]
topologyKey: "kubernetes.io/hostname"
containers:
- name: minio
image: minio/minio:RELEASE.2025-09-07T16-13-09Z-cpuv1
command:
- sh
- -c
- |
minio server \
--address ":9000" \
--console-address ":9001" \
http://minio-{0...4}.minio-headless.minio.svc.cluster.local/data
# 这里{0...4}表示是5个minio分布式节点，组合出来的域名对应在集群内部的各个minio节点
env:
- name: MINIO_ROOT_USER
value: "admin" # minio用户名
- name: MINIO_ROOT_PASSWORD
value: "Admin@20260904" # minio密码
ports:
- containerPort: 9000
name: api
- containerPort: 9001
name: console
volumeMounts:
- name: data
mountPath: /data
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: "local-storage"
resources:
requests:
storage: 20Gi
|
minio-ingress.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
| apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: minio-ingress
namespace: minio
annotations:
kubernetes.io/ingress.class: "nginx"
# 必须禁用请求体大小限制，避免上传大文件失败
nginx.ingress.kubernetes.io/proxy-body-size: "0"
# 延长超时时间
nginx.ingress.kubernetes.io/proxy-connect-timeout: "600"
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
spec:
rules:
# minio s3 api域名，要解析到ingress的对外入口ip
- host: minio-api.k8s.cn
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: minio-service
port:
number: 9000
# Web Console UI域名，要解析到ingress的对外入口ip
- host: minio-console.k8s.cn
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: minio-service
port:
number: 9001
|
将上述yaml文件在master节点中依次导入后
kubectl apply -f *.yaml
验证minio状态
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
| root@k8s-master1:~# kubectl get pod -n minio -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
minio-0 1/1 Running 0 5h 10.244.5.83 k8s-node3 <none> <none>
minio-1 1/1 Running 0 5h 10.244.7.85 k8s-node4 <none> <none>
minio-2 1/1 Running 0 5h 10.244.4.80 k8s-node2 <none> <none>
minio-3 1/1 Running 0 5h 10.244.6.53 k8s-node5 <none> <none>
minio-4 1/1 Running 0 5h 10.244.3.62 k8s-node1 <none> <none>
root@k8s-master1:~# kubectl get svc -n minio
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
minio-headless ClusterIP None <none> 9000/TCP,9001/TCP 5h
minio-service ClusterIP 10.110.164.17 <none> 9000/TCP,9001/TCP 5h
root@k8s-master1:~# kubectl get ingress -n minio
NAME CLASS HOSTS ADDRESS PORTS AGE
minio-ingress <none> minio-api.k8s.cn,minio-console.k8s.cn 10.10.0.200 80 5h
root@k8s-master1:~# kubectl get pvc -n minio
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE
data-minio-0 Bound minio-pv-3 20Gi RWO local-storage <unset> 5h
data-minio-1 Bound minio-pv-4 20Gi RWO local-storage <unset> 5h
data-minio-2 Bound minio-pv-2 20Gi RWO local-storage <unset> 5h
data-minio-3 Bound minio-pv-5 20Gi RWO local-storage <unset> 5h
data-minio-4 Bound minio-pv-1 20Gi RWO local-storage <unset> 5h
root@k8s-master1:~# curl -i http://minio-api.k8s.cn/minio/health/ready
HTTP/1.1 200 OK
Date: Fri, 04 Sep 2026 08:34:43 GMT
Content-Length: 0
Connection: keep-alive
Accept-Ranges: bytes
Strict-Transport-Security: max-age=31536000; includeSubDomains
Vary: Origin
X-Amz-Id-2: e0573483e66f84654a13984f56d550d8c757ccabcc3d613c083917b651cece3e
X-Amz-Request-Id: 18D2112C269741D9
X-Content-Type-Options: nosniff
X-Xss-Protection: 1; mode=block
root@k8s-master1:~#
|
验证上传文件
