聊聊K8S高可用集群搭建部署
K8S硬件环境准备
硬件要求如下:
- CPU: 2core+
- Memory：4GB+
- Disk：100GB+
- IP地址及主机名:
10.0.0.241 k8s-cluster241
10.0.0.242 k8s-cluster242
10.0.0.243 k8s-cluster243
- LVM卷扩展
lvextend /dev/mapper/ubuntu--vg-ubuntu--lv -l +100%FREE
resize2fs /dev/mapper/ubuntu--vg-ubuntu--lv K8S集群各主机基础优化
1.所有节点安装常用的软件包
apt update && apt -y install bind9-utils expect rsync jq psmisc net-tools lvm2 vim unzip rename tree
2.k8s-cluster241节点免密钥登录集群并同步数据
[root@k8s-cluster241 ~]# cat >> /etc/hosts <<'EOF'
10.0.0.240 apiserver-lb
10.0.0.241 k8s-cluster241
10.0.0.242 k8s-cluster242
10.0.0.243 k8s-cluster243
EOF3.配置免密码登录其他节点
[root@k8s-cluster241 ~]# cat > password_free_login.sh <<'EOF'
#!/bin/bash
# auther: Jason Yin
# 创建密钥对
ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa -q
# 声明你服务器密码，建议所有节点的密码均一致，否则该脚本需要再次进行优化
export mypasswd=1
# 定义主机列表
k8s_host_list=(k8s-cluster241 k8s-cluster242 k8s-cluster243)
# 配置免密登录，利用expect工具免交互输入
for i in ${k8s_host_list[@]};do
expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@$i
expect {
\"*yes/no*\" {send \"yes\r\"; exp_continue}
\"*password*\" {send \"$mypasswd\r\"; exp_continue}
}"
done
EOF
bash password_free_login.sh4.编写同步脚本
[root@k8s-cluster241 ~]# cat > /usr/local/sbin/data_rsync.sh <<'EOF'
#!/bin/bash
# Auther: Jason Yin
if [ $# -lt 1 ];then
echo "Usage: $0 /path/to/file(绝对路径) [mode: m|w]"
exit
fi
if [ ! -e $1 ];then
echo "[ $1 ] dir or file not find!"
exit
fi
fullpath=`dirname $1`
basename=`basename $1`
cd $fullpath
case $2 in
WORKER_NODE|w)
K8S_NODE=(k8s-cluster242 k8s-cluster243)
;;
MASTER_NODE|m)
K8S_NODE=(k8s-cluster242 k8s-cluster243)
;;
*)
K8S_NODE=(k8s-cluster242 k8s-cluster243)
;;
esac
for host in ${K8S_NODE[@]};do
tput setaf 2
echo ===== rsyncing ${host}: $basename =====
tput setaf 7
rsync -az $basename `whoami`@${host}:$fullpath
if [ $? -eq 0 ];then
echo "命令执行成功!"
fi
done
EOF
chmod +x /usr/local/sbin/data_rsync.sh
data_rsync.sh /etc/hosts5.所有节点Linux基础环境优化
systemctl disable --now NetworkManager ufw
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
ln -svf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
cat >> /etc/security/limits.conf <<'EOF'
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF
sed -i 's@#UseDNS yes@UseDNS no@g' /etc/ssh/sshd_config
sed -i 's@^GSSAPIAuthentication yes@GSSAPIAuthentication no@g' /etc/ssh/sshd_config
cat > /etc/sysctl.d/k8s.conf <<'EOF'
# 以下3个参数是containerd所依赖的内核参数
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv6.conf.all.disable_ipv6 = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
sysctl --system
cat <<EOF >> ~/.bashrc
PS1='[\[\e[34;1m\]\u@\[\e[0m\]\[\e[32;1m\]\H\[\e[0m\]\[\e[31;1m\] \W\[\e[0m\]]# '
EOF
source ~/.bashrc
free -h
6.所有节点安装ipvsadm以实现kube-proxy的负载均衡
apt -y install ipvsadm ipset sysstat conntrack
cat > /etc/modules-load.d/ipvs.conf << 'EOF'
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
br_netfilter
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF7.重复所有节点并验证模块是否加载成功
reboot
lsmod | grep --color=auto -e ip_vs -e nf_conntrack -e br_netfilter
uname -r
ifconfig
free -h 参考示例：
[root@k8s-cluster241 ~]# lsmod | grep --color=auto -e ip_vs -e nf_conntrack -e br_netfilter
br_netfilter 32768 0
bridge 311296 1 br_netfilter
ip_vs_ftp 16384 0
nf_nat 49152 1 ip_vs_ftp
ip_vs_sed 16384 0
ip_vs_nq 16384 0
ip_vs_fo 16384 0
ip_vs_sh 16384 0
ip_vs_dh 16384 0
ip_vs_lblcr 16384 0
ip_vs_lblc 16384 0
ip_vs_wrr 16384 0
ip_vs_rr 16384 0
ip_vs_wlc 16384 0
ip_vs_lc 16384 0
ip_vs 176128 25 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_v_lc,ip_vs_sed,ip_vs_ftp
nf_conntrack 172032 2 nf_nat,ip_vs
nf_defrag_ipv6 24576 2 nf_conntrack,ip_vs
nf_defrag_ipv4 16384 1 nf_conntrack
libcrc32c 16384 5 nf_conntrack,nf_nat,btrfs,raid456,ip_vs
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# free -h
total used free shared buff/cache available
Mem: 7.7Gi 334Mi 7.1Gi 1.0Mi 340Mi 7.1Gi
Swap: 0B 0B 0B
[root@k8s-cluster241 ~]#
[root@k8s-cluster242 ~]# lsmod | grep --color=auto -e ip_vs -e nf_conntrack -e br_netfilter
br_netfilter 32768 0
bridge 311296 1 br_netfilter
ip_vs_ftp 16384 0
nf_nat 49152 1 ip_vs_ftp
ip_vs_sed 16384 0
ip_vs_nq 16384 0
ip_vs_fo 16384 0
ip_vs_sh 16384 0
ip_vs_dh 16384 0
ip_vs_lblcr 16384 0
ip_vs_lblc 16384 0
ip_vs_wrr 16384 0
ip_vs_rr 16384 0
ip_vs_wlc 16384 0
ip_vs_lc 16384 0
ip_vs 176128 24 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblcip_vs_wrr,ip_vs_lc,ip_vs_sed,ip_vs_ftp
nf_conntrack 172032 2 nf_nat,ip_vs
nf_defrag_ipv6 24576 2 nf_conntrack,ip_vs
nf_defrag_ipv4 16384 1 nf_conntrack
libcrc32c 16384 5 nf_conntrack,nf_nat,btrfs,raid456,ip_vs
[root@k8s-cluster242 ~]#
[root@k8s-cluster242 ~]# free -h
total used free shared buff/cache available
Mem: 7.7Gi 315Mi 7.1Gi 1.0Mi 336Mi 7.2Gi
Swap: 0B 0B 0B
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# lsmod | grep --color=auto -e ip_vs -e nf_conntrack
ip_vs_ftp 16384 0
nf_nat 49152 1 ip_vs_ftp
ip_vs_sed 16384 0
ip_vs_nq 16384 0
ip_vs_fo 16384 0
ip_vs_sh 16384 0
ip_vs_dh 16384 0
ip_vs_lblcr 16384 0
ip_vs_lblc 16384 0
ip_vs_wrr 16384 0
ip_vs_rr 16384 0
ip_vs_wlc 16384 0
ip_vs_lc 16384 0
ip_vs 176128 25 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_vs_lc,ip_v_sed,ip_vs_ftp
nf_conntrack 172032 2 nf_nat,ip_vs
nf_defrag_ipv6 24576 2 nf_conntrack,ip_vs
nf_defrag_ipv4 16384 1 nf_conntrack
libcrc32c 16384 5 nf_conntrack,nf_nat,btrfs,raid456,ip_vs
[root@k8s-cluster243 ~]#
[root@k8s-cluster243 ~]# free -h
total used free shared buff/cache available
Mem: 7.7Gi 317Mi 7.1Gi 1.0Mi 331Mi 7.2Gi
Swap: 0B 0B 0B
[root@k8s-cluster243 ~]# 8.关机拍快照在进行下一步喔！（学习测试环境）
快照名称'k8s操作系统环境准备就绪'
部署Containerd运行时
./install-docker.sh r
ip link del docker0
3.检查Containerd的版本
[root@k8s-cluster241 ~]# ctr version
Client:
Version: v1.6.36
Revision: 88c3d9bc5b5a193f40b7c14fa996d23532d6f956
Go version: go1.22.7
Server:
Version: v1.6.36
Revision: 88c3d9bc5b5a193f40b7c14fa996d23532d6f956
UUID: 02339237-7847-4564-9733-1e6ac9618a33
[root@k8s-cluster241 ~]#
[root@k8s-cluster242 ~]# ctr version
Client:
Version: v1.6.36
Revision: 88c3d9bc5b5a193f40b7c14fa996d23532d6f956
Go version: go1.22.7
Server:
Version: v1.6.36
Revision: 88c3d9bc5b5a193f40b7c14fa996d23532d6f956
UUID: f9ba2325-9146-4217-b2a8-1e733fa7facf
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# ctr version
Client:
Version: v1.6.36
Revision: 88c3d9bc5b5a193f40b7c14fa996d23532d6f956
Go version: go1.22.7
Server:
Version: v1.6.36
Revision: 88c3d9bc5b5a193f40b7c14fa996d23532d6f956
UUID: 1c95864f-3aa1-412e-a0e7-dbfc62557c78
[root@k8s-cluster243 ~]# etcd高可用集群部署
1 下载etcd的软件包
wget https://github.com/etcd-io/etcd/releases/download/v3.5.22/etcd-v3.5.22-linux-amd64.tar.gz
svip:
[root@k8s-cluster241 ~]# wget http://192.168.21.253/Resources/Prometheus/softwares/Etcd/etcd-v3.5.22-linux-amd64.tar.gz2 解压etcd的二进制程序包到PATH环境变量路径
[root@k8s-cluster241 ~]# tar -xf etcd-v3.5.22-linux-amd64.tar.gz -C /usr/local/bin etcd-v3.5.22-linux-amd64/etcd{,ctl} --strip-components=1
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# ll /usr/local/bin/etcd*
-rwxr-xr-x 1 yinzhengjie yinzhengjie 24072344 Mar 28 06:58 /usr/local/bin/etcd*
-rwxr-xr-x 1 yinzhengjie yinzhengjie 18419864 Mar 28 06:58 /usr/local/bin/etcdctl*
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl version
etcdctl version: 3.5.22
API version: 3.5
[root@k8s-cluster241 ~]#
3 将软件包下发到所有节点
'4.准备etcd的证书文件
4.1 安装cfssl证书管理工具
[root@k8s-cluster241 ~]# wget http://192.168.21.253/Resources/Prometheus/softwares/Etcd/oldboyedu-cfssl-v1.6.5.zip
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# unzip oldboyedu-cfssl-v1.6.5.zip
Archive: oldboyedu-cfssl-v1.6.5.zip
inflating: cfssl-certinfo_1.6.5_linux_amd64
inflating: cfssljson_1.6.5_linux_amd64
inflating: cfssl_1.6.5_linux_amd64
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# rename -v "s/_1.6.5_linux_amd64//g" cfssl*
cfssl_1.6.5_linux_amd64 renamed as cfssl
cfssl-certinfo_1.6.5_linux_amd64 renamed as cfssl-certinfo
cfssljson_1.6.5_linux_amd64 renamed as cfssljson
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# mv cfssl* /usr/local/bin/
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# chmod +x /usr/local/bin/cfssl*
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# ll /usr/local/bin/cfssl*
-rwxr-xr-x 1 root root 11890840 Jun 15 2024 /usr/local/bin/cfssl*
-rwxr-xr-x 1 root root 8413336 Jun 15 2024 /usr/local/bin/cfssl-certinfo*
-rwxr-xr-x 1 root root 6205592 Jun 15 2024 /usr/local/bin/cfssljson*
[root@k8s-cluster241 ~]#
4.2 创建证书存储目录
[root@k8s-cluster241 ~]# mkdir -pv /oldboyedu/{certs,pki}/etcd
mkdir: created directory '/oldboyedu'
mkdir: created directory '/oldboyedu/certs'
mkdir: created directory '/oldboyedu/certs/etcd'
mkdir: created directory '/oldboyedu/pki'
mkdir: created directory '/oldboyedu/pki/etcd'
[root@k8s-cluster241 ~]#
4.3 生成证书的CSR文件： 证书签发请求文件，配置了一些域名，公司，单位
[root@k8s-cluster241 ~]# cd /oldboyedu/pki/etcd
[root@k8s-cluster241 etcd]#
[root@k8s-cluster241 etcd]# cat > etcd-ca-csr.json <<EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd",
"OU": "Etcd Security"
}
],
"ca": {
"expiry": "876000h"
}
}
EOF4.4 生成etcd CA证书和CA证书的key
[root@k8s-cluster241 etcd]# cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /oldboyedu/certs/etcd/etcd-ca
2025/06/13 16:15:00 [INFO] generating a new CA key and certificate from CSR
2025/06/13 16:15:00 [INFO] generate received request
2025/06/13 16:15:00 [INFO] received CSR
2025/06/13 16:15:00 [INFO] generating key: rsa-2048
2025/06/13 16:15:00 [INFO] encoded CSR
2025/06/13 16:15:00 [INFO] signed certificate with serial number 469498848898639707673864924355340868165928121853
[root@k8s-cluster241 etcd]#
[root@k8s-cluster241 etcd]# ll /oldboyedu/certs/etcd/etcd-ca*
-rw-r--r-- 1 root root 1050 Jun 13 16:15 /oldboyedu/certs/etcd/etcd-ca.csr
-rw------- 1 root root 1679 Jun 13 16:15 /oldboyedu/certs/etcd/etcd-ca-key.pem
-rw-r--r-- 1 root root 1318 Jun 13 16:15 /oldboyedu/certs/etcd/etcd-ca.pem
[root@k8s-cluster241 etcd]#
4.5 生成etcd证书的有效期为100年
[root@k8s-cluster241 etcd]# cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "876000h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "876000h"
}
}
}
}
EOF
4.6 生成证书的CSR文件： 证书签发请求文件，配置了一些域名，公司，单位
[root@k8s-cluster241 etcd]# cat > etcd-csr.json <<EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
EOF
4.7 基于自建的ectd ca证书生成etcd的证书
[root@k8s-cluster241 etcd]# cfssl gencert \
-ca=/oldboyedu/certs/etcd/etcd-ca.pem \
-ca-key=/oldboyedu/certs/etcd/etcd-ca-key.pem \
-config=ca-config.json \
--hostname=127.0.0.1,k8s-cluster241,k8s-cluster242,k8s-cluster243,10.0.0.241,10.0.0.242,10.0.0.243 \
--profile=kubernetes \
etcd-csr.json | cfssljson -bare /oldboyedu/certs/etcd/etcd-server
[root@k8s-cluster241 etcd]# ll /oldboyedu/certs/etcd/etcd-server*
-rw-r--r-- 1 root root 1139 Jun 13 16:16 /oldboyedu/certs/etcd/etcd-server.csr
-rw------- 1 root root 1679 Jun 13 16:16 /oldboyedu/certs/etcd/etcd-server-key.pem
-rw-r--r-- 1 root root 1472 Jun 13 16:16 /oldboyedu/certs/etcd/etcd-server.pem
[root@k8s-cluster241 etcd]# 4.8 将etcd证书拷贝到其他两个master节点
[root@k8s-cluster241 etcd]# data_rsync.sh /oldboyedu/certs/
===== rsyncing k8s-cluster242: certs =====
命令执行成功!
===== rsyncing k8s-cluster243: certs =====
命令执行成功!
[root@k8s-cluster241 etcd]#
[root@k8s-cluster242 ~]# tree /oldboyedu/certs/etcd/
/oldboyedu/certs/etcd/
├── etcd-ca.csr
├── etcd-ca-key.pem
├── etcd-ca.pem
├── etcd-server.csr
├── etcd-server-key.pem
└── etcd-server.pem
0 directories, 6 files
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# tree /oldboyedu/certs/etcd/
/oldboyedu/certs/etcd/
├── etcd-ca.csr
├── etcd-ca-key.pem
├── etcd-ca.pem
├── etcd-server.csr
├── etcd-server-key.pem
└── etcd-server.pem
0 directories, 6 files
[root@k8s-cluster243 ~]#
5.创建etcd集群各节点配置文件
5.1 k8s-cluster241节点的配置文件
[root@k8s-cluster241 ~]# mkdir -pv /oldboyedu/softwares/etcd
mkdir: created directory '/oldboyedu/softwares'
mkdir: created directory '/oldboyedu/softwares/etcd'
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# cat > /oldboyedu/softwares/etcd/etcd.config.yml <<'EOF'
name: 'k8s-cluster241'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://10.0.0.241:2380'
listen-client-urls: 'https://10.0.0.241:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://10.0.0.241:2380'
advertise-client-urls: 'https://10.0.0.241:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-cluster241=https://10.0.0.241:2380,k8s-cluster242=https://10.0.0.242:2380,k8s-cluster243=https://10.0.0.243:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
5.2 k8s-cluster242节点的配置文件
[root@k8s-cluster242 ~]# mkdir -pv /oldboyedu/softwares/etcd
mkdir: created directory '/oldboyedu/softwares'
mkdir: created directory '/oldboyedu/softwares/etcd'
[root@k8s-cluster242 ~]#
[root@k8s-cluster242 ~]# cat > /oldboyedu/softwares/etcd/etcd.config.yml <<'EOF'
name: 'k8s-cluster242'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://10.0.0.242:2380'
listen-client-urls: 'https://10.0.0.242:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://10.0.0.242:2380'
advertise-client-urls: 'https://10.0.0.242:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-cluster241=https://10.0.0.241:2380,k8s-cluster242=https://10.0.0.242:2380,k8s-cluster243=https://10.0.0.243:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
5.3 k8s-cluster243节点的配置文件
[root@k8s-cluster243 ~]# mkdir -pv /oldboyedu/softwares/etcd
mkdir: created directory '/oldboyedu/softwares'
mkdir: created directory '/oldboyedu/softwares/etcd'
[root@k8s-cluster243 ~]#
[root@k8s-cluster243 ~]# cat > /oldboyedu/softwares/etcd/etcd.config.yml <<'EOF'
name: 'k8s-cluster243'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://10.0.0.243:2380'
listen-client-urls: 'https://10.0.0.243:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://10.0.0.243:2380'
advertise-client-urls: 'https://10.0.0.243:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-cluster241=https://10.0.0.241:2380,k8s-cluster242=https://10.0.0.242:2380,k8s-cluster243=https://10.0.0.243:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF6.所有节点编写etcd启动脚本
cat > /usr/lib/systemd/system/etcd.service <<'EOF'
[Unit]
Description=Jason Li's Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target
[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/oldboyedu/softwares/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
Alias=etcd3.service
EOF8.查看etcd集群状态
[root@k8s-cluster241 ~]# etcdctl --endpoints="https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379" --cacert=/oldboyedu/certs/etcd/etcd-ca.pem --cert=/oldboyedu/certs/etcd/etcd-server.pem --key=/oldboyedu/certs/etcd/etcd-server-key.pem endpoint status --write-out=table
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.0.0.241:2379 | 566d563f3c9274ed | 3.5.21 | 25 kB | true | false | 2 | 9 | 9 | |
| https://10.0.0.242:2379 | b83b69ba7d246b29 | 3.5.21 | 25 kB | false | false | 2 | 9 | 9 | |
| https://10.0.0.243:2379 | 47b70f9ecb1f200 | 3.5.21 | 20 kB | false | false | 2 | 9 | 9 | |
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@k8s-cluster241 ~]#
9.验证etcd高可用集群
9.1 停止leader节点
[root@k8s-cluster241 ~]# ss -ntl | egrep "2379|2380"
LISTEN 0 16384 127.0.0.1:2379 0.0.0.0:*
LISTEN 0 16384 10.0.0.241:2379 0.0.0.0:*
LISTEN 0 16384 10.0.0.241:2380 0.0.0.0:*
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# systemctl stop etcd
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# ss -ntl | egrep "2379|2380"
[root@k8s-cluster241 ~]#
9.2 查看现有集群环境，发现新leader诞生
[root@k8s-cluster241 ~]# etcdctl --endpoints="https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379" --cacert=/oldboyedu/certs/etcd/etcd-ca.pem --cert=/oldboyedu/certs/etcd/etcd-server.pem --key=/oldboyedu/certs/etcd/etcd-server-key.pem endpoint status --write-out=table
{"level":"warn","ts":"2025-06-13T16:24:51.098673+0800","logger":"etcd-client","caller":"v3@v3.5.21/retry_interceptor.go:63","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0002f45a0/10.0.0.241:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 10.0.0.241:2379: connect: connection refused\""}
Failed to get the status of endpoint https://10.0.0.241:2379 (context deadline exceeded)
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.0.0.242:2379 | b83b69ba7d246b29 | 3.5.21 | 25 kB | true | false | 3 | 10 | 10 | |
| https://10.0.0.243:2379 | 47b70f9ecb1f200 | 3.5.21 | 20 kB | false | false | 3 | 10 | 10 | |
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl --endpoints="https://10.0.0.242:2379,https://10.0.0.243:2379" --cacert=/oldboyedu/certs/etcd/etcd-ca.pem --cert=/oldboyedu/certs/etcd/etcd-server.pem --key=/oldboyedu/certs/etcd/etcd-server-key.pem endpoint status --write-out=table
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.0.0.242:2379 | b83b69ba7d246b29 | 3.5.21 | 25 kB | true | false | 3 | 10 | 10 | |
| https://10.0.0.243:2379 | 47b70f9ecb1f200 | 3.5.21 | 20 kB | false | false | 3 | 10 | 10 | |
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@k8s-cluster241 ~]# 9.3 再将之前的leader起来
[root@k8s-cluster241 ~]# ss -ntl | egrep "2379|2380"
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# systemctl start etcd
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# ss -ntl | egrep "2379|2380"
LISTEN 0 16384 127.0.0.1:2379 0.0.0.0:*
LISTEN 0 16384 10.0.0.241:2379 0.0.0.0:*
LISTEN 0 16384 10.0.0.241:2380 0.0.0.0:*
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl --endpoints="https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379" --cacert=/oldboyedu/certs/etcd/etcd-ca.pem --cert=/oldboyedu/certs/etcd/etcd-server.pem --key=/oldboyedu/certs/etcd/etcd-server-key.pem endpoint status --write-out=table
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.0.0.241:2379 | 566d563f3c9274ed | 3.5.21 | 25 kB | false | false | 3 | 11 | 11 | |
| https://10.0.0.242:2379 | b83b69ba7d246b29 | 3.5.21 | 25 kB | true | false | 3 | 11 | 11 | |
| https://10.0.0.243:2379 | 47b70f9ecb1f200 | 3.5.21 | 20 kB | false | false | 3 | 11 | 11 | |
+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@k8s-cluster241 ~]#
10.添加别名
10.1 添加别名
[root@k8s-cluster241 ~]# vim .bashrc
...
alias etcdctl='etcdctl --endpoints="10.0.0.241:2379,10.0.0.242:2379,10.0.0.243:2379" --cacert=/oldboyedu/certs/etcd/etcd-ca.pem --cert=/oldboyedu/certs/etcd/etcd-server.pem --key=/oldboyedu/certs/etcd/etcd-server-key.pem '
...
[root@k8s-cluster241 ~]# source .bashrc
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl endpoint status --write-out=table
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 10.0.0.241:2379 | 566d563f3c9274ed | 3.5.21 | 25 kB | false | false | 3 | 11 | 11 | |
| 10.0.0.242:2379 | b83b69ba7d246b29 | 3.5.21 | 25 kB | true | false | 3 | 11 | 11 | |
| 10.0.0.243:2379 | 47b70f9ecb1f200 | 3.5.21 | 20 kB | false | false | 3 | 11 | 11 | |
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# data_rsync.sh .bashrc
===== rsyncing k8s-cluster242: .bashrc =====
命令执行成功!
===== rsyncing k8s-cluster243: .bashrc =====
命令执行成功!
[root@k8s-cluster241 ~]# 10.2 测试验证 【需要断开重连】喔！！
[root@k8s-cluster242 ~]# source ~/.bashrc
[root@k8s-cluster242 ~]#
[root@k8s-cluster242 ~]# etcdctl endpoint status --write-out=table
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 10.0.0.241:2379 | 566d563f3c9274ed | 3.5.21 | 25 kB | false | false | 3 | 11 | 11 | |
| 10.0.0.242:2379 | b83b69ba7d246b29 | 3.5.21 | 25 kB | true | false | 3 | 11 | 11 | |
| 10.0.0.243:2379 | 47b70f9ecb1f200 | 3.5.21 | 20 kB | false | false | 3 | 11 | 11 | |
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# source ~/.bashrc
[root@k8s-cluster243 ~]#
[root@k8s-cluster243 ~]# etcdctl endpoint status --write-out=table
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 10.0.0.241:2379 | 566d563f3c9274ed | 3.5.21 | 25 kB | false | false | 3 | 11 | 11 | |
| 10.0.0.242:2379 | b83b69ba7d246b29 | 3.5.21 | 25 kB | true | false | 3 | 11 | 11 | |
| 10.0.0.243:2379 | 47b70f9ecb1f200 | 3.5.21 | 20 kB | false | false | 3 | 11 | 11 | |
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@k8s-cluster243 ~]#
11.关机拍快照喔！
快照名称: 'etcd环境准备就绪'
温馨提示:
拍快照之前先重启测试下效果，观察etcd集群是否可用。然后在关机拍快照。
[root@k8s-cluster241 ~]# etcdctl endpoint status --write-out=table
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 10.0.0.241:2379 | 566d563f3c9274ed | 3.5.21 | 25 kB | true | false | 5 | 16 | 16 | |
| 10.0.0.242:2379 | b83b69ba7d246b29 | 3.5.21 | 25 kB | false | false | 5 | 16 | 16 | |
| 10.0.0.243:2379 | 47b70f9ecb1f200 | 3.5.21 | 20 kB | false | false | 5 | 16 | 16 | |
+-----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@k8s-cluster241 ~]#
etcd的基本使用
1.etcd基础操作概述
etcd的操作和zookeeper，Redis的操作类似，存储数据都是键值对
2.etcd增删改查基础操作
2.1 写入数据KEY的school，value等于oldboyedu
[root@k8s-cluster241 ~]# etcdctl put school oldboyedu
OK
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl put /class linux98
OK
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl put classroom 教室1
OK
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl put /etc/hosts 10.0.0.141 ceph141
OK
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl put /oldboyedu/docker/registry/harbor 企业级镜像仓库
OK
[root@k8s-cluster241 ~]#
2.2 查看数据
'2.3 修改数据2.3 修改数据
2.3 修改数据2.3 修改数据
2.3 修改数据
2.3 修改数据
[root@k8s-cluster241 ~]# etcdctl get school --print-value-only
oldboyedu
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl put school 老男孩IT教育
OK
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl get school --print-value-only
老男孩IT教育
[root@k8s-cluster241 ~]#
2.4 删除数据
[root@k8s-cluster241 ~]# etcdctl get "" --prefix --keys-only
/class
/etc/hosts
/oldboyedu/docker/registry/harbor
classroom
school
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl del school
1
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl del / --prefix
3
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl get "" --prefix --keys-only
classroom
[root@k8s-cluster241 ~]#
etcd集群数据备份和恢复
推荐阅读
https://etcd.io/docs/v3.5/op-guide/recovery/
https://etcd.io/docs/v3.5/op-guide/
https://etcd.io/docs/v3.5/learning/
https://etcd.io/docs/v3.5/upgrades/
1 准备测试数据【数据随机创建即可，用于模拟备份环节】
[root@k8s-cluster241 ~]# etcdctl get "" --prefix
classroom
教室1
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl put /shaopeng/linux98 嘻嘻
OK
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl put /shaopeng/linux99 哈哈
OK
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl get "" --prefix
/oldboyedu/linux98
嘻嘻
/shaopeng/linux99
哈哈
classroom
教室1
[root@k8s-cluster241 ~]#
2 创建快照用于备份数据
[root@k8s-cluster241 ~]# \etcdctl snapshot save /tmp/oldboyedu-etcd-`date +%F`.backup
{"level":"info","ts":"2025-07-31T14:32:49.948793+0800","caller":"snapshot/v3_snapshot.go:65","msg":"created temporary db file","path":"/tmp/oldboyedu-etcd-2025-07-31.backup.part"}
{"level":"info","ts":"2025-07-31T14:32:49.949734+0800","logger":"client","caller":"v3@v3.5.22/maintenance.go:212","msg":"opened snapshot stream; downloading"}
{"level":"info","ts":"2025-07-31T14:32:49.949764+0800","caller":"snapshot/v3_snapshot.go:73","msg":"fetching snapshot","endpoint":"127.0.0.1:2379"}
{"level":"info","ts":"2025-07-31T14:32:49.952514+0800","logger":"client","caller":"v3@v3.5.22/maintenance.go:220","msg":"completed snapshot read; closing"}
{"level":"info","ts":"2025-07-31T14:32:49.953291+0800","caller":"snapshot/v3_snapshot.go:88","msg":"fetched snapshot","endpoint":"127.0.0.1:2379","size":"20 kB","took":"now"}
{"level":"info","ts":"2025-07-31T14:32:49.953364+0800","caller":"snapshot/v3_snapshot.go:97","msg":"saved","path":"/tmp/oldboyedu-etcd-2025-07-31.backup"}
Snapshot saved at /tmp/oldboyedu-etcd-2025-07-31.backup
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# ll /tmp/oldboyedu-etcd-`date +%F`.backup
-rw------- 1 root root 20512 Jul 31 14:32 /tmp/oldboyedu-etcd-2025-07-31.backup
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# file /tmp/oldboyedu-etcd-`date +%F`.backup
/tmp/oldboyedu-etcd-2025-07-31.backup: data
[root@k8s-cluster241 ~]#
3.查看快照的基本信息
[root@k8s-cluster241 ~]# etcdctl snapshot status /tmp/oldboyedu-etcd-`date +%F`.backup -w table # 查看备份快照的状态
Deprecated: Use `etcdutl snapshot status` instead.
+---------+----------+------------+------------+
| HASH | REVISION | TOTAL KEYS | TOTAL SIZE |
+---------+----------+------------+------------+
| e546d7a | 11 | 20 | 20 kB |
+---------+----------+------------+------------+
[root@k8s-cluster241 ~]#
4.将快照拷贝到其他两个集群节点
[root@k8s-cluster241 ~]# scp /tmp/oldboyedu-etcd-`date +%F`.backup k8s-cluster242:/tmp
[root@k8s-cluster241 ~]# scp /tmp/oldboyedu-etcd-`date +%F`.backup k8s-cluster243:/tmp
5 删除所有数据【搞破坏】
[root@k8s-cluster241 ~]# etcdctl get "" --prefix
/oldboyedu/linux98
嘻嘻
/oldboyedu/linux99
哈哈
classroom
教室1
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl del "" --prefix
3
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl get "" --prefix
[root@k8s-cluster241 ~]#
6 停止etcd集群
[root@k8s-cluster241 ~]# systemctl stop etcd
[root@k8s-cluster242 ~]# systemctl stop etcd
[root@k8s-cluster243 ~]# systemctl stop etcd
7.各节点恢复数据 【恢复的数据目录必须为空】
[root@k8s-cluster241 ~]# etcdctl snapshot restore /tmp/oldboyedu-etcd-`date +%F`.backup --data-dir=/var/lib/etcd-2025
Deprecated: Use `etcdutl snapshot restore` instead.
2025-07-31T14:37:26+08:00 info snapshot/v3_snapshot.go:265 restoring snapshot {"path": "/tmp/oldboyedu-etcd-2025-07-31.backup", "wal-dir": "/var/lib/etcd-2025/member/wal", "data-dir": "/var/lib/etcd-2025", "snap-dir": "/var/lib/etcd-2025/member/snap", "initial-memory-map-size": 0}
2025-07-31T14:37:26+08:00 info membership/store.go:138 Trimming membership information from the backend...
2025-07-31T14:37:26+08:00 info membership/cluster.go:421 added member {"cluster-id": "cdf818194e3a8c32", "local-member-id": "0", "added-peer-id": "8e9e05c52164694d", "added-peer-peer-urls": ["http://localhost:2380"], "added-peer-is-learner": false}
2025-07-31T14:37:26+08:00 info snapshot/v3_snapshot.go:293 restored snapshot {"path": "/tmp/oldboyedu-etcd-2025-07-31.backup", "wal-dir": "/var/lib/etcd-2025/member/wal", "data-dir": "/var/lib/etcd-2025", "snap-dir": "/var/lib/etcd-2025/member/snap", "initial-memory-map-size": 0}
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# ll /var/lib/etcd-2025
total 12
drwx------ 3 root root 4096 Jul 31 14:37 ./
drwxr-xr-x 63 root root 4096 Jul 31 14:37 ../
drwx------ 4 root root 4096 Jul 31 14:37 member/
[root@k8s-cluster241 ~]#
[root@k8s-cluster242 ~]# etcdctl snapshot restore /tmp/oldboyedu-etcd-`date +%F`.backup --data-dir=/var/lib/etcd-2025
Deprecated: Use `etcdutl snapshot restore` instead.
2025-07-31T14:37:53+08:00 info snapshot/v3_snapshot.go:265 restoring snapshot {"path": "/tmp/oldboyedu-etcd-2025-07-31.backup", "wal-dir": "/var/lib/etcd-2025/member/wal", "data-dir": "/var/lib/etcd-2025", "snap-dir": "/var/lib/etcd-2025/member/snap", "initial-memory-map-size": 0}
2025-07-31T14:37:53+08:00 info membership/store.go:138 Trimming membership information from the backend...
2025-07-31T14:37:53+08:00 info membership/cluster.go:421 added member {"cluster-id": "cdf818194e3a8c32", "local-member-id": "0", "added-peer-id": "8e9e05c52164694d", "added-peer-peer-urls": ["http://localhost:2380"], "added-peer-is-learner": false}
2025-07-31T14:37:53+08:00 info snapshot/v3_snapshot.go:293 restored snapshot {"path": "/tmp/oldboyedu-etcd-2025-07-31.backup", "wal-dir": "/var/lib/etcd-2025/member/wal", "data-dir": "/var/lib/etcd-2025", "snap-dir": "/var/lib/etcd-2025/member/snap", "initial-memory-map-size": 0}
[root@k8s-cluster242 ~]#
[root@k8s-cluster242 ~]# ll /var/lib/etcd-2025
total 12
drwx------ 3 root root 4096 Jul 31 14:37 ./
drwxr-xr-x 63 root root 4096 Jul 31 14:37 ../
drwx------ 4 root root 4096 Jul 31 14:37 member/
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# etcdctl snapshot restore /tmp/oldboyedu-etcd-`date +%F`.backup --data-dir=/var/lib/etcd-2025
Deprecated: Use `etcdutl snapshot restore` instead.
2025-07-31T14:38:11+08:00 info snapshot/v3_snapshot.go:265 restoring snapshot {"path": "/tmp/oldboyedu-etcd-2025-07-31.backup", "wal-dir": "/var/lib/etcd-2025/member/wal", "data-dir": "/var/lib/etcd-2025", "snap-dir": "/var/lib/etcd-2025/member/snap", "initial-memory-map-size": 0}
2025-07-31T14:38:11+08:00 info membership/store.go:138 Trimming membership information from the backend...
2025-07-31T14:38:11+08:00 info membership/cluster.go:421 added member {"cluster-id": "cdf818194e3a8c32", "local-member-id": "0", "added-peer-id": "8e9e05c52164694d", "added-peer-peer-urls": ["http://localhost:2380"], "added-peer-is-learner": false}
2025-07-31T14:38:11+08:00 info snapshot/v3_snapshot.go:293 restored snapshot {"path": "/tmp/oldboyedu-etcd-2025-07-31.backup", "wal-dir": "/var/lib/etcd-2025/member/wal", "data-dir": "/var/lib/etcd-2025", "snap-dir": "/var/lib/etcd-2025/member/snap", "initial-memory-map-size": 0}
[root@k8s-cluster243 ~]#
[root@k8s-cluster243 ~]# ll /var/lib/etcd-2025
total 12
drwx------ 3 root root 4096 Jul 31 14:38 ./
drwxr-xr-x 63 root root 4096 Jul 31 14:38 ../
drwx------ 4 root root 4096 Jul 31 14:38 member/
[root@k8s-cluster243 ~]#
6 将恢复后的数据目录作为新的数据目录
[root@k8s-cluster241 ~]# grep "/var/lib/etcd" /oldboyedu/softwares/etcd/etcd.config.yml
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# sed -ri "s#(/var/lib/etcd)#\1-2025#g" /oldboyedu/softwares/etcd/etcd.config.yml
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# grep "/var/lib/etcd" /oldboyedu/softwares/etcd/etcd.config.yml
data-dir: /var/lib/etcd-2025
wal-dir: /var/lib/etcd-2025/wal
[root@k8s-cluster241 ~]#
[root@k8s-cluster242 ~]# sed -ri "s#(/var/lib/etcd)#\1-2025#g" /oldboyedu/softwares/etcd/etcd.config.yml
[root@k8s-cluster242 ~]# grep "/var/lib/etcd" /oldboyedu/softwares/etcd/etcd.config.yml
data-dir: /var/lib/etcd-2025
wal-dir: /var/lib/etcd-2025/wal
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# sed -ri "s#(/var/lib/etcd)#\1-2025#g" /oldboyedu/softwares/etcd/etcd.config.yml
[root@k8s-cluster243 ~]# grep "/var/lib/etcd" /oldboyedu/softwares/etcd/etcd.config.yml
data-dir: /var/lib/etcd-2025
wal-dir: /var/lib/etcd-2025/wal
[root@k8s-cluster243 ~]#
7 启动etcd集群
[root@k8s-cluster241 ~]# systemctl start etcd
[root@k8s-cluster242 ~]# systemctl start etcd
[root@k8s-cluster243 ~]# systemctl start etcd
8 验证数据是否恢复
[root@k8s-cluster243 ~]# etcdctl get "" --prefix --keys-only
/oldboyedu/linux98
/oldboyedu/linux99
classroom
[root@k8s-cluster243 ~]# etcdctl get "" --prefix
/oldboyedu/linux98
嘻嘻
/oldboyedu/linux99
哈哈
classroom
教室1
[root@k8s-cluster243 ~]#
9.测试数据是否可以正常读写
[root@k8s-cluster242 ~]# etcdctl put xixi 哈哈
OK
[root@k8s-cluster242 ~]#
[root@k8s-cluster242 ~]# etcdctl get "" --prefix --keys-only
/oldboyedu/linux98
/oldboyedu/linux99
classroom
xixi
[root@k8s-cluster242 ~]#
etcd-workbench图形化管理etcd集群
参考链接:
https://tzfun.github.io/etcd-workbench/
https://github.com/tzfun/etcd-workbench/blob/master/README_ZH.md
https://github.com/tzfun/etcd-workbench-web/blob/master/server/src/main/resources/etcd-workbench.conf1.拉取镜像
[root@k8s-cluster243 ~]# ctr i pull tzfun/etcd-workbench:1.1.4
SVIP:
[root@k8s-cluster243 ~]# wget http://192.168.21.253/Resources/Prometheus/images/etcd-workbench/oldboyedu-etcd-workbench-v1.1.4.tar.gz
[root@k8s-cluster243 ~]# ctr ns ls
NAME LABELS
[root@k8s-cluster243 ~]#
[root@k8s-cluster243 ~]# ctr i import oldboyedu-etcd-workbench-v1.1.4.tar.gz
unpacking docker.io/tzfun/etcd-workbench:1.1.4 (sha256:ddfe5e61fc4b54e02e2d7a75d209a6aeb72c8bd0b993de2b2934671371b9b93f)...done
[root@k8s-cluster243 ~]#
[root@k8s-cluster243 ~]# ctr ns ls
NAME LABELS
default
[root@k8s-cluster243 ~]#
[root@k8s-cluster243 ~]# ctr i ls
REF TYPE DIGEST SIZE PLATFORMS LABELS
docker.io/tzfun/etcd-workbench:1.1.4 application/vnd.docker.distribution.manifest.v2+json sha256:ddfe5e61fc4b54e02e2d7a75d209a6aeb72c8bd0b993de2b2934671371b9b93f 310.2 MiB linux/amd64 -
[root@k8s-cluster243 ~]# 2.运行etcd-workbench
[root@k8s-cluster243 ~]# ctr container create --net-host docker.io/tzfun/etcd-workbench:1.1.4 etcd-workbench
[root@k8s-cluster243 ~]#
[root@k8s-cluster243 ~]# ctr c ls
CONTAINER IMAGE RUNTIME
etcd-workbench docker.io/tzfun/etcd-workbench:1.1.4 io.containerd.runc.v2
[root@k8s-cluster243 ~]#
[root@k8s-cluster243 ~]# ctr t start etcd-workbench
_____ ____ ____ ____ ____ ____ ____
|_ _||_ _| |_ _||_ \ / _||_ \ / _|
| | \ \ / / | \/ | | \/ |
_ | | \ \ / / | |\ /| | | |\ /| |
| |__' | \ ' / _| |_\/_| |_ _| |_\/_| |_
`.____.' \_/ |_____||_____||_____||_____|
Powered by JVMM https://github.com/tzfun/jvmm
Framework version: 2.4.2
2025-07-31 06:52:28.881+0000 INFO org.beifengtz.etcd.server.EtcdServer Load configuration successfully
2025-07-31 06:52:29.350+0000 INFO org.beifengtz.etcd.server.service.HttpService Please access http://10.0.0.243:8002
2025-07-31 06:52:29.350+0000 INFO org.beifengtz.etcd.server.service.HttpService Http server service started on 8002 in 317 ms
2025-07-31 06:52:29.351+0000 INFO org.beifengtz.etcd.server.EtcdServer Etcd workbench version: 1.1.4
2025-07-31 06:52:29.352+0000 INFO org.beifengtz.etcd.server.EtcdServer Etcd workbench build hash: 6020e3f
3.访问etcd-workbench的webUI
http://10.0.0.243:8002如果基于docker启用认证的话，则可以使用用户名和密码登录即可。
4.拷贝证书到windows系统
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'5.上传证书并测试
彩蛋： 基于Docker部署支持认证功能:
1.准备配置文件
[root@harbor250.oldboyedu.com ~]# cat > etcd-workbench.conf <<'EOF'
[server]
# 服务监听的端口
port = 8002
# 链接超时时间
etcdExecuteTimeoutMillis = 3000
# 数据存储目录
dataDir = ./data
[auth]
# 启用认证功能
enable = true
# 指定用户名和密码
user = admin:yinzhengjie
[log]
# 指定日志的级别
level = INFO
# 日志存储目录
file = ./logs
# 日志文件的名称
fileName = etcd-workbench
# 指定日志的滚动大小
fileLimitSize = 100
# 日志打印的位置
printers = std,file
EOF
2.启动容器
[root@harbor250.oldboyedu.com ~]# docker run -d -v /root/etcd-workbench.conf:/usr/tzfun/etcd-workbench/etcd-workbench.conf --name etcd-workbench --network host tzfun/etcd-workbench:1.1.4
88e4dc60963e92f988a617727e7cf76db3e0d565096859ca63549bed7883fc46
[root@harbor250.oldboyedu.com ~]#
[root@harbor250.oldboyedu.com ~]# ss -ntl | grep 8002
LISTEN 0 4096 *:8002 *:*
[root@harbor250.oldboyedu.com ~]#
[root@harbor250.oldboyedu.com ~]# docker ps -l
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
88e4dc60963e tzfun/etcd-workbench:1.1.4 "/bin/sh -c 'java …" 9 seconds ago Up 8 seconds etcd-workbench
[root@harbor250.oldboyedu.com ~]#
下载并安装K8S软件包
wget https://dl.k8s.io/v1.33.3/kubernetes-server-linux-amd64.tar.gz
2.解压指定的软件包
[root@k8s-cluster241 ~]# tar xf kubernetes-server-linux-amd64.tar.gz --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# ll /usr/local/bin/kube*
-rwxr-xr-x 1 root root 97960120 May 15 16:40 /usr/local/bin/kube-apiserver*
-rwxr-xr-x 1 root root 90759352 May 15 16:40 /usr/local/bin/kube-controller-manager*
-rwxr-xr-x 1 root root 60121272 May 15 16:40 /usr/local/bin/kubectl*
-rwxr-xr-x 1 root root 81690916 May 15 16:40 /usr/local/bin/kubelet*
-rwxr-xr-x 1 root root 70594744 May 15 16:40 /usr/local/bin/kube-proxy*
-rwxr-xr-x 1 root root 69603512 May 15 16:40 /usr/local/bin/kube-scheduler*
[root@k8s-cluster241 ~]#
3.查看kubelet的版本
[root@k8s-cluster241 ~]# kube-apiserver --version
Kubernetes v1.33.3
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kube-proxy --version
Kubernetes v1.33.3
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kube-controller-manager --version
Kubernetes v1.33.3
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kube-scheduler --version
Kubernetes v1.33.3
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubelet --version
Kubernetes v1.33.3
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl version
Client Version: v1.33.3
Kustomize Version: v5.6.0
The connection to the server localhost:8080 was refused - did you specify the right host or port?
[root@k8s-cluster241 ~]#
4.分发软件包
[root@k8s-cluster241 ~]# for i in `ls -1 /usr/local/bin/kube*`;do data_rsync.sh $i ;done5.其他节点验证
[root@k8s-cluster242 ~]# ll /usr/local/bin/kube*
-rwxr-xr-x 1 root root 97968312 Jul 16 02:18 /usr/local/bin/kube-apiserver*
-rwxr-xr-x 1 root root 90767544 Jul 16 02:18 /usr/local/bin/kube-controller-manager*
-rwxr-xr-x 1 root root 60129464 Jul 16 02:18 /usr/local/bin/kubectl*
-rwxr-xr-x 1 root root 81703204 Jul 16 02:18 /usr/local/bin/kubelet*
-rwxr-xr-x 1 root root 70602936 Jul 16 02:18 /usr/local/bin/kube-proxy*
-rwxr-xr-x 1 root root 69611704 Jul 16 02:18 /usr/local/bin/kube-scheduler*
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# ll /usr/local/bin/kube*
-rwxr-xr-x 1 root root 97968312 Jul 16 02:18 /usr/local/bin/kube-apiserver*
-rwxr-xr-x 1 root root 90767544 Jul 16 02:18 /usr/local/bin/kube-controller-manager*
-rwxr-xr-x 1 root root 60129464 Jul 16 02:18 /usr/local/bin/kubectl*
-rwxr-xr-x 1 root root 81703204 Jul 16 02:18 /usr/local/bin/kubelet*
-rwxr-xr-x 1 root root 70602936 Jul 16 02:18 /usr/local/bin/kube-proxy*
-rwxr-xr-x 1 root root 69611704 Jul 16 02:18 /usr/local/bin/kube-scheduler*
[root@k8s-cluster243 ~]#
生成k8s组件相关证书温馨提示: 建议大家在做此步骤之前，拍个快照喔。
1.生成证书的CSR文件： 证书签发请求文件，配置了一些域名，公司，单位
[root@k8s-cluster241 ~]# mkdir -pv /oldboyedu/pki/k8s && cd /oldboyedu/pki/k8s
mkdir: created directory '/oldboyedu/pki/k8s'
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]# cat > k8s-ca-csr.json <<EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "Kubernetes",
"OU": "Kubernetes-manual"
}
],
"ca": {
"expiry": "876000h"
}
}
EOF2.生成K8S证书
[root@k8s-cluster241 k8s]# mkdir -pv /oldboyedu/certs/k8s/
mkdir: created directory '/oldboyedu/certs/k8s/'
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]# cfssl gencert -initca k8s-ca-csr.json | cfssljson -bare /oldboyedu/certs/k8s/k8s-ca
2025/07/31 16:01:22 [INFO] generating a new CA key and certificate from CSR
2025/07/31 16:01:22 [INFO] generate received request
2025/07/31 16:01:22 [INFO] received CSR
2025/07/31 16:01:22 [INFO] generating key: rsa-2048
2025/07/31 16:01:22 [INFO] encoded CSR
2025/07/31 16:01:22 [INFO] signed certificate with serial number 597707726810272531705749021812232151373399052669
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]# ll /oldboyedu/certs/k8s/k8s-ca*
-rw-r--r-- 1 root root 1070 Jul 31 16:01 /oldboyedu/certs/k8s/k8s-ca.csr
-rw------- 1 root root 1679 Jul 31 16:01 /oldboyedu/certs/k8s/k8s-ca-key.pem
-rw-r--r-- 1 root root 1363 Jul 31 16:01 /oldboyedu/certs/k8s/k8s-ca.pem
[root@k8s-cluster241 k8s]# 3.生成k8s证书的有效期为100年
[root@k8s-cluster241 k8s]# cat > k8s-ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "876000h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "876000h"
}
}
}
}
EOF4.生成apiserver证书的CSR文件： 证书签发请求文件，配置了一些域名，公司，单位
[root@k8s-cluster241 k8s]# cat > apiserver-csr.json <<EOF
{
"CN": "kube-apiserver",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "Kubernetes",
"OU": "Kubernetes-manual"
}
]
}
EOF
5.基于自建ca证书生成apiServer的证书文件
[root@k8s-cluster241 k8s]# cfssl gencert \
-ca=/oldboyedu/certs/k8s/k8s-ca.pem \
-ca-key=/oldboyedu/certs/k8s/k8s-ca-key.pem \
-config=k8s-ca-config.json \
--hostname=10.200.0.1,10.0.0.240,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.oldboyedu,kubernetes.default.svc.oldboyedu.com,10.0.0.241,10.0.0.242,10.0.0.243 \
--profile=kubernetes \
apiserver-csr.json | cfssljson -bare /oldboyedu/certs/k8s/apiserver
2025/07/31 16:03:22 [INFO] generate received request
2025/07/31 16:03:22 [INFO] received CSR
2025/07/31 16:03:22 [INFO] generating key: rsa-2048
2025/07/31 16:03:22 [INFO] encoded CSR
2025/07/31 16:03:22 [INFO] signed certificate with serial number 673748347141635635906292205037564855770852883027
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]# ll /oldboyedu/certs/k8s/apiserver*
-rw-r--r-- 1 root root 1293 Jul 31 16:03 /oldboyedu/certs/k8s/apiserver.csr
-rw------- 1 root root 1679 Jul 31 16:03 /oldboyedu/certs/k8s/apiserver-key.pem
-rw-r--r-- 1 root root 1688 Jul 31 16:03 /oldboyedu/certs/k8s/apiserver.pem
[root@k8s-cluster241 k8s]#
温馨提示:
"10.200.0.1"为咱们的svc网段的第一个地址，您需要根据自己的场景稍作修改。
"10.0.0.240"是负载均衡器的VIP地址。
"kubernetes,...,kubernetes.default.svc.oldboyedu.com"对应的是apiServer的svc解析的A记录。
"10.0.0.41,...,10.0.0.43"对应的是K8S集群的地址。
5 生成聚合证书的用于自建ca的CSR文件
聚合证书的作用就是让第三方组件(比如metrics-server等)能够拿这个证书文件和apiServer进行通信。
[root@k8s-cluster241 k8s]# cat > front-proxy-ca-csr.json <<EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
}
}
EOF
6 生成聚合证书的自建ca证书
[root@node-exporter41 pki]# cfssl gencert -initca front-proxy-ca-csr.json | cfssljson -bare /oldboyedu/certs/k8s/front-proxy-ca
2025/07/31 16:04:04 [INFO] generating a new CA key and certificate from CSR
2025/07/31 16:04:04 [INFO] generate received request
2025/07/31 16:04:04 [INFO] received CSR
2025/07/31 16:04:04 [INFO] generating key: rsa-2048
2025/07/31 16:04:04 [INFO] encoded CSR
2025/07/31 16:04:04 [INFO] signed certificate with serial number 510482660017855825441334359944401305341828692486
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]# ll /oldboyedu/certs/k8s/front-proxy-ca*
-rw-r--r-- 1 root root 891 Jul 31 16:04 /oldboyedu/certs/k8s/front-proxy-ca.csr
-rw------- 1 root root 1679 Jul 31 16:04 /oldboyedu/certs/k8s/front-proxy-ca-key.pem
-rw-r--r-- 1 root root 1094 Jul 31 16:04 /oldboyedu/certs/k8s/front-proxy-ca.pem
[root@k8s-cluster241 k8s]#
7.生成聚合证书的用于客户端的CSR文件
[root@k8s-cluster241 k8s]# cat > front-proxy-client-csr.json <<EOF
{
"CN": "front-proxy-client",
"key": {
"algo": "rsa",
"size": 2048
}
}
EOF
8 基于聚合证书的自建ca证书签发聚合证书的客户端证书
[root@k8s-cluster241 k8s]# cfssl gencert \
-ca=/oldboyedu/certs/k8s/front-proxy-ca.pem \
-ca-key=/oldboyedu/certs/k8s/front-proxy-ca-key.pem \
-config=k8s-ca-config.json \
-profile=kubernetes \
front-proxy-client-csr.json | cfssljson -bare /oldboyedu/certs/k8s/front-proxy-client
2025/07/31 16:04:26 [INFO] generate received request
2025/07/31 16:04:26 [INFO] received CSR
2025/07/31 16:04:26 [INFO] generating key: rsa-2048
2025/07/31 16:04:26 [INFO] encoded CSR
2025/07/31 16:04:26 [INFO] signed certificate with serial number 107619448349148361013766691468796306114314872515
2025/07/31 16:04:26 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]# ll /oldboyedu/certs/k8s/front-proxy-client*
-rw-r--r-- 1 root root 903 Jul 31 16:04 /oldboyedu/certs/k8s/front-proxy-client.csr
-rw------- 1 root root 1679 Jul 31 16:04 /oldboyedu/certs/k8s/front-proxy-client-key.pem
-rw-r--r-- 1 root root 1188 Jul 31 16:04 /oldboyedu/certs/k8s/front-proxy-client.pem
[root@k8s-cluster241 k8s]#9.生成kube-proxy的csr文件
[root@k8s-cluster241 k8s]# cat > kube-proxy-csr.json <<EOF
{
"CN": "system:kube-proxy",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "system:kube-proxy",
"OU": "Kubernetes-manual"
}
]
}
EOF
10.创建kube-proxy需要的证书文件
[root@k8s-cluster241 k8s]# cfssl gencert \
-ca=/oldboyedu/certs/k8s/k8s-ca.pem \
-ca-key=/oldboyedu/certs/k8s/k8s-ca-key.pem \
-config=k8s-ca-config.json \
-profile=kubernetes \
kube-proxy-csr.json | cfssljson -bare /oldboyedu/certs/k8s/kube-proxy
2025/06/16 09:02:34 [INFO] generate received request
2025/06/16 09:02:34 [INFO] received CSR
2025/06/16 09:02:34 [INFO] generating key: rsa-2048
2025/06/16 09:02:34 [INFO] encoded CSR
2025/06/16 09:02:34 [INFO] signed certificate with serial number 428484742950452765379969700730076331479633468881
2025/06/16 09:02:34 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]# ll /oldboyedu/certs/k8s/kube-proxy*
-rw-r--r-- 1 root root 1045 Jun 16 09:02 /oldboyedu/certs/k8s/kube-proxy.csr
-rw------- 1 root root 1675 Jun 16 09:02 /oldboyedu/certs/k8s/kube-proxy-key.pem
-rw-r--r-- 1 root root 1464 Jun 16 09:02 /oldboyedu/certs/k8s/kube-proxy.pem
[root@k8s-cluster241 k8s]#
11.生成controller-manager的CSR文件
[root@k8s-cluster241 k8s]# cat > controller-manager-csr.json <<EOF
{
"CN": "system:kube-controller-manager",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "system:kube-controller-manager",
"OU": "Kubernetes-manual"
}
]
}
EOF12.生成controller-manager证书文件
[root@k8s-cluster241 k8s]# cfssl gencert \
-ca=/oldboyedu/certs/k8s/k8s-ca.pem \
-ca-key=/oldboyedu/certs/k8s/k8s-ca-key.pem \
-config=k8s-ca-config.json \
-profile=kubernetes \
controller-manager-csr.json | cfssljson -bare /oldboyedu/certs/k8s/controller-manager
2025/07/31 16:05:40 [INFO] generate received request
2025/07/31 16:05:40 [INFO] received CSR
2025/07/31 16:05:40 [INFO] generating key: rsa-2048
2025/07/31 16:05:41 [INFO] encoded CSR
2025/07/31 16:05:41 [INFO] signed certificate with serial number 701177347536747357211350863493315844960643128712
2025/07/31 16:05:41 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]# ll /oldboyedu/certs/k8s/controller-manager*
-rw-r--r-- 1 root root 1082 Jul 31 16:05 /oldboyedu/certs/k8s/controller-manager.csr
-rw------- 1 root root 1679 Jul 31 16:05 /oldboyedu/certs/k8s/controller-manager-key.pem
-rw-r--r-- 1 root root 1501 Jul 31 16:05 /oldboyedu/certs/k8s/controller-manager.pem
[root@k8s-cluster241 k8s]#
13.生成scheduler的CSR文件
[root@k8s-cluster241 k8s]# cat > scheduler-csr.json <<EOF
{
"CN": "system:kube-scheduler",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "system:kube-scheduler",
"OU": "Kubernetes-manual"
}
]
}
EOF14.生成scheduler证书文件
[root@k8s-cluster241 k8s]# cfssl gencert \
-ca=/oldboyedu/certs/k8s/k8s-ca.pem \
-ca-key=/oldboyedu/certs/k8s/k8s-ca-key.pem \
-config=k8s-ca-config.json \
-profile=kubernetes \
scheduler-csr.json | cfssljson -bare /oldboyedu/certs/k8s/scheduler
2025/07/31 16:06:47 [INFO] generate received request
2025/07/31 16:06:47 [INFO] received CSR
2025/07/31 16:06:47 [INFO] generating key: rsa-2048
2025/07/31 16:06:47 [INFO] encoded CSR
2025/07/31 16:06:47 [INFO] signed certificate with serial number 212792112494494126026675764617349696176153259100
2025/07/31 16:06:47 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]# ll /oldboyedu/certs/k8s/scheduler*
-rw-r--r-- 1 root root 1058 Jul 31 16:06 /oldboyedu/certs/k8s/scheduler.csr
-rw------- 1 root root 1675 Jul 31 16:06 /oldboyedu/certs/k8s/scheduler-key.pem
-rw-r--r-- 1 root root 1476 Jul 31 16:06 /oldboyedu/certs/k8s/scheduler.pem
[root@k8s-cluster241 k8s]# 15.生成管理员的CSR文件
[root@k8s-cluster241 k8s]# cat > admin-csr.json <<EOF
{
"CN": "admin",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "system:masters",
"OU": "Kubernetes-manual"
}
]
}
EOF16.生成k8s集群管理员证书
[root@k8s-cluster241 k8s]# cfssl gencert \
-ca=/oldboyedu/certs/k8s/k8s-ca.pem \
-ca-key=/oldboyedu/certs/k8s/k8s-ca-key.pem \
-config=k8s-ca-config.json \
-profile=kubernetes \
admin-csr.json | cfssljson -bare /oldboyedu/certs/k8s/admin
2025/07/31 16:07:15 [INFO] generate received request
2025/07/31 16:07:15 [INFO] received CSR
2025/07/31 16:07:15 [INFO] generating key: rsa-2048
2025/07/31 16:07:15 [INFO] encoded CSR
2025/07/31 16:07:15 [INFO] signed certificate with serial number 401352708867554717353193111824639183965095055782
2025/07/31 16:07:15 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-cluster241 k8s]#
[root@k8s-cluster241 k8s]# ll /oldboyedu/certs/k8s/admin*
-rw-r--r-- 1 root root 1025 Jul 31 16:07 /oldboyedu/certs/k8s/admin.csr
-rw------- 1 root root 1679 Jul 31 16:07 /oldboyedu/certs/k8s/admin-key.pem
-rw-r--r-- 1 root root 1444 Jul 31 16:07 /oldboyedu/certs/k8s/admin.pem
[root@k8s-cluster241 k8s]#
17.创建ServiceAccount账号证书【api-server和controller manager组件可以基于该私钥签署所颁发的ID令牌(token)。】
[root@k8s-cluster241 k8s]# openssl genrsa -out /oldboyedu/certs/k8s/sa.key 2048
[root@k8s-cluster241 k8s]# openssl rsa -in /oldboyedu/certs/k8s/sa.key -pubout -out /oldboyedu/certs/k8s/sa.pub
[root@k8s-cluster241 k8s]# ll /oldboyedu/certs/k8s/sa*
-rw------- 1 root root 1704 Jun 16 09:15 /oldboyedu/certs/k8s/sa.key
-rw-r--r-- 1 root root 451 Jun 16 09:15 /oldboyedu/certs/k8s/sa.pub
[root@k8s-cluster241 k8s]#
温馨提示:
细心地小伙伴可能已经发现了缺少kubelet相关证书，当然我们也可以考虑创建出来，但也可以直接使用bootstrap token的方式认证
[root@k8s-cluster241 k8s]# tree /oldboyedu/
/oldboyedu/
├── certs
│ ├── etcd
│ │ ├── etcd-ca.csr
│ │ ├── etcd-ca-key.pem
│ │ ├── etcd-ca.pem
│ │ ├── etcd-server.csr
│ │ ├── etcd-server-key.pem
│ │ └── etcd-server.pem
│ └── k8s
│ ├── admin.csr
│ ├── admin-key.pem
│ ├── admin.pem
│ ├── apiserver.csr
│ ├── apiserver-key.pem
│ ├── apiserver.pem
│ ├── controller-manager.csr
│ ├── controller-manager-key.pem
│ ├── controller-manager.pem
│ ├── front-proxy-ca.csr
│ ├── front-proxy-ca-key.pem
│ ├── front-proxy-ca.pem
│ ├── front-proxy-client.csr
│ ├── front-proxy-client-key.pem
│ ├── front-proxy-client.pem
│ ├── k8s-ca.csr
│ ├── k8s-ca-key.pem
│ ├── k8s-ca.pem
│ ├── kube-proxy.csr
│ ├── kube-proxy-key.pem
│ ├── kube-proxy.pem
│ ├── sa.key
│ ├── sa.pub
│ ├── scheduler.csr
│ ├── scheduler-key.pem
│ └── scheduler.pem
├── pki
│ ├── etcd
│ │ ├── ca-config.json
│ │ ├── etcd-ca-csr.json
│ │ └── etcd-csr.json
│ └── k8s
│ ├── admin-csr.json
│ ├── apiserver-csr.json
│ ├── controller-manager-csr.json
│ ├── front-proxy-ca-csr.json
│ ├── front-proxy-client-csr.json
│ ├── k8s-ca-config.json
│ ├── k8s-ca-csr.json
│ ├── kube-proxy-csr.json
│ └── scheduler-csr.json
└── softwares
└── etcd
└── etcd.config.yml
8 directories, 45 files
[root@k8s-cluster241 k8s]#
生成k8s组件相关Kubeconfig文件并同步到其他master节点
2.1 设置一个集群
[root@k8s-cluster241 k8s]# kubectl config set-cluster yinzhengjie-k8s \
--certificate-authority=/oldboyedu/certs/k8s/k8s-ca.pem \
--embed-certs=true \
--server=https://10.0.0.240:8443 \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-controller-manager.kubeconfig
2.2 设置一个用户项
'2.3 设置一个上下文环境
[root@k8s-cluster241 k8s]# kubectl config set-context system:kube-controller-manager@kubernetes \
--cluster=yinzhengjie-k8s \
--user=system:kube-controller-manager \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-controller-manager.kubeconfig2.4 使用默认的上下文
[root@k8s-cluster241 k8s]# kubectl config use-context system:kube-controller-manager@kubernetes \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-controller-manager.kubeconfig
2.5 查看kubeconfig资源结构
[root@k8s-cluster241 k8s]# kubectl config view --kubeconfig=/oldboyedu/certs/kubeconfig/kube-controller-manager.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.0.0.240:8443
name: yinzhengjie-k8s
contexts:
- context:
cluster: yinzhengjie-k8s
user: system:kube-controller-manager
name: system:kube-controller-manager@kubernetes
current-context: system:kube-controller-manager@kubernetes
kind: Config
preferences: {}
users:
- name: system:kube-controller-manager
user:
client-certificate-data: DATA+OMITTED
client-key-data: DATA+OMITTED
[root@k8s-cluster241 k8s]# 3.生成scheduler证书及kubeconfig文件
3.1 设置一个集群
[root@k8s-cluster241 k8s]# kubectl config set-cluster yinzhengjie-k8s \
--certificate-authority=/oldboyedu/certs/k8s/k8s-ca.pem \
--embed-certs=true \
--server=https://10.0.0.240:8443 \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-scheduler.kubeconfig
3.2 设置一个用户项
[root@k8s-cluster241 k8s]# kubectl config set-credentials system:kube-scheduler \
--client-certificate=/oldboyedu/certs/k8s/scheduler.pem \
--client-key=/oldboyedu/certs/k8s/scheduler-key.pem \
--embed-certs=true \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-scheduler.kubeconfig
3.3 设置一个上下文环境
[root@k8s-cluster241 k8s]# kubectl config set-context system:kube-scheduler@kubernetes \
--cluster=yinzhengjie-k8s \
--user=system:kube-scheduler \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-scheduler.kubeconfig
3.4 使用默认的上下文
[root@k8s-cluster241 k8s]# kubectl config use-context system:kube-scheduler@kubernetes \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-scheduler.kubeconfig
3.5 查看kubeconfig资源结构
[root@k8s-cluster241 k8s]# kubectl config view --kubeconfig=/oldboyedu/certs/kubeconfig/kube-scheduler.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.0.0.240:8443
name: yinzhengjie-k8s
contexts:
- context:
cluster: yinzhengjie-k8s
user: system:kube-scheduler
name: system:kube-scheduler@kubernetes
current-context: system:kube-scheduler@kubernetes
kind: Config
preferences: {}
users:
- name: system:kube-scheduler
user:
client-certificate-data: DATA+OMITTED
client-key-data: DATA+OMITTED
[root@k8s-cluster241 k8s]#
4.生成kube-proxy证书及kubeconfig文件
4.1 设置集群
[root@k8s-cluster241 k8s]# kubectl config set-cluster yinzhengjie-k8s \
--certificate-authority=/oldboyedu/certs/k8s/k8s-ca.pem \
--embed-certs=true \
--server=https://10.0.0.240:8443 \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-proxy.kubeconfig
4.2 设置用户
[root@k8s-cluster241 k8s]# kubectl config set-credentials system:kube-proxy \
--client-certificate=/oldboyedu/certs/k8s/kube-proxy.pem \
--client-key=/oldboyedu/certs/k8s/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-proxy.kubeconfig
4.3 设置上下文
[root@k8s-cluster241 k8s]# kubectl config set-context kube-proxy@kubernetes \
--cluster=yinzhengjie-k8s \
--user=system:kube-proxy \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-proxy.kubeconfig
4.4 设置默认上下文
[root@k8s-cluster241 k8s]# kubectl config use-context kube-proxy@kubernetes \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-proxy.kubeconfig
4.5 查看kubeconfig资源结构
[root@k8s-cluster241 k8s]# kubectl config view --kubeconfig=/oldboyedu/certs/kubeconfig/kube-proxy.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.0.0.240:8443
name: yinzhengjie-k8s
contexts:
- context:
cluster: yinzhengjie-k8s
user: system:kube-proxy
name: kube-proxy@kubernetes
current-context: kube-proxy@kubernetes
kind: Config
preferences: {}
users:
- name: system:kube-proxy
user:
client-certificate-data: DATA+OMITTED
client-key-data: DATA+OMITTED
[root@k8s-cluster241 k8s]#
5.配置k8s集群管理员证书及kubeconfig文件
5.1配置k8s集群管理员证书及kubeconfig文件
[root@k8s-cluster241 k8s]# kubectl config set-cluster yinzhengjie-k8s \
--certificate-authority=/oldboyedu/certs/k8s/k8s-ca.pem \
--embed-certs=true \
--server=https://10.0.0.240:8443 \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-admin.kubeconfig5.2 设置一个用户项
[root@k8s-cluster241 k8s]# kubectl config set-credentials kube-admin \
--client-certificate=/oldboyedu/certs/k8s/admin.pem \
--client-key=/oldboyedu/certs/k8s/admin-key.pem \
--embed-certs=true \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-admin.kubeconfig
5.3 设置一个上下文环境
[root@k8s-cluster241 k8s]# kubectl config set-context kube-admin@kubernetes \
--cluster=yinzhengjie-k8s \
--user=kube-admin \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-admin.kubeconfig
5.4 使用默认的上下文
[root@k8s-cluster241 k8s]# kubectl config use-context kube-admin@kubernetes \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-admin.kubeconfig
5.5 查看kubeconfig资源结构
[root@k8s-cluster241 k8s]# kubectl config view --kubeconfig=/oldboyedu/certs/kubeconfig/kube-admin.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.0.0.240:8443
name: yinzhengjie-k8s
contexts:
- context:
cluster: shaopeng-k8s
user: kube-admin
name: kube-admin@kubernetes
current-context: kube-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kube-admin
user:
client-certificate-data: DATA+OMITTED
client-key-data: DATA+OMITTED
[root@k8s-cluster241 k8s]#
6.将K8S组件证书拷贝到其他两个master节点
6.1 拷贝kubeconfig文件【如果controller manager，scheduler和ApiServer不在同一个节点，则证书可以不用拷贝，因为证书已经写入了kubeconfig文件】
root@k8s-cluster241 k8s]# data_rsync.sh /oldboyedu/certs/kubeconfig/
===== rsyncing k8s-cluster242: kubeconfig =====
命令执行成功!
===== rsyncing k8s-cluster243: kubeconfig =====
命令执行成功!
[root@k8s-cluster241 k8s]#
6.2 拷贝证书文件【由于我们的环境api-server和controller manager，scheduler并没有单独部署，因此所有节点都得有证书，以供api-server启动时使用
[root@k8s-cluster241 ~]# data_rsync.sh /oldboyedu/certs/k8s/
===== rsyncing k8s-cluster242: k8s =====
命令执行成功!
===== rsyncing k8s-cluster243: k8s =====
命令执行成功!
[root@k8s-cluster241 ~]#
6.3 查看所有节点kubeconfig目录组织结构
[root@k8s-cluster241 k8s]# tree /oldboyedu/
/oldboyedu/
├── certs
│ ├── etcd
│ │ ├── etcd-ca.csr
│ │ ├── etcd-ca-key.pem
│ │ ├── etcd-ca.pem
│ │ ├── etcd-server.csr
│ │ ├── etcd-server-key.pem
│ │ └── etcd-server.pem
│ ├── k8s
│ │ ├── admin.csr
│ │ ├── admin-key.pem
│ │ ├── admin.pem
│ │ ├── apiserver.csr
│ │ ├── apiserver-key.pem
│ │ ├── apiserver.pem
│ │ ├── controller-manager.csr
│ │ ├── controller-manager-key.pem
│ │ ├── controller-manager.pem
│ │ ├── front-proxy-ca.csr
│ │ ├── front-proxy-ca-key.pem
│ │ ├── front-proxy-ca.pem
│ │ ├── front-proxy-client.csr
│ │ ├── front-proxy-client-key.pem
│ │ ├── front-proxy-client.pem
│ │ ├── k8s-ca.csr
│ │ ├── k8s-ca-key.pem
│ │ ├── k8s-ca.pem
│ │ ├── kube-proxy.csr
│ │ ├── kube-proxy-key.pem
│ │ ├── kube-proxy.pem
│ │ ├── sa.key
│ │ ├── sa.pub
│ │ ├── scheduler.csr
│ │ ├── scheduler-key.pem
│ │ └── scheduler.pem
│ └── kubeconfig
│ ├── kube-admin.kubeconfig
│ ├── kube-controller-manager.kubeconfig
│ ├── kube-proxy.kubeconfig
│ └── kube-scheduler.kubeconfig
├── pki
│ ├── etcd
│ │ ├── ca-config.json
│ │ ├── etcd-ca-csr.json
│ │ └── etcd-csr.json
│ └── k8s
│ ├── admin-csr.json
│ ├── apiserver-csr.json
│ ├── controller-manager-csr.json
│ ├── front-proxy-ca-csr.json
│ ├── front-proxy-client-csr.json
│ ├── k8s-ca-config.json
│ ├── k8s-ca-csr.json
│ ├── kube-proxy-csr.json
│ └── scheduler-csr.json
└── softwares
└── etcd
└── etcd.config.yml
9 directories, 49 files
[root@k8s-cluster241 k8s]#
[root@k8s-cluster242 ~]# tree /oldboyedu/
/oldboyedu/
├── certs
│ ├── etcd
│ │ ├── etcd-ca.csr
│ │ ├── etcd-ca-key.pem
│ │ ├── etcd-ca.pem
│ │ ├── etcd-server.csr
│ │ ├── etcd-server-key.pem
│ │ └── etcd-server.pem
│ ├── k8s
│ │ ├── admin.csr
│ │ ├── admin-key.pem
│ │ ├── admin.pem
│ │ ├── apiserver.csr
│ │ ├── apiserver-key.pem
│ │ ├── apiserver.pem
│ │ ├── controller-manager.csr
│ │ ├── controller-manager-key.pem
│ │ ├── controller-manager.pem
│ │ ├── front-proxy-ca.csr
│ │ ├── front-proxy-ca-key.pem
│ │ ├── front-proxy-ca.pem
│ │ ├── front-proxy-client.csr
│ │ ├── front-proxy-client-key.pem
│ │ ├── front-proxy-client.pem
│ │ ├── k8s-ca.csr
│ │ ├── k8s-ca-key.pem
│ │ ├── k8s-ca.pem
│ │ ├── kube-proxy.csr
│ │ ├── kube-proxy-key.pem
│ │ ├── kube-proxy.pem
│ │ ├── sa.key
│ │ ├── sa.pub
│ │ ├── scheduler.csr
│ │ ├── scheduler-key.pem
│ │ └── scheduler.pem
│ └── kubeconfig
│ ├── kube-admin.kubeconfig
│ ├── kube-controller-manager.kubeconfig
│ ├── kube-proxy.kubeconfig
│ └── kube-scheduler.kubeconfig
└── softwares
└── etcd
└── etcd.config.yml
6 directories, 37 files
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# tree /oldboyedu/
/oldboyedu/
├── certs
│ ├── etcd
│ │ ├── etcd-ca.csr
│ │ ├── etcd-ca-key.pem
│ │ ├── etcd-ca.pem
│ │ ├── etcd-server.csr
│ │ ├── etcd-server-key.pem
│ │ └── etcd-server.pem
│ ├── k8s
│ │ ├── admin.csr
│ │ ├── admin-key.pem
│ │ ├── admin.pem
│ │ ├── apiserver.csr
│ │ ├── apiserver-key.pem
│ │ ├── apiserver.pem
│ │ ├── controller-manager.csr
│ │ ├── controller-manager-key.pem
│ │ ├── controller-manager.pem
│ │ ├── front-proxy-ca.csr
│ │ ├── front-proxy-ca-key.pem
│ │ ├── front-proxy-ca.pem
│ │ ├── front-proxy-client.csr
│ │ ├── front-proxy-client-key.pem
│ │ ├── front-proxy-client.pem
│ │ ├── k8s-ca.csr
│ │ ├── k8s-ca-key.pem
│ │ ├── k8s-ca.pem
│ │ ├── kube-proxy.csr
│ │ ├── kube-proxy-key.pem
│ │ ├── kube-proxy.pem
│ │ ├── sa.key
│ │ ├── sa.pub
│ │ ├── scheduler.csr
│ │ ├── scheduler-key.pem
│ │ └── scheduler.pem
│ └── kubeconfig
│ ├── kube-admin.kubeconfig
│ ├── kube-controller-manager.kubeconfig
│ ├── kube-proxy.kubeconfig
│ └── kube-scheduler.kubeconfig
└── softwares
└── etcd
└── etcd.config.yml
6 directories, 37 files
[root@k8s-cluster243 ~]#
启动ApiServer组件服务
1 'k8s-cluster241'节点启动ApiServer
温馨提示:
- "--advertise-address"是对应的master节点的IP地址;
- "--service-cluster-ip-range"对应的是svc的网段
- "--service-node-port-range"对应的是svc的NodePort端口范围;
- "--etcd-servers"指定的是etcd集群地址
配置文件参考链接:
https://kubernetes.io/zh-cn/docs/reference/command-line-tools-reference/kube-apiserver/
具体实操:
1.1 创建'k8s-cluster241'节点的配置文件
[root@k8s-cluster241 ~]# cat > /usr/lib/systemd/system/kube-apiserver.service << 'EOF'
[Unit]
Description=Jason Yin's Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \
--requestheader-allowed-names=front-proxy-client \
--v=2 \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--allow_privileged=true \
--advertise-address=10.0.0.241 \
--service-cluster-ip-range=10.200.0.0/16 \
--service-node-port-range=3000-50000 \
--etcd-servers=https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379 \
--etcd-cafile=/oldboyedu/certs/etcd/etcd-ca.pem \
--etcd-certfile=/oldboyedu/certs/etcd/etcd-server.pem \
--etcd-keyfile=/oldboyedu/certs/etcd/etcd-server-key.pem \
--client-ca-file=/oldboyedu/certs/k8s/k8s-ca.pem \
--tls-cert-file=/oldboyedu/certs/k8s/apiserver.pem \
--tls-private-key-file=/oldboyedu/certs/k8s/apiserver-key.pem \
--kubelet-client-certificate=/oldboyedu/certs/k8s/apiserver.pem \
--kubelet-client-key=/oldboyedu/certs/k8s/apiserver-key.pem \
--service-account-key-file=/oldboyedu/certs/k8s/sa.pub \
--service-account-signing-key-file=/oldboyedu/certs/k8s/sa.key \
--service-account-issuer=https://kubernetes.default.svc.oldboyedu.com \
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
--authorization-mode=Node,RBAC \
--enable-bootstrap-token-auth=true \
--requestheader-client-ca-file=/oldboyedu/certs/k8s/front-proxy-ca.pem \
--proxy-client-cert-file=/oldboyedu/certs/k8s/front-proxy-client.pem \
--proxy-client-key-file=/oldboyedu/certs/k8s/front-proxy-client-key.pem \
--requestheader-allowed-names=aggregator \
--requestheader-group-headers=X-Remote-Group \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-username-headers=X-Remote-User
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF
1.2 启动服务
systemctl daemon-reload && systemctl enable --now kube-apiserver
systemctl status kube-apiserver
ss -ntl | grep 6443
1.3 etcd数据库测试验证【发现启动api-server后，该组件就直接会和etcd进行数据交互。】
[root@k8s-cluster242 ~]# etcdctl get "" --prefix --keys-only | head
/registry/apiregistration.k8s.io/apiservices/v1.
/registry/apiregistration.k8s.io/apiservices/v1.admissionregistration.k8s.io
/registry/apiregistration.k8s.io/apiservices/v1.apiextensions.k8s.io
/registry/apiregistration.k8s.io/apiservices/v1.apps
/registry/apiregistration.k8s.io/apiservices/v1.authentication.k8s.io
[root@k8s-cluster242 ~]#
[root@k8s-cluster242 ~]# etcdctl get "" --prefix --keys-only | wc -l
370
[root@k8s-cluster242 ~]#
2 'k8s-cluster242'节点启动ApiServer
温馨提示:
如果该节点api-server无法启动，请检查日志，尤其是证书文件需要api-server启动时要直接加载etcd，sa，api-server，客户端认证等相关证书。
如果证书没有同步，可以执行命令'data_rsync.sh /oldboyedu/certs/k8s/'自动进行同步。
具体实操:
2.1 创建'k8s-cluster242'节点的配置文件
[root@k8s-cluster242 ~]# cat > /usr/lib/systemd/system/kube-apiserver.service << 'EOF'
[Unit]
Description=Jason Yin's Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \
--requestheader-allowed-names=front-proxy-client \
--v=2 \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--allow_privileged=true \
--advertise-address=10.0.0.242 \
--service-cluster-ip-range=10.200.0.0/16 \
--service-node-port-range=3000-50000 \
--etcd-servers=https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379 \
--etcd-cafile=/oldboyedu/certs/etcd/etcd-ca.pem \
--etcd-certfile=/oldboyedu/certs/etcd/etcd-server.pem \
--etcd-keyfile=/oldboyedu/certs/etcd/etcd-server-key.pem \
--client-ca-file=/oldboyedu/certs/k8s/k8s-ca.pem \
--tls-cert-file=/oldboyedu/certs/k8s/apiserver.pem \
--tls-private-key-file=/oldboyedu/certs/k8s/apiserver-key.pem \
--kubelet-client-certificate=/oldboyedu/certs/k8s/apiserver.pem \
--kubelet-client-key=/oldboyedu/certs/k8s/apiserver-key.pem \
--service-account-key-file=/oldboyedu/certs/k8s/sa.pub \
--service-account-signing-key-file=/oldboyedu/certs/k8s/sa.key \
--service-account-issuer=https://kubernetes.default.svc.oldboyedu.com \
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
--authorization-mode=Node,RBAC \
--enable-bootstrap-token-auth=true \
--requestheader-client-ca-file=/oldboyedu/certs/k8s/front-proxy-ca.pem \
--proxy-client-cert-file=/oldboyedu/certs/k8s/front-proxy-client.pem \
--proxy-client-key-file=/oldboyedu/certs/k8s/front-proxy-client-key.pem \
--requestheader-allowed-names=aggregator \
--requestheader-group-headers=X-Remote-Group \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-username-headers=X-Remote-User
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF2.2 启动服务
systemctl daemon-reload && systemctl enable --now kube-apiserver
systemctl status kube-apiserver
ss -ntl | grep 6443
3.'k8s-cluster243'节点启动ApiServer
3.1 创建'k8s-cluster243'节点的配置文件
[root@k8s-cluster243 ~]# cat > /usr/lib/systemd/system/kube-apiserver.service << 'EOF'
[Unit]
Description=Jason Yin's Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \
--requestheader-allowed-names=front-proxy-client \
--v=2 \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--allow_privileged=true \
--advertise-address=10.0.0.243 \
--service-cluster-ip-range=10.200.0.0/16 \
--service-node-port-range=3000-50000 \
--etcd-servers=https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379 \
--etcd-cafile=/oldboyedu/certs/etcd/etcd-ca.pem \
--etcd-certfile=/oldboyedu/certs/etcd/etcd-server.pem \
--etcd-keyfile=/oldboyedu/certs/etcd/etcd-server-key.pem \
--client-ca-file=/oldboyedu/certs/k8s/k8s-ca.pem \
--tls-cert-file=/oldboyedu/certs/k8s/apiserver.pem \
--tls-private-key-file=/oldboyedu/certs/k8s/apiserver-key.pem \
--kubelet-client-certificate=/oldboyedu/certs/k8s/apiserver.pem \
--kubelet-client-key=/oldboyedu/certs/k8s/apiserver-key.pem \
--service-account-key-file=/oldboyedu/certs/k8s/sa.pub \
--service-account-signing-key-file=/oldboyedu/certs/k8s/sa.key \
--service-account-issuer=https://kubernetes.default.svc.oldboyedu.com \
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
--authorization-mode=Node,RBAC \
--enable-bootstrap-token-auth=true \
--requestheader-client-ca-file=/oldboyedu/certs/k8s/front-proxy-ca.pem \
--proxy-client-cert-file=/oldboyedu/certs/k8s/front-proxy-client.pem \
--proxy-client-key-file=/oldboyedu/certs/k8s/front-proxy-client-key.pem \
--requestheader-allowed-names=aggregator \
--requestheader-group-headers=X-Remote-Group \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-username-headers=X-Remote-User
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF3.2 启动服务
systemctl daemon-reload && systemctl enable --now kube-apiserver
systemctl status kube-apiserver
ss -ntl | grep 6443
3.3 再次查看数据
[root@k8s-cluster242 ~]# etcdctl get "" --prefix --keys-only | wc -l
378
[root@k8s-cluster242 ~]#
启动Controler Manager组件服务
1 所有节点创建配置文件
温馨提示:
- "--cluster-cidr"是Pod的网段地址，我们可以自行修改。
配置文件参考链接:
https://kubernetes.io/zh-cn/docs/reference/command-line-tools-reference/kube-controller-manager/
所有节点的controller-manager组件配置文件相同: (前提是证书文件存放的位置也要相同哟!)
cat > /usr/lib/systemd/system/kube-controller-manager.service << 'EOF'
[Unit]
Description=Jason Yin's Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
--v=2 \
--root-ca-file=/oldboyedu/certs/k8s/k8s-ca.pem \
--cluster-signing-cert-file=/oldboyedu/certs/k8s/k8s-ca.pem \
--cluster-signing-key-file=/oldboyedu/certs/k8s/k8s-ca-key.pem \
--service-account-private-key-file=/oldboyedu/certs/k8s/sa.key \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-controller-manager.kubeconfig \
--leader-elect=true \
--use-service-account-credentials=true \
--node-monitor-grace-period=40s \
--node-monitor-period=5s \
--controllers=*,bootstrapsigner,tokencleaner \
--allocate-node-cidrs=true \
--cluster-cidr=10.100.0.0/16 \
--requestheader-client-ca-file=/oldboyedu/certs/k8s/front-proxy-ca.pem \
--node-cidr-mask-size=24
Restart=always
RestartSec=10s
[Install]
WantedBy=multi-user.target
EOF2.启动controller-manager服务
systemctl daemon-reload
systemctl enable --now kube-controller-manager
systemctl status kube-controller-manager
ss -ntl | grep 10257
启动Scheduler组件服务
1 所有节点创建配置文件
配置文件参考链接:
https://kubernetes.io/zh-cn/docs/reference/command-line-tools-reference/kube-scheduler/
所有节点的Scheduler组件配置文件相同: (前提是证书文件存放的位置也要相同哟!)
cat > /usr/lib/systemd/system/kube-scheduler.service <<'EOF'
[Unit]
Description=Jason Yin's Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-scheduler \
--v=2 \
--leader-elect=true \
--kubeconfig=/oldboyedu/certs/kubeconfig/kube-scheduler.kubeconfig
Restart=always
RestartSec=10s
[Install]
WantedBy=multi-user.target
EOF
2.启动scheduler服务
systemctl daemon-reload
systemctl enable --now kube-scheduler
systemctl status kube-scheduler
ss -ntl | grep 10259
高可用组件haproxy+keepalived安装及验证
1 所有master【k8s-cluster24[1-3]】节点安装高可用组件温馨提示: - 对于高可用组件，其实我们也可以单独找两台虚拟机来部署，但我为了节省2台机器，就直接在master节点复用了。 - 如果在云上安装K8S则无安装高可用组件了，毕竟公有云大部分都是不支持keepalived的，可以直接使用云产品，比如阿里的"SLB"，腾讯的"ELB"等SAAS产品; - 推荐使用ELB，SLB有回环的问题，也就是SLB代理的服务器不能反向访问SLB，但是腾讯云修复了这个问题;
具体实操:
[root@k8s-cluster241 ~]# cat /etc/apt/sources.list
# 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy main restricted universe multiverse
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy-updates main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy-updates main restricted universe multiverse
deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy-backports main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy-backports main restricted universe multiverse
# 以下安全更新软件源包含了官方源与镜像站配置，如有需要可自行修改注释切换
deb http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse
# deb-src http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse
# 预发布软件源，不建议启用
# deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy-proposed main restricted universe multiverse
# # deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy-proposed main restricted universe multiverse
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# apt update
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# apt-get -y install keepalived haproxy
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# data_rsync.sh /etc/apt/sources.list
===== rsyncing k8s-cluster242: sources.list =====
命令执行成功!
===== rsyncing k8s-cluster243: sources.list =====
命令执行成功!
[root@k8s-cluster241 ~]#
[root@k8s-cluster242 ~]# apt update
[root@k8s-cluster242 ~]#
[root@k8s-cluster242 ~]# apt-get -y install keepalived haproxy
[root@k8s-cluster243 ~]# apt update
[root@k8s-cluster243 ~]# apt-get -y install keepalived haproxy
2.所有master节点配置haproxy
温馨提示:
- haproxy的负载均衡器监听地址我配置是8443，你可以修改为其他端口，haproxy会用来反向代理各个master组件的地址;
- 如果你真的修改晴一定注意上面的证书配置的kubeconfig文件，也要一起修改，否则就会出现链接集群失败的问题;
具体实操:2.1 备份配置文件
cat > /etc/haproxy/haproxy.cfg <<'EOF'
global
maxconn 2000
ulimit-n 16384
log 127.0.0.1 local0 err
stats timeout 30s
defaults
log global
mode http
option httplog
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-request 15s
timeout http-keep-alive 15s
frontend monitor-haproxy
bind *:9999
mode http
option httplog
monitor-uri /ruok
frontend yinzhengjie-k8s
bind 0.0.0.0:8443
bind 127.0.0.1:8443
mode tcp
option tcplog
tcp-request inspect-delay 5s
default_backend yinzhengjie-k8s
backend yinzhengjie-k8s
mode tcp
option tcplog
option tcp-check
balance roundrobin
default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
server k8s-cluster241 10.0.0.241:6443 check
server k8s-cluster242 10.0.0.242:6443 check
server k8s-cluster243 10.0.0.243:6443 check
EOF
3.所有master节点配置keepalived
温馨提示:
- 注意"interface"字段为你的物理网卡的名称，如果你的网卡是ens33，请将"eth0"修改为"ens33"哟;
- 注意"mcast_src_ip"各master节点的配置均不相同，修改根据实际环境进行修改哟;
- 注意"virtual_ipaddress"指定的是负载均衡器的VIP地址，这个地址也要和kubeconfig文件的Apiserver地址要一致哟;
- 注意"script"字段的脚本用于检测后端的apiServer是否健康;
- 注意"router_id"字段为节点ip，master每个节点配置自己的IP
具体实操:3.1."k8s-cluster241"节点创建配置文件
root@k8s-cluster241 ~]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.0.0.241 netmask 255.255.255.0 broadcast 10.0.0.255
inet6 fe80::20c:29ff:fe4c:af81 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:4c:af:81 txqueuelen 1000 (Ethernet)
RX packets 955748 bytes 496681590 (496.6 MB)
RX errors 0 dropped 24 overruns 0 frame 0
TX packets 873509 bytes 357603229 (357.6 MB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
...
[root@k8s-cluster241 ~]# cat > /etc/keepalived/keepalived.conf <<'EOF'
! Configuration File for keepalived
global_defs {
router_id 10.0.0.241
}
vrrp_script chk_lb {
script "/etc/keepalived/check_port.sh 8443"
interval 2
weight -20
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 251
priority 100
advert_int 1
mcast_src_ip 10.0.0.241
nopreempt
authentication {
auth_type PASS
auth_pass yinzhengjie_k8s
}
track_script {
chk_lb
}
virtual_ipaddress {
10.0.0.240
}
}
EOF3.2."k8s-cluster242"节点创建配置文件
[root@k8s-cluster242 ~]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.0.0.242 netmask 255.255.255.0 broadcast 10.0.0.255
inet6 fe80::20c:29ff:fe97:a11 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:97:0a:11 txqueuelen 1000 (Ethernet)
RX packets 544582 bytes 213769468 (213.7 MB)
RX errors 0 dropped 24 overruns 0 frame 0
TX packets 453156 bytes 75867279 (75.8 MB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
...
[root@k8s-cluster242 ~]# cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
router_id 10.0.0.242
}
vrrp_script chk_lb {
script "/etc/keepalived/check_port.sh 8443"
interval 2
weight -20
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 251
priority 100
advert_int 1
mcast_src_ip 10.0.0.242
nopreempt
authentication {
auth_type PASS
auth_pass yinzhengjie_k8s
}
track_script {
chk_lb
}
virtual_ipaddress {
10.0.0.240
}
}
EOF
3.3."k8s-cluster243"节点创建配置文件
[root@k8s-cluster243 ~]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.0.0.243 netmask 255.255.255.0 broadcast 10.0.0.255
inet6 fe80::20c:29ff:fec9:46ff prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:c9:46:ff txqueuelen 1000 (Ethernet)
RX packets 918814 bytes 774698270 (774.6 MB)
RX errors 0 dropped 24 overruns 0 frame 0
TX packets 495407 bytes 78301697 (78.3 MB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
...
[root@k8s-cluster243 ~]# cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
router_id 10.0.0.243
}
vrrp_script chk_lb {
script "/etc/keepalived/check_port.sh 8443"
interval 2
weight -20
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 251
priority 100
advert_int 1
mcast_src_ip 10.0.0.243
nopreempt
authentication {
auth_type PASS
auth_pass yinzhengjie_k8s
}
track_script {
chk_lb
}
virtual_ipaddress {
10.0.0.240
}
}
EOF
3.4.所有keepalived节点均需要创建健康检查脚本
cat > /etc/keepalived/check_port.sh <<'EOF'
#!/bin/bash
CHK_PORT=$1
if [ -n "$CHK_PORT" ];then
PORT_PROCESS=`ss -lt|grep $CHK_PORT|wc -l`
if [ $PORT_PROCESS -eq 0 ];then
echo "Port $CHK_PORT Is Not Used,End."
systemctl stop keepalived
fi
else
echo "Check Port Cant Be Empty!"
fi
EOF
chmod +x /etc/keepalived/check_port.sh
4.验证haproxy服务并验证
4.1 所有节点启动haproxy服务
systemctl enable --now haproxy
systemctl restart haproxy
systemctl status haproxy
ss -ntl | egrep "8443|9999"
4.2 基于webUI进行验证
[root@k8s-cluster241 ~]# curl http://10.0.0.241:9999/ruok
<html><body><h1>200 OK</h1>
Service ready.
</body></html>
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# curl http://10.0.0.242:9999/ruok
<html><body><h1>200 OK</h1>
Service ready.
</body></html>
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# curl http://10.0.0.243:9999/ruok
<html><body><h1>200 OK</h1>
Service ready.
</body></html>
[root@k8s-cluster241 ~]#
5.启动keepalived服务并验证
5.1.所有节点启动keepalived服务
[root@k8s-cluster241 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:4c:af:81 brd ff:ff:ff:ff:ff:ff
altname enp2s1
altname ens33
inet 10.0.0.241/24 brd 10.0.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet 10.0.0.240/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe4c:af81/64 scope link
valid_lft forever preferred_lft forever
3: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
[root@k8s-cluster241 ~]#
5.3 基于telnet验证haporxy是否正常
[root@k8s-cluster242 ~]# telnet 10.0.0.240 8443
Trying 10.0.0.240...
Connected to 10.0.0.240.
Escape character is '^]'.
[root@k8s-cluster243 ~]# ping 10.0.0.240 -c 3
PING 10.0.0.240 (10.0.0.240) 56(84) bytes of data.
64 bytes from 10.0.0.240: icmp_seq=1 ttl=64 time=0.132 ms
64 bytes from 10.0.0.240: icmp_seq=2 ttl=64 time=0.211 ms
64 bytes from 10.0.0.240: icmp_seq=3 ttl=64 time=0.202 ms
--- 10.0.0.240 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2038ms
rtt min/avg/max/mdev = 0.132/0.181/0.211/0.035 ms
[root@k8s-cluster243 ~]#
5.4 将VIP节点的haproxy停止
[root@k8s-cluster241 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:4c:af:81 brd ff:ff:ff:ff:ff:ff
altname enp2s1
altname ens33
inet 10.0.0.241/24 brd 10.0.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet 10.0.0.240/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe4c:af81/64 scope link
valid_lft forever preferred_lft forever
3: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# systemctl stop haproxy.service # 停止服务
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:4c:af:81 brd ff:ff:ff:ff:ff:ff
altname enp2s1
altname ens33
inet 10.0.0.241/24 brd 10.0.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe4c:af81/64 scope link
valid_lft forever preferred_lft forever
3: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
[root@k8s-cluster241 ~]#
5.5 观察VIP是否飘逸
[root@k8s-cluster242 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:97:0a:11 brd ff:ff:ff:ff:ff:ff
altname enp2s1
altname ens33
inet 10.0.0.242/24 brd 10.0.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe97:a11/64 scope link
valid_lft forever preferred_lft forever
3: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:c9:46:ff brd ff:ff:ff:ff:ff:ff
altname enp2s1
altname ens33
inet 10.0.0.243/24 brd 10.0.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet 10.0.0.240/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fec9:46ff/64 scope link
valid_lft forever preferred_lft forever
3: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
[root@k8s-cluster243 ~]#
5.6 如果长期ping测试的话会看到如下的提示
[root@k8s-cluster241 ~]# ping 10.0.0.240
PING 10.0.0.240 (10.0.0.240) 56(84) bytes of data.
64 bytes from 10.0.0.240: icmp_seq=1 ttl=64 time=0.016 ms
64 bytes from 10.0.0.240: icmp_seq=2 ttl=64 time=0.024 ms
64 bytes from 10.0.0.240: icmp_seq=3 ttl=64 time=0.024 ms
64 bytes from 10.0.0.240: icmp_seq=4 ttl=64 time=0.024 ms
64 bytes from 10.0.0.240: icmp_seq=5 ttl=64 time=0.020 ms
64 bytes from 10.0.0.240: icmp_seq=6 ttl=64 time=0.024 ms
64 bytes from 10.0.0.240: icmp_seq=7 ttl=64 time=0.022 ms
64 bytes from 10.0.0.240: icmp_seq=8 ttl=64 time=403 ms
From 10.0.0.242 icmp_seq=9 Redirect Host(New nexthop: 10.0.0.240)
64 bytes from 10.0.0.240: icmp_seq=9 ttl=64 time=0.275 ms
64 bytes from 10.0.0.240: icmp_seq=10 ttl=64 time=0.202 ms
64 bytes from 10.0.0.240: icmp_seq=11 ttl=64 time=0.249 ms
From 10.0.0.242 icmp_seq=12 Redirect Host(New nexthop: 10.0.0.240)
64 bytes from 10.0.0.240: icmp_seq=12 ttl=64 time=0.292 ms
64 bytes from 10.0.0.240: icmp_seq=13 ttl=64 time=0.314 ms
64 bytes from 10.0.0.240: icmp_seq=14 ttl=64 time=0.164 ms
64 bytes from 10.0.0.240: icmp_seq=15 ttl=64 time=0.158 ms
64 bytes from 10.0.0.240: icmp_seq=16 ttl=64 time=0.170 ms
64 bytes from 10.0.0.240: icmp_seq=17 ttl=64 time=0.152 ms
64 bytes from 10.0.0.240: icmp_seq=18 ttl=64 time=0.227 ms
64 bytes from 10.0.0.240: icmp_seq=19 ttl=64 time=0.141 ms
64 bytes from 10.0.0.240: icmp_seq=20 ttl=64 time=0.558 ms
64 bytes from 10.0.0.240: icmp_seq=21 ttl=64 time=0.173 ms
64 bytes from 10.0.0.240: icmp_seq=22 ttl=64 time=0.165 ms
64 bytes from 10.0.0.240: icmp_seq=23 ttl=64 time=0.178 ms
64 bytes from 10.0.0.240: icmp_seq=24 ttl=64 time=0.199 ms
^C
--- 10.0.0.240 ping statistics ---
24 packets transmitted, 24 received, +2 errors, 0% packet loss, time 23510ms
rtt min/avg/max/mdev = 0.016/16.932/402.600/80.417 ms
[root@k8s-cluster241 ~]#
5.7 可以尝试将停止的服务启动观察VIP是否飘逸回来
很明显，经过测试验证，VIP并不会飘逸回来
启动服务参考命令:
systemctl start keepalived haproxy
systemctl status keepalived haproxy
配置K8S管理节点并自动补全功能
1.测试验证
[root@k8s-cluster241 ~]# kubectl cluster-info --kubeconfig=/oldboyedu/certs/kubeconfig/kube-admin.kubeconfig
Kubernetes control plane is running at https://10.0.0.240:8443
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl get cs --kubeconfig=/oldboyedu/certs/kubeconfig/kube-admin.kubeconfig
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-0 Healthy ok
[root@k8s-cluster241 ~]#
2.所有master节点操作
mkdir -p $HOME/.kube
cp -i /oldboyedu/certs/kubeconfig/kube-admin.kubeconfig $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config
3.测试验证
[root@k8s-cluster241 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy ok
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl cluster-info
Kubernetes control plane is running at https://10.0.0.240:8443
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.
[root@k8s-cluster241 ~]#
[root@k8s-cluster242 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy ok
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy ok
[root@k8s-cluster243 ~]#
4.所有master节点配置自动补全功能
kubectl completion bash > ~/.kube/completion.bash.inc
echo source '$HOME/.kube/completion.bash.inc' >> ~/.bashrc
source ~/.bashrc5.测试验证自动补全功能
[root@k8s-cluster241 ~]# kubectl
annotate (Update the annotations on a resource)
api-resources (Print the supported API resources on the server)
api-versions (Print the supported API versions on the server, in the form of "group/version")
apply (Apply a configuration to a resource by file name or stdin)
attach (Attach to a running container)
auth (Inspect authorization)
autoscale (Auto-scale a deployment, replica set, stateful set, or replication controller)
certificate (Modify certificate resources)
cluster-info (Display cluster information)
completion (Output shell completion code for the specified shell (bash, zsh, fish, or powershell))
config (Modify kubeconfig files)
cordon (Mark node as unschedulable)
cp (Copy files and directories to and from containers)
create (Create a resource from a file or from stdin)
debug (Create debugging sessions for troubleshooting workloads and nodes)
delete (Delete resources by file names, stdin, resources and names, or by resources and label selector)
describe (Show details of a specific resource or group of resources)
diff (Diff the live version against a would-be applied version)
drain (Drain node in preparation for maintenance)
edit (Edit a resource on the server)
events (List events)
exec (Execute a command in a container)
explain (Get documentation for a resource)
expose (Take a replication controller, service, deployment or pod and expose it as a new Kubernetes service)
get (Display one or many resources)
help (Help about any command)
kustomize (Build a kustomization target from a directory or URL)
--More--
启动kubelet服务组件
1.创建Bootstrapping自动颁发kubelet证书配置
温馨提示:
- "--server"指向的是负载均衡器的IP地址，由负载均衡器对master节点进行反向代理哟。
- "--token"也可以自定义，但也要同时修改"bootstrap"的Secret的"token-id"和"token-secret"对应值哟;
1.1 设置集群
[root@k8s-cluster241 ~]# kubectl config set-cluster yinzhengjie-k8s \
--certificate-authority=/oldboyedu/certs/k8s/k8s-ca.pem \
--embed-certs=true \
--server=https://10.0.0.240:8443 \
--kubeconfig=/oldboyedu/certs/kubeconfig/bootstrap-kubelet.kubeconfig
1.2 创建用户
[root@k8s-cluster241 ~]# kubectl config set-credentials tls-bootstrap-token-user \
--token=oldboy.jasonyinzhengjie \
--kubeconfig=/oldboyedu/certs/kubeconfig/bootstrap-kubelet.kubeconfig
1.3 将集群和用户进行绑定
[root@k8s-cluster241 ~]# kubectl config set-context tls-bootstrap-token-user@kubernetes \
--cluster=yinzhengjie-k8s \
--user=tls-bootstrap-token-user \
--kubeconfig=/oldboyedu/certs/kubeconfig/bootstrap-kubelet.kubeconfig1.4.配置默认的上下文
[root@k8s-cluster241 ~]# kubectl config use-context tls-bootstrap-token-user@kubernetes \
--kubeconfig=/oldboyedu/certs/kubeconfig/bootstrap-kubelet.kubeconfig
1.5 查看kubeconfig资源结构
[root@k8s-cluster241 ~]# kubectl config view --kubeconfig=/oldboyedu/certs/kubeconfig/bootstrap-kubelet.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://10.0.0.240:8443
name: yinzhengjie-k8s
contexts:
- context:
cluster: yinzhengjie-k8s
user: tls-bootstrap-token-user
name: tls-bootstrap-token-user@kubernetes
current-context: tls-bootstrap-token-user@kubernetes
kind: Config
preferences: {}
users:
- name: tls-bootstrap-token-user
user:
token: REDACTED
[root@k8s-cluster241 ~]# 1.6 拷贝kubelet的Kubeconfig文件
[root@k8s-cluster241 ~]# data_rsync.sh /oldboyedu/certs/kubeconfig/bootstrap-kubelet.kubeconfig
2 创建bootstrap-secret授权
2.1 创建配bootstrap-secret文件用于授权
[root@k8s-cluster241 ~]# cat > bootstrap-secret.yaml <<EOF
apiVersion: v1
kind: Secret
metadata:
name: bootstrap-token-oldboy
namespace: kube-system
type: bootstrap.kubernetes.io/token
stringData:
description: "The default bootstrap token generated by 'kubelet '."
token-id: oldboy
token-secret: jasonyinzhengjie
usage-bootstrap-authentication: "true"
usage-bootstrap-signing: "true"
auth-extra-groups: system:bootstrappers:default-node-token,system:bootstrappers:worker,system:bootstrappers:ingress
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubelet-bootstrap
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:node-bootstrapper
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:bootstrappers:default-node-token
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: node-autoapprove-bootstrap
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:bootstrappers:default-node-token
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: node-autoapprove-certificate-rotation
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:nodes
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:kube-apiserver-to-kubelet
rules:
- apiGroups:
- ""
resources:
- nodes/proxy
- nodes/stats
- nodes/log
- nodes/spec
- nodes/metrics
verbs:
- "*"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:kube-apiserver
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kube-apiserver-to-kubelet
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: kube-apiserver
EOF
2.2.应用bootstrap-secret配置文件
[root@k8s-cluster241 ~]# etcdctl get "/" --prefix --keys-only | wc -l
476
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl apply -f bootstrap-secret.yaml
secret/bootstrap-token-oldboy created
clusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created
clusterrolebinding.rbac.authorization.k8s.io/node-autoapprove-bootstrap created
clusterrolebinding.rbac.authorization.k8s.io/node-autoapprove-certificate-rotation created
clusterrole.rbac.authorization.k8s.io/system:kube-apiserver-to-kubelet created
clusterrolebinding.rbac.authorization.k8s.io/system:kube-apiserver created
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# etcdctl get "/" --prefix --keys-only | wc -l
488
[root@k8s-cluster241 ~]#
3.部署worker节点之kubelet启动实战
温馨提示:
- 在"10-kubelet.con"文件中使用"--kubeconfig"指定的"kubelet.kubeconfig"文件并不存在，这个证书文件后期会自动生成;
- 对于"clusterDNS"是NDS地址，我们可以自定义，比如"10.200.0.254";
- “clusterDomain”对应的是域名信息，要和我们设计的集群保持一致，比如"yinzhengjie.com";
- "10-kubelet.conf"文件中的"ExecStart="需要写2次，否则可能无法启动kubelet;
具体实操:3.1 所有节点创建工作目录
mkdir -p /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/3.2 所有节点创建kubelet的配置文件
cat > /etc/kubernetes/kubelet-conf.yml <<'EOF'
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /oldboyedu/certs/k8s/k8s-ca.pem
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- 10.200.0.254
clusterDomain: oldboyedu.com
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/kubernetes/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
EOF
3.3 所有节点配置kubelet service
cat > /usr/lib/systemd/system/kubelet.service <<'EOF'
[Unit]
Description=JasonYin's Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=containerd.service
Requires=containerd.service
[Service]
ExecStart=/usr/local/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF3.4 所有节点配置kubelet service的配置文件
cat > /etc/systemd/system/kubelet.service.d/10-kubelet.conf <<'EOF'
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/oldboyedu/certs/kubeconfig/bootstrap-kubelet.kubeconfig --kubeconfig=/oldboyedu/certs/kubeconfig/kubelet.kubeconfig"
Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml"
Environment="KUBELET_SYSTEM_ARGS=--container-runtime-endpoint=unix:///run/containerd/containerd.sock"
Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node='' "
ExecStart=
ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_SYSTEM_ARGS $KUBELET_EXTRA_ARGS
EOF3.5 启动所有节点kubelet
systemctl daemon-reload
systemctl enable --now kubelet
systemctl status kubelet4. 在所有master节点上查看nodes信息。
4.1 'k8s-cluster241'查看worker节点列表
[root@k8s-cluster241 ~]# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-cluster241 NotReady <none> 100s v1.33.1 10.0.0.241 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
k8s-cluster242 NotReady <none> 8s v1.33.1 10.0.0.242 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
k8s-cluster243 NotReady <none> 2s v1.33.1 10.0.0.243 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
[root@k8s-cluster241 ~]#
4.2 'k8s-cluster242'查看worker节点列表
[root@k8s-cluster242 ~]# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-cluster241 NotReady <none> 2m3s v1.33.1 10.0.0.241 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
k8s-cluster242 NotReady <none> 31s v1.33.1 10.0.0.242 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
k8s-cluster243 NotReady <none> 25s v1.33.1 10.0.0.243 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
[root@k8s-cluster242 ~]# 4.3 'k8s-cluster243'查看worker节点列表
[root@k8s-cluster243 ~]# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-cluster241 NotReady <none> 2m15s v1.33.1 10.0.0.241 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
k8s-cluster242 NotReady <none> 43s v1.33.1 10.0.0.242 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
k8s-cluster243 NotReady <none> 37s v1.33.1 10.0.0.243 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
[root@k8s-cluster243 ~]# 5.可以查看到有相应的csr用户客户端的证书请求
[root@k8s-cluster243 ~]# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR REQUESTEDDURATION CONDITION
csr-2qmw7 62s kubernetes.io/kube-apiserver-client-kubelet system:bootstrap:oldboy <none> Approved,Issued
csr-qcvjf 2m40s kubernetes.io/kube-apiserver-client-kubelet system:bootstrap:oldboy <none> Approved,Issued
csr-rrp6d 69s kubernetes.io/kube-apiserver-client-kubelet system:bootstrap:oldboy <none> Approved,Issued
[root@k8s-cluster243 ~]#
启动kube-proxy组件服务
1.所有节点创建kube-proxy.conf配置文件
cat > /etc/kubernetes/kube-proxy.yml << EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
metricsBindAddress: 127.0.0.1:10249
clientConnection:
acceptConnection: ""
burst: 10
contentType: application/vnd.kubernetes.protobuf
kubeconfig: /oldboyedu/certs/kubeconfig/kube-proxy.kubeconfig
qps: 5
clusterCIDR: 10.100.0.0/16
configSyncPeriod: 15m0s
conntrack:
max: null
maxPerCore: 32768
min: 131072
tcpCloseWaitTimeout: 1h0m0s
tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
masqueradeAll: false
masqueradeBit: 14
minSyncPeriod: 0s
ipvs:
masqueradeAll: true
minSyncPeriod: 5s
scheduler: "rr"
syncPeriod: 30s
mode: "ipvs"
nodeProtAddress: null
oomScoreAdj: -999
portRange: ""
udpIdelTimeout: 250ms
EOF
2.所有节点使用systemd管理kube-proxy
cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Jason Yin's Kubernetes Proxy
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-proxy \
--config=/etc/kubernetes/kube-proxy.yml \
--v=2
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF3.启动kube-proxy服务之前查看etcd是数据统计
[root@k8s-cluster243 ~]# etcdctl get "/" --prefix --keys-only | wc -l
560
[root@k8s-cluster243 ~]#
4.所有节点启动kube-proxy
systemctl daemon-reload && systemctl enable --now kube-proxy
systemctl status kube-proxy
ss -ntl |egrep "10256|10249"5.启动kube-proxy服务之后再次查看etcd是数据统计【说明kube-proxy服务启动后，其实也会和api-server通信，并由api-server写入数据到etcd中】
[root@k8s-cluster243 ~]# etcdctl get "/" --prefix --keys-only | wc -l
566
[root@k8s-cluster243 ~]#
6.集群关机拍快照(需要部署不同的网络插件)
部署Calico实战
参考链接:
https://docs.tigera.io/calico/latest/getting-started/kubernetes/quickstart#step-2-install-calico
1.所有节点准备宿主机DNS解析文件
[root@k8s-cluster241 ~]# cat > /etc/kubernetes/resolv.conf <<EOF
nameserver 223.5.5.5
options edns0 trust-ad
search .
EOF
[root@k8s-cluster241 ~]# data_rsync.sh /etc/kubernetes/resolv.conf
2.下载资源清单
[root@k8s-cluster241 ~]# wget https://raw.githubusercontent.com/projectcalico/calico/v3.30.2/manifests/tigera-operator.yaml
[root@k8s-cluster241 ~]# wget https://raw.githubusercontent.com/projectcalico/calico/v3.30.2/manifests/custom-resources.yaml
3.导入镜像
[root@k8s-cluster241 ~]# kubectl apply -f tigera-operator.yaml
namespace/tigera-operator created
serviceaccount/tigera-operator created
clusterrole.rbac.authorization.k8s.io/tigera-operator-secrets created
clusterrole.rbac.authorization.k8s.io/tigera-operator created
clusterrolebinding.rbac.authorization.k8s.io/tigera-operator created
rolebinding.rbac.authorization.k8s.io/tigera-operator-secrets created
deployment.apps/tigera-operator created
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl get pods -A -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
tigera-operator tigera-operator-747864d56d-z6d2m 1/1 Running 0 5m52s 10.0.0.241 k8s-cluster241 <none> <none>
[root@k8s-cluster241 ~]#
4.通过创建必要的自定义资源来安装Calico。
[root@k8s-cluster241 ~]# grep 16 custom-resources.yaml
cidr: 192.168.0.0/16
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# sed -i '/16/s#192.168#10.100#' custom-resources.yaml
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# grep 16 custom-resources.yaml
cidr: 10.100.0.0/16
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# grep blockSize custom-resources.yaml
blockSize: 26
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# sed -i '/blockSize/s#26#24#' custom-resources.yaml
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# grep blockSize custom-resources.yaml
blockSize: 24
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl create -f custom-resources.yaml
installation.operator.tigera.io/default created
apiserver.operator.tigera.io/default created
goldmane.operator.tigera.io/default created
whisker.operator.tigera.io/default created
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]#
5.检查Pod是否就绪
[root@k8s-cluster241 ~]# kubectl get pods -A -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
calico-apiserver calico-apiserver-68b989c5c9-7wkpv 1/1 Running 0 68s 10.100.88.7 k8s-cluster243 <none> <none>
calico-apiserver calico-apiserver-68b989c5c9-hzgqr 1/1 Running 0 68s 10.100.99.6 k8s-cluster242 <none> <none>
calico-system calico-kube-controllers-74544dd8d8-sjc6g 1/1 Running 0 68s 10.100.88.8 k8s-cluster243 <none> <none>
calico-system calico-node-p24ql 1/1 Running 0 62s 10.0.0.242 k8s-cluster242 <none> <none>
calico-system calico-node-qlr7r 1/1 Running 0 64s 10.0.0.243 k8s-cluster243 <none> <none>
calico-system calico-node-w6cmr 1/1 Running 0 67s 10.0.0.241 k8s-cluster241 <none> <none>
calico-system calico-typha-6744756b8b-cxkxp 1/1 Running 0 67s 10.0.0.241 k8s-cluster241 <none> <none>
calico-system calico-typha-6744756b8b-mbdww 1/1 Running 0 68s 10.0.0.243 k8s-cluster243 <none> <none>
calico-system csi-node-driver-6lnsp 2/2 Running 0 65s 10.100.86.6 k8s-cluster241 <none> <none>
calico-system csi-node-driver-p299m 2/2 Running 0 61s 10.100.88.10 k8s-cluster243 <none> <none>
calico-system csi-node-driver-tstgf 2/2 Running 0 64s 10.100.99.8 k8s-cluster242 <none> <none>
calico-system goldmane-56dbdbc4c8-hdhtv 1/1 Running 0 67s 10.100.88.9 k8s-cluster243 <none> <none>
calico-system whisker-c75778546-mpnqz 2/2 Running 0 67s 10.100.99.7 k8s-cluster242 <none> <none>
tigera-operator tigera-operator-747864d56d-9v67r 1/1 Running 0 67s 10.0.0.242 k8s-cluster242 <none> <none>
[root@k8s-cluster241 ~]# 6.监控Calico Whisker中的网络流量
[root@k8s-cluster243 ~]# kubectl port-forward -n calico-system service/whisker 8081:8081 --address 0.0.0.0
Forwarding from 0.0.0.0:8081 -> 8081
7.浏览器访问测试
http://10.0.0.243:8081/flow-logs
8.检查节点的状态
[root@k8s-cluster241 ~]# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-cluster241 Ready <none> 83m v1.33.3 10.0.0.241 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
k8s-cluster242 Ready <none> 83m v1.33.3 10.0.0.242 <none> Ubuntu 22.04.4 LTS 5.15.0-119-generic containerd://1.6.36
k8s-cluster243 Ready <none> 83m v1.33.3 10.0.0.243 <none> Ubuntu 22.04.4 LTS 5.15.0-151-generic containerd://1.6.36
[root@k8s-cluster241 ~]#
测试CNI网络插件
1.编写资源清单
[root@k8s-cluster241 ~]# cat > oldboyedu-network-cni-test.yaml <<EOF
apiVersion: v1
kind: Pod
metadata:
name: xiuxian-v1
labels:
apps: v1
spec:
nodeName: k8s-cluster242
containers:
- image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1
name: xiuxian
---
apiVersion: v1
kind: Pod
metadata:
name: xiuxian-v2
labels:
apps: v2
spec:
nodeName: k8s-cluster243
containers:
- image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v2
name: xiuxian
EOF2 创建资源并测试Pod跨节点访问
[root@k8s-cluster241 ~]# kubectl apply -f oldboyedu-network-cni-test.yaml
pod/xiuxian-v1 created
pod/xiuxian-v2 created
[root@k8s-cluster241 ~]#
3.测试网络
[root@k8s-cluster241 ~]# kubectl get pods -o wide --show-labels
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES LABELS
xiuxian-v1 1/1 Running 0 5s 10.100.99.10 k8s-cluster242 <none> <none> apps=v1
xiuxian-v2 1/1 Running 0 5s 10.100.88.12 k8s-cluster243 <none> <none> apps=v2
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# curl 10.100.99.10
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>yinzhengjie apps v1</title>
<style>
div img {
width: 900px;
height: 600px;
margin: 0;
}
</style>
</head>
<body>
<h1 style="color: green">凡人修仙传 v1 </h1>
<div>
<img src="1.jpg">
<div>
</body>
</html>
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# curl 10.100.88.12
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>yinzhengjie apps v2</title>
<style>
div img {
width: 900px;
height: 600px;
margin: 0;
}
</style>
</head>
<body>
<h1 style="color: red">凡人修仙传 v2 </h1>
<div>
<img src="2.jpg">
<div>
</body>
</html>
[root@k8s-cluster241 ~]# 4.暴露服务对集群外部【可跳过】测试环境
[root@k8s-cluster241 ~]# kubectl expose po xiuxian-v1 --port=80
service/xiuxian-v1 exposed
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.200.0.1 <none> 443/TCP 19h
xiuxian-v1 ClusterIP 10.200.240.114 <none> 80/TCP 4s
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# curl 10.200.240.114
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>yinzhengjie apps v1</title>
<style>
div img {
width: 900px;
height: 600px;
margin: 0;
}
</style>
</head>
<body>
<h1 style="color: green">凡人修仙传 v1 </h1>
<div>
<img src="1.jpg">
<div>
</body>
</html>
[root@k8s-cluster241 ~]# 温馨提示:官方关于calico网络的Whisker组件流量监控是有出图效果的，可以模拟测试，但建议先部署CoreDNS组件。附加组件CoreDNS部署实战1 下载资源清单 参考链接: https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/dns/coredns
[root@k8s-cluster241 ~]# wget http://192.168.21.253/Resources/Kubernetes/Add-ons/CoreDNS/coredns.yaml.base
2 修改资源清单模板的关键字段
[root@k8s-cluster241 ~]# sed -i '/__DNS__DOMAIN__/s#__DNS__DOMAIN__#oldboyedu.com#' coredns.yaml.base
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# sed -i '/__DNS__MEMORY__LIMIT__/s#__DNS__MEMORY__LIMIT__#200Mi#' coredns.yaml.base
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# sed -i '/__DNS__SERVER__/s#__DNS__SERVER__#10.200.0.254#' coredns.yaml.base
[root@k8s-cluster241 ~]#
相关字段说明:
__DNS__DOMAIN__
DNS自定义域名，要和你实际的K8S域名对应上。
__DNS__MEMORY__LIMIT__
CoreDNS组件的内存限制。
__DNS__SERVER__
DNS服务器的svc的CLusterIP地址。
3.部署CoreDNS组件
温馨提示:
如果镜像下载失败，可以手动导入。操作如下:
wget http://192.168.21.253/Resources/Kubernetes/Add-ons/CoreDNS/oldboyedu-coredns-v1.12.0.tar.gz
ctr -n k8s.io i import oldboyedu-coredns-v1.12.0.tar.gz
5.验证DNS服务
[root@k8s-cluster241 ~]# kubectl get svc -A
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
calico-apiserver calico-api ClusterIP 10.200.64.223 <none> 443/TCP 3h40m
calico-system calico-kube-controllers-metrics ClusterIP None <none> 9094/TCP 3h19m
calico-system calico-typha ClusterIP 10.200.103.220 <none> 5473/TCP 3h40m
calico-system goldmane ClusterIP 10.200.245.83 <none> 7443/TCP 3h40m
calico-system whisker ClusterIP 10.200.160.62 <none> 8081/TCP 3h40m
default kubernetes ClusterIP 10.200.0.1 <none> 443/TCP 22h
default xiuxian-v1 ClusterIP 10.200.240.114 <none> 80/TCP 3h2m
kube-system kube-dns ClusterIP 10.200.0.254 <none> 53/UDP,53/TCP,9153/TCP 110s
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# dig @10.200.0.254 kube-dns.kube-system.svc.oldboyedu.com +short
10.200.0.254
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# dig @10.200.0.254 kubernetes.default.svc.oldboyedu.com +short
10.200.0.1
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# dig @10.200.0.254 calico-api.calico-apiserver.svc.oldboyedu.com +short
10.200.64.223
[root@k8s-cluster241 ~]# 6.部署Pod验证默认的DNS服务器
[root@k8s-cluster241 ~]# kubectl apply -f oldboyedu-network-cni-test.yaml
pod/xiuxian-v1 created
pod/xiuxian-v2 created
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
xiuxian-v1 1/1 Running 0 3s 10.100.99.11 k8s-cluster242 <none> <none>
xiuxian-v2 1/1 Running 0 3s 10.100.88.13 k8s-cluster243 <none> <none>
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl exec -it xiuxian-v1 -- sh
/ # cat /etc/resolv.conf
search default.svc.oldboyedu.com svc.oldboyedu.com oldboyedu.com
nameserver 10.200.0.254
options ndots:5
/ # 7.清除Pod环境
[root@k8s-cluster241 ~]# kubectl delete pods --all
pod "xiuxian-v1" deleted
pod "xiuxian-v2" deleted
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl get pods -o wide
No resources found in default namespace.
[root@k8s-cluster241 ~]#
部署MetallB组件实现LoadBalancer
1 配置kube-proxy代理模式为ipvs
[root@k8s-cluster241 ~]# grep mode /etc/kubernetes/kube-proxy.yml
mode: "ipvs"
[root@k8s-cluster241 ~]#
[root@k8s-cluster242 ~]# grep mode /etc/kubernetes/kube-proxy.yml
mode: "ipvs"
[root@k8s-cluster242 ~]#
[root@k8s-cluster243 ~]# grep mode /etc/kubernetes/kube-proxy.yml
mode: "ipvs"
[root@k8s-cluster243 ~]#
2 K8S集群所有节点导入镜像
[root@master231 metallb]# wget https://raw.githubusercontent.com/metallb/metallb/v0.15.2/config/manifests/metallb-native.yaml
SVIP:
[root@k8s-cluster241 ~]# wget http://192.168.21.253/Resources/Kubernetes/Add-ons/metallb/v0.15.2/metallb-native.yaml
4 部署Metallb
[root@k8s-cluster241 ~]# kubectl apply -f metallb-native.yaml
5 创建IP地址池
[root@k8s-cluster241 ~]# cat > metallb-ip-pool.yaml <<EOF
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: jasonyin2020
namespace: metallb-system
spec:
addresses:
# 注意改为你自己为MetalLB分配的IP地址，改地址，建议设置为你windows能够访问的网段。【建议设置你的虚拟机Vmnet8网段】
- 10.0.0.150-10.0.0.180
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
name: yinzhengjie
namespace: metallb-system
spec:
ipAddressPools:
- jasonyin2020
EOF
[root@k8s-cluster241 ~]# kubectl apply -f metallb-ip-pool.yaml
ipaddresspool.metallb.io/jasonyin2020 created
l2advertisement.metallb.io/yinzhengjie created
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl get ipaddresspools.metallb.io -A
NAMESPACE NAME AUTO ASSIGN AVOID BUGGY IPS ADDRESSES
metallb-system jasonyin2020 true false ["10.0.0.150-10.0.0.180"]
[root@k8s-cluster241 ~]#
6 创建LoadBalancer的Service测试验证
[root@k8s-cluster241 ~]# cat > deploy-svc-LoadBalancer.yaml <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-xiuxian
spec:
replicas: 3
selector:
matchLabels:
apps: v1
template:
metadata:
labels:
apps: v1
spec:
containers:
- name: c1
image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v3
---
apiVersion: v1
kind: Service
metadata:
name: svc-xiuxian
spec:
type: LoadBalancer
selector:
apps: v1
ports:
- port: 80
EOF
[root@k8s-cluster241 ~]# kubectl apply -f deploy-svc-LoadBalancer.yaml
deployment.apps/deploy-xiuxian created
service/svc-xiuxian created
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl get deploy,svc,po -o wide
NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR
deployment.apps/deploy-xiuxian 3/3 3 3 9s c1 registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v3 apps=v1
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
service/kubernetes ClusterIP 10.200.0.1 <none> 443/TCP 22h <none>
service/svc-xiuxian LoadBalancer 10.200.134.72 10.0.0.150 80:17862/TCP 9s apps=v1
service/xiuxian-v1 ClusterIP 10.200.240.114 <none> 80/TCP 3h12m apps=v1
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod/deploy-xiuxian-5bc4d8c6d5-hnln8 1/1 Running 0 9s 10.100.88.14 k8s-cluster243 <none> <none>
pod/deploy-xiuxian-5bc4d8c6d5-hw7w6 1/1 Running 0 9s 10.100.86.9 k8s-cluster241 <none> <none>
pod/deploy-xiuxian-5bc4d8c6d5-jr5vf 1/1 Running 0 9s 10.100.99.12 k8s-cluster242 <none> <none>
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# curl 10.0.0.150
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>yinzhengjie apps v3</title>
<style>
div img {
width: 900px;
height: 600px;
margin: 0;
}
</style>
</head>
<body>
<h1 style="color: pink">凡人修仙传 v3 </h1>
<div>
<img src="3.jpg">
<div>
</body>
</html>
[root@k8s-cluster241 ~]#
K8S附加组件helm部署
1.下载helm软件包
wget https://get.helm.sh/helm-v3.18.4-linux-amd64.tar.gz2.解压软件包
[root@k8s-cluster241 ~]# tar xf helm-v3.18.4-linux-amd64.tar.gz -C /usr/local/bin/ linux-amd64/helm --strip-components=1
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# ll /usr/local/bin/helm
-rwxr-xr-x 1 1001 fwupd-refresh 59715768 Jul 9 04:36 /usr/local/bin/helm*
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# helm version
version.BuildInfo{Version:"v3.18.4", GitCommit:"d80839cf37d860c8aa9a0503fe463278f26cd5e2", GitTreeState:"clean", GoVersion:"go1.24.4"}
[root@k8s-cluster241 ~]#
3.配置helm的自动补全功能
[root@k8s-cluster241 ~]# helm completion bash > /etc/bash_completion.d/helm
[root@k8s-cluster241 ~]# source /etc/bash_completion.d/helm
[root@k8s-cluster241 ~]# echo 'source /etc/bash_completion.d/helm' >> ~/.bashrc
[root@k8s-cluster241 ~]#
基于helm部署Dashboard
参考链接:
https://github.com/kubernetes/dashboard
1.添加Dashboard的仓库地址
[root@k8s-cluster241 ~]# helm repo add kubernetes-dashboard https://kubernetes.github.io/dashboard/
"kubernetes-dashboard" has been added to your repositories
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# helm repo list
NAME URL
kubernetes-dashboard https://kubernetes.github.io/dashboard/
[root@k8s-cluster241 ~]# 2.安装Dashboard
[root@k8s-cluster241 ~]# helm upgrade --install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard --create-namespace --namespace kubernetes-dashboard
Release "kubernetes-dashboard" does not exist. Installing it now.
Error: Get "https://github.com/kubernetes/dashboard/releases/download/kubernetes-dashboard-7.13.0/kubernetes-dashboard-7.13.0.tgz": dial tcp 20.205.243.166:443: connect: connection refused
[root@k8s-cluster241 ~]#
svip:
[root@k8s-cluster241 ~]# wget http://192.168.21.253/Resources/Kubernetes/Add-ons/dashboard/helm/v7.13.0/kubernetes-dashboard-7.13.0.tgz
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# tar xf kubernetes-dashboard-7.13.0.tgz
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# ll kubernetes-dashboard
total 56
drwxr-xr-x 4 root root 4096 Aug 1 14:55 ./
drwx------ 10 root root 4096 Aug 1 14:55 ../
-rw-r--r-- 1 root root 497 May 28 23:14 Chart.lock
drwxr-xr-x 6 root root 4096 Aug 1 14:55 charts/
-rw-r--r-- 1 root root 982 May 28 23:14 Chart.yaml
-rw-r--r-- 1 root root 948 May 28 23:14 .helmignore
-rw-r--r-- 1 root root 8209 May 28 23:14 README.md
drwxr-xr-x 10 root root 4096 Aug 1 14:55 templates/
-rw-r--r-- 1 root root 13729 May 28 23:14 values.yaml
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# helm upgrade --install mywebui kubernetes-dashboard --create-namespace --namespace kubernetes-dashboard
Release "mywebui" does not exist. Installing it now.
NAME: mywebui
LAST DEPLOYED: Fri Aug 1 14:55:46 2025
NAMESPACE: kubernetes-dashboard
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
*************************************************************************************************
*** PLEASE BE PATIENT: Kubernetes Dashboard may need a few minutes to get up and become ready ***
*************************************************************************************************
Congratulations! You have just installed Kubernetes Dashboard in your cluster.
To access Dashboard run:
kubectl -n kubernetes-dashboard port-forward svc/mywebui-kong-proxy 8443:443
NOTE: In case port-forward command does not work, make sure that kong service name is correct.
Check the services in Kubernetes Dashboard namespace using:
kubectl -n kubernetes-dashboard get svc
Dashboard will be available at:
https://localhost:8443
[root@k8s-cluster241 ~]#
3.查看部署信息
[root@k8s-cluster241 ~]# helm -n kubernetes-dashboard list
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
mywebui kubernetes-dashboard 1 2025-08-01 14:55:46.018206818 +0800 CST deployed kubernetes-dashboard-7.13.0
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl -n kubernetes-dashboard get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
mywebui-kong-5bdcb94b79-86ftr 1/1 Running 0 9m37s 10.100.86.11 k8s-cluster241 <none> <none>
mywebui-kubernetes-dashboard-api-74fbd85467-vspv9 1/1 Running 0 9m37s 10.100.88.16 k8s-cluster243 <none> <none>
mywebui-kubernetes-dashboard-auth-69d4c5864b-zpzcj 1/1 Running 0 9m37s 10.100.99.13 k8s-cluster242 <none> <none>
mywebui-kubernetes-dashboard-metrics-scraper-5c99c5ccc8-4f96n 1/1 Running 0 9m37s 10.100.86.10 k8s-cluster241 <none> <none>
mywebui-kubernetes-dashboard-web-cd678f7dd-bmjgj 1/1 Running 0 9m37s 10.100.88.15 k8s-cluster243 <none> <none>
[root@k8s-cluster241 ~]#
SVIP镜像下载地址:
http://192.168.21.253/Resources/Kubernetes/Add-ons/dashboard/helm/v7.13.0/images/
4.修改svc的类型(测试环境)
[root@k8s-cluster241 ~]# kubectl get svc -n kubernetes-dashboard mywebui-kong-proxy
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
mywebui-kong-proxy ClusterIP 10.200.64.101 <none> 443/TCP 10m
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl edit svc -n kubernetes-dashboard mywebui-kong-proxy
service/mywebui-kong-proxy edited
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl get svc -n kubernetes-dashboard mywebui-kong-proxy
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
mywebui-kong-proxy LoadBalancer 10.200.64.101 10.0.0.151 443:14400/TCP 10m
[root@k8s-cluster241 ~]#
拉取官方镜像修改法
&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
[root@k8s-cluster241 ~]# kubectl get svc -n kubernetes-dashboard mywebui-kong-proxy
Error from server (NotFound): services "mywebui-kong-proxy" not found
[root@k8s-cluster241 ~]# kubectl get svc -n kubernetes-dashboard kubernetes-dashboard-
kubernetes-dashboard-api kubernetes-dashboard-kong-proxy kubernetes-dashboard-web
kubernetes-dashboard-auth kubernetes-dashboard-metrics-scraper
[root@k8s-cluster241 ~]# kubectl get svc -n kubernetes-dashboard kubernetes-dashboard-kong-proxy
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes-dashboard-kong-proxy ClusterIP 10.200.68.140 <none> 443/TCP 28m
[root@k8s-cluster241 ~]# kubectl edit svc -n kubernetes-dashboard-kong-proxy
error: edit cancelled, no objects found
[root@k8s-cluster241 ~]# kubectl edit svc -n kubernetes-dashboard kubernetes-dashboard-kong-proxy
service/kubernetes-dashboard-kong-proxy edited
[root@k8s-cluster241 ~]# kubectl get svc -n kubernetes-dashboard kubernetes-dashboard-kong-proxy
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes-dashboard-kong-proxy LoadBalancer 10.200.68.140 10.0.0.151 443:34636/TCP 31m
[root@k8s-cluster241 ~]#
********************************************************************5.访问WebUI
https://10.0.0.151/#/login
6.创建登录账号
6.1 创建sa
[root@k8s-cluster241 ~]# kubectl create clusterrolebinding dashboard-admin --serviceaccount=default:admin --clusterrole=cluster-admin
clusterrolebinding.rbac.authorization.k8s.io/dashboard-admin created
[root@k8s-cluster241 ~]#
6.2 将sa和CLuster-admin进行绑定
[root@k8s-cluster241 ~]# kubectl create clusterrolebinding dashboard-admin --serviceaccount=default:admin --clusterrole=cluster-admin
clusterrolebinding.rbac.authorization.k8s.io/dashboard-admin created
[root@k8s-cluster241 ~]#
6.3 获取账号的token并进行webUI的登录
[root@k8s-cluster241 ~]# kubectl create token admin
eyJhbGciOiJSUzI1NiIsImtpZCI6IjFSTlY2dk5FS3BrdHkySFNnTW1nZFJSMXhibU83X0twWFhMUHBhZGRhV2sifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLm9sZGJveWVkdS5jb20iXSwiZXhwIjoxNzU0MDM1Njc1LCJpYXQiOjE3NTQwMzIwNzUsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5vbGRib3llZHUuY29tIiwianRpIjoiNjc1NTgyZDAtZTMyYS00NjkwLTllODQtZWMyMTJiY2JhYTM4Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJkZWZhdWx0Iiwic2VydmljZWFjY291bnQiOnsibmFtZSI6ImFkbWluIiwidWlkIjoiZDgwY2E0OTgtOTE0ZC00MjI4LWI3YmMtMTNlNjYyNjkzYmE1In19LCJuYmYiOjE3NTQwMzIwNzUsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpkZWZhdWx0OmFkbWluIn0.Pth4k-a23691RSdkrklTqwcfyoUyKM675q5Tkjpiw1IsWWoo1_tqm0oh7DTHqcMNtyTnQGvauLLLuKi8ANn2344z3wO_qGIl6wOL7X9qXS5stxhJUWYVA_tokcAoLgomERDy7xNFV03plJIW60g53yfP1oA7ng4z7g5AZArRy2Mf1tvkFTaiMtRK3Ovsnj9K-CGox3R3vpl1Qrkvmnrd-Z465-V61DLmrlyf6YRrSt7sLDIcjeoiEq0DKs4Jau-srAJTIdvJi0OSkVucYlxAyJx5fTPmW4LyFcsWe7tAQBZg-9p0Bu9Rr4scOAhxVDjuu7Rs4gbXLdX0iL-GkMVyfA
[root@k8s-cluster241 ~]#
部署metrics-server组件
1 下载资源清单
[root@k8s-cluster241 ~]# wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/high-availability-1.21+.yaml
2 编辑配置文件
[root@k8s-cluster241 ~]# vim high-availability-1.21+.yaml
...
114 apiVersion: apps/v1
115 kind: Deployment
116 metadata:
...
144 - args:
145 - --kubelet-insecure-tls # 不要验证Kubelets提供的服务证书的CA。不配置则会报错x509。
...
... image: registry.aliyuncs.com/google_containers/metrics-server:v0.7.2
3 部署metrics-server组件
[root@k8s-cluster241 ~]# kubectl apply -f high-availability-1.21+.yaml
serviceaccount/metrics-server created
clusterrole.rbac.authorization.k8s.io/system:aggregated-metrics-reader created
clusterrole.rbac.authorization.k8s.io/system:metrics-server created
rolebinding.rbac.authorization.k8s.io/metrics-server-auth-reader created
clusterrolebinding.rbac.authorization.k8s.io/metrics-server:system:auth-delegator created
clusterrolebinding.rbac.authorization.k8s.io/system:metrics-server created
service/metrics-server created
deployment.apps/metrics-server created
poddisruptionbudget.policy/metrics-server created
apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io created
[root@k8s-cluster241 ~]#
镜像下载地址：
http://192.168.21.253/Resources/Kubernetes/Add-ons/metrics-server/0.7.x/
4 查看镜像是否部署成功
[root@k8s-cluster241 ~]# kubectl get pods,svc -n kube-system -l k8s-app=metrics-server -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod/metrics-server-79bdcb6569-lsbs6 1/1 Running 0 34s 10.100.99.14 k8s-cluster242 <none> <none>
pod/metrics-server-79bdcb6569-mtgm8 1/1 Running 0 34s 10.100.86.12 k8s-cluster241 <none> <none>
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
service/metrics-server ClusterIP 10.200.157.24 <none> 443/TCP 34s k8s-app=metrics-server
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl -n kube-system describe svc metrics-server
Name: metrics-server
Namespace: kube-system
Labels: k8s-app=metrics-server
Annotations: <none>
Selector: k8s-app=metrics-server
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.200.157.24
IPs: 10.200.157.24
Port: https 443/TCP
TargetPort: https/TCP
Endpoints: 10.100.99.14:10250,10.100.86.12:10250
Session Affinity: None
Internal Traffic Policy: Cluster
Events: <none>
[root@k8s-cluster241 ~]#5 验证metrics组件是否正常工作
[root@k8s-cluster241 ~]# kubectl top node
NAME CPU(cores) CPU(%) MEMORY(bytes) MEMORY(%)
k8s-cluster241 100m 5% 2644Mi 33%
k8s-cluster242 111m 5% 1968Mi 25%
k8s-cluster243 80m 4% 2222Mi 28%
[root@k8s-cluster241 ~]#
[root@k8s-cluster241 ~]# kubectl top pod -A
NAMESPACE NAME CPU(cores) MEMORY(bytes)
calico-apiserver calico-apiserver-68b989c5c9-7wkpv 2m 46Mi
calico-apiserver calico-apiserver-68b989c5c9-hzgqr 2m 37Mi
calico-system calico-kube-controllers-74544dd8d8-sjc6g 3m 17Mi
calico-system calico-node-p24ql 29m 154Mi
calico-system calico-node-qlr7r 17m 154Mi
calico-system calico-node-w6cmr 21m 152Mi
calico-system calico-typha-6744756b8b-cxkxp 2m 21Mi
calico-system calico-typha-6744756b8b-mbdww 2m 22Mi
calico-system csi-node-driver-6lnsp 1m 6Mi
calico-system csi-node-driver-p299m 1m 6Mi
calico-system csi-node-driver-tstgf 1m 6Mi
calico-system goldmane-56dbdbc4c8-hdhtv 6m 15Mi
calico-system whisker-c75778546-mpnqz 0m 12Mi
default deploy-xiuxian-5bc4d8c6d5-hnln8 0m 2Mi
default deploy-xiuxian-5bc4d8c6d5-hw7w6 0m 2Mi
default deploy-xiuxian-5bc4d8c6d5-jr5vf 0m 2Mi
kube-system coredns-5578c9dc84-qvpqt 1m 13Mi
kube-system metrics-server-79bdcb6569-lsbs6 2m 13Mi
kube-system metrics-server-79bdcb6569-mtgm8 2m 14Mi
kubernetes-dashboard mywebui-kong-5bdcb94b79-86ftr 2m 96Mi
kubernetes-dashboard mywebui-kubernetes-dashboard-api-74fbd85467-vspv9 1m 12Mi
kubernetes-dashboard mywebui-kubernetes-dashboard-auth-69d4c5864b-zpzcj 1m 6Mi
kubernetes-dashboard mywebui-kubernetes-dashboard-metrics-scraper-5c99c5ccc8-4f96n 1m 7Mi
kubernetes-dashboard mywebui-kubernetes-dashboard-web-cd678f7dd-bmjgj 1m 6Mi
metallb-system controller-58fdf44d87-l7vgc 1m 22Mi
metallb-system speaker-6qgsc 3m 17Mi
metallb-system speaker-ctkb2 3m 17Mi
metallb-system speaker-nqsp2 3m 16Mi
tigera-operator tigera-operator-747864d56d-9v67r 2m 73Mi
[root@k8s-cluster241 ~]#
哈哈，到这里为止，一套高可用k8s集群测试环境部署完毕！！
本文是原创文章，采用 CC BY-NC-ND 4.0 协议，完整转载请注明来自 程序员小航
评论
匿名评论
隐私政策
你无需删除空行，直接评论以获取最佳展示效果
