K8s教程-kubernetes异常测试
node 关闭 docker
systemctl stop docker
此时kubelet也会被自动关闭
# 关闭 docker
systemctl stop docker
# nodes 状态变成 NotReady
root@k8s-master:~/k8s-test# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-node1 Ready <none> 41h v1.21.1 10.0.26.188 <none> Ubuntu 20.04.2 LTS 5.4.0-73-generic docker://20.10.6
k8s-node2 NotReady <none> 16h v1.21.1 10.0.26.187 <none> Ubuntu 20.04.2 LTS 5.4.0-73-generic docker://20.10.6
# pods 没有变化，等待状态中？
root@k8s-master:~/k8s-test# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deployment-66b6c48dd5-9b2b9 1/1 Running 0 3m9s 172.26.188.2 k8s-node1 <none> <none>
nginx-deployment-66b6c48dd5-zhxj4 1/1 Running 0 3m9s 172.26.187.2 k8s-node2 <none> <none>
# pods 探测时间：5分钟？
# 状态改为：Terminating，并在正常的node上面新增pod实例
root@k8s-master:~/k8s-test# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deployment-66b6c48dd5-4hrk6 1/1 Running 0 10s 172.26.188.3 k8s-node1 <none> <none>
nginx-deployment-66b6c48dd5-9b2b9 1/1 Running 0 7m32s 172.26.188.2 k8s-node1 <none> <none>
nginx-deployment-66b6c48dd5-zhxj4 1/1 Terminating 0 7m32s 172.26.187.2 k8s-node2 <none> <none>
# 启动 docker
systemctl start docker
systemctl start kubelet
# 状态正常了
root@k8s-master:~/k8s-test# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-node1 Ready <none> 41h v1.21.1 10.0.26.188 <none> Ubuntu 20.04.2 LTS 5.4.0-73-generic docker://20.10.6
k8s-node2 Ready <none> 16h v1.21.1 10.0.26.187 <none> Ubuntu 20.04.2 LTS 5.4.0-73-generic docker://20.10.6
# 删除：Terminating的实例，docker恢复后会收到Terminating消息，进行清理
# May 27 01:53:01 k8s-node2 systemd[1]: Removed slice libcontainer container kubepods-besteffort-podaf67c065_bf21_4005_88b8_ec72da3d8896.slice.
root@k8s-master:~/k8s-test# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deployment-66b6c48dd5-4hrk6 1/1 Running 0 3m18s 172.26.188.3 k8s-node1 <none> <none>
nginx-deployment-66b6c48dd5-9b2b9 1/1 Running 0 10m 172.26.188.2 k8s-node1 <none> <none>
# 这里 Terminating 的 不会被恢复了
# 调整调度，删除一个pod
kubectl delete pod nginx-deployment-66b6c48dd5-4hrk6
# 调度恢复
root@k8s-master:~/k8s-test# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deployment-66b6c48dd5-9b2b9 1/1 Running 0 14m 172.26.188.2 k8s-node1 <none> <none>
nginx-deployment-66b6c48dd5-d52x6 1/1 Running 0 8s 172.26.187.2 k8s-node2 <none> <none>
kill -9 docker_pid
如果kill -9 docker_pid，dockerd会被自动拉起来
# 其实关闭的是服务，但是底层shim不会被释放，所以重启dockerd进程不会影响到pods
root@k8s-node2:~# ps aux|grep dock
root 398529 0.0 0.3 113112 7516 ? Sl 01:57 0:00 /usr/bin/containerd-shim-runc-v2 -namespace moby -id c5488f3eb8518a729dc5c8be505fafea406f37051f279021dac9546c1e76f10c -address /var/run/docker/containerd/containerd.sock
root 398618 0.0 0.3 113112 7840 ? Sl 01:57 0:00 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 0307656e1ab4269198fdcdbe44b29b34700a5381a784c1f905ce7b816edaf6a0 -address /var/run/docker/containerd/containerd.sock
root 399110 11.0 2.7 768488 55392 ? Ssl 01:58 0:00 /usr/bin/dockerd
root 399125 4.0 1.3 730336 28472 ? Ssl 01:58 0:00 containerd --config /var/run/docker/containerd/containerd.toml --log-level info
node 关闭 kubectl
结论：kubectl 状态异常后，可以查看node对应pod还在运行没有
systemctl stop kubelet.service
# 一段时间后，没有收到心跳包，将状态切换为：NotReady
# 新资源将无法从 NotReady 状态的 node 申请
root@k8s-master:~/k8s-test# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-node1 Ready <none> 41h v1.21.1 10.0.26.188 <none> Ubuntu 20.04.2 LTS 5.4.0-73-generic docker://20.10.6
k8s-node2 NotReady <none> 17h v1.21.1 10.0.26.187 <none> Ubuntu 20.04.2 LTS 5.4.0-73-generic docker://20.10.6
# pods 被切换到正常 nodes上面（切换的时间比较长）
root@k8s-master:~/k8s-test# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deployment-66b6c48dd5-5g99n 1/1 Running 0 119s 172.26.188.3 k8s-node1 <none> <none>
nginx-deployment-66b6c48dd5-9b2b9 1/1 Running 0 26m 172.26.188.2 k8s-node1 <none> <none>
nginx-deployment-66b6c48dd5-d52x6 1/1 Terminating 0 11m 172.26.187.2 k8s-node2 <none> <none>
# 启动 kubelet
systemctl start kubelet.service
# 启动后 Terminating 资源被释放
root@k8s-master:~/k8s-test# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deployment-66b6c48dd5-5g99n 1/1 Running 0 3m37s 172.26.188.3 k8s-node1 <none> <none>
nginx-deployment-66b6c48dd5-9b2b9 1/1 Running 0 27m 172.26.188.2 k8s-node1 <none> <none>
# 删除(也可以新加再删除？)
root@k8s-master:~/k8s-test# kubectl delete pod nginx-deployment-66b6c48dd5-5g99n
pod "nginx-deployment-66b6c48dd5-5g99n" deleted
# 自动调整
root@k8s-master:~/k8s-test# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deployment-66b6c48dd5-69zxg 1/1 Running 0 12s 172.26.187.2 k8s-node2 <none> <none>
nginx-deployment-66b6c48dd5-9b2b9 1/1 Running 0 29m 172.26.188.2 k8s-node1 <none> <none>
- 原文作者：zaza
- 原文链接：https://zazayaya.github.io/2021/05/27/k8s-qa-test.html
- 说明：转载本站文章请标明出处，部分资源来源于网络，如有侵权请及时与我联系!