1 部署kubelet
在k8s-master01上操作
1.1 创建kubelet-bootstrap.kubeconfig
BOOTSTRAP_TOKEN=$(awk -F "," '{print $1}' /etc/kubernetes/token.csv)
kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://192.168.31.100:6443 --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config set-credentials kubelet-bootstrap --token=${BOOTSTRAP_TOKEN} --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config use-context default --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=kubelet-bootstrap
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl describe clusterrolebinding cluster-system-anonymous
kubectl describe clusterrolebinding kubelet-bootstrap
2 创建kubelet配置文件
cat > kubelet.json << "EOF"
{
"kind": "KubeletConfiguration",
"apiVersion": "kubelet.config.k8s.io/v1beta1",
"authentication": {
"x509": {
"clientCAFile": "/etc/kubernetes/ssl/ca.pem"
},
"webhook": {
"enabled": true,
"cacheTTL": "2m0s"
},
"anonymous": {
"enabled": false
}
},
"authorization": {
"mode": "Webhook",
"webhook": {
"cacheAuthorizedTTL": "5m0s",
"cacheUnauthorizedTTL": "30s"
}
},
"address": "192.168.31.34",
"port": 10250,
"readOnlyPort": 10255,
"cgroupDriver": "systemd",
"hairpinMode": "promiscuous-bridge",
"serializeImagePulls": false,
"clusterDomain": "cluster.local.",
"clusterDNS": ["10.96.0.2"]
}
EOF
2.1 创建kubelet服务启动管理文件
cat > kubelet.service << "EOF"
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=containerd.service
Requires=containerd.service
[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/bin/kubelet \
--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig \
--cert-dir=/etc/kubernetes/ssl \
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
--config=/etc/kubernetes/kubelet.json \
--cni-conf-dir=/etc/cni/net.d \
--container-runtime=remote \
--container-runtime-endpoint=unix:///run/containerd/containerd.sock \
--network-plugin=cni \
--rotate-certificates \
--pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9 \
--root-dir=/etc/cni/net.d \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
2.2 同步文件到集群节点
cp kubelet-bootstrap.kubeconfig /etc/kubernetes/
cp kubelet.json /etc/kubernetes/
cp kubelet.service /usr/lib/systemd/system/
for i in k8s-master02 k8s-master03 k8s-node01 k8s-node02;do scp kubelet-bootstrap.kubeconfig kubelet.json $i:/etc/kubernetes/;done
for i in k8s-master02 k8s-master03 k8s-node01 k8s-node02;do scp ca.pem $i:/etc/kubernetes/ssl/;done
for i in k8s-master02 k8s-master03 k8s-node01 k8s-node02;do scp kubelet.service $i:/usr/lib/systemd/system/;done
说明：
kubelet.json中address需要修改为当前主机IP地址。
2.3 创建目录及启动服务
mkdir -p /var/lib/kubelet
mkdir -p /var/log/kubernetes
systemctl daemon-reload
systemctl enable --now kubelet
systemctl status kubelet
报错
[root@k8s-master01 k8s-work]# systemctl status kubelet -l
● kubelet.service - Kubernetes Kubelet
Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; vendor preset: disabled)
Active: activating (auto-restart) (Result: exit-code) since Tue 2023-11-21 21:20:18 CST; 4s ago
Docs: https://github.com/kubernetes/kubernetes
Process: 9376 ExecStart=/usr/local/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig --cert-dir=/etc/kubernetes/ssl --kubeconfig=/etc/kubernetes/kubelet.kubeconfig --config=/etc/kubernetes/kubelet.json --container-runtime-endpoint=unix:///run/containerd/containerd.sock --rotate-certificates --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9 --root-dir=/etc/cni/net.d --v=2 (code=exited, status=1/FAILURE)
Main PID: 9376 (code=exited, status=1/FAILURE)
Nov 21 21:20:18 k8s-master01 kubelet[9376]: I1121 21:20:18.983936 9376 server.go:895] "Client rotation is on, will bootstrap in background"
Nov 21 21:20:18 k8s-master01 kubelet[9376]: E1121 21:20:18.984353 9376 bootstrap.go:241] unable to read existing bootstrap client config from /etc/kubernetes/kubelet.kubeconfig: invalid configuration: [unable to read client-cert /etc/kubernetes/ssl/kubelet-client-current.pem for default-auth due to open /etc/kubernetes/ssl/kubelet-client-current.pem: no such file or directory, unable to read client-key /etc/kubernetes/ssl/kubelet-client-current.pem for default-auth due to open /etc/kubernetes/ssl/kubelet-client-current.pem: no such file or directory]
Nov 21 21:20:18 k8s-master01 kubelet[9376]: I1121 21:20:18.985122 9376 bootstrap.go:101] "Use the bootstrap credentials to request a cert, and set kubeconfig to point to the certificate dir"
Nov 21 21:20:18 k8s-master01 kubelet[9376]: I1121 21:20:18.985241 9376 server.go:952] "Starting client certificate rotation"
Nov 21 21:20:18 k8s-master01 kubelet[9376]: I1121 21:20:18.985250 9376 certificate_manager.go:356] kubernetes.io/kube-apiserver-client-kubelet: Certificate rotation is enabled
Nov 21 21:20:18 k8s-master01 kubelet[9376]: I1121 21:20:18.985472 9376 certificate_manager.go:356] kubernetes.io/kube-apiserver-client-kubelet: Rotating certificates
Nov 21 21:20:18 k8s-master01 kubelet[9376]: I1121 21:20:18.985506 9376 dynamic_cafile_content.go:119] "Loaded a new CA Bundle and Verifier" name="client-ca-bundle::/etc/kubernetes/ssl/ca.pem"
Nov 21 21:20:18 k8s-master01 systemd[1]: kubelet.service failed.
Nov 21 21:20:18 k8s-master01 kubelet[9376]: I1121 21:20:18.985703 9376 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/etc/kubernetes/ssl/ca.pem"
Nov 21 21:20:18 k8s-master01 kubelet[9376]: E1121 21:20:18.988960 9376 run.go:74] "command failed" err="failed to run Kubelet: validate service connection: validate CRI v1 runtime API for endpoint \"unix:///run/containerd/containerd.sock\": rpc error: code = Unimplemented desc = unknown service runtime.v1.RuntimeService"
报错关键点查看containerd信息
root@k8s-master01 k8s-work]# journalctl -f -u containerd.service
-- Logs begin at Tue 2023-11-21 18:23:41 CST. --
Nov 21 21:35:47 k8s-master01 containerd[11532]: time="2023-11-21T21:35:47.984785204+08:00" level=info msg="loading plugtainerd.grpc.v1.version\"..." type=io.containerd.grpc.v1
Nov 21 21:35:47 k8s-master01 containerd[11532]: time="2023-11-21T21:35:47.984794914+08:00" level=info msg="loading plugtainerd.grpc.v1.cri\"..." type=io.containerd.grpc.v1
Nov 21 21:35:47 k8s-master01 containerd[11532]: time="2023-11-21T21:35:47.984984614+08:00" level=warning msg="failed ton io.containerd.grpc.v1.cri" error="invalid plugin config: `systemd_cgroup` only works for runtime io.containerd.runtim
Nov 21 21:35:47 k8s-master01 containerd[11532]: time="2023-11-21T21:35:47.984999254+08:00" level=info msg="loading plugtainerd.tracing.processor.v1.otlp\"..." type=io.containerd.tracing.processor.v1
Nov 21 21:35:47 k8s-master01 containerd[11532]: time="2023-11-21T21:35:47.985012723+08:00" level=info msg="skip loadingo.containerd.tracing.processor.v1.otlp\"..." error="no OpenTelemetry endpoint: skip plugin" type=io.containerd.tracing.1
Nov 21 21:35:47 k8s-master01 containerd[11532]: time="2023-11-21T21:35:47.985020034+08:00" level=info msg="loading plugtainerd.internal.v1.tracing\"..." type=io.containerd.internal.v1
Nov 21 21:35:47 k8s-master01 containerd[11532]: time="2023-11-21T21:35:47.985031233+08:00" level=info msg="skipping trasor initialization (no tracing plugin)" error="no OpenTelemetry endpoint: skip plugin"
Nov 21 21:35:47 k8s-master01 containerd[11532]: time="2023-11-21T21:35:47.985226773+08:00" level=info msg=serving... adcontainerd/containerd.sock.ttrpc
Nov 21 21:35:47 k8s-master01 containerd[11532]: time="2023-11-21T21:35:47.985249673+08:00" level=info msg=serving... adcontainerd/containerd.sock
Nov 21 21:35:47 k8s-master01 containerd[11532]: time="2023-11-21T21:35:47.985291982+08:00" level=info msg="containerd s booted in 0.022170s"
提取关键报错
time="2023-11-21T21:35:47.984984614+08:00" level=warning msg="failed ton io.containerd.grpc.v1.cri" error="invalid plugin config: `systemd_cgroup` only works for runtime io.containerd.runtim
结论
测试了各种方法发现都不行 索性来个彻底的 有知道的童鞋告诉我下哈 感激不尽
mv /etc/containerd/config.toml /root/config.toml.bak
systemctl restart containerd
# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master1 NotReady <none> 2m55s v1.21.10
k8s-master2 NotReady <none> 45s v1.21.10
k8s-master3 NotReady <none> 39s v1.21.10
k8s-worker1 NotReady <none> 5m1s v1.21.10
# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
csr-b949p 7m55s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
csr-c9hs4 3m34s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
csr-r8vhp 5m50s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
csr-zb4sr 3m40s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
说明：
确认kubelet服务启动成功后，接着到master上Approve一下bootstrap请求。
[root@k8s-master01 k8s-work]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready <none> 62m v1.28.4
k8s-master02 Ready <none> 46m v1.28.4
k8s-master03 Ready <none> 5m51s v1.28.4
k8s-node01 Ready <none> 4m58s v1.28.4
k8s-node02 Ready <none> 2m33s v1.28.4
评论