1 Kubernetes集群部署
1.1 Kubernetes软件包下载
wget --no-check-certificate https://dl.k8s.io/v1.28.4/kubernetes-server-linux-amd64.tar.gz
如果下载失败可以单独加群联系
1.2 Kubernetes软件包安装
tar -xvf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin/
[root@k8s-master01 bin]# ll
total 1147568
-rwxr-xr-x 1 root root 61353984 Nov 16 01:16 apiextensions-apiserver
-rwxr-xr-x 1 root root 49102848 Nov 16 01:16 kubeadm
-rwxr-xr-x 1 root root 58933248 Nov 16 01:16 kube-aggregator
-rwxr-xr-x 1 root root 121745408 Nov 16 01:16 kube-apiserver
-rw-r--r-- 1 root root 8 Nov 16 01:16 kube-apiserver.docker_tag
-rw------- 1 root root 127259136 Nov 16 01:16 kube-apiserver.tar
-rwxr-xr-x 1 root root 117780480 Nov 16 01:16 kube-controller-manager
-rw-r--r-- 1 root root 8 Nov 16 01:16 kube-controller-manager.docker_tag
-rw------- 1 root root 123293696 Nov 16 01:16 kube-controller-manager.tar
-rwxr-xr-x 1 root root 49885184 Nov 16 01:16 kubectl
-rwxr-xr-x 1 root root 48828416 Nov 16 01:16 kubectl-convert
-rw-r--r-- 1 root root 8 Nov 16 01:16 kubectl.docker_tag
-rw------- 1 root root 55398400 Nov 16 01:16 kubectl.tar
-rwxr-xr-x 1 root root 110850048 Nov 16 01:16 kubelet
-rwxr-xr-x 1 root root 1605632 Nov 16 01:16 kube-log-runner
-rwxr-xr-x 1 root root 55107584 Nov 16 01:16 kube-proxy
-rw-r--r-- 1 root root 8 Nov 16 01:16 kube-proxy.docker_tag
-rw------- 1 root root 74757120 Nov 16 01:16 kube-proxy.tar
-rwxr-xr-x 1 root root 56070144 Nov 16 01:16 kube-scheduler
-rw-r--r-- 1 root root 8 Nov 16 01:16 kube-scheduler.docker_tag
-rw------- 1 root root 61583360 Nov 16 01:16 kube-scheduler.tar
-rwxr-xr-x 1 root root 1527808 Nov 16 01:16 mounter
[root@k8s-master01 bin]# pwd
/data/k8s-work/kubernetes/server/bin
cp -p kube-apiserver kube-controller-manager kube-scheduler kubectl /usr/local/bin/
1.3 Kubernetes软件分发
scp -rp kube-apiserver kube-controller-manager kube-scheduler kubectl k8s-master02:/usr/local/bin/
scp -rp kube-apiserver kube-controller-manager kube-scheduler kubectl k8s-master03:/usr/local/bin/
scp -rp kubelet kube-proxy k8s-node01:/usr/local/bin
scp -rp kubelet kube-proxy k8s-node02:/usr/local/bin
1.4 在集群节点上创建目录
所有节点
mkdir -p /etc/kubernetes/
mkdir -p /etc/kubernetes/ssl
mkdir -p /var/log/kubernetes
2 部署api-server
2.1 创建apiserver证书请求文件
cd /data/k8s-work/
cat > kube-apiserver-csr.json << EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.31.32",
"192.168.31.33",
"192.168.31.34",
"192.168.31.35",
"192.168.31.36",
"192.168.31.37",
"192.168.31.38",
"192.168.31.39",
"192.168.31.40",
"192.168.31.100",
"10.96.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "kubemsb",
"OU": "CN"
}
]
}
EOF
说明：
如果 hosts 字段不为空则需要指定授权使用该证书的 IP（含VIP） 或域名列表。由于该证书被 集群使用，需要将节点的IP都填上，为了方便后期扩容可以多写几个预留的IP。
同时还需要填写 service 网络的首个IP(一般是 kube-apiserver 指定的 service-cluster-ip-range 网段的第一个IP，如 10.96.0.1)。
2.5.5.2 生成apiserver证书及token文件
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver
cat > token.csv << EOF
$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
说明：
创建TLS机制所需TOKEN
TLS Bootstraping：Master apiserver启用TLS认证后，Node节点kubelet和kube-proxy与kube-apiserver进行通信，必须使用CA签发的有效证书才可以，当Node节点很多时，这种客户端证书颁发需要大量工作，同样也会增加集群扩展复杂度。为了简化流程，Kubernetes引入了TLS bootstraping机制来自动颁发客户端证书，kubelet会以一个低权限用户自动向apiserver申请证书，kubelet的证书由apiserver动态签署。所以强烈建议在Node上使用这种方式，目前主要用于kubelet。而kube-proxy还是由我们统一颁发一个证书。
2.5.5.3 创建apiserver服务配置文件
k8s-master01
cat > /etc/kubernetes/kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
--anonymous-auth=false \
--bind-address=192.168.31.34 \
--secure-port=6443 \
--advertise-address=192.168.31.34 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--service-cluster-ip-range=10.96.0.0/16 \
--token-auth-file=/etc/kubernetes/token.csv \
--service-node-port-range=30000-32767 \
--tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \
--kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-issuer=api \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--etcd-servers=https://192.168.31.34:2379,https://192.168.31.35:2379,https://192.168.31.36:2379 \
--allow-privileged=true \
--apiserver-count=3 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--v=4"
EOF
2.5.5.4 创建apiserver服务管理配置文件
cat > /etc/systemd/system/kube-apiserver.service << "EOF"
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service
[Service]
EnvironmentFile=-/etc/kubernetes/kube-apiserver.conf
ExecStart=/usr/local/bin/kube-apiserver $KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
2.5.5.5 同步文件到集群master节点
cp ca*.pem /etc/kubernetes/ssl/
cp kube-apiserver*.pem /etc/kubernetes/ssl/
cp token.csv /etc/kubernetes/
scp /etc/kubernetes/token.csv k8s-master02:/etc/kubernetes
scp /etc/kubernetes/token.csv k8s-master03:/etc/kubernetes
scp /etc/kubernetes/ssl/kube-apiserver*.pem k8s-master02:/etc/kubernetes/ssl
scp /etc/kubernetes/ssl/kube-apiserver*.pem k8s-master03:/etc/kubernetes/ssl
scp /etc/kubernetes/ssl/ca*.pem k8s-master02:/etc/kubernetes/ssl
scp /etc/kubernetes/ssl/ca*.pem k8s-master03:/etc/kubernetes/ssl
k8s-master02
# cat > /etc/kubernetes/kube-apiserver.conf <<EOF
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
--anonymous-auth=false \
--bind-address=192.168.31.35 \
--secure-port=6443 \
--advertise-address=192.168.31.35 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--service-cluster-ip-range=10.96.0.0/16 \
--token-auth-file=/etc/kubernetes/token.csv \
--service-node-port-range=30000-32767 \
--tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \
--kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-issuer=api \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--etcd-servers=https://192.168.31.34:2379,https://192.168.31.35:2379,https://192.168.31.36:2379 \
--allow-privileged=true \
--apiserver-count=3 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--v=4"
EOF
k8s-master03
# cat > /etc/kubernetes/kube-apiserver.conf <<EOF
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
--anonymous-auth=false \
--bind-address=192.168.31.36 \
--secure-port=6443 \
--advertise-address=192.168.31.36 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--service-cluster-ip-range=10.96.0.0/16 \
--token-auth-file=/etc/kubernetes/token.csv \
--service-node-port-range=30000-32767 \
--tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \
--kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-issuer=api \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--etcd-servers=https://192.168.31.34:2379,https://192.168.31.35:2379,https://192.168.31.36:2379 \
--allow-privileged=true \
--apiserver-count=3 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--v=4"
EOF
k8s-master01
scp /etc/systemd/system/kube-apiserver.service k8s-master02:/etc/systemd/system/kube-apiserver.service
scp /etc/systemd/system/kube-apiserver.service k8s-master03:/etc/systemd/system/kube-apiserver.service
2.5.5.6 启动apiserver服务
systemctl daemon-reload
systemctl enable --now kube-apiserver
systemctl status kube-apiserver
# 测试
curl --insecure https://192.168.31.34:6443/
curl --insecure https://192.168.31.35:6443/
curl --insecure https://192.168.31.36:6443/
curl --insecure https://192.168.31.100:6443/
评论