1. nginx安装

1.1 所有的master节点创建运行nginx的用户

useradd nginx -s /sbin/nologin -M

1.2 安装依赖

mkdir -p /data/k8s-work
cd /data/k8s-work
yum -y install pcre pcre-devel openssl openssl-devel gcc gcc-c++ automake autoconf libtool make

1.3 下载nginx软件包

wget http://nginx.org/download/nginx-1.21.6.tar.gz

1.4 解压软件包

tar xf nginx-1.21.6.tar.gz

1.5 配置nginx

cd nginx-1.21.6
./configure --prefix=/usr/local/nginx/ \
            --with-pcre \
            --with-http_ssl_module \
            --with-http_stub_status_module \
            --with-stream \
            --with-http_stub_status_module \
            --with-http_gzip_static_module

1.6 编译并安装nginx

make -j 4 &&  make install 

1.7 使用systemctl管理，并设置开机启动

cat > /usr/lib/systemd/system/nginx.service <<EOF
[Unit]
Description=The nginx HTTP and reverse proxy server
After=network.target sshd-keygen.service

[Service]
Type=forking
EnvironmentFile=/etc/sysconfig/sshd
ExecStartPre=/usr/local/nginx/sbin/nginx -t -c /usr/local/nginx/conf/nginx.conf
ExecStart=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
ExecReload=/usr/local/nginx/sbin/nginx -s reload
ExecStop=/usr/local/nginx/sbin/nginx -s stop
Restart=on-failure
RestartSec=42s

[Install]
WantedBy=multi-user.target
EOF

1.8 检查nginx服务是否启动

systemctl enable --now nginx.service
systemctl status nginx 
ps -ef|grep nginx

1.9 同步nginx软件包和脚本到集群的k8s-ha02

scp -rp k8s-ha02:/usr/local/nginx/
scp -rp k8s-ha02:/usr/lib/systemd/system/nginx.service

nginx配置文件

2.1 编辑nginx配置文件

cat > /usr/local/nginx/conf/nginx.conf <<EOF
user nginx nginx;
worker_processes auto;

events {
    worker_connections  20240;
    use epoll;
}

error_log /var/log/nginx_error.log info;

stream {
    upstream kube-servers {
        hash $remote_addr consistent;
        server k8s-master01:6443 weight=5 max_fails=1 fail_timeout=3s;
        server k8s-master02:6443 weight=5 max_fails=1 fail_timeout=3s;
        server k8s-master03:6443 weight=5 max_fails=1 fail_timeout=3s;
	}

	server {
        listen 6443;
        proxy_connect_timeout 3s;
        proxy_timeout 3000s;
        proxy_pass kube-servers;
    }
}
EOF

2.2 同步nginx的配置文件到k8s-ha02

scp -rp k8s-ha02:/usr/local/nginx/conf/nginx.conf

2.3 所有节点启动nginx服务

systemctl enable --now nginx 
systemctl reload nginx

[root@k8s-ha01 nginx-1.21.6]# netstat -lntp|grep 6443
tcp        0      0 0.0.0.0:6443            0.0.0.0:*               LISTEN      5119/nginx: master  

3.部署keepalived

3.1 安装keepalived组件

yum  -y install  keepalived

3.2 修改keepalive的配置文件(根据实际环境，interface eth0可能需要修改为interface ens33)

3.2.1 编写配置文件，ha节点需要修改router_id和mcast_src_ip的值即可。

3.2.1.1 k8s-ha01节点

cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
   router_id 192.168.31.32
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh"
    interval 2
    weight -20
}
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 100
    priority 100
    advert_int 1
    mcast_src_ip 192.168.31.32
    nopreempt
    authentication {
        auth_type PASS
        auth_pass wangmanyuan.com
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        192.168.31.100
    }
}
EOF

3.2.1.1 k8s-ha02节点

cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
   router_id 192.168.31.33
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh"
    interval 2
    weight -20
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 100
    priority 99
    advert_int 2
    mcast_src_ip 192.168.31.33
    nopreempt
    authentication {
        auth_type PASS
        auth_pass wangmanyuan.com
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        192.168.31.100
    }
}
EOF

3.2.2 各节点编写健康检查脚本

cat > /etc/keepalived/check_port.sh <<\EOF
#!/bin/bash
err=0
for k in $(seq 1 3)
do
   check_code=$(pgrep nginx)
   if [[ $check_code == "" ]]; then
       err=$(expr $err + 1)
       sleep 1
       continue
   else
       err=0
       break
   fi
done

if [[ $err != "0" ]]; then
   echo "systemctl stop keepalived"
   /usr/bin/systemctl stop keepalived
   exit 1
else
   exit 0
fi
EOF

chmod +x /etc/keepalived/check_port.sh

3.3 启动keepalived

systemctl enable --now keepalived	

3.4 测试keepalived

ip a  # 查看VIP在哪个节点
systemct stop keepalived  # 停止服务，观察是否飘逸VIP

3.5 参数说明

温馨提示:
	router_id:
		节点ip，master每个节点配置自己的IP
	mcast_src_ip:
		节点IP，master每个节点配置自己的IP
	virtual_ipaddress:
		虚拟IP，即VIP。
	interface:
		指定接口的名称。
	virtual_router_id:
		有效值为0-255，可以理解为一个组ID，只有相同的ID才被确认为一个组。
		如果每个keepalived实例修改的ID不一致，则会出现各自有一个VIP的现象。
	```