- A+
所属分类：linux技术
keepalived实现nginx负载均衡机高可用
环境说明：
| 系统 | 主机名 | IP | 服务 |
|---|---|---|---|
| centos8 | master | 192.168.111.141 | nginx keepalived |
| centos8 | backup | 192.168.111.142 | nginx keepalived |
| centos8 | apache | 192.168.111.143 | httpd |
| centos8 | nginx | 192.168.111.144 | nginx |
配置web界面
apache
//修改名字 [root@localhost ~]# hostnamectl set-hostname apache [root@localhost ~]# bash [root@apache ~]# //关闭防火墙和selinux [root@apache ~]# setenforce 0 [root@apache ~]# sed -ri 's/^(SELINUX=).*/1disabled/g' /etc/selinux/config [root@apache ~]# systemctl disable --now firewalld [root@apache ~]# reboot //配置yum源 [root@apache ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo [root@apache ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo //安装apache,主页内容为apache [root@apache ~]# yum -y install httpd [root@apache ~]# echo "apache" > /var/www/html/index.html [root@apache ~]# systemctl enable --now httpd [root@apache ~]# ss -anlt State Recv-Q Send-Q Local Address:Port Peer Address:Port Process LISTEN 0 128 0.0.0.0:22 0.0.0.0:* LISTEN 0 128 *:80 *:* LISTEN 0 128 [::]:22 [::]:* [root@apache ~]# curl 192.168.111.143 apache nginx
//修改名字 [root@localhost ~]# hostnamectl set-hostname nginx [root@localhost ~]# bash [root@nginx ~]# //关闭防火墙和selinux [root@nginx ~]# setenforce 0 [root@nginx ~]# sed -ri 's/^(SELINUX=).*/1disabled/g' /etc/selinux/config [root@nginx ~]# systemctl disable --now firewalld [root@nginx ~]# reboot //配置yum源 [root@nginx ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo [root@nginx ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo //安装nginx,主页内容为nginx [root@nginx ~]# yum -y install nginx [root@nginx ~]# echo "nginx" > /usr/share/nginx/html/index.html [root@nginx ~]# systemctl enable --now nginx [root@nginx ~]# ss -anlt State Recv-Q Send-Q Local Address:Port Peer Address:Port Process LISTEN 0 128 0.0.0.0:80 0.0.0.0:* LISTEN 0 128 0.0.0.0:22 0.0.0.0:* LISTEN 0 128 [::]:80 [::]:* LISTEN 0 128 [::]:22 [::]:* [root@nginx ~]# curl 192.168.111.144 nginx 配置nginx负载均衡
master
//修改名字 [root@localhost ~]# hostnamectl set-hostname master [root@localhost ~]# bash [root@master ~]# //关闭防火墙和selinux [root@master ~]# setenforce 0 [root@master ~]# sed -ri 's/^(SELINUX=).*/1disabled/g' /etc/selinux/config [root@master ~]# systemctl disable --now firewalld [root@master ~]# reboot //配置yum源 [root@master ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo [root@master ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo //创建用户 [root@master ~]# useradd -rMs /sbin/nologin nginx //安装依赖包 [root@master ~]# dnf -y install pcre-devel openssl openssl-devel gd-devel gcc gcc-c++ make wget vim //编译安装nginx [root@backup ~]# wget http://nginx.org/download/nginx-1.22.0.tar.gz [root@master ~]# tar xf nginx-1.22.0.tar.gz [root@master ~]# cd nginx-1.22.0 [root@master nginx-1.22.0]# ./configure --prefix=/usr/local/nginx --user=nginx --group=nginx --with-debug --with-http_ssl_module --with-http_realip_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_stub_status_module [root@master nginx-1.22.0]# make -j $(grep 'processor' /proc/cpuinfo | wc -l) && make install //配置环境变量 [root@master ~]# echo "export PATH=$PATH:/usr/local/nginx/sbin" > /etc/profile.d/nginx.sh [root@master ~]# source /etc/profile.d/nginx.sh //配置system启动服务 [root@master ~]# vim /usr/lib/systemd/system/nginx.service [Unit] Description=nginx server daemon After=network.target [Service] Type=forking ExecStart=/usr/local/nginx/sbin/nginx ExecStop=/usr/local/nginx/sbin/nginx -s stop ExecReload=/bin/kill -HUP $MAINPID [Install] WantedBy=multi-user.target [root@nginx ~]# systemctl daemon-reload //启动nginx [root@master ~]# systemctl enable --now nginx [root@master ~]# ss -anlt State Recv-Q Send-Q Local Address:Port Peer Address:Port Process LISTEN 0 128 0.0.0.0:80 0.0.0.0:* LISTEN 0 128 0.0.0.0:22 0.0.0.0:* LISTEN 0 128 [::]:22 [::]:* //修改配置文件,配置负载均衡 [root@master ~]# vim /usr/local/nginx/conf/nginx.conf upstream web { //http字段内 server 192.168.111.143; server 192.168.111.144; } ......... location / { //修改配置文件 root html; proxy_pass http://web; } [root@master ~]# systemctl restart nginx //查看负载均衡效果 [root@master ~]# curl 192.168.111.141 apache [root@master ~]# curl 192.168.111.141 nginx [root@master ~]# curl 192.168.111.141 apache [root@master ~]# curl 192.168.111.141 nginx backup
//修改名字 [root@localhost ~]# hostnamectl set-hostname backup [root@localhost ~]# bash [root@backup ~]# //关闭防火墙和selinux [root@backup ~]# setenforce 0 [root@backup ~]# sed -ri 's/^(SELINUX=).*/1disabled/g' /etc/selinux/config [root@backup ~]# systemctl disable --now firewalld [root@backup ~]# reboot //配置yum源 [root@backup ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo [root@backup ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo //创建用户 [root@backup ~]# useradd -rMs /sbin/nologin nginx //安装依赖包 [root@backup ~]# dnf -y install pcre-devel openssl openssl-devel gd-devel gcc gcc-c++ make wget vim //编译安装nginx [root@backup ~]# wget http://nginx.org/download/nginx-1.22.0.tar.gz [root@backup ~]# tar xf nginx-1.22.0.tar.gz [root@backup ~]# cd nginx-1.22.0 [root@backup nginx-1.22.0]# ./configure --prefix=/usr/local/nginx --user=nginx --group=nginx --with-debug --with-http_ssl_module --with-http_realip_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_stub_status_module [root@backup nginx-1.22.0]# make -j $(grep 'processor' /proc/cpuinfo | wc -l) && make install //配置环境变量 [root@backup ~]# echo "export PATH=$PATH:/usr/local/nginx/sbin" > /etc/profile.d/nginx.sh [root@backup ~]# source /etc/profile.d/nginx.sh //配置system启动服务 [root@backup ~]# vim /usr/lib/systemd/system/nginx.service [Unit] Description=nginx server daemon After=network.target [Service] Type=forking ExecStart=/usr/local/nginx/sbin/nginx ExecStop=/usr/local/nginx/sbin/nginx -s stop ExecReload=/bin/kill -HUP $MAINPID [Install] WantedBy=multi-user.target [root@backup ~]# systemctl daemon-reload //启动nginx [root@backup ~]# systemctl enable --now nginx [root@backup ~]# ss -anlt State Recv-Q Send-Q Local Address:Port Peer Address:Port Process LISTEN 0 128 0.0.0.0:80 0.0.0.0:* LISTEN 0 128 0.0.0.0:22 0.0.0.0:* LISTEN 0 128 [::]:22 [::]:* //修改配置文件,配置负载均衡 [root@backup ~]# vim /usr/local/nginx/conf/nginx.conf upstream web { //http字段内 server 192.168.111.143; server 192.168.111.144; } ......... location / { //修改配置文件 root html; proxy_pass http://web; } [root@backup ~]# systemctl restart nginx //查看负载均衡效果 [root@backup ~]# curl 192.168.111.142 apache [root@backup ~]# curl 192.168.111.142 nginx [root@backup ~]# curl 192.168.111.142 apache [root@backup ~]# curl 192.168.111.142 nginx 配置keepalived高可用
master
//首先安装keepalived [root@master ~]# yum -y install keepalived //编辑配置文件，并启动服务 [root@master ~]# mv /etc/keepalived/keepalived.conf{,.bak} [root@master ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { router_id lb01 } vrrp_instance VI_1 { state MASTER interface ens33 virtual_router_id 51 priority 100 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 192.168.111.250 } } virtual_server 192.168.111.250 80 { delay_loop 6 lb_algo rr lb_kind DR persistence_timeout 50 protocol TCP real_server 192.168.111.141 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } real_server 192.168.111.142 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } } [root@master ~]# systemctl enable --now keepalived //通过虚拟IP访问 [root@master ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff inet 192.168.111.141/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33 valid_lft 1500sec preferred_lft 1500sec inet 192.168.111.250/32 scope global ens33 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute valid_lft forever preferred_lft forever //通过虚拟ip访问 //一定要把backup端的nginx服务关掉才能访问， [root@master ~]# curl 192.168.111.250 apache [root@master ~]# curl 192.168.111.250 nginx [root@master ~]# curl 192.168.111.250 apache [root@master ~]# curl 192.168.111.250 nginx backup
//首先安装keepalived [root@backup ~]# dnf -y install keepalived //编辑配置文件，并启动服务 [root@backup ~]# mv /etc/keepalived/keepalived.conf{,.back} [root@backup ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { router_id lb02 } vrrp_instance VI_1 { state BACKUP interface ens33 virtual_router_id 50 priority 90 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 192.168.111.250 } } virtual_server 192.168.111.250 80 { delay_loop 6 lb_algo rr lb_kind DR persistence_timeout 50 protocol TCP real_server 192.168.111.141 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } real_server 192.168.111.142 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } } [root@backup ~]# systemctl enable --now keepalived 编写脚本
master
[root@master ~]# mkdir /scripts [root@master ~]# cd /scripts/ [root@master scripts]# vim check_nginx.sh #!/bin/bash nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep 'bnginxb'|wc -l) if [ $nginx_status -lt 1 ];then systemctl stop keepalived fi [root@master scripts]# vim notify.sh #!/bin/bash VIP=$2 case "$1" in master) nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep 'bnginxb'|wc -l) if [ $nginx_status -lt 1 ];then systemctl start nginx fi ;; backup) nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep 'bnginxb'|wc -l) if [ $nginx_status -gt 0 ];then systemctl stop nginx fi ;; *) echo "Usage:$0 master|backup VIP" ;; esac [root@master scripts]# chmod +x check_haproxy.sh notify.sh backup
[root@backup ~]# mkdir /scripts [root@backup ~]# cd /scripts/ [root@backup scripts]# scp root@192.168.111.141:/scripts/notify.sh . 配置keepalived加入监控脚本的配置
master
[root@master ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { router_id lb01 } vrrp_script nginx_check { //添加 script "/scripts/check_nginx.sh" interval 1 weight -20 } vrrp_instance VI_1 { state MASTER interface ens33 virtual_router_id 51 priority 100 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 192.168.111.250 } track_script { //添加 haproxy_check } notify_master "/scripts/notify.sh master 192.168.111.250" } virtual_server 192.168.111.250 80 { delay_loop 6 lb_algo rr lb_kind DR persistence_timeout 50 protocol TCP real_server 192.168.111.141 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } real_server 192.168.111.142 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } } [root@master scripts]# systemctl restart keepalived backup
[root@backup ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { router_id lb02 } vrrp_instance VI_1 { state BACKUP interface ens33 virtual_router_id 50 priority 90 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 192.168.111.250 } notify_master "/scripts/notify.sh master 192.168.111.250" //添加 notify_backup "/scripts/notify.sh backup 192.168.111.250" } virtual_server 192.168.111.250 80 { delay_loop 6 lb_algo rr lb_kind DR persistence_timeout 50 protocol TCP real_server 192.168.111.141 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } real_server 192.168.111.142 80 { weight 1 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } } [root@backup ~]# systemctl restart keepalived 测试
模拟nginx服务故障
//master端 [root@master ~]# curl 192.168.111.250 apache [root@master ~]# curl 192.168.111.250 nginx [root@master ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff inet 192.168.111.141/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33 valid_lft 1601sec preferred_lft 1601sec inet 192.168.111.250/32 scope global ens33 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute valid_lft forever preferred_lft forever [root@master ~]# systemctl stop nginx [root@master ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff inet 192.168.111.141/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33 valid_lft 1591sec preferred_lft 1591sec inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute valid_lft forever preferred_lft forever //backup端 [root@backup ~]# systemctl start nginx //前面把服务关了这里启动一下 [root@backup ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:0c:29:07:42:65 brd ff:ff:ff:ff:ff:ff inet 192.168.111.142/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33 valid_lft 947sec preferred_lft 947sec inet 192.168.111.250/32 scope global ens33 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe07:4265/64 scope link noprefixroute valid_lft forever preferred_lft forever [root@backup ~]# curl 192.168.111.250 apache [root@backup ~]# curl 192.168.111.250 nginx [root@backup ~]# curl 192.168.111.250 apache [root@backup ~]# curl 192.168.111.250 nginx 启动master端的nginx服务
//master端 [root@master scripts]# systemctl start nginx [root@master scripts]# systemctl restart keepalived [root@master ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff inet 192.168.111.141/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33 valid_lft 1223sec preferred_lft 1223sec inet 192.168.111.250/32 scope global ens33 valid_lft forever preferred_lft forever inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute valid_lft forever preferred_lft forever //backup端 [root@backup ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:0c:29:07:42:65 brd ff:ff:ff:ff:ff:ff inet 192.168.111.142/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33 valid_lft 1632sec preferred_lft 1632sec inet6 fe80::20c:29ff:fe07:4265/64 scope link noprefixroute valid_lft forever preferred_lft forever 
