双主Keepalive与lvs实现自动监控主机状态并实时切换

Posted 2022-06-17

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了双主Keepalive与lvs实现自动监控主机状态并实时切换相关的知识，希望对你有一定的参考价值。

环境：

202.106.0.17：as Client
202.106.0.27: as router
192.168.205.37: as lvs1
192.168.205.47: as lvs2
192.168.205.57: as websrv1
192.168.205.67: as websrv2
192.168.205.77: as websrv3
192.168.205.87: as websrv4
注：所有操作系统默认停用firewalld,iptable为空,关闭selinux

版本：

OS: centos 7 1810 with mini install
keepalived
httpd

目地：

使用keepalived 监控四台web服务器，两台为一组，各使用一个虚拟IP地址，lvs1为vip1的主vip2的从，lvs2为vip2的主，vip1的从，互相主备, 并使用lvs做为负载均衡，并能自动的发现服务down机并自动移除故障主机，从而实现无故障调度。

配置router

开启路由转发功能

[root@router data]#ech 1 > /proc/sys/net/ipv4/ip_forward

增加一个10.1.1.1在router的eth1上, 由于我要到lvs1和lvs2服务器时必须有路由，由于是两个主机的虚拟Ip是很有可能飘移，所以当其中一台down时无法确定在哪台主机上，为了安全不加路由，直接用IP直连。
```
[root@router data]#ip a a 10.1.1.1/24 dev eth0:1
```
先配置四台real server

在app1的两台服务器192.168.205.57/67上运行如下脚本

[root@websrv1 data]#cat lvs_dr_rs.sh 
#!/bin/bash
vip=10.1.1.100
gateway=192.168.205.27
mask=‘24‘
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "this is `hostname`" > /var/www/html/index.html

case $1 in
start)
    echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
    ip a a $vip/$mask dev $dev #broadcast $vip up
    ip route add default via $gateway dev eth0
    #route add -host $vip dev $dev
    echo "The RS Server is Ready!"
    ;;
stop)
    systemctl restart network
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
    echo "The RS Server is Canceled!"
    ;;
*) 
    echo "Usage: $(basename $0) start|stop"
    exit 1
    ;;
esac

在app2的两台服务器192.168.205.77/87上运行如下脚本

[root@websrv3 data]#cat lvs_dr_rs.sh 
#!/bin/bash
vip=10.1.1.200
gateway=192.168.205.27
mask=‘24‘
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "this is `hostname`" > /var/www/html/index.html

case $1 in
start)
    echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
    ip address add $vip/$mask dev $dev #broadcast $vip up
    ip route add default via $gateway dev eth0
    #route add -host $vip dev $dev
    echo "The RS Server is Ready!"
    ;;
stop)
    systemctl restart network
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
    echo "The RS Server is Canceled!"
    ;;
*) 
    echo "Usage: $(basename $0) start|stop"
    exit 1
    ;;
esac

在两台lvs上做同样的动（除特别说明）

安装httpd服务，将sorry server放在本地的两台主机中

yum install httpd
echo server is under maitenance > /var/www/html/index.html

一定要加默认路由到router,如果加正确可以工作，但sorry server不会正常
```
ip route add default via 192.168.205.27 dev eth0
```
为了访问方便最好是将两个lvs服务器之间做ssh key验证，用下面的方法就不用在47上再做一次了，真接实现互认证
```
ssh-keygen
scp -r /root/.ssh 192.168.205.47:/root
```

最好将host文件加入两台主机名称解析

vi /etc/hosts
192.168.205.37 websrv1
192.168.205.47 websrv2
scp /etc/hosts 192.168.205.47:/etc

安装keepalive, 为了能看清如何加的lvs策略，我们把iplvadm也装上
```
yum install keepalived ipvsadm
```

在lvs1上修改配置文件

[root@lsv1 ~]#vi /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs 
   notification_email 
     root@localhost
   
   notification_email_from keepalive@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS1
   vrrp_mcast_group4 224.0.0.100                                                                          

vrrp_instance VI_1 
    state MASTER
    interface eth0
    virtual_router_id 37
    priority 100
    advert_int 1
    authentication 
        auth_type PASS
        auth_pass centos
    
    virtual_ipaddress 
        10.1.1.100/24 dev eth0 label eth0:0
    

vrrp_instance VI_2 
    state BACKUP
    interface eth0
    virtual_router_id 47
    priority 80
    advert_int 1
    authentication 
        auth_type PASS
        auth_pass centos
    
    virtual_ipaddress 
        10.1.1.200/24 dev eth0 label eth0:1
    

virtual_server 10.1.1.100 80 
    delay_loop 6
    lb_algo rr 
    lb_kind DR
    protocol TCP
    sorry_server 127.0.0.1 80
    real_server 192.168.205.57 80 
        weight 1
        HTTP_GET 
            url  
              path /
              status_code 200
            
            connect_timeout 1
            nb_get_retry 3
            delay_before_retry 1
        
    
    real_server 192.168.205.67 80 
        weight 1
        HTTP_GET 
            url  
              path /
              status_code 200
            
            connect_timeout 1
            nb_get_retry 3
            delay_before_retry 1
        
                                

virtual_server 10.1.1.200 80 
    delay_loop 6
    lb_algo rr 
    lb_kind DR
    protocol TCP
    sorry_server 127.0.0.1 80
    real_server 192.168.205.77 80 
        weight 1
        HTTP_GET 
            url  
              path /
              status_code 200
            
            connect_timeout 1
            nb_get_retry 3
            delay_before_retry 1
        
    
    real_server 192.168.205.87 80 
        weight 1
        HTTP_GET 
            url  
              path /
              status_code 200
            
            connect_timeout 1
            nb_get_retry 3
            delay_before_retry 1
        
    
            
11. 为了方便将lsv1的keepalive.conf复制到lvs2上，并进行修改
[root@lvs2 ~]#vi /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs 
   notification_email 
     root@localhost
   
   notification_email_from keepalive@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS2
         vrrp_mcast_group4 224.0.0.100

vrrp_instance VI_1 
    state BACKUP
    interface eth0
    virtual_router_id 37
    priority 80
    advert_int 1
    authentication 
        auth_type PASS
        auth_pass centos
    
    virtual_ipaddress 
        10.1.1.100/24 dev eth0 label eth0:0
    

vrrp_instance VI_2 
    state MASTER
    interface eth0
    virtual_router_id 47
    priority 100
    advert_int 1
    authentication 
        auth_type PASS
        auth_pass centos
    
    virtual_ipaddress 
        10.1.1.200/24 dev eth0 label eth0:1
    

virtual_server 10.1.1.100 80 
    delay_loop 6
    lb_algo rr 
    lb_kind DR
    protocol TCP
    sorry_server 127.0.0.1 80
    real_server 192.168.205.57 80 
        weight 1
        HTTP_GET 
            url  
              path /
              status_code 200
            
            connect_timeout 1
            nb_get_retry 3
            delay_before_retry 1
        
    
    real_server 192.168.205.67 80 
        weight 1
        HTTP_GET 
            url  
              path /
              status_code 200
            
            connect_timeout 1
            nb_get_retry 3
            delay_before_retry 1
        
    

virtual_server 10.1.1.200 80 
    delay_loop 6
    lb_algo rr 
    lb_kind DR
    protocol TCP
    sorry_server 127.0.0.1 80
    real_server 192.168.205.77 80 
        weight 1
        HTTP_GET 
            url  
              path /
              status_code 200
            
            connect_timeout 1
            nb_get_retry 3
            delay_before_retry 1
        
    
    real_server 192.168.205.87 80 
        weight 1
        HTTP_GET 
            url  
              path /
              status_code 200
            
            connect_timeout 1
            nb_get_retry 3
            delay_before_retry 1

起动keepalived服务
```
 systemctl start keepalived
```

在lvs1看到状态为RR调度

[root@lsv1 ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.1.1.100:80 rr
  -> 192.168.205.57:80            Route   1      0          0         
  -> 192.168.205.67:80            Route   1      0          0         
TCP  10.1.1.200:80 rr
  -> 192.168.205.77:80            Route   1      0          0         
  -> 192.168.205.87:80            Route   1      0          0

在lvs1中看到只一10.1.1.100IP，在Lvs2中会看到只有10.1.1.200IP

[root@lsv1 ~]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:56:e1:ea brd ff:ff:ff:ff:ff:ff
    inet 192.168.205.37/24 brd 192.168.205.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 10.1.1.100/24 scope global eth0:0
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe56:e1ea/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@lvs2 ~]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:37:f9:93 brd ff:ff:ff:ff:ff:ff
    inet 192.168.205.47/24 brd 192.168.205.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 10.1.1.200/24 scope global eth0:1
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe37:f993/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

测试

在client上运行一个循环进行测试

[root@client ~]#while : ;do curl 10.1.1.100; sleep 0.5; done
this is websrv1
this is websrv2
this is websrv1
this is websrv2
this is websrv1
this is websrv2
[root@client ~]#while : ;do curl 10.1.1.200; sleep 0.5; done 
this is websrv4
this is websrv3
this is websrv4
this is websrv3
this is websrv4

停掉web1，再测试，发现只会调度到web2

[root@websrv1 data]#systemctl stop httpd
[root@client ~]#while : ;do curl 10.1.1.100; sleep 0.5; done
this is websrv2
this is websrv2
this is websrv2
this is websrv2
this is websrv2

停掉web2，再测试，发现sorry server带替工作，并在lvs1上可以看到127.0.0.1加载

[root@websrv2 ~]#systemctl stop httpd
[root@client ~]#while : ;do curl 10.1.1.100; sleep 0.5; done
server is under maitenance
server is under maitenance
server is under maitenance
server is under maitenance
[root@lsv1 ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.1.1.100:80 rr
  -> 127.0.0.1:80                 Route   1      0          4         
TCP  10.1.1.200:80 rr
  -> 192.168.205.77:80            Route   1      0          0         
  -> 192.168.205.87:80            Route   1      0          0

恢复两个websrv1 和websrv2,并停掉lvs2, 发现没有影响，但可以看到两个vip全部回到lvs1上

[root@websrv1 ~]#systemctl start httpd
[root@websrv2 ~]#systemctl start httpd
[root@lvs2 ~]#systemctl stop keepalived
[root@client ~]#while : ;do curl 10.1.1.100; sleep 0.5; done
this is websrv2
this is websrv1
this is websrv2
this is websrv1
[root@client ~]#while : ;do curl 10.1.1.200; sleep 0.5; done
this is websrv3
this is websrv4
this is websrv3
this is websrv4
[root@lsv1 ~]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:56:e1:ea brd ff:ff:ff:ff:ff:ff
    inet 192.168.205.37/24 brd 192.168.205.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 10.1.1.100/24 scope global eth0:0
       valid_lft forever preferred_lft forever
    inet 10.1.1.200/24 scope global secondary eth0:1
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe56:e1ea/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

将lvs2恢复状态，因为有抢占功能又回到原来的主和备份上

[root@client ~]#while : ;do curl 10.1.1.100; sleep 0.5; done
this is websrv2
this is websrv1
this is websrv2
[root@client ~]#while : ;do curl 10.1.1.200; sleep 0.5; done
this is websrv4
this is websrv3
this is websrv4
[root@lsv1 ~]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:56:e1:ea brd ff:ff:ff:ff:ff:ff
    inet 192.168.205.37/24 brd 192.168.205.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 10.1.1.100/24 scope global eth0:0
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe56:e1ea/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@lvs2 ~]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:37:f9:93 brd ff:ff:ff:ff:ff:ff
    inet 192.168.205.47/24 brd 192.168.205.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 10.1.1.200/24 scope global eth0:1
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe37:f993/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

以上是关于双主Keepalive与lvs实现自动监控主机状态并实时切换的主要内容，如果未能解决你的问题，请参考以下文章

初识MariaDB之9——keepalive+GTID双主实现高可用

双主Keepalive与lvs实现自动监控主机状态并实时切换

环境：

版本：

目地：

配置router

先配置四台real server

在两台lvs上做同样的动（除特别说明）

测试