基础篇 | HAProxy in action

Posted 2021-05-01 5ithink

         

        HAProxy Technologies is the world’s leading provider of software load balancers and application delivery controllers (ADCs) for modern enterprises. HAProxy empowers users with the flexibility and confidence to deliver websites and applications with high availability, performance and security at any scale and in any environment. HAProxy’s range of solutions is backed by expert support and services.

安装配置

---

1.系统环境

IP：192.168.1.202[CentOS 6.9]

2.下载&解压haproxy-1.4.23.tar.gz

tar xvf haproxy-1.4.23.tar.gz

3.编译安装

cd haproxy-1.4.23

make TARGET=linux26 PREFIX=/opt/webs/app/haproxy

make install PREFIX=/opt/webs/app/haproxy

4.编辑配置

vi /opt/webs/app/haproxy/haproxy.cfg 

global
log 192.168.1.202 local3 debug
maxconn 50000
chroot /opt/webs/app/haproxy
uid root
gid root
daemon
quiet
nbproc 2
pidfile /opt/webs/app/haproxy/haproxy.pid

defaults
log global
mode http
option dontlognull
log 192.168.1.202 local3 notice

maxconn  3000
contimeout 5000
clitimeout 50000
srvtimeout 50000

#------------config redis read cluster---------------
frontend springboot-web
   bind *:9099
   default_backend springboot

backend springboot
   mode http
   option httpchk /springboot/index
   balance roundrobin
   server spingboot1 192.168.1.200:9090 check inter 3000 rise 3 fall 3

#------------config redis read cluster---------------
listen redis-cluster *:63790
   mode tcp
   #balance source
   balance roundrobin
   option dontlog-normal
   option dontlognull
   option log-health-checks
   option log-separate-errors
   option socket-stats
   option tcpka
   option tcplog
   option contstats
   option socket-stats
   log global
   #server master 192.168.1.202:6379 check inter 3000 rise 3 fall 3
   server s1 192.168.1.202:63791 check inter 3000 rise 3 fall 3
   server s2 192.168.1.202:63792 check inter 3000 rise 3 fall 3
   server s3 192.168.1.202:63793 check inter 3000 rise 3 fall 3

#------------config haproxy monitor---------------
listen haproxy-monitor 0.0.0.0:8888
   mode http
   stats refresh 10s
   stats uri /haproxy
   stats realm Haproxy \ statistic
   stats auth think:think
   stats hide-version

5.配置日志

• vi /etc/rsyslog.conf

新增 local3.*    /var/log/haproxy.log

• vi /etc/sysconfig/rsyslog

修改 SYSLOGD_OPTIONS="-r -m 0"

• service rsyslog restart 

6.运行haproxy

• 启动服务

/opt/webs/app/haproxy/sbin/haproxy -f /opt/webs/app/haproxy/haproxy.cfg

• 重启服务

/opt/webs/app/haproxy/sbin/haproxy -f /opt/webs/app/haproxy/haproxy.cfg -sf `cat /opt/webs/app/haproxy/haproxy.pid`

• 停止服务

pkill -9 haproxy

7.验证

a.haproxy stats

b.日志：tail -f /var/log/haproxy.log

8.注册系统服务

a.编辑haproxy script

vim /etc/rc.d/init.d/haproxy

#! /bin/sh
set -e
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/opt/webs/app/haproxy/sbin
PROGDIR=/opt/webs/app/haproxy/
PROGNAME=haproxy
DAEMON=$PROGDIR/sbin/$PROGNAME
CONFIG=$PROGDIR/$PROGNAME.conf
PIDFILE=$PROGDIR/$PROGNAME.pid
DESC="HAProxy daemon"
SCRIPTNAME=/etc/init.d/$PROGNAME

# Gracefully exit if the package has been removed.
test -x $DAEMON || exit 0

start()
{
       echo -n "Starting $DESC: $PROGNAME"
       $DAEMON -f $CONFIG
       echo "."
}

stop()
{
       echo -n "Stopping $DESC: $PROGNAME"
       haproxy_pid=cat $PIDFILE
       kill $haproxy_pid
       echo "."
}

restart()
{
       echo -n "Restarting $DESC: $PROGNAME"
       $DAEMON -f $CONFIG -p $PIDFILE -sf $(cat $PIDFILE)
       echo "."
}

case "$1" in
 start)
       start
       ;;
 stop)
       stop
       ;;
 restart)
       restart
       ;;
 *)
       echo "Usage: $SCRIPTNAME {start|stop|restart}" >&2
       exit 1
       ;;
esac
exit 0

b.添加执行权限

chmod u+x /etc/rc.d/init.d/haproxy

c.添加到服务启动列表

chkconfig --add haproxy

d.关闭自启服务

chkconfig haproxy off

e.验证

chkconfig --list |grep haproxy

f.服务操作命令

service haproxy [start|stop|restart]

TCP[HAProxy+Redis]

---

1.启动redis服务

• 主节点

/home/redis/single/redis-server /home/redis/single/redis.conf

• 从节点1

/home/redis/master-slaver/s1/redis-server /home/redis/master-slaver/s1/redis.conf

• 从节点2

/home/redis/master-slaver/s2/redis-server  /home/redis/master-slaver/s2/redis.conf

• 从节点3

/home/redis/master-slaver/s3/redis-server  /home/redis/master-slaver/s3/redis.conf

ps:[redis主从搭建参考]

2.验证：

a.连接redis服务

b.通过代理IP访问

haproxy tcp proxy配置

/home/redis/redis-3.2.11/src/redis-cli -h 192.168.1.202 -p 63790

c.访问主节点

/home/redis/redis-3.2.11/src/redis-cli -h 192.168.1.202 -p 6379

d.访问从节点

/home/redis/redis-3.2.11/src/redis-cli -h 192.168.1.202 -p 63791

HTTP[haproxy+web]

---

1.haproxy http proxy配置

2.后台启动springboot服务

java -jar /opt/webs/m2/com/think/springboot/2.0-SNAPSHOT/springboot-2.0-SNAPSHOT.jar &

3.验证

a.通过代理端口9099启动

b.通过服务9090端口访问

HA[haproxy+keepalived]

---

1.环境：机器A，B台[Centos6.9]

• A IP：192.168.1.200

• B IP：192.168.1.202

• VIP ：192.168.1.222

ps:A,B两台机器同时安装并启动HAProxy(同上)

2.A,B主机均安装keepalived

a.yum install keepalived

b.编辑[主机A]keepalived.conf

! Configuration File for keepalived
global_defs {
   router_id LVS_DEVEL
}

vrrp_script check_haproxy {
   script "/opt/webs/app/keepalived/check_haproxy.sh"     
   interval 2
   weight -4
}

vrrp_instance VI_1 {
   state MASTER                  
   interface eth0
   virtual_router_id 51
   priority 150
   advert_int 1

   authentication {
       auth_type PASS
       auth_pass 5ithink
   }

   track_script {                
       check_haproxy
   }

   virtual_ipaddress {
      192.168.1.222               
   }
}

c.编辑[主机B]keepalived.conf

! Configuration File for keepalived
global_defs {
   router_id LVS_DEVEL
}

vrrp_script check_haproxy {
   script "/opt/webs/app/keepalived/check_haproxy.sh"     
   interval 2
   weight -4
}

vrrp_instance VI_1 {
   state BACKUP                 
   interface eth2
   virtual_router_id 51
   priority 100
   advert_int 1

   authentication {
       auth_type PASS
       auth_pass 5ithink
   }

   track_script {                
       check_haproxy
   }

   virtual_ipaddress {
      192.168.1.222               
   }
}

d.编辑check_proxy.sh[A,B主机保持一致]

mkidr -p /opt/webs/app/keepalived

cd /opt/webs/app/keepalived/

e.vi check_proxy.sh

#!/bin/bash
if [ $(ps -C haproxy --no-header | wc -l) -eq 0 ]; then  
   /opt/webs/app/haproxy/sbin/haproxy -f /opt/webs/app/haproxy/haproxy.cfg
   sleep 3
   if [ $(ps -C haproxy --no-header | wc -l) -eq 0 ]; then
          /etc/init.d/keepalived stop
   fi   
  echo "haproxy start up success."  
fi

f.添加权限

chmod u+x check_proxy.sh

3.启动keepalived

服务器A,B均启动keepalived： service keepalived start

4.验证场景：

• A,B主机同时启动HAProxy,主机Akeepalived[master]，优先抢占绑定VIP[192.168.1.222]

a.查看日志

tail -f /var/log/messages

b.ip addr

A主机：

B主机：

c.通过vip访问haproxy stats

http://192.168.1.222:8888/haproxy

通过vip访问springboot项目

• 当A服务器haproxy进程down掉，keepalived会监控并自动重启haproxy

模拟：A服务器杀掉haproxy进程

pkill -9 haproxy

• 当主机A keepalived[master] down掉，主机B[backup]keepalived自动接管

注：通过vip[192.168.1.222]漂移与主机B绑定实现

1.模拟：停止A主机keepalived服务

service keepalived stop 

2.命令ip addr观察vip与A主机已解除绑定

3.命令ip addr观察B主机与vip[192.168.1.222]已绑定

4.验证

a.通过VIP能正常访问haproxy stats

b.通过VIP正常访问springboot项目

• 当重新启动主机A keepalived[master]，vip[192.168.1.222]会重新与主机A绑定，主机B keepalived stats状态会变为[backup]

1.模拟：启动主机A keepalived服务

service keepalived start 

2.通过ip addr命令观察主机A已经重新与vip[192.168.1.222]绑定

附a.haproxy配置详解[haproxy.conf]

#全局配置
global
    #日志级别:[err warning info debug]
    #日志设备可选:
    #[kern user mail daemon auth syslog lpr news
    #uucp cron auth2 ftp ntp audit alert cron2
    #local0 local1 local2 local3 local4 local5 local6 local7]
    log 192.168.1.200 local3 debug
    maxconn 4096
    user root
    group root
    #进程后台运行
    daemon
    #进程数：2(配合"daemon"使用)
    nbproc 2
    pidfile /opt/webs/app/haproxy/haproxy.pid
#默认配置
defaults
    #mode模式 [tcp|http|health] health：返回OK
    mode http
    #采用http日志格式
    option httplog
    #失败3重连次数
    retries 3
    #重定向正常的服务器
    option redispatch
    #负载过高关闭当前队列处理时间比较长的链接
    option abortonclose
    maxconn 4096
    #连接超时
    contimeout 5000
    #客户端超时
    clitimeout 30000
    #服务器超时
    srvtimeout 30000
    #心跳检测超时
    timeout check 2000

    #注：时间值单位默认为毫秒(ms)，可以通过加#后缀来使用其他的单位。
    #- us : microseconds. 1 microsecond = 1/1000000 second
    #- ms : milliseconds. 1 millisecond = 1/1000 second. This is the default.
    #- s : seconds. 1s = 1000ms
    #- m : minutes. 1m = 60s = 60000ms
    #- h : hours. 1h = 60m = 3600s = 3600000ms
    #- d : days. 1d = 24h = 1440m = 86400s = 86400000ms

########统计页面配置############
listen haproxy-monitor 0.0.0.0:8888
    #bind 0.0.0.0:8888
    mode http
    #日志设置
    log global
    #统计页面自动刷新时间
    stats refresh 10s
    #统计页面url
    stats uri /haproxy
    #统计页面密码框上提示文本
    stats realm Haproxy \ statistic
    #统计页面用户名和密码设置
    stats auth think:think
    #隐藏统计页面上HAProxy的版本信息
    stats hide-version

#######网站监测############
listen site_monitor
    bind 0.0.0.0:9999
    mode http
    log 192.168.1.200 local3 info
    #网站健康检测URL
    monitor-uri /site_monitor
    #定义网站down时的策略:负载均衡上的指定backend的中有效机器数<1时,返回true
    acl site_dead nbsrv(denali_server) lt 1
    acl site_dead nbsrv(tm_server) lt 1
    acl site_dead nbsrv(mms_server) lt 1
    monitor fail if site_dead
    #http:200 ok 50x fail
    monitor-net 192.168.1.222/31

########frontend配置############
frontend springboot-web
    bind 0.0.0.0:9099
    mode http
    #应用全局的日志配置
    log global
    #启用http的log
    option httplog
    #每次请求完毕后主动关闭http通道，HA-Proxy不支持keep-alive模式
    option httpclose
    #获得客户端IP
    option forwardfor

    ###########HAProxy的日志记录内容配置##########
    capture request header Host len 40
    capture request header Content-Length len 10
    capture request header Referer len 200
    capture response header Server len 40
    capture response header Content-Length len 10
    capture response header Cache-Control len 8

    ####################acl策略定义#########################
    #hdr_reg:正则表达式匹配(结果：[true|false] -i:忽略大小写)
    acl domain_policy hdr_reg(host) -i ^(www.5ithink.net|www.5ithink.cn)$
    #hdr_dom:(相等www.5ithink.cn 结果：true|false -i:忽略大小写）
    acl tm_policy hdr_dom(host) -i www.5ithink.cn
    #url_sub:包含关系（请求url包含app=）
    acl invalid_req url_sub -i app=
    #url_dir:包含路径aaa
    acl timetask_req url_dir -i aaa
    #当请求的header中Content-length等于0时返回 true
    acl missing_cl hdr_cnt(Content-length) eq 0

    ######################acl策略匹配相应###################
    #当请求中header中Content-length等于0 阻止请求返回403
    block if missing_cl
    #block表示阻止请求，返回403错误，表示:不满足策略invalid_req，或者满足策略timetask_req，则阻止请求
    block if !invalid_req || timetask_req
    #当满足denali_policy的策略时使用denali_server的backend
    use_backend denali_server if denali_policy
    #当满足tm_policy的策略时使用tm_server的backend
    use_backend tm_server if tm_policy

    #reqisetbe自定义backend关键字
    reqisetbe ^Host:\ img test1
    reqisetbe ^[^\ ]*\ /(img|css)/ test2 #[backend name]
    reqisetbe ^[^\ ]*\ /stats stats
    #以上都不满足的时候使用默认mms_server的backend
    default_backend springboot_server
    #HAProxy错误页面设置
    errorfile 400 /opt/webs/app/haproxy/errorfiles/400.http
    errorfile 403 /opt/webs/app/haproxy/errorfiles/403.http
    errorfile 408 /opt/webs/app/haproxy/errorfiles/408.http
    errorfile 500 /opt/webs/app/haproxy/errorfiles/500.http
    errorfile 502 /opt/webs/app/haproxy/errorfiles/502.http
    errorfile 503 /opt/webs/app/haproxy/errorfiles/503.http
    errorfile 504 /opt/webs/app/haproxy/errorfiles/504.http

##########backend的设置##############
backend springboot_server
    mode http
    #负载均衡策略：roundrobin|leastconn
    #roundrobin:轮询方式
    #leastconn:根据服务器当前的请求数，取当前请求数最少的服务器
    #source 根据客户端IP对服务个数求hash值
    balance roundrobin
    #允许插入serverid到cookie中，serverid后面可以定义
    cookie SERVERID
    #设置backup 第一个server优先，设置option allbackups:备份服务器权重一样
    option allbackups
    #心跳检测的URL
    option httpchk GET /springboot/index HTTP/1.1\r\nHost:www.5ithink.cn

    #服务器定义：
    #cookie 1：表示serverid为1
    #check inter 1500：检测心跳频率1500
    #rise 3：确认服务器可用确认次数
    #fall 3：确认服务器不可用次数
    #minconn 10
    #maxconn 20
    #weight：权重
    server spingboot1 192.168.1.200:9090 cookie 1 check inter 1500 rise 3 fall 3 weight 1
    server spingboot2 192.168.1.202:9090 cookie 2 check inter 1500 rise 3 fall 3 weight 2

    #server spingboot1 192.168.1.200:9090 minconn 4 maxconn 12 check inter 1500 rise 3 fall 3
    #server spingboot2 192.168.1.202:9090 minconn 10 maxconn 20 check inter 1500 rise 3 fall 3
    #备份机器配置:正常情况下备机不会使用，当主机的全部服务器都down的时候备备机会启用
    #server spingboot1 192.168.1.200:9090 check backup inter 1500 rise 3 fall 3
    #server spingboot2 192.168.1.202:9090 check backup inter 1500 rise 3 fall 3

附b:keepalived配置文件详解[keepalived.conf]

! Configuration File for keepalived
#全局定义（global definition）
global_defs {
   #发送邮件地址列表
   notification_email {
     [e-mail]
     [e-mail]
     [e-mail]
   }
   #邮件发送地址
   notification_email_from [e-mail]
   #邮件smtp_server
   smtp_server 192.168.200.1
   #邮件发送超时时间
   smtp_connect_timeout 30
   #机器标识:通常为hostname
   router_id LVS_DEVEL
}
#静态地址和路由配置
static_ipaddress
    {
        192.168.1.1/24 brd + dev eth0 scope global
        192.168.1.2/24 brd + dev eth1 scope global
    }
static_routes
    {
        src $SRC_IP to $DST_IP dev $SRC_DEVICE
        src $SRC_IP to $DST_IP via $GW dev $SRC_DEVICE
    }
#默认区域不需要配置：等效于系统里命令配置IP地址和路由：
#192.168.1.1/24 brd + dev eth0 scope global 等效于 ip addr add 192.168.1.1/24 brd + dev eth0 scope global

#VRRPD配置包括（1.VRRP同步组(synchroization group) 2.vrrp_instance 3.vrrp_script）

vrrp_sync_group VG_1 {
    group {
        http
        mysql
    }
    #切换到[master]状态,执行script
    notify_master [path]/notify_master.sh
    #切换到[backup]状态,执行script
    notify_backup [path]/notify_backup.sh
    #错误,执行script
    notify_fault "[path]/notify_fault.sh VG_1"
    notify /path/to/notify.sh
    #发送邮件
    smtp_alert
}
#vrrp_instance区域
vrrp_instance VI_1 {
   #instance(Initial)的初始状态 [MASTER|BACKUP]
   state MASTER
   #interface：实例绑定的网卡
   interface eth0
   #设置VRID 相同的VRID为一个组，决定多播的MAC地址
   virtual_router_id 51
   #设置本节点的优先级，优先级高的为master
   priority 150
   #检查间隔，默认为1秒
   advert_int 1

   #设置认证
   authentication {
       #auth type：认证方式[PASS|AH]
       auth_type PASS
       #认证密码
       auth_pass 5ithink
   }

   track_script {                #call check_haproxy.sh
       check_haproxy
   }
   #virtual ipaddress：VIP 可以设置多个IP，随state绑定或者解除vip，主要由优先级来决定，和state初始值关系不大
   virtual_ipaddress {
      192.168.1.222               #VIP
   }
}
#vrrp_script区域
vrrp_script check_haproxy {
   script "/opt/webs/app/keepalived/check_haproxy.sh"     #haproxy monitor script
   #脚本执行间隔
   interval 2
   #优先级[-10~10]
   weight -4
}

注：
#dont track primary：忽略VRRP的interface错误
#track interface：跟踪接口
#mcast src ip：发送多播数据包时的源IP地址
#garp master delay：在切换到master状态后，延迟进行ARP(gratuitous ARP)请求
#virtual routes：原理和virtual ipaddress一样，只不过这里是增加和删除路由
#lvs sync daemon interface：lvs syncd绑定的网卡
#nopreempt：设置不抢占 用在在state为backup的节点上
#preempt delay：抢占延迟
#debug：debug级别
#notify master：同sync group

参考链接：

• https://www.haproxy.com/

• http://www.keepalived.org/LVS-NAT-Keepalived-HOWTO.html