openresty用haproxy2.0实现负载均衡

Posted 2020-12-10 haozheyu

安装openresty

编译安装

yum install pcre-devel openssl-devel gcc curl wget 
wget https://openresty.org/download/openresty-1.15.8.3.tar.gz
tar -xzvf openresty-1.15.8.3.tar.gz
cd openresty-1.15.8.3
./configure
gmake
gmake install

yum 安装

yum install -y wget
wget -O /etc/yum.repos.d/openresty.repo https://openresty.org/package/centos/openresty.repo
yum install -y openresty openresty-resty

#默认安装编译参数检查
[root@localhost openresty-1.15.8.3]# openresty -V
nginx version: openresty/1.15.8.3
built by gcc 8.3.1 20190311 (Red Hat 8.3.1-3) (GCC) 
built with OpenSSL 1.1.0l  10 Sep 2019
TLS SNI support enabled
configure arguments: --prefix=/usr/local/openresty/nginx --with-cc-opt=‘-O2 -DNGX_LUA_ABORT_AT_PANIC -I/usr/local/openresty/zlib/include -I/usr/local/openresty/pcre/include -I/usr/local/openresty/openssl/include‘ --add-module=../ngx_devel_kit-0.3.1rc1 --add-module=../echo-nginx-module-0.61 --add-module=../xss-nginx-module-0.06 --add-module=../ngx_coolkit-0.2 --add-module=../set-misc-nginx-module-0.32 --add-module=../form-input-nginx-module-0.12 --add-module=../encrypted-session-nginx-module-0.08 --add-module=../srcache-nginx-module-0.31 --add-module=../ngx_lua-0.10.15 --add-module=../ngx_lua_upstream-0.07 --add-module=../headers-more-nginx-module-0.33 --add-module=../array-var-nginx-module-0.05 --add-module=../memc-nginx-module-0.19 --add-module=../redis2-nginx-module-0.15 --add-module=../redis-nginx-module-0.3.7 --add-module=../ngx_stream_lua-0.0.7 --with-ld-opt=‘-Wl,-rpath,/usr/local/openresty/luajit/lib -L/usr/local/openresty/zlib/lib -L/usr/local/openresty/pcre/lib -L/usr/local/openresty/openssl/lib -Wl,-rpath,/usr/local/openresty/zlib/lib:/usr/local/openresty/pcre/lib:/usr/local/openresty/openssl/lib‘ --with-cc=‘ccache gcc -fdiagnostics-color=always‘ --with-pcre-jit --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-http_v2_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-http_stub_status_module --with-http_realip_module --with-http_addition_module --with-http_auth_request_module --with-http_secure_link_module --with-http_random_index_module --with-http_gzip_static_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-threads --with-compat --with-stream --with-stream_ssl_preread_module --with-http_ssl_module


#安装php
yum install php-fpm php-mysql php-xml php -y
systemctl enable php-fpm.service --now

#虚拟站点配置文件
[root@localhost conf]# cat /usr/local/openresty/nginx/test/index.php 
echo hellowld
<h1>yayayayayale</h1>

[root@localhost conf]# cat /usr/local/openresty/nginx/conf.d/php.conf 
server {    
    
  #监听端口    
  listen 801;    
    
  #网站根目录    
  root /usr/local/openresty/nginx/test;    
    
  #虚拟主机名称    
  server_name 192.168.168.21;    
    
  #网站主页排序    
  index index.php index.html index.htm default.php default.htm default.html;    
    
  #网站访问、错误日志    
  access_log /usr/local/openresty/nginx/test/test.access.log;    
  error_log /usr/local/openresty/nginx/logs/test/test.error.log;    
    
  #流量限制（网站最大并发数500|单IP访问最大并发数50|每个请求流量上限1024KB）    
  #limit_conn perserver 500;    
  #limit_conn perip 50;    
  #limit_rate 1024k;    
      
  #配置错误页面    
  #error_page 404 /404.html;    
  #error_page 500 502 503 504 /50x.html;    
    
  #禁止访问文件和目录    
  location ~ ^/(.user.ini|.htaccess|.git|.svn|.project|LICENSE|README.md) {    
    return 404;    
  }    
    
  #配置资源防盗链    
  location ~ .*.(jpg|jpeg|gif|png|js|css)$ {    
    expires 30d;    
    access_log /dev/null;    
    valid_referers none blocked 192.168.168.21;    
    if ($invalid_referer) {    
      return 404;    
    }    
  }    
    
  #配置图片资源缓存时间    
  location ~ .*.(gif|jpg|jpeg|png|bmp|swf)$ {    
    expires 30d;    
    error_log off;    
    access_log /dev/null;    
  }    
      
  #设置样式资源缓存时间    
  location ~ .*.(js|css)?$ {    
    expires 12h;    
    error_log off;    
    access_log /dev/null;    
  }    
    
  #解析PHP    
  location ~* .php$ {    
    fastcgi_index index.php;    
    fastcgi_pass 127.0.0.1:9000;    
    include fastcgi_params;    
    fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;    
    fastcgi_param SCRIPT_NAME $fastcgi_script_name;    
  }
}
#主配置文件
[root@localhost conf]# cat /usr/local/openresty/nginx/conf/nginx.conf

#user  nobody;
worker_processes  1;

error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    #加载虚拟主机配置文件    
    include /usr/local/openresty/nginx/conf.d/*.conf; 
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  ‘$remote_addr - $remote_user [$time_local] "$request" ‘
    #                 ‘$status $body_bytes_sent "$http_referer" ‘
    #                ‘"$http_user_agent" "$http_x_forwarded_for"‘;

    #access_log  logs/access.log  main;
    log_format  json  ‘{"@timestamp":"$time_iso8601",‘
                      ‘"@version":"1",‘
                      ‘"client":"$remote_addr",‘
                      ‘"url":"$uri",‘
                      ‘"status":"$status",‘
                      ‘"domain":"$host",‘
                      ‘"host":"$server_addr",‘
                      ‘"size":"$body_bytes_sent",‘
                      ‘"responsentime":"$request_time",‘
                      ‘"referer":"$http_referer",‘
                      ‘"useragent":"$http_user_agent",‘
                      ‘"upstreampstatus":"$upstream_status",‘
                      ‘"upstreamaddr":"$upstream_addr",‘
                      ‘"upstreamresponsetime":"$upstream_response_time"‘
                       ‘}‘;

    access_log  logs/access_json.log  json; 
    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
            root   html;
            index  index1.html index.htm;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ .php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ .php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache‘s document root
        # concurs with nginx‘s one
        #
        #location ~ /.ht {
        #    deny  all;
        #}
    }


    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}


    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}

安装haproxy

1.1需要先安装lua5.3版本，否则后面后报错

curl -R -O http://www.lua.org/ftp/lua-5.3.5.tar.gz 
yum install readline-devel gcc -y
tar zxvf lua-5.3.5.tar.gz
cd lua-5.3.5
make linux
make INSTALL_TOP=/usr/local/lua install
yum install systemd-devel wget openssl openssl-devel -y

1.2部署haproxy。

cdwget https://www.haproxy.org/download/2.0/src/haproxy-2.0.1.tar.gztar zxvf haproxy-2.0.1.tar.gz 
cd haproxy-2.0.1
make -j $(nproc) TARGET=linux-glibc USE_OPENSSL=1 USE_ZLIB=1 USE_LUA=1 LUA_LIB=/usr/local/lua/lib/ LUA_INC=/usr/local/lua/include/ USE_PCRE=1 USE_SYSTEMD=1 
make install PREFIX=/usr/local/haproxycd

1.3创建运行用户（应该也可忽略，只是使用该用户运行，加强安全而已。我一般是直接使用root用户运行）

useradd haproxy -s /sbin/nologin

1.4创建配置文件haproxy.cfg。在安装目录/usr/local/hadproxy下新建haproxy.cfg

global　　#全局设置
　 daemon　　#以后台进程运行
　 maxconn 256　　#每个进程的最大连接数
　 nbproc 1　　#进程数，该值可以设置小于或等于cpu核心数
　 balance roundrobin #默认的负载均衡的方式,轮询方式 
  #balance source #默认的负载均衡的方式,类似nginx的ip_hash 
  #balance leastconn #默认的负载均衡的方式,最小连接 
defaults　　#默认设置
　　mode http　　#设置http(七层模式)，也可设置为tcp(四层模式)，另外还有一个Health健康监测模式。对mysql进行负载均衡的话，这里记得修改为tcp
　　timeout connect 5000ms
　　timeout client 50000ms
　　timeout server 50000ms
　　

listen admin_stats #配置haproxy管理页面
　　bind *:9999　　#访问端口为9999
　　mode http
　　option httplog
　　stats refresh 30s　　#自动刷新时间
　　stats uri /stats　　#项目名为status，ip+端口+项目名即可访问
　　stats auth admin:admin　　#配置管理用户账号密码
　　stats admin if TRUE
　　stats hide-version

frontend http-in　　#配置前端访问端口
　　bind *:1080　　#通过该端口进行负载均衡
　　default_backend servers　　#指定后端服务器

backend servers
　　server server1 172.16.1.230:80 check inter 2000 rise 3 fall 3 weight 1 maxconn 32　　

　　#建议加上check，否则后台服务器A宕机了，负载均衡还会把请求发送到该宕机服务器上，inter 2000指check检测时间为2000毫秒，rise 3检测3次均正常则表示后天服务器正常，fall 3检测3次失败，则会把该后天服务器标志宕机，不再玩该后台服务器发送请求，weight 1指权重，取消weight改为backup，则是所有后台服务器宕机后才会启用该backup后台服务器
　　server server2 172.16.1.227:80 check inter 2000 rise 3 fall 3 weight 1 maxconn 32



#参考yum 安装的配置文件
[root@node2 haproxy]# cat /etc/haproxy/haproxy.cfg |grep -v "^#"

global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the ‘-r‘ option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

frontend  main *:5000
    acl url_static       path_beg       -i /static /images /javascript /stylesheets
    acl url_static       path_end       -i .jpg .gif .png .css .js

    use_backend static          if url_static
    default_backend             app

backend static
    balance     roundrobin
    server      static 127.0.0.1:4331 check

backend app
    balance     roundrobin
    server  app1 127.0.0.1:5001 check
    server  app2 127.0.0.1:5002 check
    server  app3 127.0.0.1:5003 check
    server  app4 127.0.0.1:5004 check
    
    

1.5使用指定配置文件运行haproxy。建议都haproxy和hadproxy.cfg都使用绝对路径

#/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg

1.6验证安装的haproxy版本

[root@localhost ~]# /usr/local/haproxy/sbin/haproxy -v
HA-Proxy version 2.0.1 2019/06/26 - https://haproxy.org/

1.7配置开机自启

#修改haproxy cfg
[root@localhost haproxy]# cat /usr/local/haproxy/haproxy.cfg 
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the ‘-r‘ option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /usr/local/haproxy
    pidfile     /usr/local/haproxy/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /usr/local/haproxy/socket

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

frontend  main 
    bind 0.0.0.0:8000
    acl url_static       path_beg       -i /static /images /javascript /stylesheets
    acl url_static       path_end       -i .jpg .gif .png .css .js

    use_backend static          if url_static
    default_backend             app

backend static
    balance     roundrobin
    server      static 127.0.0.1:4331 check

backend app
    balance     roundrobin
    server  app1 127.0.0.1:80 check
    server  app2 127.0.0.1:801 check
    #server  app3 127.0.0.1:5003 check
    #server  app4 127.0.0.1:5004 check
[root@localhost haproxy]# 

[root@localhost haproxy]# cat /usr/lib/systemd/system/haproxy.service 
[Unit]
Description=haporxy load bulancer
After=syslog.target network.target

[Service]
ExecStartPre=/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg -c -q
ExecStart=/usr/local/haproxy/sbin/haproxy -Ws -f /usr/local/haproxy/haproxy.cfg -p /usr/local/haproxy/haproxy.pid
ExecReload=/bin/kill -USER2 $MAINPID

[Install]
WantedBy=multi-user.target

systemctl enable haproxy --now

1.8测试效果

其中228的80端口上我启用了httpd，页面内容为this is 228.另外一台230上的80端口也是httpd，页面为默认内容。使用谷歌浏览器访问172.16.1.227，每次刷新的内容都和上一次不一样

curl 127.0.0.1:8000

[root@localhost haproxy]# curl 192.168.1.119:8000
<!DOCTYPE html>
<html>
<head>
<title>Welcome to OpenResty!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
<body>
<h1>Welcome to OpenResty!</h1>
<p>If you see this page, the OpenResty web platform is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="https://openresty.org/">openresty.org</a>.<br/>
Commercial support is available at
<a href="https://openresty.com/">openresty.com</a>.</p>

<p><em>Thank you for flying OpenResty.</em></p>
</body>
</html>

[root@localhost haproxy]# curl 192.168.1.119:8000
echo hellowld
<h1>yayayayayale</h1>