lb-nginx

Posted 2020-12-11 看萝卜在飘

使用roles部署负载均衡的nginx

环境

外网IP	内网IP	主机名
10.0.0.5	172.16.1.5	lb01 （负载均衡）
10.0.0.6	172.16.1.6	lb02
10.0.0.7	172.16.1.7	web01(服务器)
10.0.0.8	172.16.1.8	web02
10.0.0.9	172.16.1.9	web03
10.0.0.31	172.16.1.31	nfs （共享存储）
10.0.0.41	172.16.1.41	backup
10.0.0.51	172.16.1.51	db01 （数据库）
10.0.0.52	172.16.1.52	db02
10.0.0.53	172.16.1.53	db03（代理机）
10.0.0.54	172.16.1.54	db04（代理机）
10.0.0.61	172.16.1.61	m01 （跳板机）
10.0.0.71	172.16.1.71	zabbix

流程分析

1.安装ansible
2.优化ansible
3.推送公钥
4.开启防火墙
5.开启80 443 873 nfs等端口和服务白名单
6.关闭selinux
7.创建同一的用户
	1.安装nginx
	2.拷贝nginx配置文件
	3.拷贝nginx虚拟主机配置及include文件
	4.启动nginx

推送公钥

1.创建密钥对
[root@m01 ~]# ssh-keygen
2.推送公钥
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.5
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.6
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.7
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.8
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.9
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.31
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.41
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.51
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.52
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.53
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.54
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.61
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.71
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.81

ansible优化

1.下载
[root@m01 ~]#  yum install -y ansible
2.优化
[root@m01 ~]#  vim /etc/ansible/ansible.cfg		#改为
host_key_checking = False

配置主机清单

[root@m01 ~]# vim /root/ansible/hosts 
#[]标签名任意，但是最好不要用特殊符号（- | &）和大写字母，中文（不能是nginx）
#端口是22的时候可以省略
[web_group]
172.16.1.7 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘
172.16.1.8 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘
172.16.1.9 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘

[db_group]
172.16.1.51 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘
172.16.1.52 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘
172.16.1.53 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘
172.16.1.54 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘

[nfs_group]
172.16.1.31 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘

[redis_group]
172.16.1.81 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘

[lb_group]
172.16.1.5 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘
172.16.1.6 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘

[backup_group]
172.16.1.41 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘

[zabbix_group]
172.16.1.71 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘

[m01_group]
172.16.1.61 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘

[mtj_group]
172.16.1.202 ansible_ssh_port=22 asible_ssh_user=root ansible_ssh_pass=‘1‘

nginx配置文件

user  {{ ww_w }};
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  ‘$remote_addr - $remote_user [$time_local] "$request" ‘
                      ‘$status $body_bytes_sent "$http_referer" ‘
                      ‘"$http_user_agent" "$http_x_forwarded_for"‘;

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}

nginx server

upstream {{ansible_fqdn}} {
{% for n in range(10) %}
        server 172.16.1.{{ n }}:{{ 80 }};
{% endfor %}
}
server {
	listen 80;
	server_name {{wp_com}} {{zh_com}};

    location / {
        proxy_pass http://{{ansible_fqdn}};    
        include proxy_params;
    }
}
----------------------------------------------------------------------------
#编辑params

# 客户端的请求头部信息，带着域名来找我，我也带着域名去找下一级（代理机或者代理服务器）
proxy_set_header Host $host;
# 显示客户端的真实ip（和代理的所有IP）
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
	
#nginx代理与后端服务器连接超时时间(代理连接超时)
proxy_connect_timeout 60s;
#nginx代理等待后端服务器的响应时间
proxy_read_timeout 60s;
	#后端服务器数据回传给nginx代理超时时间
proxy_send_timeout 60s;
	
#nignx会把后端返回的内容先放到缓冲区当中，然后再返回给客户端,边收边传, 不是全部接收完再传给客户端
proxy_buffering on;
#设置nginx代理保存用户头信息的缓冲区大小
proxy_buffer_size 4k;
#proxy_buffer_size 8k;
#proxy_buffers 缓冲区
proxy_buffers 8 4k;
#proxy_buffers 8 8k;
#使用http 1.1协议版本
proxy_http_version 1.1;

#错误页面重定向
proxy_next_upstream error timeout http_500 http_502 http_503 http_504 http_404;

keepalived启动脚本

#优化keepalived
keepalived.service 

[Unit]
Description=LVS and VRRP High Availability Monitor
After=syslog.target network-online.target

[Service]
Type=forking
PIDFile=/var/run/keepalived.pid
#KillMode=process
EnvironmentFile=-/etc/sysconfig/keepalived
ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS
ExecReload=/bin/kill -HUP $MAINPID

[Install]
WantedBy=multi-user.target

keepalived配置文件之jinjia模板

global_defs {
    router_id {{ ansible_fqdn }}
}

{% if ansible_fqdn == "lb01" %}
vrrp_script check {
    script "check_nginx_php.sh"
    interval 5
}
{% endif %}

vrrp_instance VI_1 {
{% if ansible_fqdn == "lb01" %}
    state MASTER
    priority 150
{% else %}
    state BACKUP
    priority 100
{% endif %}

    interface eth0
    virtual_router_id 50
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {         
        "{{ vi_ip }}"
    }
}

创建角色

[root@m01 roles]# ansible-galaxy init nginx_lb

编辑tasks目录

1.安装负载均衡的nginx
[root@m01 nginx_lb]# vim tasks/install.yml 
- name: check {{ nginx_packages_name }}
  shell: "ls /tmp/nginx-1.18.0"
  ignore_errors: yes
  register: check_nginx_packages

- name: jieya {{ nginx_packages_name }}
  unarchive:
    src: "{{ nginx_packages_name }}"
    dest: "{{nginx_packages_pos}}"
  when: check_nginx_packages != 0

- name: check nginx
  shell: "rpm -q nginx"
  ignore_errors: yes
  register: check_nginx

- name: Install Nginx Server
  yum:
    name:
      - "/tmp/nginx-1.18.0/nginx-1.18.0-1.el7.ngx.x86_64.rpm"
  when: check_nginx.rc != 0
2.拷贝nginx主配置文件和server
[root@m01 nginx_lb]# vim tasks/copy.yml 
- name: copy nginx.conf server
  template:
    src: "{{ item.src }}"
    dest: "{{ item.dest }}"
  loop:
    - { src: "nginx.conf.j2",dest: "/etc/nginx/nginx.conf" }
    - { src: "nginx.server.j2",dest: "/etc/nginx/conf.d/server.conf" }

- name: copy proxy_params
  copy:
    src: "{{ item.src }}"
    dest: "{{ item.dest }}"
  with_items:
    - { src: "proxy_params",dest: "/etc/nginx/" }
  notify:
    - "reload nginx"
3.启动nginx
[root@m01 nginx_lb]# vim tasks/start.yml 
- name: start nginx
  service:
    name: nginx
    state: started
4.include
[root@m01 nginx_lb]# vim tasks/main.yml 
- include: install.yml
- include: copy.yml
- include: start.yml
5.编辑files目录
[root@m01 nginx_lb]# vim files/keepalived.service 
[root@m01 nginx_lb]# vim files/proxy_params
[root@m01 nginx_lb]# rz		nginx-1.18.0.tar.gz
6.编辑template目录
[root@m01 nginx_lb]# vim templates/nginx.conf.j2 
[root@m01 nginx_lb]# vim templates/nginx.server.j2
7.编辑handlers
[root@m01 nginx_lb]# vim handlers/main.yml 
- name: reload nginx
  service:
    name: nginx
    state: reloaded
8.变量
[root@m01 nginx_lb]# vim vars/main.yml 
#统一ww_w用户
ww_w: www
#使用变量定义域名
yuming_com: "cs.wp.com cs.zh.com"
#nginx包名
nginx_packages_name: "nginx-1.18.0.tar.gz"
#nginx压缩包解压到目标机的位置
nginx_packages_pos: "/tmp"

编辑入口文件

[root@m01 roles]# vim site.yml 
- hosts: all
  roles:
    #- { role: base }
    #- { role: rsync_client,when: ansible_fqdn is match ‘web*‘ }
    #- { role: rsync_client,when: ansible_fqdn is match ‘nfs*‘ }
    #- { role: rsync_server,when: ansible_fqdn is match ‘backup*‘ }
    #- { role: nfs_server,when: ansible_fqdn is match ‘nfs*‘ }
    #- { role: nfs_client,when: ansible_fqdn is match ‘web*‘ }
    #- { role: mount_server,when: ansible_fqdn is match ‘nfs*‘ }
    #- { role: mount_client,when: ansible_fqdn is match ‘web*‘ }
    #- { role: sersync,when: ansible_fqdn is match ‘web*‘ }
    #- { role: nginx_web,when: ansible_fqdn is match ‘web*‘ }
    - { role: nginx_lb,when: ansible_fqdn is match ‘lb*‘ }

执行

[root@m01 roles]# ansible-playbook site.yml