NFS单点故障及深度优化方案

Posted 2020-09-08

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了NFS单点故障及深度优化方案相关的知识，希望对你有一定的参考价值。

####主机：
backup nfs01

####系统环境：

[[email protected] ~]# cat /etc/redhat-release 
CentOS release 6.8 (Final)
[[email protected] ~]# uname -r
2.6.32-642.el6.x86_64
[[email protected] ~]# /etc/init.d/iptables status
iptables: Firewall is not running.
[[email protected] ~]# getenforce
Disabled

####安装包
yum install keepalived -y

-------------------------------------------------
-------------------------------------------------
####keepalived配置文件
###backup

global_defs {
   router_id LVS_02
}

vrrp_script check_nfs {           
script "/server/scripts/check_nfs.sh"  
interval 2   
weight 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth1
    virtual_router_id 100
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
     172.16.1.3/16 dev eth1 label eth1:1
    }
  track_script {            
  check_nfs
  }
}

---------------------------------------------------
---------------------------------------------------
###nfs01

global_defs {
   router_id LVS_01
}

vrrp_script check_nfs {           
script "/server/scripts/check_nfs.sh"  
interval 2   
weight 2
}

vrrp_instance VI_1 {
    state MASTER
    interface eth1
    virtual_router_id 100
    priority 150
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
     172.16.1.3/16 dev eth1 label eth1:1
    }
  track_script {            
  check_nfs
  }
}

----------------------------------------------
----------------------------------------------

####修改backup rsyncd.conf配置文件

######rsync_config_______________start
##rsyncd.conf start##
uid = www
gid = www
use chroot = no
max connections = 200
timeout = 300
pid file = /var/run/rsyncd.pid
lock file = /var/run/rsync.lock
log file = /var/log/rsyncd.log
ignore errors
read only = false
list = false
hosts allow = 172.16.1.0/24
#hosts deny = 0.0.0.0/32
auth users = rsync_backup
secrets file = /etc/rsync.password
[backup]
path = /backup/
[nfsbackup]
path = /nfsbackup/
[web01backup]
path = /web01backup/
[data]
path = /data
#rsync_config_______________end

####修改rsync匿名用户为www uid为888，添加一个模块/data
--------------------------------------------------
--------------------------------------------------

####添加用户

useradd -s /sbin/nologin -M -u888 www
id www
mkdir /data

chown -R www.www /web01backup /nfsbackup /backup /data

####修改nfs01的sersync配置文件，将rsync的模块改为data

<?xml version="1.0" encoding="ISO-8859-1"?>
<head version="2.5">
    <host hostip="localhost" port="8008"></host>
    <debug start="false"/>
    <fileSystem xfs="false"/>
    <filter start="false">
    <exclude expression="(.*)\.svn"></exclude>
    <exclude expression="(.*)\.gz"></exclude>
    <exclude expression="^info/*"></exclude>
    <exclude expression="^static/*"></exclude>
    </filter>
    <inotify>
    <delete start="true"/>
    <createFolder start="true"/>
    <createFile start="false"/>
    <closeWrite start="true"/>
    <moveFrom start="true"/>
    <moveTo start="true"/>
    <attrib start="false"/>
    <modify start="false"/>
    </inotify>

    <sersync>
    <localpath watch="/data">
        <remote ip="172.16.1.41" name="data"/>
        <!--<remote ip="192.168.8.39" name="tongbu"/>-->
        <!--<remote ip="192.168.8.40" name="tongbu"/>-->
    </localpath>
    <rsync>
        <commonParams params="-az --delete"/>
        <auth start="true" users="rsync_backup" passwordfile="/etc/rsync.password"/>
        <userDefinedPort start="false" port="874"/><!-- port=874 -->
        <timeout start="false" time="100"/><!-- timeout=100 -->
        <ssh start="false"/>
    </rsync>
    <failLog path="/var/log/rsync_fail_log.sh" timeToExecute="60"/><!--default every 60mins execute once-->
    <crontab start="false" schedule="600"><!--600mins-->
        <crontabfilter start="false">
        <exclude expression="*.php"></exclude>
        <exclude expression="info/*"></exclude>
        </crontabfilter>
    </crontab>
    <plugin start="false" name="command"/>
    </sersync>

    <plugin name="command">
    <param prefix="/bin/sh" suffix="" ignoreError="true"/>    <!--prefix /opt/tongbu/mmm.sh suffix-->
    <filter start="false">
        <include expression="(.*)\.php"/>
        <include expression="(.*)\.sh"/>
    </filter>
    </plugin>

    <plugin name="socket">
    <localpath watch="/opt/tongbu">
        <deshost ip="192.168.138.20" port="8009"/>
    </localpath>
    </plugin>
    <plugin name="refreshCDN">
    <localpath watch="/data0/htdocs/cms.xoyo.com/site/">
        <cdninfo domainname="ccms.chinacache.com" port="80" username="xxxx" passwd="xxxx"/>
        <sendurl base="http://pic.xoyo.com/cms"/>
        <regexurl regex="false" match="cms.xoyo.com/site([/a-zA-Z0-9]*).xoyo.com/images"/>
    </localpath>
    </plugin>
</head>

-------------------------------------------------------------------------
####在backup启动rpcbind nfs服务，共享/data目录

[[email protected] data]# cat /etc/exports 
#share /data by oldboy for bingbing at 20160425
/data/www 172.16.1.0/24(rw,sync,all_squash,anonuid=888,anongid=888)
/data/bbs 172.16.1.0/24(rw,sync,all_squash,anonuid=888,anongid=888)
/data/blog 172.16.1.0/24(rw,sync,all_squash,anonuid=888,anongid=888)

####启动服务

chkconfig rpcbind on
chkconfig nfs on

####将web服务的挂载ip改为172.16.1.3

umount -lf /application/nginx-1.10.2/html/www/uploads
umount -lf /application/nginx-1.10.2/html/bbs
umount -lf /application/nginx-1.10.2/html/blog/wp-content

####NFS客户端内核优化

cat >>/etc/sysctl.conf<<EOF
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
EOF
sysctl -p

####客户端挂载优化

mount -t nfs -o nosuid,noexec,nodev,noatime,nodiratime,rsize=131072,wsize=131072 172.16.1.3:/data/www /application/nginx-1.10.2/html/www/uploads

mount -t nfs -o nosuid,noexec,nodev,noatime,nodiratime,rsize=131072,wsize=131072 172.16.1.3:/data/bbs /application/nginx-1.10.2/html/bbs

mount -t nfs -o nosuid,noexec,nodev,noatime,nodiratime,rsize=131072,wsize=131072 172.16.1.3:/data/blog /application/nginx-1.10.2/html/blog/wp-content

####解决nfs服务停止keepalived不跳到backup的问题，编写脚本

[[email protected] storage]# cat check_nfs.sh 
#!/bin/bash
if [ `showmount -e localhost|wc -l` -lt 4 ];then
   /etc/init.d/keepalived stop
fi

####启动服务，加入开机自启动

/etc/init.d/keepalived start
chkconfig keepalived on

以上是关于NFS单点故障及深度优化方案的主要内容，如果未能解决你的问题，请参考以下文章

day35-NFS优化及优缺点

Nginx集群（负载均衡）

十个网络优化改造案例之一交换机由单机改造为双机热备

NFS服务的简介及常见故障解决方法

5）NFS存储实时复制原理

nfs