mogilefs杂记

Posted 2020-10-27

mogilefs集群

nginx反代mogilefs

1、编译安装Nginx

1.1 安装编译环境

安装编译环境：
yum install gcc gcc-c++ automake pcre pcre-devel zlip zlib-devel openssl openssl-devel patch 

1.2 编译安装

[[email protected] ~]# ls
nginx_mogilefs_module-1.0.4.tar.gz  tengine-2.0.1.tar.gz

展开压缩包：
[[email protected] ~]# tar xf nginx_mogilefs_module-1.0.4.tar.gz 
[[email protected] ~]# tar xf tengine-2.0.1.tar.gz 

1.2.1 创建用户和组
[[email protected] ~]# groupadd -r nginx
[[email protected] ~]# useradd -r -g nginx nginx

1.2.2 编译安装
[[email protected] tengine-2.0.1]# ./configure --prefix=/usr --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx/nginx.pid --lock-path=/var/lock/nginx.lock --user=nginx --group=nginx --with-http_ssl_module --with-http_flv_module --with-http_stub_status_module --with-http_gzip_static_module --with-pcre --with-debug --add-module=../nginx_mogilefs_module-1.0.4

[[email protected] tengine-2.0.1]# make

报错：
../nginx_mogilefs_module-1.0.4/ngx_http_mogilefs_module.c: In function ‘ngx_http_mogilefs_create_spare_location’:
../nginx_mogilefs_module-1.0.4/ngx_http_mogilefs_module.c:1532:39: error: variable ‘pclcf’ set but not used [-Werror=unused-but-set-variable]
     ngx_http_core_loc_conf_t  *clcf, *pclcf, *rclcf;

配合文件去除Werror
[[email protected] tengine-2.0.1]# vim objs/Makefile 

CC =    cc
CFLAGS =  -pipe  -O -W -Wall -Wpointer-arith -Wno-unused-parameter -g    

[[email protected] tengine-2.0.1]# make install

2、配置启动脚本

[[email protected] tengine-2.0.1]# vim /usr/lib/systemd/system/nginx.service

[Unit]
Description=The nginx HTTP and reverse proxy server
After=network.target remote-fs.target nss-lookup.target

[Service]
Type=forking
PIDFile=/var/run/nginx/nginx.pid
ExecStartPre=/usr/sbin//nginx -t
ExecStart=/usr/sbin/nginx
ExecReload=/bin/kill -s HUP $MAINPID
KillMode=process
KillSignal=SIGQUIT
TimeoutStopSec=5
PrivateTmp=true

[Install]
WanteBy=multi-user.target

——————
[[email protected] tengine-2.0.1]# systemctl daemon-reload

启动
[[email protected] tengine-2.0.1]# systemctl start nginx.service

3、nginx配置文件

3.1 nginx配置

备份配置文件
[[email protected] nginx]# cp nginx.conf nginx.conf.bak

关闭VIM搜索高亮：
:nohl

配置：

[[email protected] nginx]# vim nginx.conf
server {
    listen       80;
    server_name  localhost;

    #charset koi8-r;

    #access_log  logs/host.access.log  main;

    location / {
        root   html;
        index  index.html index.htm;
    }

    location ~([^\/]+)$ {
        mogilefs_tracker 10.201.106.132:7001;
        mogilefs_domain img;
        mogilefs_methods GET;
        mogilefs_noverify on;

        mogilefs_pass {
            proxy_pass $mogilefs_path;
            proxy_hide_header Content-Type;
            proxy_buffering off;
        }

    #error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   html;
    }

    # proxy the php scripts to Apache listening on 127.0.0.1:80
    #
    #location ~ \.php$ {
    #    proxy_pass   http://127.0.0.1;
    #}

    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
    #
    #location ~ \.php$ {
    #    root           html;
    #    fastcgi_pass   127.0.0.1:9000;
        root   html;
        index  index.html index.htm;
    }

    location /images {
        mogilefs_tracker 10.201.106.132:7001;
        mogilefs_domain imgs;
        mogilefs_methods GET;
        mogilefs_noverify on;

        mogilefs_pass {
            proxy_pass $mogilefs_path;
            proxy_hide_header Content-Type;
            proxy_buffering off;
        }
    }

语法测试：
[[email protected] nginx]# nginx -t
the configuration file /etc/nginx/nginx.conf syntax is ok
configuration file /etc/nginx/nginx.conf test is successful

重新载入服务：
[[email protected] nginx]# systemctl reload nginx.service

访问测试：
http://10.201.106.131/images/1.jpg

3.2 更改路径测试

[[email protected] ~]# vim /etc/nginx/nginx.conf

location /images/ {

}
[[email protected] ~]# systemctl reload nginx.service

tracker配置
[[email protected] ~]# mogdelete --trackers=10.201.106.132:7001 --domain=imgs --key=‘/1.jpg‘
[[email protected] ~]# mogdelete --trackers=10.201.106.132:7001 --domain=imgs --key=‘/2.jpg‘

去掉斜线，重新上传：
[[email protected] ~]# mogupload --trackers=10.201.106.132:7001 --domain=imgs --class=jpeg --key=‘2.jpg‘ --file=‘/root/2.jpg‘
[[email protected] ~]# mogupload --trackers=10.201.106.132:7001 --domain=imgs --class=jpeg --key=‘1.jpg‘ --file=‘/root/2.jpg‘

测试：
http://10.201.106.131/images/1.jpg

4、增加tracerk节点，消除tracerk单点隐患

4.1 拷贝tracerk节点的配置文件到另外两个节点

[[email protected] ~]# scp /etc/mogilefs/mogilefsd.conf master3:/etc/mogilefs/
[email protected]‘s password: 
mogilefsd.conf                                                        100% 1464     1.4KB/s   00:00    
[[email protected] ~]# 
[[email protected] ~]# scp /etc/mogilefs/mogilefsd.conf master4:/etc/mogilefs/
[email protected]‘s password: 
mogilefsd.conf     

[[email protected] ~]# systemctl start mogilefsd.service
[[email protected] ~]# systemctl start mogilefsd.service

4.2 测试访问

[[email protected] ~]# moglistkeys --trackers=10.201.106.133:7001 --domain=imgs
1.jpg
2.jpg
[[email protected] ~]# moglistkeys --trackers=10.201.106.134:7001 --domain=imgs
1.jpg
2.jpg

4.3 nginx设置负载均衡

[[email protected] ~]# vim /etc/nginx/nginx.conf

upstream trackers {
        server 10.201.106.132:7001 weight=1;
        server 10.201.106.133:7001 weight=1;
        server 10.201.106.134:7001 weight=1;

        #以下配置只在TNGINX有用
        check interval=1000 rise=2 fall=5 timeout=1000;
        # check_keepalive_request 100;
        # check_http_send "GET / HTTP/1.0\r\n\r\n";
        # check_http_expect_alive http_2xx http_3xx;
}       

server {
    location /images/ {
        mogilefs_tracker trackers;

[[email protected] ~]# nginx -t
the configuration file /etc/nginx/nginx.conf syntax is ok
configuration file /etc/nginx/nginx.conf test is successful
[[email protected] ~]# systemctl reload nginx.service

4.4 访问测试

http://10.201.106.131/images/1.jpg
http://10.201.106.131/images/2.jpg

4.5 打开nginx状态页，查看后端健康情况

[[email protected] ~]# vim /etc/nginx/nginx.conf  

server {

    location /status {
        check_status;

        access_log off;
        #allow SOME.IP.ADD.RESS;
        #deny all;
    }

[[email protected] ~]# nginx -t
the configuration file /etc/nginx/nginx.conf syntax is ok
configuration file /etc/nginx/nginx.conf test is successful
[[email protected] ~]# systemctl reload nginx.service

测试：
http://10.201.106.131/status

关闭一个节点，测试状态页面：
[[email protected] ~]# systemctl stop mogilefsd.service

5、