记一次nginx 504排查

Posted 2023-02-22 Java_Chuck

有一个导出接口，由于里面组装数据会很慢，直接表现为504 Gateway Time-out，时间为2min,梳理了下调用的链路，nginx -> spring gateway -> business server,里面打开了nginx的配置，如下：

#user  nobody;
worker_processes  4;

error_log  /app/openresty/nginx/logs/error.log  error;
pid        /app/openresty/nginx/logs/nginx.pid;
#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;


events 
    worker_connections 65536 ;



http 
    include       mime.types;
    include       custom_upstream.conf;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr|$remote_user|[$time_local]|"$request"'
                      '|$status|$request_time|$body_bytes_sent|"$http_referer"'
                      '|"$http_user_agent"|"$http_x_forwarded_for"|$upstream_response_time|$upstream_status';    
    access_log  /app/openresty/nginx/logs/access.log  main;
    
    server_tokens   off;
    sendfile        on;
    tcp_nopush      on;
    tcp_nodelay     on;
    client_max_body_size 100m;
    client_header_buffer_size 100m;
    keepalive_timeout  120;
    proxy_send_timeout 120;
    proxy_read_timeout 120;
    proxy_buffering off;
    gzip on;

    server 
        listen       80;
        server_name  localhost;

        listen 443 ssl;
        ssl_certificate_key /app/openresty/nginx/conf/key/sf.key;
        ssl_certificate /app/openresty/nginx/conf/key/sf.crt;

        #charset koi8-r;

        include       custom_location.conf;

        #location / 
        #    root   html;
        #    index  index.html index.htm;
        #

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
		location = /50x.html 
            root   html;
        

        location = /nstats 
            check_status;
            access_log      off;
            allow           10.116.0.0/16;
            allow           10.110.0.0/16;
            allow           10.117.0.0/16;
            allow           10.150.0.0/16;
            allow           100.0.0.0/8;
            allow           10.0.0.0/8;
            deny            all;
        
    

	location / 
		root   html;
		index  index.html index.htm;
	
	

	location ^~ /foo 
		proxy_pass http://bar:8080;
		proxy_set_header Host $host;
		proxy_set_header X-Real-IP $remote_addr;
		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
		proxy_set_header Cookie $http_cookie;
		index index.html index.jsp login.jsp index.htm;    
	

配置经过处理，已替换成foo，bar

果然在配置中找到了2min的配置，我们的接口是一个导出的接口，自然是跟响应有关，说干就干，那就简单了，改成30min分钟。

    keepalive_timeout  120;
    proxy_send_timeout 120;
    proxy_read_timeout 120;

为了不影响其他接口，这里我们新增了一个精确匹配的路由

    location = /foo/downloaddatafile

        proxy_pass http://bar:8080;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Cookie $http_cookie;
        keepalive_timeout  120;
        proxy_send_timeout 120;
        proxy_read_timeout 1800;        
    

重启一波，点击导出，依然是504，这次的超时竟然还变短了，1min.what???这是什么情况。

只能打开浏览器的respose,很幸运是有东西的，提示我们去检查nginx的error.log.

好的，那我们就去看看吧，关键信息：[error] 33#0: *703 upstream timed out (110: Connection timed out)

度娘告诉我们需要改缓冲配置。

 

    location = /aom-asset-repair/repair/downloaddatafile
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Cookie $http_cookie;
        keepalive_timeout  120;
        proxy_send_timeout 120;
        proxy_read_timeout 1800;
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_buffer_size 64k;
        proxy_buffers   4 32k;
        proxy_busy_buffers_size 64k;
        proxy_temp_file_write_size 64k;
        proxy_pass http://bar:8080;
    

写在最后:

1. proxy_pass配置的顺序为啥会影响其他配置项生效？

2.导出这种慢操作，建议优化成异步处理，后台异步处理时，可以使用多线程。