Nginx 高并发下报错 connect() failed (110: Connection timed out) while connecting to upstream

Posted 2023-05-17

参考技术A

背景
在对应用服务进行压力测试时，nginx在持续压测请求1min左右后开始报错，花了一些时间对报错的原因进行排查，并最终定位到问题，现将过程总结下。

压测工具
这里压测使用的是 siege , 其非常容易指定并发访问数以及并发时间，以及有非常清晰的结果反馈，成功访问数，失败数，吞吐率等性能结果。

压测指标
单接口压测，并发100，持续1min。

压测工具 报错

Nginx error.log 报错

排查问题

发现在TCP的连接有两个异常点

关于这两点开始进行分析：

从 TIME-WAIT 定义中分析得知，当压测工具关闭连接后，实际上Nginx所在机器连接并未立刻CLOSED，而是进入TIME-WAIT状态，网上可以搜到非常多讲解TIME-WAIT过多导致丢包的情况，与我在压测时所遇到情况一样。

net.ipv4.tcp_max_tw_buckets = 5000 5000表示系统同时保持TIME_WAIT套接字的最大数量，如果超过这个数字，TIME_WAIT套接字将立刻被清除并打印警告信息。

优化方案
参照在网上搜索获取的信息，调整Linux内核参数优化：

参考资料：

Java 高并发下的实践

一、使用的技术

HashMap

ConcurrentHashMap

Lock

ReadWriteLock

synchronized

 

二、一百万并发下的组合

ConcurrentLockMap

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package cn.edu.scau.mk.map;

import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

/**
 *
 * @author MK
 */
public class ConcurrentLockMap implements In {

    ConcurrentHashMap<Integer, Integer> map = new ConcurrentHashMap<>();
    Lock lock = new ReentrantLock();

    @Override
    public void add(int a) {
        lock.lock();
        try {
            if (map.containsKey(a)) {
                map.put(a, 1 + map.get(a));
            } else {
                map.put(a, 1);
            }
        } finally {
            lock.unlock();
        }
    }

    @Override
    public int get(int a) {
        int as = map.get(a);

        return as;
    }

}

View Code

ConcurrentSynchronizedMap

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package cn.edu.scau.mk.map;

import java.util.concurrent.ConcurrentHashMap;

/**
 *
 * @author MK
 */
public class ConcurrentSynchronizedMap implements In  {

    ConcurrentHashMap<Integer, Integer> map = new ConcurrentHashMap<>();

    public void add(int a) {

        if (map.containsKey(a)) {
            synchronized((Integer)a){
              map.put(a, 1 + map.get(a));
            }
        } else {
            map.put(a, 1);
        }

    }

    public int get(int a) {
        int as = map.get(a);
        return as;
    }

}

View Code

LockHashMap

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package cn.edu.scau.mk.map;

import java.util.HashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

/**
 *
 * @author MK
 */
public class LockHashMap implements In {

    HashMap<Integer, Integer> map = new HashMap<>();
    Lock lock = new ReentrantLock();

    @Override
    public void add(int a) {
        lock.lock();
        try {
            if (map.containsKey(a)) {
                map.put(a, 1 + map.get(a));
            } else {
                map.put(a, 1);
            }
        } finally {
            lock.unlock();
        }
    }

    @Override
    public int get(int a) {
        int as = map.get(a);

        return as;
    }

}

View Code

ReadWriteHashMap

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package cn.edu.scau.mk.map;

import java.util.HashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;

/**
 *
 * @author MK
 */
public class ReadWriteHashMap implements In {

    HashMap<Integer, Integer> map = new HashMap<>();
    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
    //390

    @Override
    public void add(int a) {
        lock.writeLock().lock();
        try {
            if (map.containsKey(a)) {
                map.put(a, 1 + map.get(a));
            } else {
                map.put(a, 1);
            }
        } finally {
            lock.writeLock().unlock();
        }
    }

    @Override
    public int get(int a) {
        int as = 0;
        lock.readLock().lock();
        try {
            as = map.get(a);
        } finally {
            lock.readLock().unlock();
        }
        return as;
    }

}

View Code

SynchronizedHashMap

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package cn.edu.scau.mk.map;

import java.util.HashMap;

/**
 *
 * @author MK
 */
public class SynchronizedHashMap implements In {

    HashMap<Integer, Integer> map = new HashMap<>();
  

    @Override
    public synchronized void add(int a) {
       
            if (map.containsKey(a)) {
                map.put(a, 1 + map.get(a));
            } else {
                map.put(a, 1);
            }
        
    }

    @Override
    public synchronized int get(int a) {
        int as = map.get(a);
       
        return as;
    }

}

View Code

Adder

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package cn.edu.scau.mk.map;

import java.util.Random;

/**
 *
 * @author MK
 */
public class Adder implements Runnable {

    In in;
    Random random = new Random();

    public Adder(In in) {
        this.in=in;
    }

    @Override
    public void run() {

        for (int i = 0; i < 1000; i++) {
            //in.add(random.nextInt());
            in.add(i);
            
        }

    }
}

View Code

Getter

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package cn.edu.scau.mk.map;

import java.util.Random;

/**
 *
 * @author MK
 */
public class Getter implements Runnable {

    In in;
    Random random = new Random();

    public Getter(In in) {
        this.in=in;
    }

    @Override
    public void run() {

        for (int i = 0; i < 1000; i++) {
            //in.add(random.nextInt());
            in.get(i);
            
        }

    }
}

View Code

TestDemo

package cn.edu.scau.mk.map;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

/**
 *
 * @author MK
 */
public class TestDemo {

    /*
        add:66945 get:68174
        add:26896 get:27872
        add:128779 get:127131
        add:99832 get:102323
        add:70762 get:70493
     */
    public static void main(String[] args) {
        In in = null;

        in = new ReadWriteHashMap();
        operate(in);
        in = new ConcurrentSynchronizedMap();
        operate(in);
        in = new ConcurrentLockMap();
        operate(in);
        in = new SynchronizedHashMap();
        operate(in);
        in = new LockHashMap();
        operate(in);
    }

    public static void operate(In in) {
        int poolSize = 1000000;
        Adder add = new Adder(in);
        ExecutorService threadPool = Executors.newFixedThreadPool(8);
        long start = System.currentTimeMillis();
        for (int i = 0; i < poolSize; i++) {
            threadPool.execute(add);
        }
        threadPool.shutdown();
        try {//等待直到所有任务完成
            threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
            long end = System.currentTimeMillis() - start;
            System.out.print("add:" + end);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        Getter get = new Getter(in);
        threadPool = Executors.newFixedThreadPool(8);
        start = System.currentTimeMillis();
        for (int i = 0; i < poolSize; i++) {
            threadPool.execute(add);
        }
        threadPool.shutdown();
        try {//等待直到所有任务完成
            threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
            long end = System.currentTimeMillis() - start;
            System.out.println(" get:" + end);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}

View Code

 

三、输出结果

ReadWriteHashMap                                   add:66945 get:68174
ConcurrentSynchronizedMap                    add:26896 get:27872
ConcurrentLockMap                                  add:128779 get:127131
SynchronizedHashMap                             add:99832 get:102323
LockHashMap                                           add:70762 get:70493