apache无法启动报错No space left on device

Posted 2021-02-06 reblue520

apache无法启动报错No space left on device

故障现象:apache无法启动ipcs信号量很多

# service httpd start
Starting httpd : [Failed]

当达到极限信号量：

# ipcs -s | wc -l
32004

而：

# cat /proc/sys/kernel/msgmni
32000

# cat /proc/sys/kernel/sem
250 24000 32 1024

原因
这些错误意味着有缺乏在系统内处理的通信资源，例如信号量或共享存储器段。

解析度
登录到通过服务器的SSH。

增加的限制sysctl 配置：

# mkdir -p /etc/sysctl.d/
# touch /etc/sysctl.d/99-zz_plesk_semaphores.conf
# chmod 755 /etc/sysctl.d/ /etc/sysctl.d/99-zz_plesk_semaphores.conf
# chown root:root /etc/sysctl.d/ /etc/sysctl.d/99-zz_plesk_semaphores.conf

这样的 /etc/sysctl.d/99-zz_plesk_semaphores.conf配置文件必须包含：

kernel.msgmni = 64000
kernel.sem = 250 256000 32 1024

负荷了新的sysctl设置 sysctl

# sysctl -p

检查哪些用户用尽信号灯：

# ipcs -s | awk ‘{print $3}‘ | uniq -c
4 root
33 httpd
234 somesoftware

阿帕奇保持清洁信号灯：

# for i in `ipcs -s | awk ‘/httpd/ {print $2}‘`; do (ipcrm -s $i); done

在某些情况下，其他软件滥用严重限制信号，禁用该软件，清理作进一步调查其信号量和接触软件供应商：

# for i in `ipcs -s | awk ‘/httpd/ {print $2}‘`; do (ipcrm -s $i); done

for i in `ipcs -s | awk ‘/httpd/{ next; } {print $2}‘`; do (ipcrm -s $i); done

执行清理后问题依旧，反复执行也无效

[[email protected]:~]# ipcs -s

------ Semaphore Arrays --------
key semid owner perms nsems
0x00000000 0 root 600 1
0x00000000 65537 root 600 1
0x00000000 131074 apache 600 1
0x00000000 163843 apache 600 1
0x7a004a4e 196612 zabbix 600 13
0x00000000 229381 apache 600 1
0x00000000 262150 apache 600 1
0x00000000 294919 apache 600 1
0x00000000 327688 apache 600 1

[[email protected]:~]# ipcs -s | grep apache | perl -e ‘while (<STDIN>) { @a=split(/s+/); print `ipcrm sem $a[1]`}‘

resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted

for i in `ipcs -s | awk ‘/httpd/{ next; } {print $2}‘`; do (ipcrm -s $i); done
[[email protected]:~]# ipcs -s

------ Semaphore Arrays --------
key semid owner perms nsems
0x00000000 0 root 600 1
0x00000000 65537 root 600 1
0x7a004a4e 196612 zabbix 600 13

日志：

[[email protected]:~]# tail -f /var/log/messages
Dec 4 04:47:36 server nrpe[36326]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36328]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36330]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36332]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36334]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:48:07 server nrpe[36711]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:48:07 server nrpe[36713]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:48:07 server nrpe[36715]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:49:04 server nrpe[37324]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:49:04 server nrpe[37333]: Host 192.168.254.63 is not allowed to talk to us!

# 原因是因为切分、监控等程序造成的，全部kill就可以了
[[email protected]:~]# ps -ef|grep apache|grep -v grep
root 18475 18474 0 16:50 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 18477 18475 0 16:50 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 26621 26618 0 17:00 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 26623 26621 0 17:00 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 34354 34352 0 17:10 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 34355 34354 0 17:10 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 41177 41175 0 17:20 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 41178 41177 0 17:20 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 47964 47962 0 17:30 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 47965 47964 0 17:30 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 49905 49904 0 14:20 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 49907 49905 0 14:20 ? 00:00:03 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 54555 54554 0 17:40 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 54557 54555 0 17:40 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 56774 56773 0 14:30 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 56775 56774 0 14:30 ? 00:00:03 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 61962 61960 0 17:50 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 61963 61962 0 17:50 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 70159 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/error.log.%Y-%m-%d 86400
root 70160 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/i-job.chinasoft.com_error.log.%Y-%m-%d 86400
root 70161 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/support.chinasoft.com.old_error.log.%Y-%m-%d 86400
root 70162 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/dlcbs.chinasoft.com_error.log.%Y-%m-%d 86400
root 70163 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/access.log.%Y-%m-%d 86400
root 70164 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/i-job.chinasoft.com_access.log.%Y-%m-%d 86400
root 70165 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/dlcbs.chinasoft.com_access.log.%Y-%m-%d 86400
apache 70166 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70167 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70168 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70169 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70170 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70171 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70172 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70173 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70174 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70175 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
root 84551 84548 0 06:00 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 84553 84551 0 06:00 ? 00:00:23 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 97069 97068 0 15:30 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 97071 97069 0 15:30 ? 00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 109607 109605 0 15:50 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 109608 109607 0 15:50 ? 00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 115643 115642 0 16:00 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 115645 115643 0 16:00 ? 00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh

[[email protected]:~]# ps -ef|grep apache|grep -v grep|awk ‘{print $2}‘|xargs kill -9
[root:~]# ps -ef|grep apache
root 70590 70566 0 17:58 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/access.log.%Y-%m-%d 86400
root 70591 70566 0 17:58 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/i-job.chinasoft.com_access.log.%Y-%m-%d 86400
root 70592 70566 0 17:58 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/dlcbs.chinasoft.com_access.log.%Y-%m-%d 86400
apache 70608 70566 0 17:58 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70609 70566 0 17:58 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70610 70566 0 17:58 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
root 70612 35669 0 17:58 pts/0 00:00:00 grep apache

[root:~]# killall httpd

[root:~]# ipcs -s

------ Semaphore Arrays --------
key semid owner perms nsems

# 删除信号

for i in `ipcs -s | awk ‘/httpd/{ next; } {print $2}‘`; do (ipcrm -s $i); done