From d4ba00434c4925446a7aa2faa0ea782ae2496a47 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 1 Apr 2021 11:00:58 +0200 Subject: [PATCH] VR: fix rsyslog compresses log files but not release disk space in VR (#4869) We had critical issue with VR recently. The VRs of shared network or vpc stops working after some days. After investigation, I found that the disk space is full ``` root@r-10-VM:~# df Filesystem 1K-blocks Used Available Use% Mounted on /dev/vda5 2086316 2069932 0 100% / ``` logrotate/ryslog has compresses the log files, but space is not released. see `lsof |grep deleted` ``` root@r-10-VM:~# lsof |grep deleted rsyslogd 960 root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) rsyslogd 960 962 in:imuxso root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 962 in:imuxso root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 962 in:imuxso root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 962 in:imuxso root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) rsyslogd 960 963 in:imklog root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 963 in:imklog root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 963 in:imklog root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 963 in:imklog root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) rsyslogd 960 964 in:imfile root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 964 in:imfile root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 964 in:imfile root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 964 in:imfile root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) rsyslogd 960 965 in:imudp root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 965 in:imudp root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 965 in:imudp root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 965 in:imudp root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) rsyslogd 960 966 rs:main root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 966 rs:main root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 966 rs:main root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 966 rs:main root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) ``` workaround: restarting rsyslog to release the space. ``` systemctl restart rsyslog ``` The root cause is, the following command does not work in 4.15 template ``` root@r-10-VM:~# invoke-rc.d rsyslog rotate [FAIL] Closing open files: rsyslogd failed! ``` Fix: use `/usr/lib/rsyslog/rsyslog-rotate` instead ``` root@r-10-VM:~# /usr/lib/rsyslog/rsyslog-rotate root@r-10-VM:~# cat /usr/lib/rsyslog/rsyslog-rotate if [ -d /run/systemd/system ]; then systemctl kill -s HUP rsyslog.service else invoke-rc.d rsyslog rotate > /dev/null fi ``` --- systemvm/debian/etc/logrotate.d/rsyslog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/systemvm/debian/etc/logrotate.d/rsyslog b/systemvm/debian/etc/logrotate.d/rsyslog index 9291494481e..cef88191606 100644 --- a/systemvm/debian/etc/logrotate.d/rsyslog +++ b/systemvm/debian/etc/logrotate.d/rsyslog @@ -6,7 +6,7 @@ notifempty compress postrotate - /usr/sbin/invoke-rc.d rsyslog rotate > /dev/null + /usr/lib/rsyslog/rsyslog-rotate endscript } @@ -30,6 +30,6 @@ compress sharedscripts postrotate - /usr/sbin/invoke-rc.d rsyslog rotate > /dev/null + /usr/lib/rsyslog/rsyslog-rotate endscript }