UMOUNT NAS报错No space left on device和was not found in /proc/mounts

UMOUNT NAS报错No space left on device和was not found in /proc/mounts

NAS_IP:/sysidc 表示NAS的存储设备。

[root@mvxl0280 /]# df -h

Filesystem Size Used Avail Use% Mounted on

/dev/mapper/VolGroup-lv_root 22G 20G 275M 99% /

tmpfs 7.8G 0 7.8G 0% /dev/shm

/dev/sda1 485M 37M 423M 9% /boot

/dev/mapper/vg_01-lv_01 98G 72G 21G 78% /eam_app

NAS_IP:/sysidc 3.0T 2.2T 743G 76% /mnt

## mount命令不加参数显示的结果和/etc/mtab文件记录内容一样。

[root@mvxl0280 /]# mount

/dev/mapper/VolGroup-lv_root on / type ext4 (rw)

proc on /proc type proc (rw)

sysfs on /sys type sysfs (rw)

devpts on /dev/pts type devpts (rw,gid=5,mode=620)

tmpfs on /dev/shm type tmpfs (rw)

/dev/sda1 on /boot type ext4 (rw)

/dev/mapper/vg_01-lv_01 on /eam_app type ext4 (rw)

none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)

sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw)

NAS_IP:/sysidc on /mnt type nfs (rw,addr=NAS_IP)

[root@mvxl0280 /]# cat /proc/mounts

rootfs / rootfs rw 0 0

proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0

sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0

devtmpfs /dev devtmpfs rw,nosuid,relatime,size=8157104k,nr_inodes=2039276,mode=755 0 0

devpts /dev/pts devpts rw,relatime,gid=5,mode=620,ptmxmode=000 0 0

tmpfs /dev/shm tmpfs rw,nosuid,nodev,relatime 0 0

/dev/mapper/VolGroup-lv_root / ext4 rw,relatime,barrier=1,data=ordered 0 0

/proc/bus/usb /proc/bus/usb usbfs rw,relatime 0 0

/dev/sda1 /boot ext4 rw,relatime,barrier=1,data=ordered 0 0

/dev/mapper/vg_01-lv_01 /eam_app ext4 rw,relatime,barrier=1,data=ordered 0 0

none /proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0

sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0

/etc/auto.misc /misc autofs rw,relatime,fd=7,pgrp=2080,timeout=300,minproto=5,maxproto=5,indirect 0 0

-hosts /net autofs rw,relatime,fd=13,pgrp=2080,timeout=300,minproto=5,maxproto=5,indirect 0 0

NAS_IP:/sysidc /mnt nfs rw,relatime,vers=3,rsize=32768,wsize=32768,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.16.17.171,mountvers=3,mountport=1234,mountproto=udp,local_lock=none,addr=NAS_IP 0 0

umount NAS存储提示【No space left on device】

[root@mvxl0280 ~]# umount /mnt

can't create lock file /etc/mtab~9811: No space left on device (use -n flag to override)

/mnt was not found in /proc/mounts

NAS空间突然变小，从3.0T变成22G，其实此时看到的/mnt虽然挂载点是NAS"NAS_IP:/sysidc"，但其实是"/"目录的挂载点。

[root@mvxl0280 ~]# df -h

Filesystem Size Used Avail Use% Mounted on

/dev/mapper/VolGroup-lv_root 22G 20G 346M 99% /

tmpfs 7.8G 0 7.8G 0% /dev/shm

/dev/sda1 485M 37M 423M 9% /boot

/dev/mapper/vg_01-lv_01 98G 72G 21G 78% /eam_app

NAS_IP:/sysidc 22G 20G 346M 99% /mnt

其实umount执行之后，NAS存储实际已经成功umount，通过查看/mnt目录就知道已经是空的。

[root@mvxl0280 ~]# ls -l /mnt

total 0

貌似是没有卸载成功，但是此时再执行一次带【-f】的umount，依然报错。

[root@mvxl0280 ~]# umount -f /mnt

/mnt was not found in /proc/mounts

/mnt was not found in /proc/mounts

重新mount NAS存储也提示【No space left on device】

[root@mvxl0280 /]# mount NAS_IP:/sysidc /mnt

can't create lock file /etc/mtab~10608: No space left on device (use -n flag to override)

网上说可以用umount的【-l】参数，其实也没有效果。

[root@mvxl0280 /]# umount -l /mnt/

can't create lock file /etc/mtab~10512: No space left on device (use -n flag to override)

can't create lock file /etc/mtab~10513: No space left on device (use -n flag to override)

但是mount可以查看到NAS存储"NAS_IP:/sysidc"已经挂载在/mnt下。

[root@mvxl0280 ~]# mount

/dev/mapper/VolGroup-lv_root on / type ext4 (rw)

proc on /proc type proc (rw)

sysfs on /sys type sysfs (rw)

devpts on /dev/pts type devpts (rw,gid=5,mode=620)

tmpfs on /dev/shm type tmpfs (rw)

/dev/sda1 on /boot type ext4 (rw)

/dev/mapper/vg_01-lv_01 on /eam_app type ext4 (rw)

none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)

sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw)

NAS_IP:/sysidc on /mnt type nfs (rw,addr=NAS_IP)

而/proc/mounts【/proc/self/mounts的软连接】却看不到挂载点信息，因此报错【/mnt was not found in /proc/mounts】。

[root@mvxl0280 ~]# ls -l /proc/mounts

lrwxrwxrwx 1 root root 11 Feb 5 09:06 /proc/mounts -> self/mounts

[root@mvxl0280 ~]# file /proc/mounts

/proc/mounts: symbolic link to `self/mounts'

[root@mvxl0280 ~]# ls -l /proc/self

lrwxrwxrwx 1 root root 64 Feb 5 08:57 /proc/self -> 9879

[root@mvxl0280 ~]# ls -l /proc/self/mounts

-r--r--r-- 1 root root 0 Feb 5 09:07 /proc/self/mounts

[root@mvxl0280 ~]# more /proc/self/mounts

rootfs / rootfs rw 0 0

proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0

sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0

devtmpfs /dev devtmpfs rw,nosuid,relatime,size=8157104k,nr_inodes=2039276,mode=755 0 0

devpts /dev/pts devpts rw,relatime,gid=5,mode=620,ptmxmode=000 0 0

tmpfs /dev/shm tmpfs rw,nosuid,nodev,relatime 0 0

/dev/mapper/VolGroup-lv_root / ext4 rw,relatime,barrier=1,data=ordered 0 0

/proc/bus/usb /proc/bus/usb usbfs rw,relatime 0 0

/dev/sda1 /boot ext4 rw,relatime,barrier=1,data=ordered 0 0

/dev/mapper/vg_01-lv_01 /eam_app ext4 rw,relatime,barrier=1,data=ordered 0 0

none /proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0

sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0

/etc/auto.misc /misc autofs rw,relatime,fd=7,pgrp=2080,timeout=300,minproto=5,maxproto=5,indirect 0 0

-hosts /net autofs rw,relatime,fd=13,pgrp=2080,timeout=300,minproto=5,maxproto=5,indirect 0 0

经过对比分析，发现是"/"根目录空间不足导致，而使用空间最多是一个/home/wasadmin/deploy的程序备份目录，在跟开发人员确认之后，这个目录下的程序文件是定期备份，但是没有部署相关的定期清理脚本，而导致空间被用完。

[root@mvxl0280 home]# du -sh *

60K monitor

13G wasadmin

/home/wasadmin是用户的home目录，使用13G空间，而该目录下只有2个常规目录。其中client_ffdc这个目录才204K。那么deploy占用了剩余的其他12G多的空间。

[root@mvxl0280 home]# cd wasadmin

[root@mvxl0280 wasadmin]# ls

client_ffdc deploy

[root@mvxl0280 wasadmin]# du -sh client_ffdc/

204K client_ffdc/

在删除/home/wasadmin/deploy目录下的部分备份程序之后，"/"目录的空间已经足够，再次执行umount NAS的命令，可以成功执行。

[wasadmin@mvxl0280 ~]$ df -h

Filesystem Size Used Avail Use% Mounted on

/dev/mapper/VolGroup-lv_root 22G 8.9G 12G 45% /

tmpfs 7.8G 0 7.8G 0% /dev/shm

/dev/sda1 485M 37M 423M 9% /boot

/dev/mapper/vg_01-lv_01 98G 68G 26G 73% /eam_app

NAS_IP:/sysidc 3.0T 2.2T 743G 76% /mnt

[root@mvxl0280 wasadmin]# umount /mnt

[root@mvxl0280 wasadmin]# df -h

Filesystem Size Used Avail Use% Mounted on

/dev/mapper/VolGroup-lv_root 22G 8.9G 12G 45% /

tmpfs 7.8G 0 7.8G 0% /dev/shm

/dev/sda1 485M 37M 423M 9% /boot

/dev/mapper/vg_01-lv_01 98G 68G 26G 73% /eam_app