Tag: fsck

RHEL 5 server stopping boot process at message SELinux: Disabled at runtime. type=1404 audt(1358286571.639.2): selinux=0 auid=4294967295 ses=4294967295

Boot was stopping at the following message

SELinux: Disabled at runtime.
type=1404 audt(1358286571.639.2): selinux=0 auid=4294967295 ses=4294967295


To solve this problem, follow the document Why RHEL5 boot hangs after “SELinux: Disabled at runtime?

  • Disable console redirection.
  • Remove serial console entry from GRUB kernel line.

Rebooted the server
Edited grub line removing any mention of console, at the end of line inserted init=/bin/bash. Pressed then b to boot with these modified line
After getting prompt

# mount -o rw,remount /
# cp -p /etc/fstab /etc/fstab.20171211

Edited /etc/fstab and commented any filesystem that is not from the operating system

Rebooted the server

After the server is online, edited /boot/grub/menu.lst and removed all mention of console on kernel lines

root@linux:/boot/grub # vi menu.lst
# grub.conf generated by anaconda
#
# Note that you do not have to rerun grub after making changes to this file
# NOTICE: You have a /boot partition. This means that
# all kernel and initrd paths are relative to /boot/, eg.
# root (hd0,0)
# kernel /vmlinuz-version ro root=/dev/rootvg/rootlv
# initrd /initrd-version.img
#boot=/dev/sda
default=0
timeout=5
#splashimage=(hd0,0)/grub/splash.xpm.gz
#hiddenmenu

serial –unit=0 –speed=115200
terminal –timeout=10 console serial

title Red Hat Enterprise Linux Server (2.6.18-408.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-408.el5 ro root=/dev/rootvg/rootlv rhgb noquiet crashkernel=256M@16M log_buf_len=3M elevator=noop nmi_watchdog=0
initrd /initrd-2.6.18-408.el5.img
title Red Hat Enterprise Linux Server (2.6.18-407.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-407.el5 ro root=/dev/rootvg/rootlv rhgb noquiet crashkernel=256M@16M log_buf_len=3M elevator=noop nmi_watchdog=0
initrd /initrd-2.6.18-407.el5.img
title Red Hat Enterprise Linux Server (2.6.18-398.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-398.el5 ro root=/dev/rootvg/rootlv rhgb noquiet crashkernel=256M@16M log_buf_len=3M elevator=noop nmi_watchdog=0
initrd /initrd-2.6.18-398.el5.img

Uncomment the filesystems you had just commented and then run fsck on them

root@linux:~ # vi /etc/fstab
/dev/rootvg/rootlv / ext4 defaults,nobarrier 1 1
/dev/rootvg/optlv /opt ext4 defaults,nobarrier 1 2
/dev/rootvg/tmplv /tmp ext4 defaults,nobarrier 1 2
/dev/rootvg/auditlv /audit ext4 defaults,nobarrier 1 2
/dev/rootvg/usrlv /usr ext4 defaults,nobarrier 1 2
/dev/rootvg/userlv /usr/users ext4 defaults,nobarrier 1 2
/dev/rootvg/varlv /var ext4 defaults,nobarrier 1 2
/dev/rootvg/crashlv /var/crash ext4 defaults,nobarrier 1 2
LABEL=/boot /boot ext3 defaults 1 2
tmpfs /dev/shm tmpfs defaults 0 0
devpts /dev/pts devpts gid=5,mode=620 0 0
sysfs /sys sysfs defaults 0 0
proc /proc proc defaults 0 0
/dev/rootvg/swaplv swap swap defaults 0 0

## Files Systems Oracle ##
##/dev/mapper/softwarevg-softwarelv /usr/software ext4 defaults,nobarrier 1 2
##/dev/mapper/dcj200_vg-dcj200_data /usr/oradata/oradcj200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dcj200_vg-dcj200_arch /usr/oradata/archive/oradcj200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dit200_vg-dit200_data /usr/oradata/oradit200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dit200_vg-dit200_arch /usr/oradata/archive/oradit200 ext4 defaults,nobarrier 1 2
##/dev/mapper/drj200_vg-drj200_data /usr/oradata/oradrj200 ext4 defaults,nobarrier 1 2
##/dev/mapper/drj200_vg-drj200_arch /usr/oradata/archive/oradrj200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dsl200_vg-dsl200_data /usr/oradata/oradsl200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dsl200_vg-dsl200_arch /usr/oradata/archive/oradsl200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dvt200_vg-dvt200_data /usr/oradata/oradvt200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dvt200_vg-dvt200_arch /usr/oradata/archive/oradvt200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dbh200_vg-dbh200_data /usr/oradata/oradbh200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dbh200_vg-dbh200_arch /usr/oradata/archive/oradbh200 ext4 defaults,nobarrier 1 2

Forcing to run fsck

fsck -t ext4 -y -f /dev/mapper/softwarevg-softwarelv
fsck -t ext4 -y -f /dev/mapper/dcj200_vg-dcj200_data
fsck -t ext4 -y -f /dev/mapper/dcj200_vg-dcj200_arch
fsck -t ext4 -y -f /dev/mapper/dit200_vg-dit200_data
fsck -t ext4 -y -f /dev/mapper/dit200_vg-dit200_arch
fsck -t ext4 -y -f /dev/mapper/drj200_vg-drj200_data
fsck -t ext4 -y -f /dev/mapper/drj200_vg-drj200_arch
fsck -t ext4 -y -f /dev/mapper/dsl200_vg-dsl200_data
fsck -t ext4 -y -f /dev/mapper/dsl200_vg-dsl200_arch
fsck -t ext4 -y -f /dev/mapper/dvt200_vg-dvt200_data
fsck -t ext4 -y -f /dev/mapper/dvt200_vg-dvt200_arch
fsck -t ext4 -y -f /dev/mapper/dbh200_vg-dbh200_data
fsck -t ext4 -y -f /dev/mapper/dbh200_vg-dbh200_arch

Mount filesystems

mount /usr/software
mount /usr/oradata/oradcj200
mount /usr/oradata/archive/oradcj200
mount /usr/oradata/oradit200
mount /usr/oradata/archive/oradit200
mount /usr/oradata/oradrj200
mount /usr/oradata/archive/oradrj200
mount /usr/oradata/oradsl200
mount /usr/oradata/archive/oradsl200
mount /usr/oradata/oradvt200
mount /usr/oradata/archive/oradvt200
mount /usr/oradata/oradbh200
mount /usr/oradata/archive/oradbh200

ORA-00206: error in writing (block 42, # blocks 1) of control file

DBA notified me of this error

Wed Nov 25 13:46:11 BRST 2015
Errors in file /usr/software/oracle/admin/orarvt015/bdump/rvt015_lgwr_2388.trc:
ORA-00206: error in writing (block 42, # blocks 1) of control file
ORA-00202: control file: ‘/usr/oradata/orarvt015/control2/control02.ctl’
ORA-27061: waiting for async I/Os failed
Linux-x86_64 Error: 5: Input/output error

Umount the filesystem

root@linux:~ # umount /usr/oradata/orarvt015

I ran fsck

root@linux:~ # fsck -t ext3 /dev/mapper/oradatavg-dat.orarvt015new
fsck 1.38 (30-Jun-2005)
e2fsck 1.38 (30-Jun-2005)
/dev/mapper/oradatavg-dat.orarvt015new: recovering journal
/dev/mapper/oradatavg-dat.orarvt015new: clean, 137/209715200 files, 318872229/419428352 blocks

Then I mounted the filesystem again

root@linux:~ # mount /usr/oradata/orarvt015

Showing filesystem size

root@linux:~ # df -h /usr/oradata/orarvt015
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/oradatavg-dat.orarvt015new
1.6T 1.2T 304G 80% /usr/oradata/orarvt015

It solved the problem

fsck in Linux: Superblock last write time is in the future. FIXED

When rebooting the server and it shows this message:

Superblock last write time is in the future. FIXED

Move the file /etc/adjtime and set the hardware clock using the command below:

root@linux:~ # mv /etc/adjtime /etc/adjtime.bak

root@linux:~ # hwclock
Fri 27 Jun 2014 11:27:05 AM BRT -0.574168 seconds

root@linux:~ # hwclock —systohc —localtime

root@linux:~ # hwclock -w

root@linux:~ # hwclock
Fri 27 Jun 2014 11:30:17 AM BRT -0.616638 seconds

Also check if your system is using /etc/localtime as a file or link

root@linux:~ # ls -l /etc/localtime
-rw-r–r– 1 root root 2001 Aug 19 2010 /etc/localtime

root@linux:~ # ls -l /etc/localtime
lrwxrwxrwx 1 root root 31 Oct 13 2011 /etc/localtime -> /usr/share/zoneinfo/Brazil/East

The time will be different on boot if you use a separate /usr partition because /etc/localtime will point to an invalid file at startup

root@linux:~ # df -h /usr/share/zoneinfo/Brazil
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/rootvg-usrlv
5.0G 2.5G 2.3G 52% /usr

Copy the file from /usr/share/zoneinfo to /etc/localtime

root@linux:~ # rm /etc/localtime
root@linux:~ # cp -p /usr/share/zoneinfo/Brazil/East /etc/localtime

Source: After reboot, the Linux system time is always wrong
Using the hwclock program

Skip or force fsck when rebooting a Linux server

If you issue this command, the system will create a file /fastboot

root@linux:~ # shutdown -rf now

or

root@linux:~ # touch /fastboot
root@linux:~ # shutdown -r now

You can also pass the arguments to grub prompt to skip fsck on boot

grub> kernel /vmlinuz-2.6.16.60-0.66.1-smp root=/dev/rootvg/rootlv vga=0x317 resume=/dev/rootvg/swaplv splash=silent showopts fastboot rootdelay=10

Create a /forcefsck file or reboot your computer with -F option to force a fsck on boot

root@linux:~ # shutdown -rF now

or

root@linux:~ # touch /forcefsck
root@linux:~ # shutdown -r now

Running fsck and automatically fix any errors encountered

Tried to mount a filesystem but the OS requested to run fsck

root@solaris:/ # mount /1a
mount: the state of /dev/md/dsk/d7 is not okay
and it was attempted to be mounted read/write
mount: Please run fsck and try again

To run fsck and try to automatically fix any errors encountered, pass the flag -y

root@solaris:/ # fsck -y /dev/md/dsk/d7
** /dev/md/rdsk/d7
** Last Mounted on /1a
** Phase 1 – Check Blocks and Sizes
** Phase 2 – Check Pathnames
** Phase 3a – Check Connectivity
** Phase 3b – Verify Shadows/ACLs
** Phase 4 – Check Reference Counts
UNREF FILE I=1762708 OWNER=patrol MODE=100644
SIZE=263127 MTIME=Jan 25 15:40 2009
RECONNECT? yes

LINK COUNT FILE I=1762708 OWNER=patrol MODE=100644
SIZE=263127 MTIME=Jan 25 15:40 2009 COUNT 0 SHOULD BE 1
ADJUST? yes

UNREF FILE I=2376356 OWNER=patrol MODE=10700
SIZE=0 MTIME=Jan 25 15:37 2009
CLEAR? yes

UNREF FILE I=2977939 OWNER=spectrum MODE=100600
SIZE=0 MTIME=Jan 25 15:37 2009
CLEAR? yes

UNREF FILE I=2977940 OWNER=spectrum MODE=100600
SIZE=0 MTIME=Jan 25 15:37 2009
CLEAR? yes

UNREF FILE I=2977941 OWNER=spectrum MODE=100600
SIZE=0 MTIME=Jan 25 15:37 2009
CLEAR? yes

** Phase 5 – Check Cylinder Groups

CORRECT BAD CG SUMMARIES FOR CG 34? yes

CORRECTED SUPERBLOCK SUMMARIES FOR CG 34
CORRECTED SUMMARIES FOR CG 34
FILE BITMAP WRONG
FIX? yes

FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 69
CORRECTED SUMMARIES FOR CG 69
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 146
CORRECTED SUMMARIES FOR CG 146
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 147
CORRECTED SUMMARIES FOR CG 147
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 165
CORRECTED SUMMARIES FOR CG 165
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 168
CORRECTED SUMMARIES FOR CG 168
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 175
CORRECTED SUMMARIES FOR CG 175
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 202
CORRECTED SUMMARIES FOR CG 202
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 224
CORRECTED SUMMARIES FOR CG 224
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 233
CORRECTED SUMMARIES FOR CG 233
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 264
CORRECTED SUMMARIES FOR CG 264
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 269
CORRECTED SUMMARIES FOR CG 269
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 275
CORRECTED SUMMARIES FOR CG 275
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 286
CORRECTED SUMMARIES FOR CG 286
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 311
CORRECTED SUMMARIES FOR CG 311
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 316
CORRECTED SUMMARIES FOR CG 316
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 380
CORRECTED SUMMARIES FOR CG 380
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 386
CORRECTED SUMMARIES FOR CG 386
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 388
CORRECTED SUMMARIES FOR CG 388
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 395
CORRECTED SUPERBLOCK SUMMARIES FOR CG 495
CORRECTED SUPERBLOCK SUMMARIES FOR CG 516
CORRECTED SUMMARIES FOR CG 516
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 519
CORRECTED SUMMARIES FOR CG 519
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 539
CORRECTED SUMMARIES FOR CG 539
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 546
CORRECTED SUMMARIES FOR CG 546
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 576
CORRECTED SUMMARIES FOR CG 576
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 578
CORRECTED SUMMARIES FOR CG 578
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 579
CORRECTED SUMMARIES FOR CG 579
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 588
CORRECTED SUMMARIES FOR CG 588
FRAG BITMAP WRONG (CORRECTED)
CORRECTED SUPERBLOCK SUMMARIES FOR CG 631
CORRECTED SUMMARIES FOR CG 631
FRAG BITMAP WRONG (CORRECTED)
CORRECT GLOBAL SUMMARY
SALVAGE? yes

90655 files, 5690552 used, 50760198 free (12494 frags, 6343463 blocks, 0.0% fragmentation)

***** FILE SYSTEM WAS MODIFIED *****

Since fsck encountered errors, we pass once again and see if more errors are encountered

root@solaris:/ # fsck -y /dev/md/dsk/d7
** /dev/md/rdsk/d7
** Last Mounted on /1a
** Phase 1 – Check Blocks and Sizes
** Phase 2 – Check Pathnames
** Phase 3a – Check Connectivity
** Phase 3b – Verify Shadows/ACLs
** Phase 4 – Check Reference Counts
** Phase 5 – Check Cylinder Groups
90655 files, 5690552 used, 50760198 free (12494 frags, 6343463 blocks, 0.0% fragmentation)

Since no more errors were encountered, we try to mount the filesystem once again

root@solaris:/ # mount /1a
root@solaris:/ # df -k /1a
Filesystem kbytes used avail capacity Mounted on
/dev/md/dsk/d7 56450750 5690552 50195691 11% /1a

Couldn’t mount AIX file system. First information says that media is not formatted

I had a problem mounting a file system after the server rebooted. At first it reported that the logical volume wasn’t formatted or the format is incorrect. Then it asked to run fsck.

root@aix5:/ # mount /fallback
Replaying log for /dev/fallback.
mount: 0506-324 Cannot mount /dev/fallback on /fallback: The media is not formatted or the format is not correct.
0506-342 The superblock on /dev/fallback is dirty.  Run a full fsck to fix.

I just ran fsck on the logical volume and then mounted the file system.

root@aix5:/ # fsck /dev/fallback

****************
The current volume is: /dev/fallback
**Phase 1 – Check Blocks, Files/Directories, and Directory Entries
**Phase 2 – Count links
**Phase 3 – Duplicate Block Rescan and Directory Connectedness
**Phase 4 – Report Problems
**Phase 5 – Check Connectivity
**Phase 7 – Verify File/Directory Allocation Maps
**Phase 8 – Verify Disk Allocation Maps
15728640 kilobytes total disk space.
63 kilobytes in 30 directories.
7664455 kilobytes in 438 user files.
8061172 kilobytes are available for use.
File system is clean.
Superblock is marked dirty; FIX? y
All observed inconsistencies have been repaired.

root@aix5:/ # mount /fallback

root@aix5:/ # df -g /fallback
Filesystem    GB blocks      Free %Used    Iused %Iused Mounted on
/dev/fallback      15.00      7.69   49%      470     1% /fallback