Advertisements

RHEL 5 server stopping boot process at message SELinux: Disabled at runtime. type=1404 audt(1358286571.639.2): selinux=0 auid=4294967295 ses=4294967295

Boot was stopping at the following message

SELinux: Disabled at runtime.
type=1404 audt(1358286571.639.2): selinux=0 auid=4294967295 ses=4294967295


To solve this problem, follow the document Why RHEL5 boot hangs after “SELinux: Disabled at runtime?

  • Disable console redirection.
  • Remove serial console entry from GRUB kernel line.

Rebooted the server
Edited grub line removing any mention of console, at the end of line inserted init=/bin/bash. Pressed then b to boot with these modified line
After getting prompt

# mount -o rw,remount /
# cp -p /etc/fstab /etc/fstab.20171211

Edited /etc/fstab and commented any filesystem that is not from the operating system

Rebooted the server

After the server is online, edited /boot/grub/menu.lst and removed all mention of console on kernel lines

root@linux:/boot/grub # vi menu.lst
# grub.conf generated by anaconda
#
# Note that you do not have to rerun grub after making changes to this file
# NOTICE: You have a /boot partition. This means that
# all kernel and initrd paths are relative to /boot/, eg.
# root (hd0,0)
# kernel /vmlinuz-version ro root=/dev/rootvg/rootlv
# initrd /initrd-version.img
#boot=/dev/sda
default=0
timeout=5
#splashimage=(hd0,0)/grub/splash.xpm.gz
#hiddenmenu

serial –unit=0 –speed=115200
terminal –timeout=10 console serial

title Red Hat Enterprise Linux Server (2.6.18-408.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-408.el5 ro root=/dev/rootvg/rootlv rhgb noquiet crashkernel=256M@16M log_buf_len=3M elevator=noop nmi_watchdog=0
initrd /initrd-2.6.18-408.el5.img
title Red Hat Enterprise Linux Server (2.6.18-407.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-407.el5 ro root=/dev/rootvg/rootlv rhgb noquiet crashkernel=256M@16M log_buf_len=3M elevator=noop nmi_watchdog=0
initrd /initrd-2.6.18-407.el5.img
title Red Hat Enterprise Linux Server (2.6.18-398.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-398.el5 ro root=/dev/rootvg/rootlv rhgb noquiet crashkernel=256M@16M log_buf_len=3M elevator=noop nmi_watchdog=0
initrd /initrd-2.6.18-398.el5.img

Uncomment the filesystems you had just commented and then run fsck on them

root@linux:~ # vi /etc/fstab
/dev/rootvg/rootlv / ext4 defaults,nobarrier 1 1
/dev/rootvg/optlv /opt ext4 defaults,nobarrier 1 2
/dev/rootvg/tmplv /tmp ext4 defaults,nobarrier 1 2
/dev/rootvg/auditlv /audit ext4 defaults,nobarrier 1 2
/dev/rootvg/usrlv /usr ext4 defaults,nobarrier 1 2
/dev/rootvg/userlv /usr/users ext4 defaults,nobarrier 1 2
/dev/rootvg/varlv /var ext4 defaults,nobarrier 1 2
/dev/rootvg/crashlv /var/crash ext4 defaults,nobarrier 1 2
LABEL=/boot /boot ext3 defaults 1 2
tmpfs /dev/shm tmpfs defaults 0 0
devpts /dev/pts devpts gid=5,mode=620 0 0
sysfs /sys sysfs defaults 0 0
proc /proc proc defaults 0 0
/dev/rootvg/swaplv swap swap defaults 0 0

## Files Systems Oracle ##
##/dev/mapper/softwarevg-softwarelv /usr/software ext4 defaults,nobarrier 1 2
##/dev/mapper/dcj200_vg-dcj200_data /usr/oradata/oradcj200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dcj200_vg-dcj200_arch /usr/oradata/archive/oradcj200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dit200_vg-dit200_data /usr/oradata/oradit200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dit200_vg-dit200_arch /usr/oradata/archive/oradit200 ext4 defaults,nobarrier 1 2
##/dev/mapper/drj200_vg-drj200_data /usr/oradata/oradrj200 ext4 defaults,nobarrier 1 2
##/dev/mapper/drj200_vg-drj200_arch /usr/oradata/archive/oradrj200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dsl200_vg-dsl200_data /usr/oradata/oradsl200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dsl200_vg-dsl200_arch /usr/oradata/archive/oradsl200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dvt200_vg-dvt200_data /usr/oradata/oradvt200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dvt200_vg-dvt200_arch /usr/oradata/archive/oradvt200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dbh200_vg-dbh200_data /usr/oradata/oradbh200 ext4 defaults,nobarrier 1 2
##/dev/mapper/dbh200_vg-dbh200_arch /usr/oradata/archive/oradbh200 ext4 defaults,nobarrier 1 2

Forcing to run fsck

fsck -t ext4 -y -f /dev/mapper/softwarevg-softwarelv
fsck -t ext4 -y -f /dev/mapper/dcj200_vg-dcj200_data
fsck -t ext4 -y -f /dev/mapper/dcj200_vg-dcj200_arch
fsck -t ext4 -y -f /dev/mapper/dit200_vg-dit200_data
fsck -t ext4 -y -f /dev/mapper/dit200_vg-dit200_arch
fsck -t ext4 -y -f /dev/mapper/drj200_vg-drj200_data
fsck -t ext4 -y -f /dev/mapper/drj200_vg-drj200_arch
fsck -t ext4 -y -f /dev/mapper/dsl200_vg-dsl200_data
fsck -t ext4 -y -f /dev/mapper/dsl200_vg-dsl200_arch
fsck -t ext4 -y -f /dev/mapper/dvt200_vg-dvt200_data
fsck -t ext4 -y -f /dev/mapper/dvt200_vg-dvt200_arch
fsck -t ext4 -y -f /dev/mapper/dbh200_vg-dbh200_data
fsck -t ext4 -y -f /dev/mapper/dbh200_vg-dbh200_arch

Mount filesystems

mount /usr/software
mount /usr/oradata/oradcj200
mount /usr/oradata/archive/oradcj200
mount /usr/oradata/oradit200
mount /usr/oradata/archive/oradit200
mount /usr/oradata/oradrj200
mount /usr/oradata/archive/oradrj200
mount /usr/oradata/oradsl200
mount /usr/oradata/archive/oradsl200
mount /usr/oradata/oradvt200
mount /usr/oradata/archive/oradvt200
mount /usr/oradata/oradbh200
mount /usr/oradata/archive/oradbh200

Advertisements

UXMON: Inode utilization of /var/opt exceeds 90 threshold

root@linux:/var/opt # df -i .
Filesystem Inodes IUsed IFree IUse% Mounted on
/dev/mapper/vgroot-lv_var_opt
327680 308929 18751 95% /var/opt

I found a script at this page – How to Free Inode Usage?

Create a file and then run the script to find which directory are holding the most files

#!/bin/bash
# count_em – count files in all subdirectories under current directory.
echo ‘echo $(ls -a “$1” | wc -l) $1’ >/tmp/count_em_$$
chmod 700 /tmp/count_em_$$
find . -mount -type d -print0 | xargs -0 -n1 /tmp/count_em_$$ | sort -n
rm -f /tmp/count_em_$$

Most files are inside /var/opt/OV/log/OpC

root@linux:/var/opt/ # ./inodes.sh

457 ./osit/acf/log
599 ./newid
849 ./osit/linux/log/bdf.statistics
5066 ./erm/save
172726 ./OV/log/OpC

Deleted the files that I saw were unnecessary and it decreased inode consumption

root@linux:/var/opt # df -i .
Filesystem Inodes IUsed IFree IUse% Mounted on
/dev/mapper/vgroot-lv_var_opt
327680 181506 146174 56% /var/opt

Installing Insight Diagnostics Online Edition for Linux on Red Hat Enterprise Linux 6

Download HPE System Management Homepage for Linux (AMD64/EM64T) hpsmh-7.6.1-9.x86_64.rpm
http://h20565.www2.hpe.com/hpsc/swd/public/detail?sp4ts.oid=8230346&swItemId=MTX_ee014bd76e7a4eef958b8d174d&swEnvOid=4103

You also need to download Insight Diagnostics Online Edition for Linux (x86-64) hpdiags-10.50.2007-2076.linux.x86_64.rpm https://support.hpe.com/hpsc/swd/public/detail?swItemId=MTX_b0c4c90c82504b16abdd8fe7dd&swEnvOid=4103

Install both packages

root@linux:/tmp # rpm -ivh hpsmh-7.6.1-9.x86_64.rpm hpdiags-10.50.2007-2076.linux.x86_64.rpm
warning: hpsmh-7.6.1-9.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID 26c2b797: NOKEY
warning: hpdiags-10.50.2007-2076.linux.x86_64.rpm: Header V4 RSA/SHA256 Signature, key ID c5399b1d: NOKEY
Preparing… ########################################### [100%]
Detected Red Hat Enterprise Linux AS/ES/WS/SERVER 6
Created hpsmh user and group…
1:hpsmh ########################################### [ 50%]

**********************************************************
* System Management Homepage installed successfully with *
* default configuration values. To change the default *
* configuration values, type the following command at *
* the root prompt: *
* *
* /opt/hp/hpsmh/sbin/smhconfig *
* *
**********************************************************

2:hpdiags ########################################### [100%]

Stopping hpsmhd: [ OK ]
Starting hpsmhd: [ OK ]

It installs in /opt/hp/hpdiags

root@linux:/opt/hp/hpdiags # ./hpdiags -p -rd -o /tmp/`hostname`_diag
HP Insight Diagnostics Version 10.50.2007A (x86_64)
Copyright (c) 2002, 2015 Hewlett-Packard Development Company, L.P.
System configuration capture running (Cannot open /etc/hba.conf
System configuration capture completed (100%)
completed (100%)

root@linux:/opt/hp/hpdiags # ./hpdiags -h
HP Insight Diagnostics Version 10.50.2007A (x86_64)
Copyright (c) 2002, 2015 Hewlett-Packard Development Company, L.P.
Syntax: hpdiags [options]
Valid options:

-l languageCode options: en, fr, it, de, es, ja, zh (Default: en)

-v [Level] Verbosity level of output between 1 and 5.
where 1 is the most terse level and 5 is the most verbose.
The default value is set to 5.

-s mmhhddMMDD Schedule automatic execution of Insight Diagnostics.
mm=minute(0-59), hh=hour(0-23), dd=day(1-31),
MM=month(1-12), DD=weekday(0-6)
All fields must be two digits (e.g. 02 instead of 2).
Use a single ‘*’ for don’t care fields.
NOTE: If you use ‘*’ you must use double-quotes around the
date/time value. (e.g. -s “0023***”)

-u Remove scheduled automatic execution of Insight Diagnostics.

-c FILE1 [FILE2] Compare a survey file to the current configuration
or, if two files are specified, compare the files.

-t Output to the terminal. May not be used with -o.

-p Specify output file format as plain text, instead of XML.
Only applies to Survey captures and comparisons.
Off by default.

-o FILE Specify output file name. By default, a file name is formed
using the current date and time in this format:
“surveyYYYY-MM-DD-hh-mm-ss.xml”
(YYYY=year, MM=month, DD=day, hh=hour, mm=minute, ss=second)
May not be used with -t.

-f Overwrite output file if it already exists.
NOTE: If no -o option is specified this option is ignored.

-a Append output file if it already exists.
NOTE: If no -o option is specified this option is ignored.
The -a option may only be used with plain text (-p).
This is because appending to XML files would result in XML parsing errors.

-rd Run a diagnosis of all diagnosable devices.

-? , -h Display this help message.

Installing Ansible for CentOS 6 using the EPEL repository

I’m going to install ansible from Extra Packages for Enterprise Linux (or EPEL)

On CentOS, you need to install the repository file that it is included in the extras repository

root@linux:~ # yum list epel-release
Loaded plugins: fastestmirror, security
Loading mirror speeds from cached hostfile
* base: centos.xpg.com.br
* extras: centos.xpg.com.br
* updates: mirror.ufscar.br
Available Packages
epel-release.noarch 6-8 extras

Install the package epel-release

root@linux:~ # yum install -y epel-release
Loaded plugins: fastestmirror, security
Setting up Install Process
Loading mirror speeds from cached hostfile
* base: centos.xpg.com.br
* extras: centos.xpg.com.br
* updates: mirror.ufscar.br
Resolving Dependencies
–> Running transaction check
—> Package epel-release.noarch 0:6-8 will be installed
–> Finished Dependency Resolution

Dependencies Resolved

================================================================================
Package Arch Version Repository Size
================================================================================
Installing:
epel-release noarch 6-8 extras 14 k

Transaction Summary
================================================================================
Install 1 Package(s)

Total download size: 14 k
Installed size: 22 k
Downloading Packages:
epel-release-6-8.noarch.rpm | 14 kB 00:00
Running rpm_check_debug
Running Transaction Test
Transaction Test Succeeded
Running Transaction
Installing : epel-release-6-8.noarch 1/1
Verifying : epel-release-6-8.noarch 1/1

Installed:
epel-release.noarch 0:6-8

Complete!

Install ansible and all the dependencies

root@linux:~ # yum install -y ansible
Loaded plugins: fastestmirror, security
Setting up Install Process
Loading mirror speeds from cached hostfile
epel/metalink | 2.4 kB 00:00
* base: centos.xpg.com.br
* epel: mirror.globo.com
* extras: centos.xpg.com.br
* updates: mirror.ufscar.br
epel | 4.3 kB 00:00
epel/primary_db | 5.9 MB 00:05
Resolving Dependencies
–> Running transaction check
—> Package ansible.noarch 0:2.3.2.0-1.el6 will be installed
–> Processing Dependency: sshpass for package: ansible-2.3.2.0-1.el6.noarch
–> Processing Dependency: python2-jmespath for package: ansible-2.3.2.0-1.el6.noarch
–> Processing Dependency: python-simplejson for package: ansible-2.3.2.0-1.el6.noarch
–> Processing Dependency: python-keyczar for package: ansible-2.3.2.0-1.el6.noarch
–> Processing Dependency: python-jinja2-26 for package: ansible-2.3.2.0-1.el6.noarch
–> Processing Dependency: python-httplib2 for package: ansible-2.3.2.0-1.el6.noarch
–> Processing Dependency: python-crypto2.6 for package: ansible-2.3.2.0-1.el6.noarch
–> Processing Dependency: PyYAML for package: ansible-2.3.2.0-1.el6.noarch
–> Running transaction check
—> Package PyYAML.x86_64 0:3.10-3.1.el6 will be installed
–> Processing Dependency: libyaml-0.so.2()(64bit) for package: PyYAML-3.10-3.1.el6.x86_64
—> Package python-crypto2.6.x86_64 0:2.6.1-2.el6 will be installed
—> Package python-httplib2.noarch 0:0.7.7-1.el6 will be installed
—> Package python-jinja2-26.noarch 0:2.6-3.el6 will be installed
–> Processing Dependency: python-babel >= 0.8 for package: python-jinja2-26-2.6-3.el6.noarch
–> Processing Dependency: python-markupsafe for package: python-jinja2-26-2.6-3.el6.noarch
—> Package python-keyczar.noarch 0:0.71c-1.el6 will be installed
–> Processing Dependency: python-pyasn1 for package: python-keyczar-0.71c-1.el6.noarch
—> Package python-simplejson.x86_64 0:2.0.9-3.1.el6 will be installed
—> Package python2-jmespath.noarch 0:0.9.0-2.el6 will be installed
—> Package sshpass.x86_64 0:1.06-1.el6 will be installed
–> Running transaction check
—> Package libyaml.x86_64 0:0.1.3-4.el6_6 will be installed
—> Package python-babel.noarch 0:0.9.4-5.1.el6 will be installed
—> Package python-markupsafe.x86_64 0:0.9.2-4.el6 will be installed
—> Package python-pyasn1.noarch 0:0.0.12a-1.el6 will be installed
–> Finished Dependency Resolution

Dependencies Resolved

================================================================================
Package Arch Version Repository Size
================================================================================
Installing:
ansible noarch 2.3.2.0-1.el6 epel 5.9 M
Installing for dependencies:
PyYAML x86_64 3.10-3.1.el6 base 157 k
libyaml x86_64 0.1.3-4.el6_6 base 52 k
python-babel noarch 0.9.4-5.1.el6 base 1.4 M
python-crypto2.6 x86_64 2.6.1-2.el6 epel 513 k
python-httplib2 noarch 0.7.7-1.el6 epel 70 k
python-jinja2-26 noarch 2.6-3.el6 epel 527 k
python-keyczar noarch 0.71c-1.el6 epel 219 k
python-markupsafe x86_64 0.9.2-4.el6 base 22 k
python-pyasn1 noarch 0.0.12a-1.el6 base 70 k
python-simplejson x86_64 2.0.9-3.1.el6 base 126 k
python2-jmespath noarch 0.9.0-2.el6 epel 39 k
sshpass x86_64 1.06-1.el6 epel 20 k

Transaction Summary
================================================================================
Install 13 Package(s)

Total download size: 9.1 M
Installed size: 40 M
Downloading Packages:
(1/13): PyYAML-3.10-3.1.el6.x86_64.rpm | 157 kB 00:00
(2/13): ansible-2.3.2.0-1.el6.noarch.rpm | 5.9 MB 00:05
(3/13): libyaml-0.1.3-4.el6_6.x86_64.rpm | 52 kB 00:00
(4/13): python-babel-0.9.4-5.1.el6.noarch.rpm | 1.4 MB 00:01
(5/13): python-crypto2.6-2.6.1-2.el6.x86_64.rpm | 513 kB 00:00
(6/13): python-httplib2-0.7.7-1.el6.noarch.rpm | 70 kB 00:00
(7/13): python-jinja2-26-2.6-3.el6.noarch.rpm | 527 kB 00:00
(8/13): python-keyczar-0.71c-1.el6.noarch.rpm | 219 kB 00:00
(9/13): python-markupsafe-0.9.2-4.el6.x86_64.rpm | 22 kB 00:00
(10/13): python-pyasn1-0.0.12a-1.el6.noarch.rpm | 70 kB 00:00
(11/13): python-simplejson-2.0.9-3.1.el6.x86_64.rpm | 126 kB 00:00
(12/13): python2-jmespath-0.9.0-2.el6.noarch.rpm | 39 kB 00:00
(13/13): sshpass-1.06-1.el6.x86_64.rpm | 20 kB 00:00
——————————————————————————–
Total 878 kB/s | 9.1 MB 00:10
warning: rpmts_HdrFromFdno: Header V3 RSA/SHA256 Signature, key ID 0608b895: NOKEY
Retrieving key from file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-6
Importing GPG key 0x0608B895:
Userid : EPEL (6) <epel@fedoraproject.org>
Package: epel-release-6-8.noarch (@extras)
From : /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-6
Running rpm_check_debug
Running Transaction Test
Transaction Test Succeeded
Running Transaction
Installing : python-markupsafe-0.9.2-4.el6.x86_64 1/13
Installing : python-httplib2-0.7.7-1.el6.noarch 2/13
Installing : python-pyasn1-0.0.12a-1.el6.noarch 3/13
Installing : python-keyczar-0.71c-1.el6.noarch 4/13
Installing : python-crypto2.6-2.6.1-2.el6.x86_64 5/13
Installing : python-simplejson-2.0.9-3.1.el6.x86_64 6/13
Installing : python2-jmespath-0.9.0-2.el6.noarch 7/13
Installing : libyaml-0.1.3-4.el6_6.x86_64 8/13
Installing : PyYAML-3.10-3.1.el6.x86_64 9/13
Installing : sshpass-1.06-1.el6.x86_64 10/13
Installing : python-babel-0.9.4-5.1.el6.noarch 11/13
Installing : python-jinja2-26-2.6-3.el6.noarch 12/13
Installing : ansible-2.3.2.0-1.el6.noarch 13/13
Verifying : PyYAML-3.10-3.1.el6.x86_64 1/13
Verifying : python-babel-0.9.4-5.1.el6.noarch 2/13
Verifying : sshpass-1.06-1.el6.x86_64 3/13
Verifying : python-keyczar-0.71c-1.el6.noarch 4/13
Verifying : libyaml-0.1.3-4.el6_6.x86_64 5/13
Verifying : python2-jmespath-0.9.0-2.el6.noarch 6/13
Verifying : python-simplejson-2.0.9-3.1.el6.x86_64 7/13
Verifying : ansible-2.3.2.0-1.el6.noarch 8/13
Verifying : python-jinja2-26-2.6-3.el6.noarch 9/13
Verifying : python-crypto2.6-2.6.1-2.el6.x86_64 10/13
Verifying : python-pyasn1-0.0.12a-1.el6.noarch 11/13
Verifying : python-httplib2-0.7.7-1.el6.noarch 12/13
Verifying : python-markupsafe-0.9.2-4.el6.x86_64 13/13

Installed:
ansible.noarch 0:2.3.2.0-1.el6

Dependency Installed:
PyYAML.x86_64 0:3.10-3.1.el6
libyaml.x86_64 0:0.1.3-4.el6_6
python-babel.noarch 0:0.9.4-5.1.el6
python-crypto2.6.x86_64 0:2.6.1-2.el6
python-httplib2.noarch 0:0.7.7-1.el6
python-jinja2-26.noarch 0:2.6-3.el6
python-keyczar.noarch 0:0.71c-1.el6
python-markupsafe.x86_64 0:0.9.2-4.el6
python-pyasn1.noarch 0:0.0.12a-1.el6
python-simplejson.x86_64 0:2.0.9-3.1.el6
python2-jmespath.noarch 0:0.9.0-2.el6
sshpass.x86_64 0:1.06-1.el6

Complete!

Check the UUID for eth0

If you cloned a VM and need to reconfigure the network configuration for eth0, it will ask for the UUID of eth0

root@linux:~ # cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
TYPE=Ethernet
UUID=ffdc3088-dd65-4a81-b1e1-02f0a7252528
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=dhcp
HWADDR=08:00:27:9A:74:85
DEFROUTE=yes
PEERDNS=yes
PEERROUTES=yes
IPV4_FAILURE_FATAL=yes
IPV6INIT=no
NAME=”System eth0″

To view the current UUID for eth0 on your system use uuidgen

root@linux:~ # uuidgen eth0
ffdc3088-dd65-4a81-b1e1-02f0a7252528

View the MAC address using ip address list

UXMON: Service /application/pkg/zones-proxy-client status is maintenance, check with svcs -xv

ATTENTION, RMC LEVEL 1 AGENT: This ticket will be automatically worked by the Automation Bus. Pls. ensure your Ticket List/View includes the “Assignee” column, monitor this ticket until the user “ABOPERATOR” is no longer assigned, BEFORE you start work on this ticket.
Node : localzone.setaoffice.com
Node Type : Sun SPARC (HTTPS)
Severity : minor
OM Server Time: 2017-09-24 22:32:27
Message : UXMON: Service /application/pkg/zones-proxy-client status is maintenance, check with svcs -xv
Msg Group : OS
Application : svcsmon
Object : svcs
Event Type :
not_found

Instance Name :
not_found

Instruction : The svcsmon has detected solaris service status

Please, for details, browse the /var/opt/OV/log/OpC/svcs_mon.log
The configuration file uses to be /var/opt/OV/conf/OpC/svcs_mon.cfg

On global zone

root@globalzone:~ # pkg publisher
PUBLISHER TYPE STATUS P LOCATION
solaris origin online F file:///net/192.168.252.12/export/IPS-repos/solaris11/repo/
solaris origin online F file:///var/scmuidrs/idr2142.1.p5p/
solaris origin online F file:///var/scmuidrs/idr2160.2.p5p/
solaris origin online F file:///var/scmuidrs/idr2193.2.p5p/
solaris origin online F file:///var/scmuidrs/idr2194.1.p5p/
solaris origin online F file:///var/scmuidrs/idr2238.1.p5p/
exa-family origin online F file:///net/192.168.252.12/export/IPS-repos/exafamily/repo/
ha-cluster origin online F file:///net/192.168.252.12/export/IPS-repos/osc4/repo/

root@globalzone:~ # svcadm disable zones-proxyd system-repository; svcadm enable system-repository zones-proxyd; sleep 30
root@globalzone:~ #

On local zone

root@localzone:~# svcs -a | grep /application/pkg/zones-proxy-client
maintenance 8:08:15 svc:/application/pkg/zones-proxy-client:default

root@localzone:~# svcadm disable svc:/application/pkg/zones-proxy-client:default

root@localzone:~# svcadm enable svc:/application/pkg/zones-proxy-client:default

root@localzone:~# svcs -a | grep /application/pkg/zones-proxy-client
online 14:09:45 svc:/application/pkg/zones-proxy-client:default

Linux Fibre HBA in status Linkdown

root@linux:~ # systool -c fc_host -v
Class = “fc_host”

Class Device = “host1”
Class Device path = “/sys/class/fc_host/host1”
fabric_name = “0x100000051e36219e”
issue_lip =
node_name = “0x50060b0000c29e39”
port_id = “0x1ebe01”
port_name = “0x50060b0000c29e38”
port_state = “Online”
port_type = “NPort (fabric via point-to-point)”
speed = “4 Gbit”
supported_classes = “Class 3”
supported_speeds = “1 Gbit, 2 Gbit, 4 Gbit”
symbolic_name = “QMH2462 FW:v5.06.03 DVR:v8.03.07.15.05.09-k”
system_hostname = “”
tgtid_bind_type = “wwpn (World Wide Port Name)”
uevent =

Device = “host1”
Device path = “/sys/devices/pci0000:00/0000:00:09.0/0000:09:00.0/host1”
fw_dump =
nvram = “ISP ”
optrom_ctl =
optrom =
reset =
uevent =
vpd = “▒8”

Class Device = “host2”
Class Device path = “/sys/class/fc_host/host2”
fabric_name = “0x50060b0000c29e3b”
issue_lip =
node_name = “0x50060b0000c29e3b”
port_id = “0x000000”
port_name = “0x50060b0000c29e3a”
port_state = “Linkdown
port_type = “Unknown
speed = “unknown
supported_classes = “Class 3”
supported_speeds = “1 Gbit, 2 Gbit, 4 Gbit”
symbolic_name = “QMH2462 FW:v5.06.03 DVR:v8.03.07.15.05.09-k”
system_hostname = “”
tgtid_bind_type = “wwpn (World Wide Port Name)”
uevent =

Device = “host2”
Device path = “/sys/devices/pci0000:00/0000:00:09.0/0000:09:00.1/host2”
fw_dump =
nvram = “ISP ”
optrom_ctl =
optrom =
reset =
uevent =
vpd = “▒8”

Perform diagnostic steps below to isolate initialization issue. Engaged hardware vendor to check the card

Resetting HP iLO through command line

From my Linux server, I ssh to the iLO IP address

emerson@linux:~ $ ssh Administrator@172.22.4.251
Warning: Permanently added ‘172.22.4.251’ (RSA) to the list of known hosts.
Administrator@172.22.4.251’s password:
User:Administrator logged-in to ILOBRC6087BS6.(172.22.4.251 / FE80::9657:A5FF:FE4E:3852)

iLO Advanced 2.40 at Dec 02 2015
Server Name: linux.setaoffice.com
Server Power: On

Type cd /Map1

</>hpiLO-> cd /Map1

status=0
status_tag=COMMAND COMPLETED
Mon Aug 28 16:37:16 2017

/Map1

Type reset

hpiLO-> reset

status=0
status_tag=COMMAND COMPLETED
Mon Aug 28 16:37:27 2017

Resetting iLO.

CLI session stopped
Received disconnect from 172.22.4.251: 11: Client Disconnect

Oracle SMF Oracle Configuration Manager (OCM) svc:/system/ocm:default

Listing status of ocm service

root@solaris:~ # svcs svc:/system/ocm:default
STATE STIME FMRI
disabled Jul_17 svc:/system/ocm:default

Listing SMF

root@solaris:~ # svcs -l svc:/system/ocm:default
fmri svc:/system/ocm:default
name Oracle Configuration Manager (OCM)
enabled false
state disabled
next_state none
state_time Mon Jul 17 04:48:06 2017
logfile /var/svc/log/system-ocm:default.log
restarter svc:/system/svc/restarter:default
contract_id
manifest /etc/svc/profile/generic.xml
manifest /lib/svc/manifest/system/ocm.xml
dependency require_all/none svc:/milestone/multi-user-server:default (online)
dependency require_all/error svc:/milestone/network:default (online)
dependency require_all/none svc:/system/cryptosvc (online)

Oracle Configuration Manager is used to collect client configuration information and upload it to the Oracle repository
When enabling OCM, it goes to maintenance mode

root@solaris:~ # svcadm enable svc:/system/ocm:default

root@solaris:~ # svcs -v svc:/system/ocm:default
STATE NSTATE STIME CTID FMRI
maintenance – 10:58:25 1081435 svc:/system/ocm:default

There are two errors in this installation. There is no proxy setup and it is missing user ocm

root@solaris:~ # cat /var/svc/log/system-ocm:default.log
[ Aug 28 10:58:09 Enabled. ]
[ Aug 28 10:58:09 Executing start method (“/lib/svc/method/svc-ocm start”). ]
/lib/svc/method/svc-ocm: starting…
OCM not registered
Collector running in connected mode
Begin anonymous registration…
Starting response file generation…
Can not create response file: Unknown Host: ccr.oracle.com: unknown error
Failed to create response file…
Failed to generate anonymous response file…
Unable to contact ccr.oracle.com. Please set your system proxy
in order to allow this system to contact Oracle for better
serviceability. See the configCCR(1M) manual page on home to set
the proxy server for Oracle Configuration Manager.

svc:/system/ocm:default has been temporarily disabled.

[ Aug 28 10:58:20 Method “start” exited with status 0. ]
[ Aug 28 10:58:20 Stopping because service disabled. ]
[ Aug 28 10:58:20 Executing stop method (“/lib/svc/method/svc-ocm stop”). ]
Stopping scheduler…
su: Unknown id: ocm
[ Aug 28 10:58:25 Method “stop” exited with status 95. ]

Solaris – UXMON: [ID 377184 daemon.error] SUNW-MSG-ID: ZFS-8000-D3, TYPE: Fault, VER: 1, SEVERITY: Major

UXMON: [ID 377184 daemon.error] SUNW-MSG-ID: ZFS-8000-D3, TYPE: Fault, VER: 1, SEVERITY: Major

Node : solaris.setaoffice.com
Node Type : Sun SPARC (HTTPS)
Severity : major
OM Server Time: 2017-08-12 10:27:31
Message : UXMON: [ID 377184 daemon.error] SUNW-MSG-ID: ZFS-8000-D3, TYPE: Fault, VER: 1, SEVERITY: Major
Msg Group : OS
Application : SOL_mon
Object : FMT
Event Type :
not_found

Instance Name :
not_found

Instruction : “The Fault Management agent has identified a HW or OS related problem with the severity presented by the ticket.
The problem(s) can be viewed and managed with the command – fmdump
To get a better understanding of the problem and on how to resolve it, locate the event that generated
the ticket in the syslog file /var/adm/messages, a URL will be found (http://sun.com/msg/xxx-nnnn-yy),
follow the link using your Oracle portal account for instructions.”

After running fmadm faulty, we see that there is a problem with a zpool. Run zpool status -x and then we see pool prd027_software is having problems

root@solaris:~ # zpool status prd027_software
pool: prd027_software
state: ONLINE
status: One or more devices has experienced an error resulting in data
corruption. Applications may be affected.
action: Restore the file in question if possible. Otherwise restore the
entire pool from backup.
Run ‘zpool status -v’ to see device specific details.
see: http://support.oracle.com/msg/ZFS-8000-8A
scan: none requested
config:

NAME STATE READ WRITE CKSUM
prd027_software ONLINE 0 0 14.7K
c0t600507680191818C1000000000000BE9d0 ONLINE 0 0 0
c0t600507680191818C1000000000000BEAd0 ONLINE 0 0 0
c0t600507680191818C1000000000000BEBd0 ONLINE 0 0 0
c0t600507680191818C1000000000000BECd0 ONLINE 0 0 0

errors: 3 data errors, use ‘-v’ for a list

Run zpool scrub prd027_software

root@solaris:~ # zpool scrub prd027_software

root@solaris:~ # zpool status -xv
pool: prd027_software
state: ONLINE
status: One or more devices has experienced an error resulting in data
corruption. Applications may be affected.
action: Restore the file in question if possible. Otherwise restore the
entire pool from backup.
see: http://support.oracle.com/msg/ZFS-8000-8A
scan: scrub in progress since Wed Dec 31 21:00:00 1969
50.7M scanned out of 1.08T at 25.3M/s, 12h25m to go
0 repaired, 0.00% done
config:

NAME STATE READ WRITE CKSUM
prd027_software ONLINE 0 0 14.7K
c0t600507680191818C1000000000000BE9d0 ONLINE 0 0 0
c0t600507680191818C1000000000000BEAd0 ONLINE 0 0 0
c0t600507680191818C1000000000000BEBd0 ONLINE 0 0 0
c0t600507680191818C1000000000000BECd0 ONLINE 0 0 0

errors: Permanent errors have been detected in the following files:

/zones/prd027/root/usr/software/best1/Patrol3/Solaris-2-10-sparc-64/best1/7.4.00/bgs/monitor/log/prd027-bgsagent_6767.als
prd027_software/software027:<0x198c996>
prd027_software/software027:<0x198de4e>

After the pool is scanned, check if there is still a problem

root@solaris:~ # zpool status -xv
all pools are healthy

Repairing fmadm entries

root@solaris:~ # fmadm faulty|grep “Aug”
Aug 12 11:23:22 82fe93a5-8120-657b-9e61-e33252b84d30 ZFS-8000-D3 Major
Aug 12 11:22:01 74c61e33-7c56-4aca-d707-a32ce06a9bd8 ZFS-8000-CS Major

root@solaris:~ # fmadm repair 82fe93a5-8120-657b-9e61-e33252b84d30
fmadm: recorded repair to 82fe93a5-8120-657b-9e61-e33252b84d30

root@solaris:~ # fmadm repair 74c61e33-7c56-4aca-d707-a32ce06a9bd8
fmadm: recorded repair to 74c61e33-7c56-4aca-d707-a32ce06a9bd8

root@solaris:~ # fmadm faulty
root@solaris:~ #

%d bloggers like this: