Advertisements

Author Archive: Emerson Takahashi

Creating a new logical drive with two physical drives mirrored on HPE Proliant

Checking disks

root@linux:~ # hpacucli ctrl all show config

Smart Array P410i in Slot 0 (Embedded) (sn: 5001438013682F50)

array A (SAS, Unused Space: 0 MB)

logicaldrive 1 (136.7 GB, RAID 1, OK)

physicaldrive 1I:1:1 (port 1I:box 1:bay 1, SAS, 146 GB, OK)
physicaldrive 1I:1:3 (port 1I:box 1:bay 3, SAS, 146 GB, OK)

array B (SAS, Unused Space: 0 MB)

logicaldrive 2 (136.7 GB, RAID 1, OK)

physicaldrive 1I:1:2 (port 1I:box 1:bay 2, SAS, 146 GB, OK)
physicaldrive 1I:1:4 (port 1I:box 1:bay 4, SAS, 146 GB, OK)

array C (SAS, Unused Space: 0 MB)

logicaldrive 3 (136.7 GB, RAID 1, OK)

physicaldrive 2I:1:5 (port 2I:box 1:bay 5, SAS, 146 GB, OK)
physicaldrive 2I:1:6 (port 2I:box 1:bay 6, SAS, 146 GB, OK)

SEP (Vendor ID PMCSIERA, Model SRC 8x6G) 250 (WWID: 5001438013682F5F)

Listing logical drives

root@linux:~ # hpacucli ctrl slot=0 logicaldrive all show status

logicaldrive 1 (136.7 GB, RAID 1): OK
logicaldrive 2 (136.7 GB, RAID 1): OK
logicaldrive 3 (136.7 GB, RAID 1): OK

Showing physical drives

root@linux:~ # hpacucli ctrl slot=0 pd all show

Smart Array P410i in Slot 0 (Embedded)

array A

physicaldrive 1I:1:1 (port 1I:box 1:bay 1, SAS, 146 GB, OK)
physicaldrive 1I:1:3 (port 1I:box 1:bay 3, SAS, 146 GB, OK)

array B

physicaldrive 1I:1:2 (port 1I:box 1:bay 2, SAS, 146 GB, OK)
physicaldrive 1I:1:4 (port 1I:box 1:bay 4, SAS, 146 GB, OK)

array C

physicaldrive 2I:1:5 (port 2I:box 1:bay 5, SAS, 146 GB, OK)
physicaldrive 2I:1:6 (port 2I:box 1:bay 6, SAS, 146 GB, OK)

Two disks were inserted

Checking status again

root@linux:~ # hpacucli ctrl slot=0 pd all show

Smart Array P410i in Slot 0 (Embedded)

array A

physicaldrive 1I:1:1 (port 1I:box 1:bay 1, SAS, 146 GB, OK)
physicaldrive 1I:1:3 (port 1I:box 1:bay 3, SAS, 146 GB, OK)

array B

physicaldrive 1I:1:2 (port 1I:box 1:bay 2, SAS, 146 GB, OK)
physicaldrive 1I:1:4 (port 1I:box 1:bay 4, SAS, 146 GB, OK)

array C

physicaldrive 2I:1:5 (port 2I:box 1:bay 5, SAS, 146 GB, OK)
physicaldrive 2I:1:6 (port 2I:box 1:bay 6, SAS, 146 GB, OK)

unassigned

physicaldrive 2I:1:7 (port 2I:box 1:bay 7, SAS, 146 GB, OK)
physicaldrive 2I:1:8 (port 2I:box 1:bay 8, SAS, 146 GB, OK)

Creating a new logical drive with mirror

root@linux:~ # hpacucli ctrl slot=0 create type=ld drives=2I:1:7,2I:1:8 raid=1

root@linux:~ # hpacucli ctrl all show config

Smart Array P410i in Slot 0 (Embedded) (sn: 5001438013682F50)

array A (SAS, Unused Space: 0 MB)

logicaldrive 1 (136.7 GB, RAID 1, OK)

physicaldrive 1I:1:1 (port 1I:box 1:bay 1, SAS, 146 GB, OK)
physicaldrive 1I:1:3 (port 1I:box 1:bay 3, SAS, 146 GB, OK)

array B (SAS, Unused Space: 0 MB)

logicaldrive 2 (136.7 GB, RAID 1, OK)

physicaldrive 1I:1:2 (port 1I:box 1:bay 2, SAS, 146 GB, OK)
physicaldrive 1I:1:4 (port 1I:box 1:bay 4, SAS, 146 GB, OK)

array C (SAS, Unused Space: 0 MB)

logicaldrive 3 (136.7 GB, RAID 1, OK)

physicaldrive 2I:1:5 (port 2I:box 1:bay 5, SAS, 146 GB, OK)
physicaldrive 2I:1:6 (port 2I:box 1:bay 6, SAS, 146 GB, OK)

array D (SAS, Unused Space: 0 MB)

logicaldrive 4 (136.7 GB, RAID 1, OK)

physicaldrive 2I:1:7 (port 2I:box 1:bay 7, SAS, 146 GB, OK)
physicaldrive 2I:1:8 (port 2I:box 1:bay 8, SAS, 146 GB, OK)

SEP (Vendor ID PMCSIERA, Model SRC 8x6G) 250 (WWID: 5001438013682F5F)

Advertisements

Checking HBA status on AIX

Storage team checked and it was not showing light on the fiber

Index Slot Port Address Media Speed State Proto
===================================================
298 3 42 28a280 id N4 In_Sync FC
314 4 42 28b280 id N4 In_Sync FC

298 3/42 28a280 Online F 4 G AN A_S4_VIT_aix_HBA5C93 Emulex HBA 10:00:00:00:c9:86:5c:93
314 4/42 28b280 Online F 4 G AN A_S4_VIT_aix_HBA5C92 Emulex HBA 10:00:00:00:c9:86:5c:92

Listing HBA cards on AIX

root@aix:/ # lsdev -Cc adapter |grep fcs
fcs0 Available 01-00 4Gb FC PCI Express Adapter (df1000fe)
fcs1 Available 01-01 4Gb FC PCI Express Adapter (df1000fe)
fcs2 Available 06-00 4Gb FC PCI Express Adapter (df1000fe)
fcs3 Available 06-01 4Gb FC PCI Express Adapter (df1000fe)
fcs4 Available 0B-00 4Gb FC PCI Express Adapter (df1000fe)
fcs5 Available 0B-01 4Gb FC PCI Express Adapter (df1000fe)
fcs6 Available 0I-00 4Gb FC PCI Express Adapter (df1000fe)
fcs7 Available 0I-01 4Gb FC PCI Express Adapter (df1000fe)
fcs8 Available 0K-00 4Gb FC PCI Express Adapter (df1000fe)
fcs9 Available 0K-01 4Gb FC PCI Express Adapter (df1000fe)

Listing the WWPN of the HBA cards

root@aix:/ # lsdev -Cc adapter |grep fcs | awk ‘{print “echo “$1” `lscfg -vl “$1” | grep \”Network Address\”`”‘} | sh
fcs0 Network Address………….10000000C986EF2E
fcs1 Network Address………….10000000C986EF2F
fcs2 Network Address………….10000000C9865C92
fcs3 Network Address………….10000000C9865C93
fcs4 Network Address………….10000000C986ECDC
fcs5 Network Address………….10000000C986ECDD
fcs6 Network Address………….10000000C986F158
fcs7 Network Address………….10000000C986F159
fcs8 Network Address………….10000000C9865B8C
fcs9 Network Address………….10000000C9865B8D

Shows that the cable is connected

root@aix:/ # lsattr -El fscsi2 | grep attach
attach switch How this adapter is CONNECTED False
root@aix:/ # lsattr -El fscsi3 | grep attach
attach switch How this adapter is CONNECTED False

Fibre was replaced

Checking HBA link status

root@aix:/ # fcstat fcs2

FIBRE CHANNEL STATISTICS REPORT: fcs2

Device Type: 4Gb FC PCI Express Adapter (df1000fe) (adapter/pciex/df1000fe)
Serial Number: 1C910080BA
Option ROM Version: 02E8277F
ZA: Z1F2.70A5
World Wide Node Name: 0x20000000C9865C92
World Wide Port Name: 0x10000000C9865C92

FC-4 TYPES:
Supported: 0x0000012000000000000000000000000000000000000000000000000000000000
Active: 0x0000010000000000000000000000000000000000000000000000000000000000
Class of Service: 3
Port Speed (supported): 4 GBIT
Port Speed (running): 4 GBIT
Port FC ID: 0x28b280
Port Type: Fabric
Attention Type: Link Up
Topology: Point to Point or Fabric

Seconds Since Last Reset: 11518131

Transmit Statistics Receive Statistics
——————- ——————
Frames: 391415147 1672088546
Words: 100836128512 731260150528

LIP Count: 0
NOS Count: 124979
Error Frames: 0
Dumped Frames: 0
Link Failure Count: 3
Loss of Sync Count: 2
Loss of Signal: 1
Primitive Seq Protocol Error Count: 0
Invalid Tx Word Count: 1420
Invalid CRC Count: 0

IP over FC Adapter Driver Information
No DMA Resource Count: 0
No Adapter Elements Count: 0

FC SCSI Adapter Driver Information
No DMA Resource Count: 0
No Adapter Elements Count: 0
No Command Resource Count: 0

IP over FC Traffic Statistics
Input Requests: 0
Output Requests: 0
Control Requests: 0
Input Bytes: 0
Output Bytes: 0

FC SCSI Traffic Statistics
Input Requests: 143103337
Output Requests: 57507565
Control Requests: 1604694
Input Bytes: 2879322009348
Output Bytes: 387479621632
root@aix:/ # fcstat fcs3

FIBRE CHANNEL STATISTICS REPORT: fcs3

Device Type: 4Gb FC PCI Express Adapter (df1000fe) (adapter/pciex/df1000fe)
Serial Number: 1C910080BA
Option ROM Version: 02E8277F
ZA: Z1F2.70A5
World Wide Node Name: 0x20000000C9865C93
World Wide Port Name: 0x10000000C9865C93

FC-4 TYPES:
Supported: 0x0000012000000000000000000000000000000000000000000000000000000000
Active: 0x0000010000000000000000000000000000000000000000000000000000000000
Class of Service: 3
Port Speed (supported): 4 GBIT
Port Speed (running): 4 GBIT
Port FC ID: 0x28a280
Port Type: Fabric
Attention Type: Link Up
Topology: Point to Point or Fabric

Seconds Since Last Reset: 11518140

Transmit Statistics Receive Statistics
——————- ——————
Frames: 4294967295 4294967295
Words: 1099511627520 1099511627520

LIP Count: 0
NOS Count: 305070
Error Frames: 0
Dumped Frames: 0
Link Failure Count: 3
Loss of Sync Count: 310
Loss of Signal: 2
Primitive Seq Protocol Error Count: 0
Invalid Tx Word Count: 633
Invalid CRC Count: 0

IP over FC Adapter Driver Information
No DMA Resource Count: 0
No Adapter Elements Count: 0

FC SCSI Adapter Driver Information
No DMA Resource Count: 0
No Adapter Elements Count: 0
No Command Resource Count: 0

IP over FC Traffic Statistics
Input Requests: 0
Output Requests: 0
Control Requests: 0
Input Bytes: 0
Output Bytes: 0

FC SCSI Traffic Statistics
Input Requests: 262598606
Output Requests: 731955157
Control Requests: 736433
Input Bytes: 68507685800533
Output Bytes: 191832750846466
root@aix:/ # lsattr -El fscsi2
attach switch How this adapter is CONNECTED False
dyntrk no Dynamic Tracking of FC Devices True+
fc_err_recov delayed_fail FC Fabric Event Error RECOVERY Policy True+
scsi_id 0x28b280 Adapter SCSI ID False
sw_fc_class 3 FC Class for Fabric True

Red Hat Enteprise Linux/CENTOS 6 with Mellanox network interface stopped working after reboot: mlx4_en.ko needs unknown symbol

In a upgrade, after installing the kernel-2.6.32-696.20.1.el6.x86_64 on a Red Hat 6 it showed the following message

mlx4_en.ko needs unknown symbol mlx4_SET_PORT_BEACON

The network interface were not working after the reboot

4 network interfaces that stopped working – eth0, eth1, eth2, eth3

They use the Mellanox driver

root@linux:~ # ethtool -i eth0
driver: mlx4_en
version: 4.1-1.0.2 (27 Jun 2017)
firmware-version: 2.35.5100
bus-info: 0000:04:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: yes

root@linux:~ # modinfo mlx4_en
filename: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-en/drivers/net/ethernet/mellanox/mlx4/mlx4_en.ko
version: 2.3-2.0.1 (Dec 7 2014)
license: Dual BSD/GPL
description: Mellanox ConnectX HCA Ethernet driver
author: Liran Liss, Yevgeny Petrilin
srcversion: 86DD1A5FE5CF17268AFBB0D
depends: mlx4_core,compat,ptp
vermagic: 2.6.32-504.el6.x86_64 SMP mod_unload modversions
parm: inline_thold:threshold for using inline data (uint)
parm: udp_rss:Enable RSS for incoming UDP traffic (uint)
parm: num_lro:Dummy module parameter to prevent loading issues (uint)
parm: pfctx:Priority based Flow Control policy on TX[7:0]. Per priority bit mask (uint)
parm: pfcrx:Priority based Flow Control policy on RX[7:0]. Per priority bit mask (uint)

You need to install the Mellanox driver in order for the network interfaces to work

The server is a HPE Proliant DL380 Gen9. Download both files to install and copy them to the server. If necessary, boot the old working kernel

HPE Mellanox RoCE (RDMA over Converged Ethernet) Driver for Red Hat Enterprise Linux 6 Update 6 (x86_64)

HPE Mellanox RoCE (RDMA over Converged Ethernet) Driver for Red Hat Enterprise Linux 6 Update 7 (x86_64)

HPE Mellanox RoCE (RDMA over Converged Ethernet) Driver for Red Hat Enterprise Linux 6 Update 8 (x86_64)

HPE Mellanox RoCE (RDMA over Converged Ethernet) Driver for Red Hat Enterprise Linux 6 Update 9 (x86_64)

Copied both files to /tmp

kmod-mlnx-ofa_kernel-4.1-OFED.4.1.1.0.2.1.gc22af88.rhel6u9.x86_64.rpm
mlnx-ofa_kernel-4.1-OFED.4.1.1.0.2.1.gc22af88.rhel6u9.x86_64.rpm

In my case I had to remove the following packages and dependencies

# yum remove librdmacm libibverbs
# yum remove mlnx-en-utils kmod-mlnx-en

Then install both packages

root@linux:/tmp # rpm -ivh mlnx-ofa_kernel-4.1-OFED.4.1.1.0.2.1.gc22af88.rhel6u9.x86_64.rpm kmod-mlnx-ofa_kernel-4.1-OFED.4.1.1.0.2.1.gc22af88.rhel6u9.x86_64.rpm
warning: mlnx-ofa_kernel-4.1-OFED.4.1.1.0.2.1.gc22af88.rhel6u9.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID 26c2b797: NOKEY
Preparing… ########################################### [100%]
1:mlnx-ofa_kernel ########################################### [ 50%]
Configured /etc/security/limits.conf
2:kmod-mlnx-ofa_kernel ########################################### [100%]
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/kernel/drivers/net/mlx4/mlx4_en.ko needs unknown symbol mlx4_test_interrupts
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/kernel/drivers/net/mlx4/mlx4_en.ko needs unknown symbol mlx4_set_stats_bitmap
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/kernel/drivers/infiniband/ulp/ipoib/ib_ipoib.ko needs unknown symbol ib_query_device
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/kernel/drivers/infiniband/ulp/ipoib/ib_ipoib.ko needs unknown symbol ib_alloc_pd
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_rq_tracked
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_q_counter
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_nic_vport_qkey_viol_cntr
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_sq_tracked
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_attach_nvmf_ns
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_nic_vport_update_local_lb
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_modify_hca_vport_context
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_rl_add_rate
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_create_auto_grouped_flow_table
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_rqt
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_hca_vport_node_guid
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_vendor_id
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_rq
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_alloc_q_counter
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_dct
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_set_delay_drop
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_dealloc_q_counter
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_arm_dct
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_tis
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_tir
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_cmd_create_vport_lag
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_unregister_interface
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_min_inline
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_dealloc_transport_domain
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_nic_vport_system_image_guid
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_lag_get_roce_netdev
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_add_flow_rules
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_destroy_flow_table
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_get_flow_namespace
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_tir
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_put_uars_page
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_get_uars_page
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_nic_vport_node_guid
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_detach_nvmf_ns
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_nvmf_backend_ctrl
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_del_flow_rules
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_rdma_netdev_alloc
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_modify_sq
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_hca_vport_system_image_guid
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_vport_counter
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_nvmf_backend_ctrl
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_max_mtu
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_hca_vport_gid
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_nic_vport_disable_roce
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_create_lag_demux_flow_table
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_sq_tracked
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_rl_remove_rate
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_mkey_cb
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_eth_proto_oper
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_rq_tracked
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_set_dc_cnak_trace
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_rqt
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_dct_query
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_rdma_netdev_free
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_vl_hw_cap
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_hca_vport_pkey
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_ib_proto_oper
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_hca_vport_context
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_oper_mtu
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_register_interface
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_nic_vport_enable_roce
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_modify_rq
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_tis
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_modify_tis
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_link_width_oper
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_modify_cq_moderation
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_dct
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_ib_ppcnt
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_alloc_bfreg
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_vector2eqn
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_lag_is_active
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_free_bfreg
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_sq
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_alloc_transport_domain
WARNING: /lib/modules/2.6.32-573.18.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_cmd_destroy_vport_lag
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/kernel/drivers/net/mlx4/mlx4_en.ko needs unknown symbol mlx4_test_interrupts
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/kernel/drivers/net/mlx4/mlx4_en.ko needs unknown symbol mlx4_set_stats_bitmap
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/kernel/drivers/infiniband/ulp/ipoib/ib_ipoib.ko needs unknown symbol ib_query_device
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/kernel/drivers/infiniband/ulp/ipoib/ib_ipoib.ko needs unknown symbol ib_alloc_pd
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_rq_tracked
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_q_counter
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_nic_vport_qkey_viol_cntr
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_sq_tracked
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_attach_nvmf_ns
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_nic_vport_update_local_lb
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_modify_hca_vport_context
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_rl_add_rate
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_create_auto_grouped_flow_table
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_rqt
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_hca_vport_node_guid
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_vendor_id
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_rq
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_alloc_q_counter
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_dct
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_set_delay_drop
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_dealloc_q_counter
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_arm_dct
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_tis
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_tir
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_cmd_create_vport_lag
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_unregister_interface
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_min_inline
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_dealloc_transport_domain
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_nic_vport_system_image_guid
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_lag_get_roce_netdev
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_add_flow_rules
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_destroy_flow_table
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_get_flow_namespace
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_tir
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_put_uars_page
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_get_uars_page
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_nic_vport_node_guid
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_detach_nvmf_ns
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_nvmf_backend_ctrl
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_del_flow_rules
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_rdma_netdev_alloc
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_modify_sq
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_hca_vport_system_image_guid
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_vport_counter
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_nvmf_backend_ctrl
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_max_mtu
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_hca_vport_gid
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_nic_vport_disable_roce
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_create_lag_demux_flow_table
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_sq_tracked
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_rl_remove_rate
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_mkey_cb
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_eth_proto_oper
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_create_rq_tracked
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_set_dc_cnak_trace
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_rqt
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_dct_query
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_rdma_netdev_free
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_vl_hw_cap
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_hca_vport_pkey
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_ib_proto_oper
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_hca_vport_context
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_oper_mtu
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_register_interface
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_nic_vport_enable_roce
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_modify_rq
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_tis
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_modify_tis
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_query_port_link_width_oper
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_modify_cq_moderation
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_destroy_dct
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_ib_ppcnt
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_alloc_bfreg
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_vector2eqn
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_lag_is_active
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_free_bfreg
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_query_sq
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_core_alloc_transport_domain
WARNING: /lib/modules/2.6.32-573.22.1.el6.x86_64/weak-updates/mlnx-ofa_kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko needs unknown symbol mlx5_cmd_destroy_vport_lag

Reboot the server

root@linux:~ # shutdown -r now

Broadcast message from root (pts/0) (Wed Feb 21 16:13:26 2018):

The system is going down for reboot NOW!

HPOM – Checking fmadm faulty and there is a pool without name

Node : sc02-app03.setaoffice.com
Node Type : Sun SPARC (HTTPS)
Severity : major
OM Server Time: 2018-02-06 11:11:01
Message : UXMON: [ID 377184 daemon.error] SUNW-MSG-ID: ZFS-8000-D3, TYPE: Fault, VER: 1, SEVERITY: Major
Msg Group : OS
Application : SOL_mon
Object : FMT
Event Type :
not_found

Instance Name :
not_found

Instruction : “The Fault Management agent has identified a HW or OS related problem with the severity presented by the ticket.
The problem(s) can be viewed and managed with the command – fmdump
To get a better understanding of the problem and on how to resolve it, locate the event that generated
the ticket in the syslog file /var/adm/messages, a URL will be found (http://sun.com/msg/xxx-nnnn-yy),
follow the link using your Oracle portal account for instructions.”
EventDataSource :

Checking fmadm faulty I see a pool without a name

root@sc02-app03:~# fmadm faulty
————— ———————————— ————– ———
TIME EVENT-ID MSG-ID SEVERITY
————— ———————————— ————– ———
Feb 06 15:59:15 4577aa7a-2b00-6eb6-f139-bf2848542fb2 ZFS-8000-HC Major

Problem Status : open
Diag Engine : zfs-diagnosis / 1.0
System
Manufacturer : unknown
Name : –
Part_Number : unknown
Serial_Number : unknown

System Component
Manufacturer : Oracle-Corporation
Name : ORCL,SPARC-T5-8
Part_Number : unknown
Serial_Number : unknown
Host_ID : (null)

—————————————-
Suspect 1 of 1 :
Problem class : fault.fs.zfs.io_failure_wait
Certainty : 100%
Affects : zfs://pool=9fbf9b5d11236d0a
Status : faulted but still in service

Resource
FMRI : “zfs://pool=9fbf9b5d11236d0a”
Status : faulted but still in service

Description : ZFS pool ” has experienced currently unrecoverable I/O failures.

Response : No automated response will occur.

Impact : Read and write I/Os cannot be serviced.

Action : Use ‘fmadm faulty’ to provide a more detailed view of this event.
Make sure the affected devices are connected, then run ‘zpool
clear’. Please refer to the associated reference document at
http://support.oracle.com/msg/ZFS-8000-HC for the latest service
procedures and policies regarding this diagnosis.

To solve this problem, you need to reboot the server

HPOM – Can’t open perl script “/var/opt/OV/bin/instrumentation/UXMONperfmon”: No such file or directory

root@linux:~ # /var/opt/OV/bin/instrumentation/UXMONbroker -help

GD UXMON monitoring package
Broker utility

usage: /var/opt/OV/bin/instrumentation/UXMONbroker [-h | –help] [-x ] [ -d ] [-l ] [-c ] [-f]
[ –col -p param ]
[ -t ] [ -p ] [-b ]
[ –os ]
[ -v ]

-h : this (help) message
–help : this (help) message

-x : triggers the execution of the module passed as parameter
-d : Allows execution activating debug
-check : Same as -x but output is redirected standard output, no logfile used
-l : output the logfile used by the module passed as parameter
-c : output the preferred config file used by the module

-t : output the TEMPORAL folder to be used if needed
-b : output the folder where the commands or instrumentation are located
-perl : output the perl runtime to be used

–col : Execute the collecting information of module
-p : Parameter passed to the recollection
–os : Show the OS name

-v : Version of UXMON package
-f : force the execution of the module bypass interval setting
supported modules are:
actmon, sshdmon, tdfmon, uxmon, nfsmon, selfcheck, swapmon, evm, mpmon, mdmon, cronmon, bondmon, rcmon, volmon, scmon, loopmon, dmesg, advfsmon, ntpmon, hwmon, bootmon, nicmon, perfmon, psmon, lpmon, vcmon, ktsmon, sgmon, dfmon.
This is the interface to the OVO templates. Templates will call this command
to get executed the different modules available, or retrieve configuration
information about the UXMON and the platform

root@linux:~ # /var/opt/OV/bin/instrumentation/UXMONbroker -check perfmon
Can’t open perl script “/var/opt/OV/bin/instrumentation/UXMONperfmon”: No such file or directory

root@linux:~ # ls -l /var/opt/OV/log/OpC/perf_mon.log
-rw-rw-r– 1 root bin 0 Feb 1 23:00 /var/opt/OV/log/OpC/perf_mon.log

How to clear fmadm faulty entries in Solaris 10

Clear fmadm log

root@sc02-app04:~ # fmadm faulty
————— ———————————— ————– ———
TIME EVENT-ID MSG-ID SEVERITY
————— ———————————— ————– ———
Feb 06 10:17:04 0ad5260a-f9e0-ef7a-c91f-efcc76b9b164 ZFS-8000-HC Major

Host : sc02-app04
Platform : ORCL,SPARC-T5-8 Chassis_id :
Product_sn :

Fault class : fault.fs.zfs.io_failure_wait
Affects : zfs://pool=prd171
faulted but still in service
Problem in : zfs://pool=prd171
faulted but still in service

Description : The ZFS pool has experienced currently unrecoverable I/O
failures.

Response : No automated response will be taken.

Impact : Read and write I/Os cannot be serviced.

Action : Use ‘fmadm faulty’ to provide a more detailed view of this event.
Make sure the affected devices are connected, then run ‘zpool
clear’. Please refer to the associated reference document at
http://sun.com/msg/ZFS-8000-HC for the latest service procedures
and policies regarding this diagnosis.

————— ———————————— ————– ———
TIME EVENT-ID MSG-ID SEVERITY
————— ———————————— ————– ———
Feb 06 10:17:03 fe2302d7-99c9-c0c8-d54b-92495cc94fc9 ZFS-8000-D3 Major

Host : sc02-app04
Platform : ORCL,SPARC-T5-8 Chassis_id :
Product_sn :

Fault class : fault.fs.zfs.device
Affects : zfs://pool=prd171/vdev=5d2cdc446e947471
faulted and taken out of service
Problem in : zfs://pool=prd171/vdev=5d2cdc446e947471
faulted and taken out of service

Description : A ZFS device failed.

Response : No automated response will occur.

Impact : Fault tolerance of the pool may be compromised.

Action : Run ‘zpool status -x’ for more information. Please refer to the
associated reference document at http://sun.com/msg/ZFS-8000-D3
for the latest service procedures and policies regarding this
diagnosis.

root@sc02-app04:~ # fmadm repair 0ad5260a-f9e0-ef7a-c91f-efcc76b9b164
fmadm: recorded repair to 0ad5260a-f9e0-ef7a-c91f-efcc76b9b164
root@sc02-app04:~ # fmadm repair fe2302d7-99c9-c0c8-d54b-92495cc94fc9
fmadm: recorded repair to fe2302d7-99c9-c0c8-d54b-92495cc94fc9

Clear ereports and resource cache

root@sc02-app04:~ # cd /var/fm/fmd
root@sc02-app04:/var/fm/fmd # rm e* f* c*/eft/* r*/*

Clearing out FMA files with no reboot needed

root@sc02-app04:~ # svcadm disable -s svc:/system/fmd:default
root@sc02-app04:~ # cd /var/fm/fmd
root@sc02-app04:/var/fm/fmd # find /var/fm/fmd -type f -exec ls {} \;
/var/fm/fmd/topo/90ab82b5-08eb-6f9f-9a9a-af2975a2808b/hc-topology.xml
/var/fm/fmd/topo/6b4eba63-3576-e155-ac3b-8f6609f0b968/hc-topology.xml
/var/fm/fmd/topo/1badc01d-82b9-6203-9440-9dd440aedaca/hc-topology.xml
/var/fm/fmd/topo/f32b13d0-63a1-4b5a-e811-bfda6bddcba1/hc-topology.xml
/var/fm/fmd/topo/c4824832-ced3-672a-ec69-a9490f94d2c0/hc-topology.xml
/var/fm/fmd/ckpt/etm/etm
/var/fm/fmd/ckpt/zfs-diagnosis/zfs-diagnosis
root@sc02-app04:/var/fm/fmd # find /var/fm/fmd -type f -exec rm {} \;
root@sc02-app04:/var/fm/fmd # svcadm enable svc:/system/fmd:default

Checking fmadm faulty

root@sc02-app04:~ # fmadm faulty
root@sc02-app04:~ #

Reset the fmd serd modules

root@sc02-app04:~ # fmadm reset cpumem-diagnosis
fmadm: failed to reset module cpumem-diagnosis: specified module is not loaded in fault manager
root@sc02-app04:~ # fmadm reset cpumem-retire
fmadm: cpumem-retire module has been reset
root@sc02-app04:~ # fmadm reset eft
fmadm: eft module has been reset
root@sc02-app04:~ # fmadm reset io-retire
fmadm: io-retire module has been reset

Source: https://saifulaziz.com/2011/12/26/how-to-clear-fmadm-log-or-fma-faults-log/

Linux LVM: Device /dev/mapper/ECP_data_disk_001 has size of 1048576000 sectors which is smaller than corresponding PV size of 1174372352 sectors. Was device resized?

Running LVM command and it shows an unrelated disk that is complaining it has a unexpected size.

This problem was happening because it is a cluster node and LUN was resized.

On the node that showed the message, I didn’t run any commands to resize it

root@linux02:~ # pvs | grep PP0_data_disk_001
Device /dev/mapper/ECP_data_disk_001 has size of 1048576000 sectors which is smaller than corresponding PV size of 1174372352 sectors. Was device resized?

Run the commands to resize the LUN on the node

root@linux02:~ # multipath -ll ECP_data_disk_001 | grep ready | awk ‘{print “echo 1 > /sys/block/”$3″/device/rescan”‘} | bash
root@linux02:~ # multipathd -k’resize map ECP_data_disk_001′
ok
root@linux02:~ # pvresize /dev/mapper/ECP_data_disk_001
Physical volume “/dev/mapper/ECP_data_disk_001” changed
1 physical volume(s) resized / 0 physical volume(s) not resized

EXT4-fs: : Can’t allocate: Allocation context details

Node : linux.setaoffice.com
Node Type : Intel/AMD x64(HTTPS)
Severity : minor
OM Server Time: 2017-12-29 15:04:09
Message : EXT4-fs: : Can’t allocate: Allocation context details:
Msg Group : OS
Application : dmsg_mon
Object : EXT4
Event Type :
not_found

Instance Name :
not_found

Instruction : No
EventDataSource :

This problem happens because there is a filesystem nearly full

root@linux:~ # df -hP /osprov
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/vg01-ospvol 405G 379G 4.4G 99% /osprov

root@linux:~ # vgs vg01
VG #PV #LV #SN Attr VSize VFree
vg01 1 4 0 wz–n- 599.88G 11.25G

Checking the logs, I discovered that the device being affected is a snapshot from a filesystem

root@linux:/var/log # zgrep “Can’t allocate: Allocation context details” messages* | grep -v CMD
messages.1.gz:Dec 8 20:00:16 linux kernel: EXT4-fs: Can’t allocate: Allocation context details:
messages.1.gz:Dec 29 20:00:17 linux kernel: EXT4-fs: Can’t allocate: Allocation context details:
messages.2.gz:Nov 10 20:00:17 linux kernel: EXT4-fs: Can’t allocate: Allocation context details:
messages.2.gz:Nov 24 20:00:17 linux kernel: EXT4-fs: Can’t allocate: Allocation context details:

Getting ” EXT4-fs: Can’t allocate: Allocation context details” errors in logs while using ext4 with Red Hat Enterprise Linux 5.6 and higher

I’ve added message to file /var/opt/OV/conf/OpC/dmsg_mon.cfg to ignore this alarm

root@linux:/var/opt/OV/conf/OpC # vi dmsg_mon.cfg
EXT4-fs: Can’t allocate: Allocation context details

How to uncompress a .xz file in Red Hat Enterprise Linux

You have generated a sosreport and now want to take a look at the files. To extract the files you need to have installed xz

root@linux:~ # rpm -qi xz
Name : xz
Version : 5.2.2
Release : 1.el7
Architecture: x86_64
Install Date: Thu 05 Jan 2017 12:36:49 PM EST
Group : Applications/File
Size : 798066
License : LGPLv2+
Signature : RSA/SHA256, Thu 05 May 2016 10:16:16 AM EDT, Key ID 199e2f91fd431d51
Source RPM : xz-5.2.2-1.el7.src.rpm
Build Date : Fri 04 Mar 2016 07:57:32 AM EST
Build Host : x86-024.build.eng.bos.redhat.com
Relocations : (not relocatable)
Packager : Red Hat, Inc.
Vendor : Red Hat, Inc.
URL : http://tukaani.org/xz/
Summary : LZMA compression utilities
Description :
XZ Utils are an attempt to make LZMA compression easy to use on free (as in
freedom) operating systems. This is achieved by providing tools and libraries
which are similar to use than the equivalents of the most popular existing
compression algorithms.

LZMA is a general purpose compression algorithm designed by Igor Pavlov as
part of 7-Zip. It provides high compression ratio while keeping the
decompression speed fast.

Then run unxz and extract the tarball with tar

root@linux:/var/tmp # unxz sosreport-linux.setaoffice.com-20171212113141.tar.xz
root@linux:/var/tmp # tar -xf sosreport-linux.setaoffice.com-20171212113141.tar

UXMON: SY1_log2_disk_001 – Only one path detected, no path redundancy

Also see:
UXMON: mpathb – Only one path detected, no path redundancy
UXMON: volumegroup – Only one path detected, no path redundancy

ATTENTION, RMC LEVEL 1 AGENT: This ticket will be automatically worked by the Automation Bus. Pls. ensure your Ticket List/View includes the “Assignee” column, monitor this ticket until the user “ABOPERATOR” is no longer assigned, BEFORE you start work on this ticket.
Node : linux.setaoffice.com
Node Type : Intel/AMD x64(HTTPS)
Severity : major
OM Server Time: 2017-12-11 03:28:31
Message : UXMON: SY1_log2_disk_001 – Only one path detected, no path redundancy
Msg Group : OS
Application : mpmon
Object : mp
Event Type :
not_found

Instance Name :
not_found

Instruction : The multipathd -k”show map $device topology” command shows more details

Please check /var/opt/OV/log/OpC/mp_mon.log for more details
EventDataSource :

When running a LVM command it is showing that several PVs are duplicate.

root@linux:~ # pvs
Found duplicate PV s3zEE42awIhydJ05hfUzJulPsN8WS266: using /dev/mapper/SY1_disknew_001 not /dev/sdbm
Using duplicate PV /dev/mapper/SY1_disknew_001 from subsystem DM, replacing /dev/sdbm
Found duplicate PV zTZlnwYgW69xzUcTi0riu5euTPiWWnRs: using /dev/mapper/swap_disk_001p1 not /dev/sdbo1
Using duplicate PV /dev/mapper/swap_disk_001p1 from subsystem DM, ignoring /dev/sdbo1
Found duplicate PV hnuWyziFeXRhJ71YpLJX1cLAjOvgtl0q: using /dev/mapper/SCR_DATA_disk_001p1 not /dev/sdbn1
Using duplicate PV /dev/mapper/SCR_DATA_disk_001p1 from subsystem DM, replacing /dev/sdbn1
Found duplicate PV EeqL0LDdMohfnagni16NlRUig3eugbap: using /dev/mapper/SY1_log2_disk_001p1 not /dev/sdbp1
Using duplicate PV /dev/mapper/SY1_log2_disk_001p1 from subsystem DM, ignoring /dev/sdbp1
Found duplicate PV sMAYQABFedTStD589d5ZcDu1ZNtXfXyh: using /dev/mapper/SY1_log1_disk_001p1 not /dev/sdbq1
Using duplicate PV /dev/mapper/SY1_log1_disk_001p1 from subsystem DM, ignoring /dev/sdbq1
Found duplicate PV qFBDDPa7FtN7F97tpUDeKv0cBO94WBr3: using /dev/mapper/SCR_disk_002 not /dev/sdbr
Using duplicate PV /dev/mapper/SCR_disk_002 from subsystem DM, ignoring /dev/sdbr
Found duplicate PV RZvwCnQn48G2A9IifNOLZT9l9ZYdE7yu: using /dev/mapper/SY1_arch_disk_001p1 not /dev/sdbs1
Using duplicate PV /dev/mapper/SY1_arch_disk_001p1 from subsystem DM, ignoring /dev/sdbs1
Found duplicate PV qWPh0v3E1ADowxUuuC8kP8BFJ1bwBsmL: using /dev/mapper/SY1_disk_001p1 not /dev/sdbt1
Using duplicate PV /dev/mapper/SY1_disk_001p1 from subsystem DM, ignoring /dev/sdbt1
Found duplicate PV hi5F4g9FPojaD9J7KH5vHCzRKgd4jQfJ: using /dev/mapper/MR2_log2_disk_001p1 not /dev/sdbu1
Using duplicate PV /dev/mapper/MR2_log2_disk_001p1 from subsystem DM, ignoring /dev/sdbu1
Found duplicate PV KA1AkbCOgfHFGuKazmHAacwqBNPmEyV8: using /dev/mapper/MR2_log1_disk_001p1 not /dev/sdbv1
Using duplicate PV /dev/mapper/MR2_log1_disk_001p1 from subsystem DM, ignoring /dev/sdbv1
Found duplicate PV dy0YaJoSrlZdTM9isfY1QGS6kWBTzs6i: using /dev/mapper/MR2_data_disk_001p1 not /dev/sdbw1
Using duplicate PV /dev/mapper/MR2_data_disk_001p1 from subsystem DM, ignoring /dev/sdbw1
Found duplicate PV khSEnsJ9CC4epw0uOAsaFwlbIKS32Qyj: using /dev/mapper/MR2_arch_disk_001p1 not /dev/sdbx1
Using duplicate PV /dev/mapper/MR2_arch_disk_001p1 from subsystem DM, ignoring /dev/sdbx1
Found duplicate PV PmTTDGWvEvDM8AMUHvqcoRJeA9g4r78D: using /dev/mapper/MRC_log2_disk_001p1 not /dev/sdbi1
Using duplicate PV /dev/mapper/MRC_log2_disk_001p1 from subsystem DM, ignoring /dev/sdbi1
Found duplicate PV ffv1TgDxRIEIyhRoEh46fMGeFuJ1tVtv: using /dev/mapper/MR2_disk_001p1 not /dev/sdby1
Using duplicate PV /dev/mapper/MR2_disk_001p1 from subsystem DM, ignoring /dev/sdby1
Found duplicate PV tCeMdOB1dFuJomhgB061M7MeZNEwptk3: using /dev/mapper/MRC_log1_disk_001p1 not /dev/sdbj1
Using duplicate PV /dev/mapper/MRC_log1_disk_001p1 from subsystem DM, ignoring /dev/sdbj1
Found duplicate PV inuuS3f39VjSuh6q7r5pEZSjzRgRhi5e: using /dev/mapper/sap_disk_001p1 not /dev/sdbz1
Using duplicate PV /dev/mapper/sap_disk_001p1 from subsystem DM, ignoring /dev/sdbz1
Found duplicate PV Ea4CVYCMjYc3eE129uYlUlZKdzhmYNpQ: using /dev/mapper/MRC_data_disk_001p1 not /dev/sdbk1
Using duplicate PV /dev/mapper/MRC_data_disk_001p1 from subsystem DM, ignoring /dev/sdbk1
Found duplicate PV Lx5viH3geNo0zSO07sHcbNlJ9nVmcGPA: using /dev/mapper/MRC_arch_disk_001p1 not /dev/sdbl1
Using duplicate PV /dev/mapper/MRC_arch_disk_001p1 from subsystem DM, ignoring /dev/sdbl1
PV VG Fmt Attr PSize PFree
/dev/mapper/MR2_arch_disk_001p1 vgMR2oraarch lvm2 a–u 19.98g 0
/dev/mapper/MR2_data_disk_001p1 vgMR2data lvm2 a–u 299.98g 99.98g
/dev/mapper/MR2_disk_001p1 vgMR2 lvm2 a–u 99.98g 20.87g
/dev/mapper/MR2_log1_disk_001p1 vgMR2log1 lvm2 a–u 9.98g 7.98g
/dev/mapper/MR2_log2_disk_001p1 vgMR2log2 lvm2 a–u 9.98g 7.98g
/dev/mapper/MRC_arch_disk_001p1 vgMRCoraarch lvm2 a–u 19.98g 0
/dev/mapper/MRC_data_disk_001p1 vgMRCdata lvm2 a–u 299.98g 39.98g
/dev/mapper/MRC_disk_002 vgMRC lvm2 a–u 149.98g 31.88g
/dev/mapper/MRC_log1_disk_001p1 vgMRClog1 lvm2 a–u 9.98g 7.98g
/dev/mapper/MRC_log2_disk_001p1 vgMRClog2 lvm2 a–u 9.98g 7.98g
/dev/mapper/SCR_ARCH_disk_001p1 vgSCRarch lvm2 a–u 29.96g 4.57g
/dev/mapper/SCR_DATA_disk_001p1 vgSCRdata lvm2 a–u 109.96g 8.40g
/dev/mapper/SCR_LOG1_disk_001p1 vgSCRlog1 lvm2 a–u 4.96g 984.00m
/dev/mapper/SCR_LOG2_disk_001p1 vgSCRlog2 lvm2 a–u 4.96g 984.00m
/dev/mapper/SCR_disk_001p1 vgSCR lvm2 a–u 49.96g 3.96g
/dev/mapper/SCR_disk_002 vgSCR lvm2 a–u 49.98g 34.98g
/dev/mapper/SY1_arch_disk_001p1 vgSY1oraarch lvm2 a–u 149.98g 0
/dev/mapper/SY1_data_disk_002 vgSY1data lvm2 a–u 2.64t 0
/dev/mapper/SY1_disk_001p1 vgSY1 lvm2 a–u 49.98g 10.52g
/dev/mapper/SY1_disknew_001 vgSY1 lvm2 a–u 59.98g 44.98g
/dev/mapper/SY1_interf_disk_001 vgSY1interface lvm2 a–u 49.98g 1008.00m
/dev/mapper/SY1_log1_disk_001p1 vgSY1log1 lvm2 a–u 19.98g 9.98g
/dev/mapper/SY1_log2_disk_001p1 vgSY1log2 lvm2 a–u 19.98g 9.98g
/dev/mapper/sap_disk_001p1 vgSAPlocal lvm2 a–u 49.98g 4.79g
/dev/mapper/swap_disk_001p1 vgswap lvm2 a–u 223.98g 0
/dev/sda2 vgroot lvm2 a–u 278.86g 198.27g

Removing disk paths that are duplicate

echo 1 > /sys/block/sdbm/device/delete
echo 1 > /sys/block/sdbo/device/delete
echo 1 > /sys/block/sdbn/device/delete
echo 1 > /sys/block/sdbp/device/delete
echo 1 > /sys/block/sdbq/device/delete
echo 1 > /sys/block/sdbr/device/delete
echo 1 > /sys/block/sdbs/device/delete
echo 1 > /sys/block/sdbt/device/delete
echo 1 > /sys/block/sdbu/device/delete
echo 1 > /sys/block/sdbv/device/delete
echo 1 > /sys/block/sdbw/device/delete
echo 1 > /sys/block/sdbx/device/delete
echo 1 > /sys/block/sdbi/device/delete
echo 1 > /sys/block/sdby/device/delete
echo 1 > /sys/block/sdbj/device/delete
echo 1 > /sys/block/sdbz/device/delete

Checking status

root@linux:~ # pvs
PV VG Fmt Attr PSize PFree
/dev/mapper/MR2_arch_disk_001p1 vgMR2oraarch lvm2 a–u 19.98g 0
/dev/mapper/MR2_data_disk_001p1 vgMR2data lvm2 a–u 299.98g 99.98g
/dev/mapper/MR2_disk_001p1 vgMR2 lvm2 a–u 99.98g 20.87g
/dev/mapper/MR2_log1_disk_001p1 vgMR2log1 lvm2 a–u 9.98g 7.98g
/dev/mapper/MR2_log2_disk_001p1 vgMR2log2 lvm2 a–u 9.98g 7.98g
/dev/mapper/MRC_arch_disk_001p1 vgMRCoraarch lvm2 a–u 19.98g 0
/dev/mapper/MRC_data_disk_001p1 vgMRCdata lvm2 a–u 299.98g 39.98g
/dev/mapper/MRC_disk_002 vgMRC lvm2 a–u 149.98g 31.88g
/dev/mapper/MRC_log1_disk_001p1 vgMRClog1 lvm2 a–u 9.98g 7.98g
/dev/mapper/MRC_log2_disk_001p1 vgMRClog2 lvm2 a–u 9.98g 7.98g
/dev/mapper/SCR_ARCH_disk_001p1 vgSCRarch lvm2 a–u 29.96g 4.57g
/dev/mapper/SCR_DATA_disk_001p1 vgSCRdata lvm2 a–u 109.96g 8.40g
/dev/mapper/SCR_LOG1_disk_001p1 vgSCRlog1 lvm2 a–u 4.96g 984.00m
/dev/mapper/SCR_LOG2_disk_001p1 vgSCRlog2 lvm2 a–u 4.96g 984.00m
/dev/mapper/SCR_disk_001p1 vgSCR lvm2 a–u 49.96g 3.96g
/dev/mapper/SCR_disk_002 vgSCR lvm2 a–u 49.98g 34.98g
/dev/mapper/SY1_arch_disk_001p1 vgSY1oraarch lvm2 a–u 149.98g 0
/dev/mapper/SY1_data_disk_002 vgSY1data lvm2 a–u 2.64t 0
/dev/mapper/SY1_disk_001p1 vgSY1 lvm2 a–u 49.98g 10.52g
/dev/mapper/SY1_disknew_001 vgSY1 lvm2 a–u 59.98g 44.98g
/dev/mapper/SY1_interf_disk_001 vgSY1interface lvm2 a–u 49.98g 1008.00m
/dev/mapper/SY1_log1_disk_001p1 vgSY1log1 lvm2 a–u 19.98g 9.98g
/dev/mapper/SY1_log2_disk_001p1 vgSY1log2 lvm2 a–u 19.98g 9.98g
/dev/mapper/sap_disk_001p1 vgSAPlocal lvm2 a–u 49.98g 4.79g
/dev/mapper/swap_disk_001p1 vgswap lvm2 a–u 223.98g 0
/dev/sda2 vgroot lvm2 a–u 278.86g 198.27g

Checking multipath devices. Some devices have only one path

root@linux:~ # multipath -ll
SCR_disk_001 (350002acb5a22374a) dm-26 3PARdata,VV
size=50G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:21 sdy 65:128 active ready running
`- 1:0:0:21 sdcb 68:240 active ready running
SY1_log2_disk_001 (350002ac17cd5374a) dm-6 3PARdata,VV
size=20G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:6 sdi 8:128 active ready running
MRC_arch_disk_001 (350002ac15504374a) dm-3 3PARdata,VV
size=20G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:3 sdf 8:80 active ready running
MR2_data_disk_001 (350002ac17cd8374a) dm-14 3PARdata,VV
size=300G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:13 sdp 8:240 active ready running
MR2_disk_001 (350002ac17cd6374a) dm-18 3PARdata,VV
size=100G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:15 sds 65:32 active ready running
SY1_log1_disk_001 (350002ac17cd4374a) dm-7 3PARdata,VV
size=20G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:7 sdj 8:144 active ready running
MR2_log2_disk_001 (350002ac17ce7374a) dm-11 3PARdata,VV
size=10G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:11 sdn 8:208 active ready running
MRC_disk_002 (350002ac2904a374a) dm-32 3PARdata,VV
size=150G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:23 sdaa 65:160 active ready running
`- 1:0:0:23 sdcd 69:16 active ready running
SCR_ARCH_disk_001 (350002acb5a2c374a) dm-16 3PARdata,VV
size=30G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:17 sdu 65:64 active ready running
`- 1:0:0:17 sdca 68:224 active ready running
MR2_log1_disk_001 (350002ac17ce5374a) dm-12 3PARdata,VV
size=10G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:12 sdo 8:224 active ready running
SY1_arch_disk_001 (350002ac17cca374a) dm-9 3PARdata,VV
size=150G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:9 sdl 8:176 active ready running
SY1_disknew_001 (350002ac4c566374a) dm-4 3PARdata,VV
size=60G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:4 sdg 8:96 active ready running
SCR_LOG2_disk_001 (350002acb5a39374a) dm-17 3PARdata,VV
size=5.0G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:18 sdv 65:80 active ready running
`- 1:0:0:18 sdq 65:0 active ready running
MR2_arch_disk_001 (350002ac17cd7374a) dm-15 3PARdata,VV
size=20G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:14 sdr 65:16 active ready running
SCR_LOG1_disk_001 (350002acb5a38374a) dm-19 3PARdata,VV
size=5.0G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:19 sdw 65:96 active ready running
`- 1:0:0:19 sdav 66:240 active ready running
MRC_data_disk_001 (350002ac15505374a) dm-2 3PARdata,VV
size=300G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:2 sde 8:64 active ready running
sap_disk_001 (350002ac1fc92374a) dm-13 3PARdata,VV
size=50G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:16 sdt 65:48 active ready running
SY1_disk_001 (350002ac17cbd374a) dm-10 3PARdata,VV
size=50G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:10 sdm 8:192 active ready running
SY1_interf_disk_001 (350002ac0ce50374a) dm-33 3PARdata,VV
size=50G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:24 sdab 65:176 active ready running
`- 1:0:0:24 sdce 69:32 active ready running
MRC_log2_disk_001 (350002ac1551a374a) dm-71 3PARdata,VV
size=10G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:0 sdc 8:32 active ready running
SY1_data_disk_002 (350002ac23826374a) dm-31 3PARdata,VV
size=2.6T features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:22 sdz 65:144 active ready running
`- 1:0:0:22 sdcc 69:0 active ready running
MRC_log1_disk_001 (350002ac15519374a) dm-72 3PARdata,VV
size=10G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:1 sdd 8:48 active ready running
SCR_DATA_disk_001 (350002acb5a37374a) dm-23 3PARdata,VV
size=110G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:20 sdx 65:112 active ready running
SCR_disk_002 (350002ac19e73374a) dm-8 3PARdata,VV
size=50G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:8 sdk 8:160 active ready running
swap_disk_001 (350002ac155d0374a) dm-5 3PARdata,VV
size=224G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
`- 0:0:0:5 sdh 8:112 active ready running

Recognize new LUNs and run multipath -v3

root@linux:~ # systool -av -c fc_host | grep “Class Device =” | awk -F’=’ {‘print $2’} | awk -F'”‘ {‘print “echo \”- – -\” > /sys/class/scsi_host/”$2″/scan”‘} | sh
root@linux:~ # multipath -v3

Paths restored

root@linux:~ # multipath -ll
SCR_disk_001 (350002acb5a22374a) dm-26 3PARdata,VV
size=50G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:21 sdy 65:128 active ready running
`- 1:0:0:21 sdcb 68:240 active ready running
SY1_log2_disk_001 (350002ac17cd5374a) dm-6 3PARdata,VV
size=20G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:6 sdi 8:128 active ready running
`- 1:0:0:6 sdai 66:32 active ready running
MRC_arch_disk_001 (350002ac15504374a) dm-3 3PARdata,VV
size=20G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:3 sdf 8:80 active ready running
`- 1:0:0:3 sdaf 65:240 active ready running
MR2_data_disk_001 (350002ac17cd8374a) dm-14 3PARdata,VV
size=300G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:13 sdp 8:240 active ready running
`- 1:0:0:13 sdap 66:144 active ready running
MR2_disk_001 (350002ac17cd6374a) dm-18 3PARdata,VV
size=100G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:15 sds 65:32 active ready running
`- 1:0:0:15 sdar 66:176 active ready running
SY1_log1_disk_001 (350002ac17cd4374a) dm-7 3PARdata,VV
size=20G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:7 sdj 8:144 active ready running
`- 1:0:0:7 sdaj 66:48 active ready running
MR2_log2_disk_001 (350002ac17ce7374a) dm-11 3PARdata,VV
size=10G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:11 sdn 8:208 active ready running
`- 1:0:0:11 sdan 66:112 active ready running
MRC_disk_002 (350002ac2904a374a) dm-32 3PARdata,VV
size=150G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:23 sdaa 65:160 active ready running
`- 1:0:0:23 sdcd 69:16 active ready running
SCR_ARCH_disk_001 (350002acb5a2c374a) dm-16 3PARdata,VV
size=30G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:17 sdu 65:64 active ready running
`- 1:0:0:17 sdca 68:224 active ready running
MR2_log1_disk_001 (350002ac17ce5374a) dm-12 3PARdata,VV
size=10G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:12 sdo 8:224 active ready running
`- 1:0:0:12 sdao 66:128 active ready running
SY1_arch_disk_001 (350002ac17cca374a) dm-9 3PARdata,VV
size=150G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:9 sdl 8:176 active ready running
`- 1:0:0:9 sdal 66:80 active ready running
SY1_disknew_001 (350002ac4c566374a) dm-4 3PARdata,VV
size=60G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:4 sdg 8:96 active ready running
`- 1:0:0:4 sdag 66:0 active ready running
SCR_LOG2_disk_001 (350002acb5a39374a) dm-17 3PARdata,VV
size=5.0G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:18 sdv 65:80 active ready running
`- 1:0:0:18 sdq 65:0 active ready running
MR2_arch_disk_001 (350002ac17cd7374a) dm-15 3PARdata,VV
size=20G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:14 sdr 65:16 active ready running
`- 1:0:0:14 sdaq 66:160 active ready running
SCR_LOG1_disk_001 (350002acb5a38374a) dm-19 3PARdata,VV
size=5.0G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:19 sdw 65:96 active ready running
`- 1:0:0:19 sdav 66:240 active ready running
MRC_data_disk_001 (350002ac15505374a) dm-2 3PARdata,VV
size=300G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:2 sde 8:64 active ready running
`- 1:0:0:2 sdae 65:224 active ready running
sap_disk_001 (350002ac1fc92374a) dm-13 3PARdata,VV
size=50G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:16 sdt 65:48 active ready running
`- 1:0:0:16 sdas 66:192 active ready running
SY1_disk_001 (350002ac17cbd374a) dm-10 3PARdata,VV
size=50G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:10 sdm 8:192 active ready running
`- 1:0:0:10 sdam 66:96 active ready running
SY1_interf_disk_001 (350002ac0ce50374a) dm-33 3PARdata,VV
size=50G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:24 sdab 65:176 active ready running
`- 1:0:0:24 sdce 69:32 active ready running
MRC_log2_disk_001 (350002ac1551a374a) dm-71 3PARdata,VV
size=10G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:0 sdc 8:32 active ready running
`- 1:0:0:0 sdac 65:192 active ready running
SY1_data_disk_002 (350002ac23826374a) dm-31 3PARdata,VV
size=2.6T features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:22 sdz 65:144 active ready running
`- 1:0:0:22 sdcc 69:0 active ready running
MRC_log1_disk_001 (350002ac15519374a) dm-72 3PARdata,VV
size=10G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:1 sdd 8:48 active ready running
`- 1:0:0:1 sdad 65:208 active ready running
SCR_DATA_disk_001 (350002acb5a37374a) dm-23 3PARdata,VV
size=110G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:20 sdx 65:112 active ready running
`- 1:0:0:20 sdat 66:208 active ready running
SCR_disk_002 (350002ac19e73374a) dm-8 3PARdata,VV
size=50G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:8 sdk 8:160 active ready running
`- 1:0:0:8 sdak 66:64 active ready running
swap_disk_001 (350002ac155d0374a) dm-5 3PARdata,VV
size=224G features=’0′ hwhandler=’0′ wp=rw
`-+- policy=’round-robin 0′ prio=1 status=active
|- 0:0:0:5 sdh 8:112 active ready running
`- 1:0:0:5 sdah 66:16 active ready running

Problem solved

root@linux:~ # /var/opt/OV/bin/instrumentation/UXMONbroker -check mpmon
Fri Dec 15 11:39:25 2017 : INFO : UXMONmpmon is running now, pid=52191
mv: `/dev/null’ and `/dev/null’ are the same file
Fri Dec 15 11:39:26 2017 : INFO : UXMONmpmon end, pid=52191

%d bloggers like this: