[zfs-discuss] call trace

Jerker Nyberg jerker at Update.UU.SE
Mon Dec 5 04:08:57 EST 2011


On Fri, 25 Nov 2011, e-t172 wrote:

> On 2011-11-25 11:02, Fajar A. Nugraha wrote:
>> For zfs to work properly:
>> - a drive needs to be report quickly that it failed
>> - zfs needs to know that the drive failed, and kick it out of the pool
>> (possibly activating hot spare, if available)
>> 
>> The problem is some drives (your HDD might be among those) will retry
>> for a LOOOONG time reading/writing to a bad sector before if it gives
>> up. And linux's driver will also spend lots of time retrying.
>
> Maybe reducing /dev/block/*/device/timeout would alleviate this issue?

Sorry, but that does not seem to help. I include some of the details about 
my system below. If any other information I can provide can be of any 
interest let me know.

--jerker


[root at esc1 ~]# cat tank.create.sh
zpool create -f tank mirror disk/by-id/ata-WDC_WD30EZRS-00J99B0_WD-WCAWZ0051307 \
                             disk/by-id/ata-WDC_WD30EZRS-00J99B0_WD-WCAWZ0049593 \
                      cache  disk/by-id/ata-INTEL_SSDSA2CW120G3_CVPR11660611120LGN-part3 \
                      log    disk/by-id/ata-INTEL_SSDSA2CW120G3_CVPR11660611120LGN-part4
zfs set dedup=on tank
zfs set compression=on tank
zfs create -o dedup=on -o compression=on tank/fish

zpool add tank mirror disk/by-id/ata-Hitachi_HDS723030ALA640_MK0311YHG29L5A \
                       disk/by-id/ata-Hitachi_HDS723030ALA640_MK0311YHG5B6BA

[root at esc1 ~]# uname -a
Linux esc1 2.6.32-131.17.1.el6.x86_64 #1 SMP Wed Oct 5 17:19:54 CDT 2011 x86_64 x86_64 x86_64 GNU/Linux

[root at esc1 ~]# rpm -q kernel spl zfs
kernel-2.6.32-71.29.1.el6.x86_64
kernel-2.6.32-131.17.1.el6.x86_64
spl-0.6.0-rc6.x86_64
zfs-0.6.0-rc6.x86_64

[root at esc1 ~]# zpool list
NAME   SIZE  ALLOC   FREE    CAP  DEDUP  HEALTH  ALTROOT
tank  5.44T  2.69T  2.75T    49%  1.92x  ONLINE  -

[root at esc1 ~]# zpool status
   pool: tank
  state: ONLINE
  scan: scrub repaired 0 in 0h0m with 0 errors on Fri Nov 11 12:19:39 2011
config:

         NAME                                                STATE     READ 
WRITE CKSUM
         tank                                                ONLINE       0     0     0
           mirror-0                                          ONLINE       0     0     0
             ata-WDC_WD30EZRS-00J99B0_WD-WCAWZ0051307        ONLINE       0     0     0
             ata-WDC_WD30EZRS-00J99B0_WD-WCAWZ0049593        ONLINE       0     0     0
           mirror-2                                          ONLINE       0     0     0
             ata-Hitachi_HDS723030ALA640_MK0311YHG29L5A      ONLINE       0     0     0
             ata-Hitachi_HDS723030ALA640_MK0311YHG5B6BA      ONLINE       0     0     0
         logs
           ata-INTEL_SSDSA2CW120G3_CVPR11660611120LGN-part4  ONLINE       0     0     0
         cache
           ata-INTEL_SSDSA2CW120G3_CVPR11660611120LGN-part3  ONLINE       0     0     0

errors: No known data errors

[root at esc1 ~]# for i in /dev/sd[a-z] ; do echo $i $(smartctl -H $i|grep result) ; done
/dev/sda SMART overall-health self-assessment test result: PASSED
/dev/sdb SMART overall-health self-assessment test result: PASSED
/dev/sdc SMART overall-health self-assessment test result: PASSED
/dev/sdd SMART overall-health self-assessment test result: PASSED
/dev/sde SMART overall-health self-assessment test result: PASSED
/dev/sdf SMART overall-health self-assessment test result: PASSED
/dev/sdg SMART overall-health self-assessment test result: PASSED

[root at esc1 ~]# edac-util
edac-util: No errors to report.

[root at esc1 ~]# cat set.sys.block.dev.device.timeout.sh
#!/bin/bash
for i in /sys/block/sd[b-z]/device/timeout ; do
      echo 7 >$i
done

[root at esc1 ~]# cat /sys/block/sd*/device/timeout
30
7
7
7
7
7
7

[root at esc1 ~]# grep 'Dec  [1-3]' /var/log/messages-20111205 | grep -v dhclient
Dec  1 14:47:42 esc1 kernel: ata7.00: exception Emask 0x0 SAct 0xff SErr 0x0 action 0x6 frozen
Dec  1 14:47:42 esc1 kernel: ata7.00: failed command: WRITE FPDMA QUEUED
Dec  1 14:47:42 esc1 kernel: ata7.00: cmd 61/06:00:fe:ea:48/00:00:24:01:00/40 tag 0 ncq 3072 out
Dec  1 14:47:42 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 14:47:42 esc1 kernel: ata7.00: status: { DRDY }
Dec  1 14:47:42 esc1 kernel: ata7.00: failed command: WRITE FPDMA QUEUED
Dec  1 14:47:42 esc1 kernel: ata7.00: cmd 61/05:08:92:eb:48/00:00:24:01:00/40 tag 1 ncq 2560 out
Dec  1 14:47:42 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 14:47:42 esc1 kernel: ata7.00: status: { DRDY }
Dec  1 14:47:42 esc1 kernel: ata7.00: failed command: WRITE FPDMA QUEUED
Dec  1 14:47:42 esc1 kernel: ata7.00: cmd 61/04:10:eb:2e:49/00:00:24:01:00/40 tag 2 ncq 2048 out
Dec  1 14:47:42 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 14:47:42 esc1 kernel: ata7.00: status: { DRDY }
Dec  1 14:47:42 esc1 kernel: ata7.00: failed command: WRITE FPDMA QUEUED
Dec  1 14:47:42 esc1 kernel: ata7.00: cmd 61/06:18:02:2f:49/00:00:24:01:00/40 tag 3 ncq 3072 out
Dec  1 14:47:42 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 14:47:42 esc1 kernel: ata7.00: status: { DRDY }
Dec  1 14:47:42 esc1 kernel: ata7.00: failed command: WRITE FPDMA QUEUED
Dec  1 14:47:42 esc1 kernel: ata7.00: cmd 61/17:20:27:2f:49/00:00:24:01:00/40 tag 4 ncq 11776 out
Dec  1 14:47:42 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 14:47:42 esc1 kernel: ata7.00: status: { DRDY }
Dec  1 14:47:42 esc1 kernel: ata7.00: failed command: WRITE FPDMA QUEUED
Dec  1 14:47:42 esc1 kernel: ata7.00: cmd 61/05:28:53:2f:49/00:00:24:01:00/40 tag 5 ncq 2560 out
Dec  1 14:47:42 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 14:47:42 esc1 kernel: ata7.00: status: { DRDY }
Dec  1 14:47:42 esc1 kernel: ata7.00: failed command: WRITE FPDMA QUEUED
Dec  1 14:47:42 esc1 kernel: ata7.00: cmd 61/02:30:5e:2f:49/00:00:24:01:00/40 tag 6 ncq 1024 out
Dec  1 14:47:42 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 14:47:42 esc1 kernel: ata7.00: status: { DRDY }
Dec  1 14:47:42 esc1 kernel: ata7.00: failed command: WRITE FPDMA QUEUED
Dec  1 14:47:42 esc1 kernel: ata7.00: cmd 61/1e:38:67:2f:49/00:00:24:01:00/40 tag 7 ncq 15360 out
Dec  1 14:47:42 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 14:47:42 esc1 kernel: ata7.00: status: { DRDY }
Dec  1 14:47:42 esc1 kernel: ata7: hard resetting link
Dec  1 14:47:42 esc1 kernel: ata7: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
Dec  1 14:47:42 esc1 kernel: ata7.00: configured for UDMA/133
Dec  1 14:47:42 esc1 kernel: ata7.00: device reported invalid CHS sector 0
Dec  1 14:47:42 esc1 kernel: ata7.00: device reported invalid CHS sector 0
Dec  1 14:47:42 esc1 kernel: ata7.00: device reported invalid CHS sector 0
Dec  1 14:47:42 esc1 kernel: ata7.00: device reported invalid CHS sector 0
Dec  1 14:47:42 esc1 kernel: ata7.00: device reported invalid CHS sector 0
Dec  1 14:47:42 esc1 kernel: ata7.00: device reported invalid CHS sector 0
Dec  1 14:47:42 esc1 kernel: ata7.00: device reported invalid CHS sector 0
Dec  1 14:47:42 esc1 kernel: ata7.00: device reported invalid CHS sector 0
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Sense Key : Aborted Command [current] [descriptor]
Dec  1 14:47:42 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 14:47:42 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 14:47:42 esc1 kernel:        00 00 00 00 
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Add. Sense: No additional sense information
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] CDB: Write(16): 8a 00 00 00 00 01 24 48 ea fe 00 00 00 06 00 00
Dec  1 14:47:42 esc1 kernel: end_request: I/O error, dev sdb, sector 4903725822
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Sense Key : Aborted Command [current] [descriptor]
Dec  1 14:47:42 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 14:47:42 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 14:47:42 esc1 kernel:        00 00 00 00 
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Add. Sense: No additional sense information
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] CDB: Write(16): 8a 00 00 00 00 01 24 48 eb 92 00 00 00 05 00 00
Dec  1 14:47:42 esc1 kernel: end_request: I/O error, dev sdb, sector 4903725970
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Sense Key : Aborted Command [current] [descriptor]
Dec  1 14:47:42 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 14:47:42 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 14:47:42 esc1 kernel:        00 00 00 00 
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Add. Sense: No additional sense information
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] CDB: Write(16): 8a 00 00 00 00 01 24 49 2e eb 00 00 00 04 00 00
Dec  1 14:47:42 esc1 kernel: end_request: I/O error, dev sdb, sector 4903743211
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Sense Key : Aborted Command [current] [descriptor]
Dec  1 14:47:42 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 14:47:42 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 14:47:42 esc1 kernel:        00 00 00 00 
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Add. Sense: No additional sense information
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] CDB: Write(16): 8a 00 00 00 00 01 24 49 2f 02 00 00 00 06 00 00
Dec  1 14:47:42 esc1 kernel: end_request: I/O error, dev sdb, sector 4903743234
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Sense Key : Aborted Command [current] [descriptor]
Dec  1 14:47:42 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 14:47:42 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 14:47:42 esc1 kernel:        00 00 00 00 
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Add. Sense: No additional sense information
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] CDB: Write(16): 8a 00 00 00 00 01 24 49 2f 27 00 00 00 17 00 00
Dec  1 14:47:42 esc1 kernel: end_request: I/O error, dev sdb, sector 4903743271
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Sense Key : Aborted Command [current] [descriptor]
Dec  1 14:47:42 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 14:47:42 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 14:47:42 esc1 kernel:        00 00 00 00 
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Add. Sense: No additional sense information
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] CDB: Write(16): 8a 00 00 00 00 01 24 49 2f 53 00 00 00 05 00 00
Dec  1 14:47:42 esc1 kernel: end_request: I/O error, dev sdb, sector 4903743315
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Sense Key : Aborted Command [current] [descriptor]
Dec  1 14:47:42 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 14:47:42 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 14:47:42 esc1 kernel:        00 00 00 00 
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Add. Sense: No additional sense information
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] CDB: Write(16): 8a 00 00 00 00 01 24 49 2f 5e 00 00 00 02 00 00
Dec  1 14:47:42 esc1 kernel: end_request: I/O error, dev sdb, sector 4903743326
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Sense Key : Aborted Command [current] [descriptor]
Dec  1 14:47:42 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 14:47:42 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 14:47:42 esc1 kernel:        00 00 00 00 
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] Add. Sense: No additional sense information
Dec  1 14:47:42 esc1 kernel: sd 6:0:0:0: [sdb] CDB: Write(16): 8a 00 00 00 00 01 24 49 2f 67 00 00 00 1e 00 00
Dec  1 14:47:42 esc1 kernel: end_request: I/O error, dev sdb, sector 4903743335
Dec  1 14:47:42 esc1 kernel: ata7: EH complete
Dec  1 22:21:27 esc1 kernel: ata8.00: exception Emask 0x0 SAct 0x3ff SErr 0x0 action 0x6 frozen
Dec  1 22:21:27 esc1 kernel: ata8.00: failed command: WRITE FPDMA QUEUED
Dec  1 22:21:27 esc1 kernel: ata8.00: cmd 61/05:00:5b:29:5b/00:00:26:01:00/40 tag 0 ncq 2560 out
Dec  1 22:21:27 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 22:21:27 esc1 kernel: ata8.00: status: { DRDY }
Dec  1 22:21:27 esc1 kernel: ata8.00: failed command: WRITE FPDMA QUEUED
Dec  1 22:21:27 esc1 kernel: ata8.00: cmd 61/05:08:90:29:5b/00:00:26:01:00/40 tag 1 ncq 2560 out
Dec  1 22:21:27 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 22:21:27 esc1 kernel: ata8.00: status: { DRDY }
Dec  1 22:21:27 esc1 kernel: ata8.00: failed command: WRITE FPDMA QUEUED
Dec  1 22:21:27 esc1 kernel: ata8.00: cmd 61/06:10:1b:29:5b/00:00:26:01:00/40 tag 2 ncq 3072 out
Dec  1 22:21:27 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 22:21:27 esc1 kernel: ata8.00: status: { DRDY }
Dec  1 22:21:27 esc1 kernel: ata8.00: failed command: WRITE FPDMA QUEUED
Dec  1 22:21:27 esc1 kernel: ata8.00: cmd 61/09:18:65:29:5b/00:00:26:01:00/40 tag 3 ncq 4608 out
Dec  1 22:21:27 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 22:21:27 esc1 kernel: ata8.00: status: { DRDY }
Dec  1 22:21:27 esc1 kernel: ata8.00: failed command: WRITE FPDMA QUEUED
Dec  1 22:21:27 esc1 kernel: ata8.00: cmd 61/04:20:79:29:5b/00:00:26:01:00/40 tag 4 ncq 2048 out
Dec  1 22:21:27 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 22:21:27 esc1 kernel: ata8.00: status: { DRDY }
Dec  1 22:21:27 esc1 kernel: ata8.00: failed command: WRITE FPDMA QUEUED
Dec  1 22:21:27 esc1 kernel: ata8.00: cmd 61/09:28:52:29:5b/00:00:26:01:00/40 tag 5 ncq 4608 out
Dec  1 22:21:27 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 22:21:27 esc1 kernel: ata8.00: status: { DRDY }
Dec  1 22:21:27 esc1 kernel: ata8.00: failed command: WRITE FPDMA QUEUED
Dec  1 22:21:27 esc1 kernel: ata8.00: cmd 61/05:30:60:29:5b/00:00:26:01:00/40 tag 6 ncq 2560 out
Dec  1 22:21:27 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 22:21:27 esc1 kernel: ata8.00: status: { DRDY }
Dec  1 22:21:27 esc1 kernel: ata8.00: failed command: WRITE FPDMA QUEUED
Dec  1 22:21:27 esc1 kernel: ata8.00: cmd 61/05:38:74:29:5b/00:00:26:01:00/40 tag 7 ncq 2560 out
Dec  1 22:21:27 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 22:21:27 esc1 kernel: ata8.00: status: { DRDY }
Dec  1 22:21:27 esc1 kernel: ata8.00: failed command: WRITE FPDMA QUEUED
Dec  1 22:21:27 esc1 kernel: ata8.00: cmd 61/0e:40:7d:29:5b/00:00:26:01:00/40 tag 8 ncq 7168 out
Dec  1 22:21:27 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 22:21:27 esc1 kernel: ata8.00: status: { DRDY }
Dec  1 22:21:27 esc1 kernel: ata8.00: failed command: WRITE FPDMA QUEUED
Dec  1 22:21:27 esc1 kernel: ata8.00: cmd 61/05:48:9a:29:5b/00:00:26:01:00/40 tag 9 ncq 2560 out
Dec  1 22:21:27 esc1 kernel:         res 40/00:00:00:00:00/00:00:00:00:00/00 Emask 0x4 (timeout)
Dec  1 22:21:27 esc1 kernel: ata8.00: status: { DRDY }
Dec  1 22:21:27 esc1 kernel: ata8: hard resetting link
Dec  1 22:21:27 esc1 kernel: ata8: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
Dec  1 22:21:27 esc1 kernel: ata8.00: configured for UDMA/133
Dec  1 22:21:27 esc1 kernel: ata8.00: device reported invalid CHS sector 0
Dec  1 22:21:27 esc1 kernel: ata8.00: device reported invalid CHS sector 0
Dec  1 22:21:27 esc1 kernel: ata8.00: device reported invalid CHS sector 0
Dec  1 22:21:27 esc1 kernel: ata8.00: device reported invalid CHS sector 0
Dec  1 22:21:27 esc1 kernel: ata8.00: device reported invalid CHS sector 0
Dec  1 22:21:27 esc1 kernel: ata8.00: device reported invalid CHS sector 0
Dec  1 22:21:27 esc1 kernel: ata8.00: device reported invalid CHS sector 0
Dec  1 22:21:27 esc1 kernel: ata8.00: device reported invalid CHS sector 0
Dec  1 22:21:27 esc1 kernel: ata8.00: device reported invalid CHS sector 0
Dec  1 22:21:27 esc1 kernel: ata8.00: device reported invalid CHS sector 0
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] Sense Key : Aborted Command [current] [descriptor]
Dec  1 22:21:27 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 22:21:27 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 22:21:27 esc1 kernel:        00 00 00 00 
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] Add. Sense: No additional sense information
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] CDB: Write(16): 8a 00 00 00 00 01 26 5b 29 5b 00 00 00 05 00 00
Dec  1 22:21:27 esc1 kernel: end_request: I/O error, dev sdc, sector 4938475867
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] Sense Key : Aborted Command [current] [descriptor]
Dec  1 22:21:27 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 22:21:27 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 22:21:27 esc1 kernel:        00 00 00 00 
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] Add. Sense: No additional sense information
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] CDB: Write(16): 8a 00 00 00 00 01 26 5b 29 90 00 00 00 05 00 00
Dec  1 22:21:27 esc1 kernel: end_request: I/O error, dev sdc, sector 4938475920
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] Sense Key : Aborted Command [current] [descriptor]
Dec  1 22:21:27 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 22:21:27 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 22:21:27 esc1 kernel:        00 00 00 00 
Dec  1 22:21:27 esc1 kernel: sd 7:0:0:0: [sdc] Add. Sense: No additional sense information
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] CDB: Write(16): 8a 00 00 00 00 01 26 5b 29 1b 00 00 00 06 00 00
Dec  1 22:21:28 esc1 kernel: end_request: I/O error, dev sdc, sector 4938475803
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Sense Key : Aborted Command [current] [descriptor]
Dec  1 22:21:28 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 22:21:28 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 22:21:28 esc1 kernel:        00 00 00 00 
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Add. Sense: No additional sense information
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] CDB: Write(16): 8a 00 00 00 00 01 26 5b 29 65 00 00 00 09 00 00
Dec  1 22:21:28 esc1 kernel: end_request: I/O error, dev sdc, sector 4938475877
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Sense Key : Aborted Command [current] [descriptor]
Dec  1 22:21:28 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 22:21:28 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 22:21:28 esc1 kernel:        00 00 00 00 
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Add. Sense: No additional sense information
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] CDB: Write(16): 8a 00 00 00 00 01 26 5b 29 79 00 00 00 04 00 00
Dec  1 22:21:28 esc1 kernel: end_request: I/O error, dev sdc, sector 4938475897
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Sense Key : Aborted Command [current] [descriptor]
Dec  1 22:21:28 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 22:21:28 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 22:21:28 esc1 kernel:        00 00 00 00 
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Add. Sense: No additional sense information
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] CDB: Write(16): 8a 00 00 00 00 01 26 5b 29 52 00 00 00 09 00 00
Dec  1 22:21:28 esc1 kernel: end_request: I/O error, dev sdc, sector 4938475858
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Sense Key : Aborted Command [current] [descriptor]
Dec  1 22:21:28 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 22:21:28 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 22:21:28 esc1 kernel:        00 00 00 00 
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Add. Sense: No additional sense information
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] CDB: Write(16): 8a 00 00 00 00 01 26 5b 29 60 00 00 00 05 00 00
Dec  1 22:21:28 esc1 kernel: end_request: I/O error, dev sdc, sector 4938475872
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Sense Key : Aborted Command [current] [descriptor]
Dec  1 22:21:28 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 22:21:28 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 22:21:28 esc1 kernel:        00 00 00 00 
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Add. Sense: No additional sense information
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] CDB: Write(16): 8a 00 00 00 00 01 26 5b 29 74 00 00 00 05 00 00
Dec  1 22:21:28 esc1 kernel: end_request: I/O error, dev sdc, sector 4938475892
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Sense Key : Aborted Command [current] [descriptor]
Dec  1 22:21:28 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 22:21:28 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 22:21:28 esc1 kernel:        00 00 00 00 
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Add. Sense: No additional sense information
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] CDB: Write(16): 8a 00 00 00 00 01 26 5b 29 7d 00 00 00 0e 00 00
Dec  1 22:21:28 esc1 kernel: end_request: I/O error, dev sdc, sector 4938475901
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Sense Key : Aborted Command [current] [descriptor]
Dec  1 22:21:28 esc1 kernel: Descriptor sense data with sense descriptors (in hex):
Dec  1 22:21:28 esc1 kernel:        72 0b 00 00 00 00 00 0c 00 0a 80 00 00 00 00 00 
Dec  1 22:21:28 esc1 kernel:        00 00 00 00 
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] Add. Sense: No additional sense information
Dec  1 22:21:28 esc1 kernel: sd 7:0:0:0: [sdc] CDB: Write(16): 8a 00 00 00 00 01 26 5b 29 9a 00 00 00 05 00 00
Dec  1 22:21:28 esc1 kernel: end_request: I/O error, dev sdc, sector 4938475930
Dec  1 22:21:28 esc1 kernel: ata8: EH complete
Dec  2 06:40:01 esc1 auditd[1841]: Audit daemon rotating log files
Dec  2 08:49:41 esc1 kernel: ata7: sata_mv: attempting PIO w/multiple DRQ: this may fail due to h/w errata
Dec  2 08:49:52 esc1 kernel: ata7: sata_mv: attempting PIO w/multiple DRQ: this may fail due to h/w errata
Dec  2 21:42:53 esc1 kernel: INFO: task rsync:4478 blocked for more than 120 seconds.
Dec  2 21:42:53 esc1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Dec  2 21:42:53 esc1 kernel: rsync         D 0000000000000002     0  4478   4473 0x00000080
Dec  2 21:42:53 esc1 kernel: ffff88016c14fb78 0000000000000082 0000000341f7f3c0 0000000000000001
Dec  2 21:42:53 esc1 kernel: ffff88021915cc00 0000000000000086 ffff88016c14faf8 ffff880222729da0
Dec  2 21:42:53 esc1 kernel: ffff88022978bab8 ffff88016c14ffd8 000000000000f598 ffff88022978bab8
Dec  2 21:42:53 esc1 kernel: Call Trace:
Dec  2 21:42:53 esc1 kernel: [<ffffffff8108e38e>] ? prepare_to_wait_exclusive+0x4e/0x80
Dec  2 21:42:53 esc1 kernel: [<ffffffffa02b9158>] cv_wait_common+0x78/0xe0 [spl]
Dec  2 21:42:53 esc1 kernel: [<ffffffff8108e180>] ? autoremove_wake_function+0x0/0x40
Dec  2 21:42:53 esc1 kernel: [<ffffffffa02b91f3>] __cv_wait+0x13/0x20 [spl]
Dec  2 21:42:53 esc1 kernel: [<ffffffffa03b2e6b>] txg_wait_open+0x7b/0xa0 [zfs]
Dec  2 21:42:53 esc1 kernel: [<ffffffffa03797cd>] dmu_tx_wait+0xed/0xf0 [zfs]
Dec  2 21:42:53 esc1 kernel: [<ffffffffa03e794e>] zfs_write+0x3be/0xca0 [zfs]
Dec  2 21:42:53 esc1 kernel: [<ffffffffa03f9262>] zpl_write_common+0x52/0x70 [zfs]
Dec  2 21:42:53 esc1 kernel: [<ffffffffa03f92e8>] zpl_write+0x68/0xa0 [zfs]
Dec  2 21:42:53 esc1 kernel: [<ffffffff81205346>] ? security_file_permission+0x16/0x20
Dec  2 21:42:53 esc1 kernel: [<ffffffff81172858>] vfs_write+0xb8/0x1a0
Dec  2 21:42:53 esc1 kernel: [<ffffffff810d1b52>] ? audit_syscall_entry+0x272/0x2a0
Dec  2 21:42:53 esc1 kernel: [<ffffffff81173291>] sys_write+0x51/0x90
Dec  2 21:42:53 esc1 kernel: [<ffffffff8100b172>] system_call_fastpath+0x16/0x1b
Dec  3 03:32:53 esc1 kernel: [<ffffffff8111fe5d>] __alloc_pages_nodemask+0x40d/0x8b0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81159a52>] kmem_getpages+0x62/0x170
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115a66a>] fallback_alloc+0x1ba/0x270
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115a0bf>] ? cache_grow+0x2cf/0x320
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115a3e9>] ____cache_alloc_node+0x99/0x160
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b4a13>] ? kmem_alloc_debug+0xb3/0x130 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115aff9>] __kmalloc+0x199/0x230
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b4a13>] kmem_alloc_debug+0xb3/0x130 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa036a1a0>] ddt_get_dedup_stats+0x40/0x80 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa036a25a>] ddt_get_dedup_dspace+0x2a/0x40 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03af410>] spa_update_dspace+0x30/0x50 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03a2232>] spa_sync+0x562/0x9a0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffff8108e196>] ? autoremove_wake_function+0x16/0x40
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03b34b5>] txg_sync_thread+0x225/0x3b0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03b3290>] ? txg_sync_thread+0x0/0x3b0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03b3290>] ? txg_sync_thread+0x0/0x3b0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b54e8>] thread_generic_wrapper+0x68/0x80 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b5480>] ? thread_generic_wrapper+0x0/0x80 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffff8108de16>] kthread+0x96/0xa0
Dec  3 03:32:53 esc1 kernel: [<ffffffff8100c1ca>] child_rip+0xa/0x20
Dec  3 03:32:53 esc1 kernel: [<ffffffff8108dd80>] ? kthread+0x0/0xa0
Dec  3 03:32:53 esc1 kernel: [<ffffffff8100c1c0>] ? child_rip+0x0/0x20
Dec  3 03:32:53 esc1 kernel: INFO: task crond:1994 blocked for more than 120 seconds.
Dec  3 03:32:53 esc1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Dec  3 03:32:53 esc1 kernel: crond         D 0000000000000001     0  1994      1 0x00000080
Dec  3 03:32:53 esc1 kernel: ffff88010e591830 0000000000000086 0000000000000000 0000000000000001
Dec  3 03:32:53 esc1 kernel: 0000000000000000 0000000000000082 ffff88010e5917d0 ffff880222729da0
Dec  3 03:32:53 esc1 kernel: ffff8801279a1b38 ffff88010e591fd8 000000000000f598 ffff8801279a1b38
Dec  3 03:32:53 esc1 kernel: Call Trace:
Dec  3 03:32:53 esc1 kernel: [<ffffffff8108e38e>] ? prepare_to_wait_exclusive+0x4e/0x80
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b9158>] cv_wait_common+0x78/0xe0 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffff8108e180>] ? autoremove_wake_function+0x0/0x40
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b91f3>] __cv_wait+0x13/0x20 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03b2e6b>] txg_wait_open+0x7b/0xa0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03797cd>] dmu_tx_wait+0xed/0xf0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa037988a>] dmu_tx_assign+0x6a/0x410 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03e981f>] zfs_inactive+0xef/0x1e0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03f9e4e>] zpl_clear_inode+0xe/0x10 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118cb7f>] clear_inode+0x8f/0x110
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118cc40>] dispose_list+0x40/0x120
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118cf94>] shrink_icache_memory+0x274/0x2e0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81125a3a>] shrink_slab+0x13a/0x1a0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81127dcb>] do_try_to_free_pages+0x2fb/0x520
Dec  3 03:32:53 esc1 kernel: [<ffffffff8111f856>] ? free_pcppages_bulk+0x2b6/0x390
Dec  3 03:32:53 esc1 kernel: [<ffffffff811281df>] try_to_free_pages+0x9f/0x130
Dec  3 03:32:53 esc1 kernel: [<ffffffff8112e586>] ? next_online_pgdat+0x26/0x50
Dec  3 03:32:53 esc1 kernel: [<ffffffff81129320>] ? isolate_pages_global+0x0/0x380
Dec  3 03:32:53 esc1 kernel: [<ffffffff8111fe5d>] __alloc_pages_nodemask+0x40d/0x8b0
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115473a>] alloc_pages_current+0xaa/0x110
Dec  3 03:32:53 esc1 kernel: [<ffffffff8111e36e>] __get_free_pages+0xe/0x50
Dec  3 03:32:53 esc1 kernel: [<ffffffff81064ed4>] copy_process+0xe4/0x1300
Dec  3 03:32:53 esc1 kernel: [<ffffffff81262cb5>] ? _atomic_dec_and_lock+0x55/0x80
Dec  3 03:32:53 esc1 kernel: [<ffffffff81066184>] do_fork+0x94/0x480
Dec  3 03:32:53 esc1 kernel: [<ffffffff81177f66>] ? sys_newstat+0x36/0x50
Dec  3 03:32:53 esc1 kernel: [<ffffffff810d1b52>] ? audit_syscall_entry+0x272/0x2a0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81009578>] sys_clone+0x28/0x30
Dec  3 03:32:53 esc1 kernel: [<ffffffff8100b493>] stub_clone+0x13/0x20
Dec  3 03:32:53 esc1 kernel: [<ffffffff8100b172>] ? system_call_fastpath+0x16/0x1b
Dec  3 03:32:53 esc1 kernel: INFO: task rsync:4478 blocked for more than 120 seconds.
Dec  3 03:32:53 esc1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Dec  3 03:32:53 esc1 kernel: rsync         D 0000000000000001     0  4478   4473 0x00000080
Dec  3 03:32:53 esc1 kernel: ffff88016c14fb78 0000000000000082 00000000a70a8ea0 0000000000000001
Dec  3 03:32:53 esc1 kernel: ffff88006a4b5a40 0000000000000086 ffff88016c14faf8 ffff880222729da0
Dec  3 03:32:53 esc1 kernel: ffff88022978bab8 ffff88016c14ffd8 000000000000f598 ffff88022978bab8
Dec  3 03:32:53 esc1 kernel: Call Trace:
Dec  3 03:32:53 esc1 kernel: [<ffffffff8108e38e>] ? prepare_to_wait_exclusive+0x4e/0x80
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b9158>] cv_wait_common+0x78/0xe0 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffff8108e180>] ? autoremove_wake_function+0x0/0x40
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b91f3>] __cv_wait+0x13/0x20 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03b2e6b>] txg_wait_open+0x7b/0xa0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03797cd>] dmu_tx_wait+0xed/0xf0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03e794e>] zfs_write+0x3be/0xca0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03f9262>] zpl_write_common+0x52/0x70 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03f92e8>] zpl_write+0x68/0xa0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffff81205346>] ? security_file_permission+0x16/0x20
Dec  3 03:32:53 esc1 kernel: [<ffffffff81172858>] vfs_write+0xb8/0x1a0
Dec  3 03:32:53 esc1 kernel: [<ffffffff810d1b52>] ? audit_syscall_entry+0x272/0x2a0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81173291>] sys_write+0x51/0x90
Dec  3 03:32:53 esc1 kernel: [<ffffffff8100b172>] system_call_fastpath+0x16/0x1b
Dec  3 03:32:53 esc1 kernel: INFO: task rsync:11194 blocked for more than 120 seconds.
Dec  3 03:32:53 esc1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Dec  3 03:32:53 esc1 kernel: rsync         D 0000000000000003     0 11194  11193 0x00000080
Dec  3 03:32:53 esc1 kernel: ffff8801bf899588 0000000000000082 0000000000000000 0000000000000001
Dec  3 03:32:53 esc1 kernel: 0000000000000000 0000000000000082 ffff8801bf899528 ffff880222729da0
Dec  3 03:32:53 esc1 kernel: ffff8802297d30b8 ffff8801bf899fd8 000000000000f598 ffff8802297d30b8
Dec  3 03:32:53 esc1 kernel: Call Trace:
Dec  3 03:32:53 esc1 kernel: [<ffffffff8108e38e>] ? prepare_to_wait_exclusive+0x4e/0x80
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b9158>] cv_wait_common+0x78/0xe0 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffff8108e180>] ? autoremove_wake_function+0x0/0x40
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b91f3>] __cv_wait+0x13/0x20 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03b2e6b>] txg_wait_open+0x7b/0xa0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03797cd>] dmu_tx_wait+0xed/0xf0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa037988a>] dmu_tx_assign+0x6a/0x410 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03e981f>] zfs_inactive+0xef/0x1e0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03f9e4e>] zpl_clear_inode+0xe/0x10 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118cb7f>] clear_inode+0x8f/0x110
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118cc40>] dispose_list+0x40/0x120
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118cf94>] shrink_icache_memory+0x274/0x2e0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81125a3a>] shrink_slab+0x13a/0x1a0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81127dcb>] do_try_to_free_pages+0x2fb/0x520
Dec  3 03:32:53 esc1 kernel: [<ffffffff811281df>] try_to_free_pages+0x9f/0x130
Dec  3 03:32:53 esc1 kernel: [<ffffffff81129320>] ? isolate_pages_global+0x0/0x380
Dec  3 03:32:53 esc1 kernel: [<ffffffff8111fe5d>] __alloc_pages_nodemask+0x40d/0x8b0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81159a52>] kmem_getpages+0x62/0x170
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115a66a>] fallback_alloc+0x1ba/0x270
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115a0bf>] ? cache_grow+0x2cf/0x320
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115a3e9>] ____cache_alloc_node+0x99/0x160
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115b1ab>] kmem_cache_alloc+0x11b/0x190
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118ac37>] d_alloc+0x27/0x1b0
Dec  3 03:32:53 esc1 kernel: [<ffffffff811807bc>] do_lookup+0x16c/0x220
Dec  3 03:32:53 esc1 kernel: [<ffffffff81180dd9>] __link_path_walk+0x569/0x820
Dec  3 03:32:53 esc1 kernel: [<ffffffff8120e314>] ? inode_has_perm+0x54/0xa0
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118175a>] path_walk+0x6a/0xe0
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118192b>] do_path_lookup+0x5b/0xa0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81181ab7>] user_path_at+0x57/0xa0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81262cb5>] ? _atomic_dec_and_lock+0x55/0x80
Dec  3 03:32:53 esc1 kernel: [<ffffffff81177b94>] ? cp_new_stat+0xe4/0x100
Dec  3 03:32:53 esc1 kernel: [<ffffffff81177dbc>] vfs_fstatat+0x3c/0x80
Dec  3 03:32:53 esc1 kernel: [<ffffffff81177e6e>] vfs_lstat+0x1e/0x20
Dec  3 03:32:53 esc1 kernel: [<ffffffff81177e94>] sys_newlstat+0x24/0x50
Dec  3 03:32:53 esc1 kernel: [<ffffffff810d1b52>] ? audit_syscall_entry+0x272/0x2a0
Dec  3 03:32:53 esc1 kernel: [<ffffffff8100b172>] system_call_fastpath+0x16/0x1b
Dec  3 03:32:53 esc1 kernel: INFO: task rsync:11432 blocked for more than 120 seconds.
Dec  3 03:32:53 esc1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Dec  3 03:32:53 esc1 kernel: rsync         D 0000000000000002     0 11432  11431 0x00000080
Dec  3 03:32:53 esc1 kernel: ffff88002cb79588 0000000000000082 0000000000000000 ffff88002cb7954c
Dec  3 03:32:53 esc1 kernel: 0000000000000000 ffff88012fc23480 ffff880028315f80 000000010e03b5dd
Dec  3 03:32:53 esc1 kernel: ffff8801290e1078 ffff88002cb79fd8 000000000000f598 ffff8801290e1078
Dec  3 03:32:53 esc1 kernel: Call Trace:
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b9158>] cv_wait_common+0x78/0xe0 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffff8108e180>] ? autoremove_wake_function+0x0/0x40
Dec  3 03:32:53 esc1 kernel: [<ffffffffa02b91f3>] __cv_wait+0x13/0x20 [spl]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03b2e6b>] txg_wait_open+0x7b/0xa0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03797cd>] dmu_tx_wait+0xed/0xf0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa037988a>] dmu_tx_assign+0x6a/0x410 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03e981f>] zfs_inactive+0xef/0x1e0 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffffa03f9e4e>] zpl_clear_inode+0xe/0x10 [zfs]
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118cb7f>] clear_inode+0x8f/0x110
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118cc40>] dispose_list+0x40/0x120
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118cf94>] shrink_icache_memory+0x274/0x2e0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81125a3a>] shrink_slab+0x13a/0x1a0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81127dcb>] do_try_to_free_pages+0x2fb/0x520
Dec  3 03:32:53 esc1 kernel: [<ffffffff811281df>] try_to_free_pages+0x9f/0x130
Dec  3 03:32:53 esc1 kernel: [<ffffffff81129320>] ? isolate_pages_global+0x0/0x380
Dec  3 03:32:53 esc1 kernel: [<ffffffff8111fe5d>] __alloc_pages_nodemask+0x40d/0x8b0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81159a52>] kmem_getpages+0x62/0x170
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115a66a>] fallback_alloc+0x1ba/0x270
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115a0bf>] ? cache_grow+0x2cf/0x320
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115a3e9>] ____cache_alloc_node+0x99/0x160
Dec  3 03:32:53 esc1 kernel: [<ffffffff8115b1ab>] kmem_cache_alloc+0x11b/0x190
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118ac37>] d_alloc+0x27/0x1b0
Dec  3 03:32:53 esc1 kernel: [<ffffffff811807bc>] do_lookup+0x16c/0x220
Dec  3 03:32:53 esc1 kernel: [<ffffffff81180dd9>] __link_path_walk+0x569/0x820
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118175a>] path_walk+0x6a/0xe0
Dec  3 03:32:53 esc1 kernel: [<ffffffff8118192b>] do_path_lookup+0x5b/0xa0
Dec  3 03:32:53 esc1 kernel: [<ffffffff81181ab7>] user_path_at+0x57/0xa0
Dec  3 03:32:53 esc1 kernel: [<ffffffff8106dd17>] ? current_fs_time+0x27/0x30
Dec  3 03:32:53 esc1 kernel: [<ffffffff81177b94>] ? cp_new_stat+0xe4/0x100
Dec  3 03:32:53 esc1 kernel: [<ffffffff81177dbc>] vfs_fstatat+0x3c/0x80
Dec  3 03:32:53 esc1 kernel: [<ffffffff81177e6e>] vfs_lstat+0x1e/0x20
Dec  3 03:32:53 esc1 kernel: [<ffffffff81177e94>] sys_newlstat+0x24/0x50
Dec  3 03:32:53 esc1 kernel: [<ffffffff810d1b52>] ? audit_syscall_entry+0x272/0x2a0
Dec  3 03:32:53 esc1 kernel: [<ffffffff8100b172>] system_call_fastpath+0x16/0x1b
Dec  3 03:34:53 esc1 kernel: INFO: task kswapd0:58 blocked for more than 120 seconds.
Dec  3 03:34:53 esc1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Dec  3 03:34:53 esc1 kernel: kswapd0       D 0000000000000001     0    58      2 0x00000000
Dec  3 03:34:53 esc1 kernel: ffff8801297dd9e0 0000000000000046 0000000100000000 0000000000000001
Dec  3 03:34:53 esc1 kernel: 0000000000000000 0000000000000086 ffff8801297dd980 ffff88010e5917d0
Dec  3 03:34:53 esc1 kernel: ffff8801297d70f8 ffff8801297ddfd8 000000000000f598 ffff8801297d70f8
Dec  3 03:34:53 esc1 kernel: Call Trace:
Dec  3 03:34:53 esc1 kernel: [<ffffffff8108e38e>] ? prepare_to_wait_exclusive+0x4e/0x80
Dec  3 03:34:53 esc1 kernel: [<ffffffffa02b9158>] cv_wait_common+0x78/0xe0 [spl]
Dec  3 03:34:53 esc1 kernel: [<ffffffff8108e180>] ? autoremove_wake_function+0x0/0x40
Dec  3 03:34:53 esc1 kernel: [<ffffffffa02b91f3>] __cv_wait+0x13/0x20 [spl]
Dec  3 03:34:53 esc1 kernel: [<ffffffffa03b2e6b>] txg_wait_open+0x7b/0xa0 [zfs]
Dec  3 03:34:53 esc1 kernel: [<ffffffffa03797cd>] dmu_tx_wait+0xed/0xf0 [zfs]
Dec  3 03:34:53 esc1 kernel: [<ffffffffa037988a>] dmu_tx_assign+0x6a/0x410 [zfs]
Dec  3 03:34:53 esc1 kernel: [<ffffffffa03e981f>] zfs_inactive+0xef/0x1e0 [zfs]
Dec  3 03:34:53 esc1 kernel: [<ffffffffa03f9e4e>] zpl_clear_inode+0xe/0x10 [zfs]
Dec  3 03:34:53 esc1 kernel: [<ffffffff8118cb7f>] clear_inode+0x8f/0x110
Dec  3 03:34:53 esc1 kernel: [<ffffffff8118cc40>] dispose_list+0x40/0x120
Dec  3 03:34:53 esc1 kernel: [<ffffffff8118cf94>] shrink_icache_memory+0x274/0x2e0
Dec  3 03:34:53 esc1 kernel: [<ffffffff81125a3a>] shrink_slab+0x13a/0x1a0
Dec  3 03:34:53 esc1 kernel: [<ffffffff81128d2d>] balance_pgdat+0x58d/0x7f0
Dec  3 03:34:53 esc1 kernel: [<ffffffff81129320>] ? isolate_pages_global+0x0/0x380
Dec  3 03:34:53 esc1 kernel: [<ffffffff811290c4>] kswapd+0x134/0x390
Dec  3 03:34:53 esc1 kernel: [<ffffffff8108e180>] ? autoremove_wake_function+0x0/0x40
Dec  3 03:34:53 esc1 kernel: [<ffffffff81128f90>] ? kswapd+0x0/0x390
Dec  3 03:34:53 esc1 kernel: [<ffffffff8108de16>] kthread+0x96/0xa0
Dec  3 03:34:53 esc1 kernel: [<ffffffff8100c1ca>] child_rip+0xa/0x20
Dec  3 03:34:53 esc1 kernel: [<ffffffff8108dd80>] ? kthread+0x0/0xa0
Dec  3 03:34:53 esc1 kernel: [<ffffffff8100c1c0>] ? child_rip+0x0/0x20
Dec  3 03:34:53 esc1 kernel: INFO: task kswapd1:59 blocked for more than 120 seconds.
Dec  3 03:34:53 esc1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Dec  3 03:34:53 esc1 kernel: kswapd1       D 0000000000000002     0    59      2 0x00000000
Dec  3 03:34:53 esc1 kernel: ffff8801297df9e0 0000000000000046 0000000000000000 0000000000000001
Dec  3 03:34:53 esc1 kernel: 0000000000000000 0000000000000086 ffff8801297df980 ffff880222729da0
Dec  3 03:34:53 esc1 kernel: ffff8801297d66b8 ffff8801297dffd8 000000000000f598 ffff8801297d66b8
Dec  3 03:34:53 esc1 kernel: Call Trace:
Dec  3 03:34:53 esc1 kernel: [<ffffffff8108e38e>] ? prepare_to_wait_exclusive+0x4e/0x80
Dec  3 03:34:53 esc1 kernel: [<ffffffffa02b9158>] cv_wait_common+0x78/0xe0 [spl]
Dec  3 03:34:53 esc1 kernel: [<ffffffff8108e180>] ? autoremove_wake_function+0x0/0x40
Dec  3 03:34:53 esc1 kernel: [<ffffffffa02b91f3>] __cv_wait+0x13/0x20 [spl]
Dec  3 03:34:53 esc1 kernel: [<ffffffffa03b2e6b>] txg_wait_open+0x7b/0xa0 [zfs]
Dec  3 03:34:53 esc1 kernel: [<ffffffffa03797cd>] dmu_tx_wait+0xed/0xf0 [zfs]
Dec  3 03:34:53 esc1 kernel: [<ffffffffa037988a>] dmu_tx_assign+0x6a/0x410 [zfs]
Dec  3 03:34:53 esc1 kernel: [<ffffffffa03e981f>] zfs_inactive+0xef/0x1e0 [zfs]
Dec  3 03:34:53 esc1 kernel: [<ffffffffa03f9e4e>] zpl_clear_inode+0xe/0x10 [zfs]
Dec  3 03:34:53 esc1 kernel: [<ffffffff8118cb7f>] clear_inode+0x8f/0x110
Dec  3 03:34:53 esc1 kernel: [<ffffffff8118cc40>] dispose_list+0x40/0x120
Dec  3 03:34:53 esc1 kernel: [<ffffffff8118cf94>] shrink_icache_memory+0x274/0x2e0
Dec  3 03:34:53 esc1 kernel: [<ffffffff81125a3a>] shrink_slab+0x13a/0x1a0
Dec  3 03:34:53 esc1 kernel: [<ffffffff81128d2d>] balance_pgdat+0x58d/0x7f0
Dec  3 03:34:53 esc1 kernel: [<ffffffff81129320>] ? isolate_pages_global+0x0/0x380
Dec  3 03:34:53 esc1 kernel: [<ffffffff811290c4>] kswapd+0x134/0x390
Dec  3 03:34:53 esc1 kernel: [<ffffffff8108e180>] ? autoremove_wake_function+0x0/0x40
Dec  3 03:34:53 esc1 kernel: [<ffffffff81128f90>] ? kswapd+0x0/0x390
Dec  3 03:34:53 esc1 kernel: [<ffffffff8108de16>] kthread+0x96/0xa0
Dec  3 03:34:53 esc1 kernel: [<ffffffff8100c1ca>] child_rip+0xa/0x20
Dec  3 03:34:53 esc1 kernel: [<ffffffff8108dd80>] ? kthread+0x0/0xa0
Dec  3 03:34:53 esc1 kernel: [<ffffffff8100c1c0>] ? child_rip+0x0/0x20

[root at esc1 ~]# dmesg | grep ata
Command line: ro root=UUID=fa793615-6e9e-4749-9a79-aaa89745d0cf rd_NO_LUKS rd_NO_LVM rd_NO_MD rd_NO_DM LANG=en_US.UTF-8 SYSFONT=latarcy  eb-sun16 KEYBOARDTYPE=pc KEYTABLE=sv-latin1 crashkernel=auto
  BIOS-e820: 00000000cff00000 - 00000000cff07000 (ACPI data)
   #0 [0000000000 - 0000001000]   BIOS data page ==> [0000000000 - 0000001000]
   #0 [0000000000 - 0000001000]   BIOS data page
Kernel command line: ro root=UUID=fa793615-6e9e-4749-9a79-aaa89745d0cf rd_NO_LUKS rd_NO_LVM rd_NO_MD rd_NO_DM LANG=en_US.UTF-8 SYSFONT=  tarcyrheb-sun16 KEYBOARDTYPE=pc KEYTABLE=sv-latin1 crashkernel=129M at 0M
Memory: 8042828k/9175040k available (5013k kernel code, 787852k absent, 344360k reserved, 7291k data, 1232k init)
libata version 3.00 loaded.
ksign: Installing public key data
Write protecting the kernel read-only data: 10240k
scsi0 : pata_serverworks
scsi1 : pata_serverworks
ata1: PATA max UDMA/66 cmd 0x1f0 ctl 0x3f6 bmdma 0x2000 irq 14
ata2: PATA max UDMA/66 cmd 0x170 ctl 0x376 bmdma 0x2008 irq 15
sata_svw 0000:01:0e.0: version 2.3
sata_svw 0000:01:0e.0: PCI INT A -> Link[LNKS] -> GSI 11 (level, low) -> IRQ 11
scsi2 : sata_svw
scsi3 : sata_svw
scsi4 : sata_svw
scsi5 : sata_svw
ata3: SATA max UDMA/133 mmio m8192 at 0xd8300000 port 0xd8300000 irq 11
ata4: SATA max UDMA/133 mmio m8192 at 0xd8300000 port 0xd8300100 irq 11
ata5: SATA max UDMA/133 mmio m8192 at 0xd8300000 port 0xd8300200 irq 11
ata6: SATA max UDMA/133 mmio m8192 at 0xd8300000 port 0xd8300300 irq 11
ata3: SATA link up 1.5 Gbps (SStatus 113 SControl 300)
ata3.00: ATA-8: INTEL SSDSA2CW120G3, 4PC10302, max UDMA/133
ata3.00: 234441648 sectors, multi 16: LBA48 NCQ (depth 0/32)
ata3.00: configured for UDMA/133
ata4: SATA link down (SStatus 4 SControl 300)
ata5: SATA link down (SStatus 4 SControl 300)
ata6: SATA link down (SStatus 4 SControl 300)
sata_mv 0000:03:00.0: version 1.28
sata_mv 0000:03:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19
sata_mv 0000:03:00.0: Gen-IIE 32 slots 4 ports SCSI mode IRQ via INTx
sata_mv 0000:03:00.0: setting latency timer to 64
scsi6 : sata_mv
scsi7 : sata_mv
scsi8 : sata_mv
scsi9 : sata_mv
ata7: SATA max UDMA/133 mmio m1048576 at 0xd8200000 port 0xd8222000 irq 19
ata8: SATA max UDMA/133 mmio m1048576 at 0xd8200000 port 0xd8224000 irq 19
ata9: SATA max UDMA/133 mmio m1048576 at 0xd8200000 port 0xd8226000 irq 19
ata10: SATA max UDMA/133 mmio m1048576 at 0xd8200000 port 0xd8228000 irq 19
sata_mv 0000:06:00.0: PCI INT A -> GSI 21 (level, low) -> IRQ 21
sata_mv 0000:06:00.0: Gen-IIE 32 slots 4 ports SCSI mode IRQ via INTx
sata_mv 0000:06:00.0: setting latency timer to 64
scsi10 : sata_mv
scsi11 : sata_mv
scsi12 : sata_mv
scsi13 : sata_mv
ata11: SATA max UDMA/133 mmio m1048576 at 0xd8500000 port 0xd8522000 irq 21
ata12: SATA max UDMA/133 mmio m1048576 at 0xd8500000 port 0xd8524000 irq 21
ata13: SATA max UDMA/133 mmio m1048576 at 0xd8500000 port 0xd8526000 irq 21
ata14: SATA max UDMA/133 mmio m1048576 at 0xd8500000 port 0xd8528000 irq 21
ata11: SATA link down (SStatus 0 SControl 300)
ata7: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
ata7.00: ATA-8: WDC WD30EZRS-00J99B0, 80.00A80, max UDMA/133
ata7.00: 5860533168 sectors, multi 0: LBA48 NCQ (depth 31/32)
ata7.00: configured for UDMA/133
ata8: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
ata8.00: ATA-8: WDC WD30EZRS-00J99B0, 80.00A80, max UDMA/133
ata8.00: 5860533168 sectors, multi 0: LBA48 NCQ (depth 31/32)
ata8.00: configured for UDMA/133
ata9: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
ata9.00: ATA-8: Hitachi HDS723030ALA640, MKAOA3B0, max UDMA/133
ata9.00: 5860533168 sectors, multi 0: LBA48 NCQ (depth 31/32)
ata9.00: configured for UDMA/133
ata10: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
ata10.00: ATA-8: Hitachi HDS723030ALA640, MKAOA3B0, max UDMA/133
ata10.00: 5860533168 sectors, multi 0: LBA48 NCQ (depth 31/32)
ata10.00: configured for UDMA/133
ata12: SATA link down (SStatus 0 SControl 300)
ata13: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
ata13.00: ATA-8: Hitachi HDS723030ALA640, MKAOA3B0, max UDMA/133
ata13.00: 5860533168 sectors, multi 0: LBA48 NCQ (depth 31/32)
ata13.00: configured for UDMA/133
ata14: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
ata14.00: ATA-8: Hitachi HDS723030ALA640, MKAOA3B0, max UDMA/133
ata14.00: 5860533168 sectors, multi 0: LBA48 NCQ (depth 31/32)
ata14.00: configured for UDMA/133
EXT4-fs (sda1): mounted filesystem with ordered data mode
EXT4-fs (sdf1): mounted filesystem with ordered data mode
EXT4-fs (sdg1): mounted filesystem with ordered data mode
[root at esc1 ~]#



More information about the zfs-discuss mailing list