bugGluster - Bugs: bug #25169, client crashes when afr self-heals...

 
 

bug #25169: client crashes when afr self-heals a directory.

Submitted by:  swankier <swankier>
Submitted on:  Wed 24 Dec 2008 08:34:06 PM UTC  
 
Category: GlusterFSSeverity: 3 - Normal
Priority: 5 - NormalItem Group: Crash
Status: FixedPrivacy: Public
Assigned to: Amar Tumballi <amarts>Originator Name: Amar Tumballi
Open/Closed: ClosedRelease: tla
Operating System: GNU/LinuxReproducibility: Every Time

Add a New Comment(Rich Markup)
   

You are not logged in

Please log in, so followups can be emailed to you.

 

Thu 25 Dec 2008 03:07:57 AM UTC, comment #4:

ls will hang on directory read.

=========

web1:/www/photo_rolls_ite/mysql# strace ls -lah
execve("/bin/ls", ["ls", "-lah"], [/* 17 vars */]) = 0
uname({sys="Linux", node="web1", ...}) = 0
brk(0) = 0x515000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2ab0aaaeb000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=12956, ...}) = 0
mmap(NULL, 12956, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2ab0aaaed000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/librt.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`(\0\0\0"..., 640) = 640
fstat(3, {st_mode=S_IFREG|0644, st_size=35752, ...}) = 0
mmap(NULL, 1084424, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2ab0aabec000
mprotect(0x2ab0aabf3000, 1055752, PROT_NONE) = 0
mmap(0x2ab0aacf3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x2ab0aacf3000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libacl.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\32\0"..., 640) = 640
fstat(3, {st_mode=S_IFREG|0644, st_size=26088, ...}) = 0
mmap(NULL, 1072824, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2ab0aacf5000
mprotect(0x2ab0aacfb000, 1048248, PROT_NONE) = 0
mmap(0x2ab0aadfa000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x2ab0aadfa000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libselinux.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260H\0\0"..., 640) = 640
fstat(3, {st_mode=S_IFREG|0644, st_size=88616, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2ab0aadfb000
mmap(NULL, 1140256, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2ab0aadfc000
mprotect(0x2ab0aae11000, 1054240, PROT_NONE) = 0
mmap(0x2ab0aaf10000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x2ab0aaf10000
mmap(0x2ab0aaf12000, 1568, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2ab0aaf12000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\305"..., 640) = 640
lseek(3, 624, SEEK_SET) = 624
read(3, "\4\0\0\0\20\0\0\0\1\0\0\0GNU\0\0\0\0\0\2\0\0\0\6\0\0\0"..., 32) = 32
fstat(3, {st_mode=S_IFREG|0755, st_size=1286104, ...}) = 0
mmap(NULL, 2344904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2ab0aaf13000
mprotect(0x2ab0ab034000, 1161160, PROT_NONE) = 0
mmap(0x2ab0ab134000, 98304, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x121000) = 0x2ab0ab134000
mmap(0x2ab0ab14c000, 14280, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2ab0ab14c000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200U\0\0"..., 640) = 640
fstat(3, {st_mode=S_IFREG|0644, st_size=96493, ...}) = 0
mmap(NULL, 1131368, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2ab0ab150000
mprotect(0x2ab0ab160000, 1065832, PROT_NONE) = 0
mmap(0x2ab0ab25f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf000) = 0x2ab0ab25f000
mmap(0x2ab0ab261000, 13160, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2ab0ab261000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libattr.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\20\0"..., 640) = 640
fstat(3, {st_mode=S_IFREG|0644, st_size=15568, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2ab0ab265000
mmap(NULL, 1062352, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2ab0ab266000
mprotect(0x2ab0ab26a000, 1045968, PROT_NONE) = 0
mmap(0x2ab0ab369000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x2ab0ab369000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libdl.so.2", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\20\0\0"..., 640) = 640
fstat(3, {st_mode=S_IFREG|0644, st_size=10392, ...}) = 0
mmap(NULL, 1057000, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2ab0ab36a000
mprotect(0x2ab0ab36c000, 1048808, PROT_NONE) = 0
mmap(0x2ab0ab46b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x2ab0ab46b000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libsepol.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`>\0\0\0"..., 640) = 640
fstat(3, {st_mode=S_IFREG|0644, st_size=241032, ...}) = 0
mmap(NULL, 1330464, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2ab0ab46d000
mprotect(0x2ab0ab4a7000, 1092896, PROT_NONE) = 0
mmap(0x2ab0ab5a7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3a000) = 0x2ab0ab5a7000
mmap(0x2ab0ab5a8000, 40224, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2ab0ab5a8000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2ab0ab5b2000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2ab0ab5b3000
mprotect(0x2ab0ab134000, 86016, PROT_READ) = 0
arch_prctl(ARCH_SET_FS, 0x2ab0ab5b3250) = 0
munmap(0x2ab0aaaed000, 12956) = 0
set_tid_address(0x2ab0ab5b32e0) = 13727
rt_sigaction(SIGRTMIN, {0x2ab0ab155230, [], SA_RESTORER|SA_SIGINFO, 0x2ab0ab15c410}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x2ab0ab1551d0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x2ab0ab15c410}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
access("/etc/selinux/", F_OK) = -1 ENOENT (No such file or directory)
brk(0) = 0x515000
brk(0x536000) = 0x536000
open("/proc/mounts", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2ab0aaaed000
read(3, "rootfs / rootfs rw 0 0\nnone /sys"..., 1024) = 593
read(3, "", 1024) = 0
close(3) = 0
munmap(0x2ab0aaaed000, 4096) = 0
open("/usr/lib/locale/locale-archive", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1209120, ...}) = 0
mmap(NULL, 1209120, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2ab0ab5b4000
close(3) = 0
ioctl(1, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=52, ws_col=203, ws_xpixel=1218, ws_ypixel=728}) = 0
open(".", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=2048, ...}) = 0
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
getdents64(3,

swankier <swankier>
Wed 24 Dec 2008 10:18:10 PM UTC, comment #3:

Thanks for the report.. It should be fixed in 'glusterfs--mainline--3.0--patch-814', should be available in next RC release.

Amar Tumballi <amarts>
Project MemberIn charge of this item.
Wed 24 Dec 2008 09:01:54 PM UTC, comment #2:

Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 47316265798368 (LWP 11621)]
0x00002b08ad4953c3 in unify_chown (frame=0x52b5d0, this=<value optimized out>, loc=0x5231f0, uid=65534, gid=65533) at unify.c:1825
1825 STACK_WIND (frame,
(gdb)

swankier <swankier>
Wed 24 Dec 2008 08:58:11 PM UTC, comment #1:

I changed line unify.c:2762 to:

if (trav && trav->key && ZR_FILE_CONTENT_REQUEST(trav->key)) {

as per bulde's recommendation.

The crash is now:

===========

2008-12-24 15:54:45 W [afr-self-heal-common.c:843:sh_missing_entries_lookup_cbk] array: path /shared_images/fredmeyer on subvolume nas1 => -1 (No such file or directory)
2008-12-24 15:54:45 W [afr-self-heal-entry.c:1129:afr_sh_entry_impunge_mkdir] array: creating directory /shared_images/fredmeyer/logs mode=040777 on nas1
2008-12-24 15:54:45 W [afr-self-heal-entry.c:1129:afr_sh_entry_impunge_mkdir] array: creating directory /shared_images/fredmeyer/photo_roll mode=040775 on nas1
2008-12-24 15:54:45 W [afr-self-heal-entry.c:1129:afr_sh_entry_impunge_mkdir] array: creating directory /shared_images/fredmeyer/order mode=040775 on nas1
pending frames:
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)
frame : type(1) op(LOOKUP)

Signal received: 11
configuration details:argp 1
backtrace 1
dlfcn 1
fdatasync 1
libpthread 1
llistxattr 1
setfsid 1
spinlock 1
epoll.h 1
xattr.h 1
tv_nsec 1
package-string: glusterfs 1.4.0tla
/lib/libc.so.6[0x2b3836254110]
/usr/local/lib/glusterfs/1.4.0tla/xlator/cluster/unify.so(unify_chown+0x2b3)[0x2b38365873c3]
/usr/local/lib/glusterfs/1.4.0tla/xlator/cluster/afr.so(afr_sh_entry_impunge_newfile_cbk+0x101)[0x2b38366b9bd1]
/usr/local/lib/glusterfs/1.4.0tla/xlator/cluster/unify.so(unify_mkdir_cbk+0xc7)[0x2b383658caf7]
/usr/local/lib/glusterfs/1.4.0tla/xlator/protocol/client.so(client_mkdir_cbk+0x184)[0x2b3836478c84]
/usr/local/lib/glusterfs/1.4.0tla/xlator/protocol/client.so(protocol_client_pollin+0xc1)[0x2b383646a221]
/usr/local/lib/glusterfs/1.4.0tla/xlator/protocol/client.so(notify+0x13a)[0x2b38364706fa]
/usr/local/lib/glusterfs/1.4.0tla/transport/socket.so[0x2aaaaaaae10c]
/usr/local/lib/libglusterfs.so.0[0x2b3835efe46b]
/usr/local/sbin/glusterfs(main+0x8f9)[0x403579]
/lib/libc.so.6(__libc_start_main+0xda)[0x2b38362414ca]
/usr/local/sbin/glusterfs[0x40231a]
---------

swankier <swankier>
Wed 24 Dec 2008 08:34:06 PM UTC, original submission:

stepping with gdb, the crash occurs on unify.c:2762

===========

008-12-24 15:07:43 W [afr.c:2005:init] array: You have specified subvolume 'nas2' as the 'favorite child'. This means that if a discrepancy in the content or attributes (ownership, permission, etc.) of a file is detected among the subvolumes, the file on 'nas2' will be considered the definitive version and its contents will OVERWRITE the contents of the file on other subvolumes. All versions of the file except that on 'nas2' WILL BE LOST.
2008-12-24 15:07:53 W [afr-self-heal-common.c:843:sh_missing_entries_lookup_cbk] array: path /shared_images/fps on subvolume nas1 => -1 (No such file or directory)
pending frames:
frame : type(1) op(LOOKUP)

Signal received: 11
configuration details:argp 1
backtrace 1
dlfcn 1
fdatasync 1
libpthread 1
llistxattr 1
setfsid 1
spinlock 1
epoll.h 1
xattr.h 1
tv_nsec 1
package-string: glusterfs 1.4.0tla
/lib/libc.so.6[0x2ab315def110]
/usr/local/lib/glusterfs/1.4.0tla/xlator/cluster/unify.so(unify_setxattr+0x28d)[0x2ab316126a5d]
/usr/local/lib/glusterfs/1.4.0tla/xlator/cluster/afr.so(afr_sh_metadata_sync+0x384)[0x2ab31624f874]
/usr/local/lib/glusterfs/1.4.0tla/xlator/cluster/afr.so(afr_sh_metadata_getxattr_cbk+0x80)[0x2ab3162503b0]
/usr/local/lib/glusterfs/1.4.0tla/xlator/protocol/client.so(client_getxattr_cbk+0xe4)[0x2ab31600a964]
/usr/local/lib/glusterfs/1.4.0tla/xlator/protocol/client.so(protocol_client_pollin+0xc1)[0x2ab316005221]
/usr/local/lib/glusterfs/1.4.0tla/xlator/protocol/client.so(notify+0x13a)[0x2ab31600b6fa]
/usr/local/lib/glusterfs/1.4.0tla/transport/socket.so[0x2aaaaaaae10c]
/usr/local/lib/libglusterfs.so.0[0x2ab315a9946b]
/usr/local/sbin/glusterfs(main+0x8f9)[0x403579]
/lib/libc.so.6(__libc_start_main+0xda)[0x2ab315ddc4ca]
/usr/local/sbin/glusterfs[0x40231a]
---------

===========

web1:/www# cat /usr/local/etc/glusterfs/glusterfs-client.vol
volume nas1brick1
type protocol/client
option transport-type tcp/client
option remote-host 10.5.1.180
option remote-subvolume nas1brick1p
end-volume

volume nas1brick2
type protocol/client
option transport-type tcp/client
option remote-host 10.5.1.180
option remote-subvolume nas1brick2p
end-volume

volume nas1-ns
type protocol/client
option transport-type tcp/client
option remote-host 10.5.1.180
option remote-subvolume nas1-nsp
end-volume

volume nas1
type cluster/unify
option namespace nas1-ns
option scheduler rr
subvolumes nas1brick1 nas1brick2
end-volume

volume nas2
type protocol/client
option transport-type tcp/client
option remote-host 10.10.1.185
option remote-subvolume nas2
end-volume

volume array
type cluster/afr
subvolumes nas2 nas1
option favorite-child nas2
end-volume

swankier <swankier>

 

(Note: upload size limit is set to 16384 kB, after insertion of the required escape characters.)

Attach File(s):
   
   
Comment:
   

No files currently attached

 

Depends on the following items: None found

Items that depend on this one: None found

 

Carbon-Copy List
  • -unavailable- added by gowda (Updated the item)
  • -unavailable- added by amarts (Posted a comment)
  • -unavailable- added by swankier (Submitted the item)
  •  

    Do you think this task is very important?
    If so, you can click here to add your encouragement to it.
    This task has 0 encouragements so far.

    Only logged-in users can vote.

     

    Please enter the title of George Orwell's famous dystopian book (it's a date):

     

     

    Follow 4 latest changes.

    Date Changed By Updated Field Previous Value => Replaced By
    Mon 22 Jun 2009 09:51:26 AM UTCgowdaOpen/ClosedOpen=>Closed
    Wed 24 Dec 2008 10:18:10 PM UTCamartsStatusNone=>Fixed
      Assigned toNone=>amarts
      Originator Name=>Amar Tumballi

    Back to the top


    Powered by Savane 3.1-cleanup1