Hello,

Something has happend to our linux box (suse) and I can't figure out which ACL is breaking (or if that's the case)

If I execute
# man man
execve: Permission denied"

if I strace man man
execve("/usr/bin/man", ["man", "man"], [/* 58 vars */]) = 0
uname({sys="Linux", node="central", ...}) = 0
brk(0) = 0x804a000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=86771, ...}) = 0
old_mmap(NULL, 86771, PROT_READ, MAP_PRIVATE, 3, 0) = 0x40018000
close(3) = 0
open("/lib/tls/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0pO\1\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1394943, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x4002e000
old_mmap(NULL, 1150108, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x4002f000
madvise(0x4002f000, 1150108, MADV_SEQUENTIAL|0x1) = 0
old_mmap(0x40142000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x113000) = 0x40142000
old_mmap(0x40146000, 7324, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x40146000
close(3) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40148000
mprotect(0x40142000, 4096, PROT_READ) = 0
set_thread_area({entry_number:-1 -> 6, base_addr:0x401486c0, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
munmap(0x40018000, 86771) = 0
brk(0) = 0x804a000
brk(0x806b000) = 0x806b000
open("/usr/lib/locale/locale-archive", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/locale.alias", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=2528, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40018000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2528
read(3, "", 4096) = 0
close(3) = 0
munmap(0x40018000, 4096) = 0
open("/usr/lib/locale/en_US.UTF-8/LC_CTYPE", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib/locale/en_US.utf8/LC_CTYPE", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=208464, ...}) = 0
mmap2(NULL, 208464, PROT_READ, MAP_PRIVATE, 3, 0) = 0x40149000
close(3) = 0
open("/usr/lib/gconv/gconv-modules.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=21544, ...}) = 0
mmap2(NULL, 21544, PROT_READ, MAP_SHARED, 3, 0) = 0x4017c000
close(3) = 0
getuid32() = 0
geteuid32() = 0
getgid32() = 0
socket(PF_FILE, SOCK_STREAM, 0) = 3
fcntl64(3, F_GETFL) = 0x2 (flags O_RDWR)
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(3, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = 0
poll([{fd=3, events=POLLOUT|POLLERR|POLLHUP, revents=POLLOUT}], 1, 5000) = 1
writev(3, [{"\2\0\0\0\v\0\0\0\7\0\0\0", 12}, {"passwd\0", 7}], 2) = 19
poll([{fd=3, events=POLLIN|POLLERR|POLLHUP, revents=POLLIN|POLLHUP}], 1, 5000) = 1
recvmsg(3, {msg_name(0)=NULL, msg_iov(1)=[{"passwd\0", 7}], msg_controllen=16, {cmsg_len=16, cmsg_level=SOL_SOCKET, cmsg_type=SCM_RIGHTS, {4}}, msg_flags=0}, 0) = 7
fstat64(4, {st_mode=S_IFREG|0600, st_size=217016, ...}) = 0
pread(4, "\1\0\0\0h\0\0\0\0\0\0\0\1\0\0\0[L\210E\0\0\0\0\323\0\0"..., 104, 0) = 104
mmap2(NULL, 217016, PROT_READ, MAP_SHARED, 4, 0) = 0x40182000
close(4) = 0
close(3) = 0
setregid32(62, 62) = 0
open("/proc/sys/kernel/ngroups_max", O_RDONLY) = 3
read(3, "65536\n", 31) = 6
close(3) = 0
socket(PF_FILE, SOCK_STREAM, 0) = 3
fcntl64(3, F_GETFL) = 0x2 (flags O_RDWR)
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(3, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = 0
poll([{fd=3, events=POLLOUT|POLLERR|POLLHUP, revents=POLLOUT}], 1, 5000) = 1
writev(3, [{"\2\0\0\0\f\0\0\0\6\0\0\0", 12}, {"group\0", 6}], 2) = 18
poll([{fd=3, events=POLLIN|POLLERR|POLLHUP, revents=POLLIN|POLLHUP}], 1, 5000) = 1
recvmsg(3, {msg_name(0)=NULL, msg_iov(1)=[{"group\0", 6}], msg_controllen=16, {cmsg_len=16, cmsg_level=SOL_SOCKET, cmsg_type=SCM_RIGHTS, {4}}, msg_flags=0}, 0) = 6
fstat64(4, {st_mode=S_IFREG|0600, st_size=217016, ...}) = 0
pread(4, "\1\0\0\0h\0\0\0\2\0\0\0\1\0\0\0[L\210E\0\0\0\0\323\0\0"..., 104, 0) = 104
mmap2(NULL, 217016, PROT_READ, MAP_SHARED, 4, 0) = 0x401b7000
close(4) = 0
close(3) = 0
setgroups32(1, [62]) = 0
setreuid32(13, 13) = 0
execve("/usr/lib/man-db/man", ["man", "man"], [/* 58 vars */]) = -1 EACCES (Permission denied)
dup(2) = 3
fcntl64(3, F_GETFL) = 0x8001 (flags O_WRONLY|O_LARGEFILE)
close(3) = 0
write(2, "execve: Permission denied\n", 26execve: Permission denied
) = 26
exit_group(-22) = ?

if I # cat /boot/config-2.6.11.4-21.14-smp | grep ACL
CONFIG_EXT2_FS_POSIX_ACL=y
CONFIG_EXT3_FS_POSIX_ACL=y
CONFIG_REISERFS_FS_POSIX_ACL=y
CONFIG_JFS_POSIX_ACL=y
CONFIG_FS_POSIX_ACL=y
CONFIG_XFS_POSIX_ACL=y
CONFIG_TMPFS_POSIX_ACL=y
CONFIG_NFS_ACL=y
CONFIG_NFSD_ACL=y
CONFIG_NFS_ACL_SUPPORT=y
CONFIG_GENERIC_ACL=y

The disk in question
/dev/sda6 / reiserfs acl,user_xattr 1 1

I have looked at other answers, but nothing seems to work. Help?
Blake

Yeah, now I have a 2nd server (2.4 kernel) that apache is now no longer allowing any connections (403 errors) even after cleaning, rebuilding apache and rebooting. Similar situation:

a custom http multi-part form processor in apache was running, and then suddenly the system gets in this locked state.

Any ideas?

Sure enough, that 2.4 Kernel (redhat 9) box now also doesn't like to fork. Rebooting doesn't fix the problem, I suspect something has been overwritten, but what system config stuff to look at? What variables would stop the system from forking?

Try to boot on a clean machine with the same settings and compare the strace output. Sorry I don't have any clue but that's what I would do.
By the way you you have to do:
strace -f to have all information (not sure you use it) and then well, -o /tmp/strace to make post analysis.
Good luck