Apache segfaults after months of running on Ubuntu

gimpy530 · 05-13-2011, 10:42 AM

OS: Ubuntu Server x64 10.04
Apache: 2.2.14-5ubuntu8.4

I have a virtual server which has been running for months with no problems then one day I saw that it crashed and I had to power cycle it. Since then I have been unable to start Apache as it segfaults no matter options I give it. Before I try to reinstall Apache, why is this segfaulting?

Code:

execve("/usr/sbin/apache2", ["/usr/sbin/apache2"], [/* 21 vars */]) = 0
brk(0)                                  = 0x7f9f48f8d000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f47e0d000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=74019, ...}) = 0
mmap(NULL, 74019, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f9f47dfa000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libpcre.so.3", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\25\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=186440, ...}) = 0
mmap(NULL, 2281640, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f479c2000
mprotect(0x7f9f479ef000, 2093056, PROT_NONE) = 0
mmap(0x7f9f47bee000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2c000) = 0x7f9f47bee000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/usr/lib/libaprutil-1.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\211\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=142912, ...}) = 0
mmap(NULL, 2238208, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f4779f000
mprotect(0x7f9f477c0000, 2097152, PROT_NONE) = 0
mmap(0x7f9f479c0000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21000) = 0x7f9f479c0000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/usr/lib/libapr-1.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\311\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=217920, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f47df9000
mmap(NULL, 2313256, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f4756a000
mprotect(0x7f9f4759e000, 2093056, PROT_NONE) = 0
mmap(0x7f9f4779d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x33000) = 0x7f9f4779d000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libpthread.so.0", O_RDONLY)  = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360Y\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=135745, ...}) = 0
mmap(NULL, 2212736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f4734d000
mprotect(0x7f9f47365000, 2093056, PROT_NONE) = 0
mmap(0x7f9f47564000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f9f47564000
mmap(0x7f9f47566000, 13184, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f9f47566000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libc.so.6", O_RDONLY)        = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\355\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1572232, ...}) = 0
mmap(NULL, 3680296, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f46fca000
mprotect(0x7f9f47144000, 2093056, PROT_NONE) = 0
mmap(0x7f9f47343000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x179000) = 0x7f9f47343000
mmap(0x7f9f47348000, 18472, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f9f47348000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libuuid.so.1", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\25\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=19008, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f47df8000
mmap(NULL, 2113920, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f46dc5000
mprotect(0x7f9f46dc9000, 2093056, PROT_NONE) = 0
mmap(0x7f9f46fc8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f9f46fc8000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/librt.so.1", O_RDONLY)       = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220!\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=31744, ...}) = 0
mmap(NULL, 2128848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f46bbd000
mprotect(0x7f9f46bc4000, 2093056, PROT_NONE) = 0
mmap(0x7f9f46dc3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f9f46dc3000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libcrypt.so.1", O_RDONLY)    = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\n\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=43296, ...}) = 0
mmap(NULL, 2326976, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f46984000
mprotect(0x7f9f4698d000, 2097152, PROT_NONE) = 0
mmap(0x7f9f46b8d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x9000) = 0x7f9f46b8d000
mmap(0x7f9f46b8f000, 184768, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f9f46b8f000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libdl.so.2", O_RDONLY)       = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14696, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f47df7000
mmap(NULL, 2109696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f46780000
mprotect(0x7f9f46782000, 2097152, PROT_NONE) = 0
mmap(0x7f9f46982000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f9f46982000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libexpat.so.1", O_RDONLY)    = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`<\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=165960, ...}) = 0
mmap(NULL, 2261072, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f9f46557000
mprotect(0x7f9f4657d000, 2097152, PROT_NONE) = 0
mmap(0x7f9f4677d000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x26000) = 0x7f9f4677d000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f47df6000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9f47df4000
arch_prctl(ARCH_SET_FS, 0x7f9f47df4740) = 0
mprotect(0x7f9f4677d000, 8192, PROT_READ) = 0
mprotect(0x7f9f46982000, 4096, PROT_READ) = 0
mprotect(0x7f9f46b8d000, 4096, PROT_READ) = 0
mprotect(0x7f9f46dc3000, 4096, PROT_READ) = 0
mprotect(0x7f9f46fc8000, 4096, PROT_READ) = 0
mprotect(0x7f9f47343000, 16384, PROT_READ) = 0
mprotect(0x7f9f47564000, 4096, PROT_READ) = 0
mprotect(0x7f9f4779d000, 4096, PROT_READ) = 0
mprotect(0x7f9f479c0000, 4096, PROT_READ) = 0
mprotect(0x7f9f47bee000, 4096, PROT_READ) = 0
mprotect(0x7f9f4807f000, 12288, PROT_READ) = 0
mprotect(0x7f9f47e0f000, 4096, PROT_READ) = 0
munmap(0x7f9f47dfa000, 74019)           = 0
set_tid_address(0x7f9f47df4a10)         = 1580
set_robust_list(0x7f9f47df4a20, 0x18)   = 0
futex(0x7ffff484e27c, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffff484e27c, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, NULL, 7f9f47df4740) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0x7f9f47352870, [], SA_RESTORER|SA_SIGINFO, 0x7f9f4735c8f0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f9f47352900, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f9f4735c8f0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
--- SIGSEGV (Segmentation fault) @ 0 (0) ---
+++ killed by SIGSEGV +++

rylan76 · 05-16-2011, 02:17 PM

Are you SURE you do not have bad RAM on that board?

I once had almost exactly the same problem, only it started with Skype (of all things) and then spread. I got segfaults that looked almost exactly like those you have there.

Turned out I had a bad RAM chip. It was slowly corrupting my entire Linux installation. Somehow it scrambled the Skype executable first, but shortly after some of my other executables started segfaulting too.

E. g. first check if you do not have bad RAM - and first fix that, before doing a reinstall or recompile. (I reinstalled immediately when Skype started segfaulting, only to have it start happening AGAIN days later in another binary - forcing me to first buy a replacement DRAM chip, THEN reinstall AGAIN...!)

So I replace the DRAM chip, and that same system has been working fine for about 3 years now... as a LAMP development server.

Testing is easy (at least, if you have a Fedora distro) - if I remember right, you can just pop in the distro DVD and there is an option or command you can issue to run a mem checker - not sure if / how you do it with your distro.

gimpy530 · 05-17-2011, 03:10 PM

Ubuntu includes memtest86 too and I already have that running before I saw your post. I doubted a hardware issue since this was within a VM but I now saw data corruption on two separate RAID arrays on the host (even on different controllers) so RAM seems plausible. This isn't typical of what I have seen RAM die before as I've yet to get a kernel panic on the host.