Hey guys, we use Apache for loadbalancing on production, after updating httpd to the latest release we get a lot of Segmentation faults. **** version **** Linux debian 8.6 3.16.0-4-amd64 #1 SMP Debian 3.16.36-1+deb8u2 (2016-10-19) x86_64 GNU/Linux Server version: Apache/2.4.23 (Unix) Server built: Sep 13 2016 08:22:30 Server's Module Magic Number: 20120211:61 Server loaded: APR 1.5.2, APR-UTIL 1.5.4 Compiled using: APR 1.5.2, APR-UTIL 1.5.4 Architecture: 64-bit Server MPM: worker threaded: yes (fixed thread count) forked: yes (variable process count) Server compiled with.... -D APR_HAS_SENDFILE -D APR_HAS_MMAP -D APR_HAVE_IPV6 (IPv4-mapped addresses enabled) -D APR_USE_SYSVSEM_SERIALIZE -D APR_USE_PTHREAD_SERIALIZE -D SINGLE_LISTEN_UNSERIALIZED_ACCEPT -D APR_HAS_OTHER_CHILD -D AP_HAVE_RELIABLE_PIPED_LOGS -D DYNAMIC_MODULE_LIMIT=256 -D HTTPD_ROOT="/opt/build/loadbalancer/apache" -D SUEXEC_BIN="/opt/build/loadbalancer/apache/bin/suexec" -D DEFAULT_PIDLOG="logs/httpd.pid" -D DEFAULT_SCOREBOARD="logs/apache_runtime_status" -D DEFAULT_ERRORLOG="logs/error_log" -D AP_TYPES_CONFIG_FILE="conf/mime.types" -D SERVER_CONFIG_FILE="conf/httpd.conf" **** version **** **** coredump **** [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/opt/build/loadbalancer/apache/bin/httpd -k start'. Program terminated with signal SIGSEGV, Segmentation fault. #0 allocator_alloc (in_size=in_size@entry=8152, allocator=0x0) at /opt/build/loadbalancer/../unpack/apr-1.5.2/memory/unix/apr_pools.c:241 241 /opt/build/loadbalancer/../unpack/apr-1.5.2/memory/unix/apr_pools.c: No such file or directory. (gdb) Bt full #0 allocator_alloc (in_size=in_size@entry=8152, allocator=0x0) at /opt/build/loadbalancer/../unpack/apr-1.5.2/memory/unix/apr_pools.c:241 node = <optimized out> ref = <optimized out> max_index = <optimized out> i = <optimized out> size = 8192 index = <optimized out> #1 apr_allocator_alloc (allocator=0x0, size=size@entry=8152) at /opt/build/loadbalancer/../unpack/apr-1.5.2/memory/unix/apr_pools.c:438 No locals. #2 0x00007f5db8f9a7ae in apr_bucket_alloc (size=96, size@entry=64, list=0x7f5d740d1658) at /opt/build/loadbalancer/../unpack/apr-util-1.5.4/buckets/apr_buckets_alloc.c:140 node = <optimized out> active = 0x7f5d64086db0 endp = <optimized out> #3 0x00007f5db8f9b4da in apr_bucket_simple_copy (a=a@entry=0x7f5d740d1a58, b=b@entry=0x7f5db0c8db98) at /opt/build/loadbalancer/../unpack/apr-util-1.5.4/buckets/apr_buckets_simple.c:22 No locals. #4 0x00007f5db8f9b586 in apr_bucket_simple_split (a=0x7f5d740d1a58, point=0) at /opt/build/loadbalancer/../unpack/apr-util-1.5.4/buckets/apr_buckets_simple.c:37 b = 0x7f5db8b5446a <apr_socket_sendv+138> point = 0 a = 0x7f5d740d1a58 #5 0x00007f5db8f9b43a in apr_bucket_shared_split (a=<optimized out>, point=<optimized out>) at /opt/build/loadbalancer/../unpack/apr-util-1.5.4/buckets/apr_buckets_refcount.c:25 r = 0x7f5d740d1698 rv = -1 #6 0x00000000004483c7 in writev_nonblocking (s=0x7f5dac10c1d0, vec=0x7f5db0c8dca0, nvec=11, bb=0x7f5dac10c980, cumulative_bytes_written=0x7f5dac10c8a8, c=0x7f5dac10c3e8) at /opt/build/unpack/httpd-2.4.23/server/core_filters.c:801 bucket = 0x7f5d740d1a58 n = 0 rv = 0 arv = <optimized out> bytes_written = <optimized out> bytes_to_write = 22832 i = 8 offset = 8 old_timeout = 720000000 #7 0x00000000004484cc in send_brigade_nonblocking (s=0x0, bb=0x7f5dac10c980, bytes_written=0x7f5d64087578, c=0x7f5d64087618) at /opt/build/unpack/httpd-2.4.23/server/core_filters.c:704 vec = {{iov_base = 0x7f5d54066c68, iov_len = 4086}, {iov_base = 0x7f5d44036b18, iov_len = 4106}, {iov_base = 0x47c2ca, iov_len = 2}, {iov_base = 0x7f5d54079218, iov_len = 6}, { iov_base = 0x7f5d5406ac88, iov_len = 3886}, {iov_base = 0x7f5d44040b68, iov_len = 4306}, {iov_base = 0x47c2ca, iov_len = 2}, {iov_base = 0x7f5d54079998, iov_len = 6}, {iov_base = 0x10d2, iov_len = 3694}, {iov_base = 0x0, iov_len = 2736}, {iov_base = 0x47c2ca, iov_len = 2}, {iov_base = 0x7f5d54079e98, iov_len = 6}, {iov_base = 0x7f5d44042b78, iov_len = 4286}, { iov_base = 0x7f5d4c088dd8, iov_len = 3906}, {iov_base = 0x47c2ca, iov_len = 2}, {iov_base = 0x7f5d5407a938, iov_len = 6}} nvec = 11 #8 0x00000000004491e1 in send_brigade_blocking (c=<optimized out>, bytes_written=<optimized out>, bb=<optimized out>, s=<optimized out>) at /opt/build/unpack/httpd-2.4.23/server/core_filters.c:733 No locals. #9 ap_core_output_filter (f=0x0, new_bb=0x7f5dac10c980) at /opt/build/unpack/httpd-2.4.23/server/core_filters.c:542 c = 0x7f5dac10c3e8 bytes_in_brigade = 0 non_file_bytes_in_brigade = 0 eor_buckets_in_brigade = 1678275960 morphing_bucket_in_brigade = 8152 #10 0x00000000004626b1 in ap_process_request (r=0x7f5d4c090e70) at /opt/build/unpack/httpd-2.4.23/modules/http/http_request.c:451 bb = 0x7f5dac10c980 c = 0x7f5dac10c3e8 rv = -1 #11 0x000000000045e9b5 in ap_process_http_sync_connection (c=0x7f5dac10c3e8) at /opt/build/unpack/httpd-2.4.23/modules/http/http_core.c:210 keep_alive_timeout = 5000000 r = 0x7f5d4c090e70 cs = 0x0 csd = 0x0 mpm_state = 0 #12 ap_process_http_connection (c=0x7f5dac10c3e8) at /opt/build/unpack/httpd-2.4.23/modules/http/http_core.c:251 No locals. ---Type <return> to continue, or q <return> to quit---#13 0x0000000000456d30 in ap_run_process_connection (c=0x7f5dac10c3e8) at /opt/build/unpack/httpd-2.4.23/server/connection.c:42 pHook = <optimized out> n = 0 rv = -1 #14 0x0000000000468f81 in process_socket (bucket_alloc=<optimized out>, my_thread_num=<optimized out>, my_child_num=<optimized out>, sock=<optimized out>, p=<optimized out>, thd=<optimized out>) at /opt/build/unpack/httpd-2.4.23/server/mpm/worker/worker.c:631 current_conn = 0x7f5dac10c3e8 conn_id = 140040295465960 sbh = 0x7f5dac10c3e0 #15 worker_thread (thd=0x0, dummy=0x1fd8) at /opt/build/unpack/httpd-2.4.23/server/mpm/worker/worker.c:992 process_slot = 8 thread_slot = 0 csd = 0x7f5dac10c1d0 bucket_alloc = 0x9 last_ptrans = 0x7f5dac10c3e8 ptrans = 0x7f5dac10c148 is_idle = -1408187952 #16 0x00007f5db84e10a4 in start_thread (arg=0x7f5db0c8e700) at pthread_create.c:309 __res = <optimized out> pd = 0x7f5db0c8e700 now = <optimized out> unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140040374642432, -8558836165728189939, 0, 140040520843360, 64, 140040374642432, 8611874569776611853, 8611893242517733901}, mask_was_saved = 0}}, priv = { pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} not_first_call = <optimized out> pagesize_m1 = <optimized out> sp = <optimized out> freesize = <optimized out> __PRETTY_FUNCTION__ = "start_thread" #17 0x00007f5db801262d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 No locals. **** coredump ****
Just to confirm, you have installed all -debuginfo / -dbg packages for httpd and apr/apr-util applicable to your environment? If this is your own build, ensure you are building with -g for symbolic references. It might be useful to try reproducing this with an -O0 -g build of httpd/apr/apr-util where the various references haven't been optimized away.
Thank you for the feedback. We don't build ours httpd with -g. We make an new build and waiting of the next segfault.
Hey guys, we have today an segfault on our infrastructure. I hope the new dump is more helpful. **** coredump **** Core was generated by `/opt/build/loadbalancer/apache/bin/httpd -k start'. Program terminated with signal SIGSEGV, Segmentation fault. #0 0x0000000000454ac1 in ap_add_common_vars (r=0x0) at /opt/build/unpack/httpd-2.4.23/server/util_script.c:197 197 if (conf->cgi_pass_auth == AP_CGI_PASS_AUTH_ON) { (gdb) frame 0 #0 0x0000000000454ac1 in ap_add_common_vars (r=0x0) at /opt/build/unpack/httpd-2.4.23/server/util_script.c:197 197 if (conf->cgi_pass_auth == AP_CGI_PASS_AUTH_ON) { (gdb) list 192 * in the environment with "ps -e". But, if you must... 193 */ 194 #ifndef SECURITY_HOLE_PASS_AUTHORIZATION 195 else if (!strcasecmp(hdrs[i].key, "Authorization") 196 || !strcasecmp(hdrs[i].key, "Proxy-Authorization")) { 197 if (conf->cgi_pass_auth == AP_CGI_PASS_AUTH_ON) { 198 add_unless_null(e, http2env(r, hdrs[i].key), hdrs[i].val); 199 } 200 } 201 #endif (gdb) bt #0 0x0000000000454ac1 in ap_add_common_vars (r=0x0) at /opt/build/unpack/httpd-2.4.23/server/util_script.c:197 #1 0x0000000000000006 in ?? () #2 0x00007f5c4c068cd8 in ?? () #3 0x00000000000003ea in ?? () #4 0x00007f5c2c088d78 in ?? () #5 0x0000000000001c16 in ?? () #6 0x0000000000499592 in ?? () #7 0x0000000000000002 in ?? () #8 0x00007f5c2c08d678 in ?? () #9 0x0000000000000006 in ?? () #10 0x00007f5c38004928 in ?? () #11 0x0000000000000322 in ?? () #12 0x00007f5c4c060c68 in ?? () #13 0x0000000000001cde in ?? () #14 0x0000000000499592 in ?? () #15 0x0000000000000002 in ?? () #16 0x00007f5c2c08ddf8 in ?? () #17 0x0000000000000006 in ?? () #18 0x00007f5c4c078d58 in ?? () #19 0x000000000000025a in ?? () #20 0x00007f5c4c072d28 in ?? () #21 0x0000000000001da6 in ?? () #22 0x0000000000499592 in ?? () #23 0x0000000000000002 in ?? () #24 0x00007f5c4c075558 in ?? () #25 0x0000000000000004 in ?? () #26 0x00007f5c38006938 in ?? () #27 0x00000000000004b2 in ?? () #28 0x0000000178fd0c60 in ?? () #29 0x00007f5c78fd0bb0 in ?? () #30 0x000000000000000f in ?? () #31 0x00000000900dc0b8 in ?? () #32 0x2064656e696c6365 in ?? () #33 0x00007f5c440e62b8 in ?? () #34 0x00007f5c78fd0c70 in ?? () #35 0x0000000000454d72 in ap_add_common_vars (r=0x1b4e) at /opt/build/unpack/httpd-2.4.23/server/util_script.c:282 ---Type <return> to continue, or q <return> to quit--- #36 0x00000000004544c2 in getsfunc_BRIGADE (buf=<optimized out>, len=<optimized out>, arg=<optimized out>) at /opt/build/unpack/httpd-2.4.23/server/util_script.c:756 #37 0x00007f5c78fd0cd0 in ?? () #38 0x00007f5c900dc070 in ?? () #39 0x00007f5c900dbcc8 in ?? () #40 0x0000000000000000 in ?? () (gdb) bt full #0 0x0000000000454ac1 in ap_add_common_vars (r=0x0) at /opt/build/unpack/httpd-2.4.23/server/util_script.c:197 e = 0x0 s = 0x7f5c900dc2c0 c = 0x7f5c900dbab0 conf = 0x7f5c900dc188 env_temp = <optimized out> hdrs_arr = 0x7f5c78fd1700 hdrs = <optimized out> i = -1642676128 rport = <optimized out> q = <optimized out> #1 0x0000000000000006 in ?? () No symbol table info available. #2 0x00007f5c4c068cd8 in ?? () No symbol table info available. #3 0x00000000000003ea in ?? () No symbol table info available. #4 0x00007f5c2c088d78 in ?? () No symbol table info available. #5 0x0000000000001c16 in ?? () No symbol table info available. #6 0x0000000000499592 in ?? () No symbol table info available. #7 0x0000000000000002 in ?? () No symbol table info available. #8 0x00007f5c2c08d678 in ?? () No symbol table info available. #9 0x0000000000000006 in ?? () No symbol table info available. #10 0x00007f5c38004928 in ?? () No symbol table info available. #11 0x0000000000000322 in ?? () No symbol table info available. #12 0x00007f5c4c060c68 in ?? () No symbol table info available. #13 0x0000000000001cde in ?? () ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #14 0x0000000000499592 in ?? () No symbol table info available. #15 0x0000000000000002 in ?? () No symbol table info available. #16 0x00007f5c2c08ddf8 in ?? () No symbol table info available. #17 0x0000000000000006 in ?? () No symbol table info available. #18 0x00007f5c4c078d58 in ?? () No symbol table info available. #19 0x000000000000025a in ?? () No symbol table info available. #20 0x00007f5c4c072d28 in ?? () No symbol table info available. #21 0x0000000000001da6 in ?? () No symbol table info available. #22 0x0000000000499592 in ?? () No symbol table info available. #23 0x0000000000000002 in ?? () No symbol table info available. #24 0x00007f5c4c075558 in ?? () No symbol table info available. #25 0x0000000000000004 in ?? () No symbol table info available. #26 0x00007f5c38006938 in ?? () No symbol table info available. #27 0x00000000000004b2 in ?? () No symbol table info available. #28 0x0000000178fd0c60 in ?? () No symbol table info available. #29 0x00007f5c78fd0bb0 in ?? () No symbol table info available. #30 0x000000000000000f in ?? () No symbol table info available. #31 0x00000000900dc0b8 in ?? () ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #32 0x2064656e696c6365 in ?? () No symbol table info available. #33 0x00007f5c440e62b8 in ?? () No symbol table info available. #34 0x00007f5c78fd0c70 in ?? () No symbol table info available. #35 0x0000000000454d72 in ap_add_common_vars (r=0x1b4e) at /opt/build/unpack/httpd-2.4.23/server/util_script.c:282 e = 0x2 s = 0x7f5c900dc2c0 c = 0x7f5c900dbab0 conf = 0x7f5c900dc188 env_temp = 0x6e6f6974617a6972 <error: Cannot access memory at address 0x6e6f6974617a6972> hdrs_arr = 0x7f5c2c08da38 hdrs = <optimized out> i = <optimized out> rport = <optimized out> q = <optimized out> #36 0x00000000004544c2 in getsfunc_BRIGADE (buf=<optimized out>, len=<optimized out>, arg=<optimized out>) at /opt/build/unpack/httpd-2.4.23/server/util_script.c:756 bb = <optimized out> dst_end = 0x1 <error: Cannot access memory at address 0x1> dst = <optimized out> e = <optimized out> rv = <optimized out> done = <optimized out> #37 0x00007f5c78fd0cd0 in ?? () No symbol table info available. #38 0x00007f5c900dc070 in ?? () No symbol table info available. #39 0x00007f5c900dbcc8 in ?? () No symbol table info available. #40 0x0000000000000000 in ?? () No symbol table info available. (gdb) list 202 else 203 add_unless_null(e, http2env(r, hdrs[i].key), hdrs[i].val); 204 } 205 206 env_temp = apr_table_get(r->subprocess_env, "PATH"); 207 if (env_temp == NULL) { 208 env_temp = getenv("PATH"); 209 } 210 if (env_temp == NULL) { 211 env_temp = DEFAULT_PATH; (gdb) **** coredump ****
Today we had another segementation fault on the same server. But this time the coredump looks quite different. (I've masked possible sensitive informations using X): **** coredump **** #0 0x00007f5c9ce40274 in ?? () No symbol table info available. #1 0x00007f5c897ef5a0 in ?? () No symbol table info available. #2 0x00007f5c9d4b18bb in ?? () No symbol table info available. #3 0x00007f5c3c0d6040 in ?? () No symbol table info available. #4 0x000000aa81800000 in ?? () No symbol table info available. #5 0x00007f5c2801aa30 in ?? () No symbol table info available. #6 0x00007f5c30020b28 in ?? () No symbol table info available. #7 0x00007f5c897ef590 in ?? () No symbol table info available. #8 0x00007f5c9d4b208a in ?? () No symbol table info available. #9 0x00007f5c50109160 in ?? () No symbol table info available. #10 0x0000000000aa800e in ?? () No symbol table info available. #11 0x00007f5c48006918 in ?? () No symbol table info available. #12 0x0000000000c14d38 in ?? () No symbol table info available. #13 0x00007f5c897ef5c0 in ?? () No symbol table info available. #14 0x0000000000438133 in ap_vrprintf (r=0x7f5c897ef540, fmt=<optimized out>, va=<optimized out>) at /opt/unpack/httpd-2.4.23/server/protocol.c:1658 written = <optimized out> vd = {vbuff = { curpos = 0x7f5c3c0d7a08 "=cnp_jQuery(d),s=p.attr(\"name\"),J=p.val();if(!(g&&this.isEmptyString(J))){var W=p.closest(\"form\"),\nH={};if(s===\"ACCOUNT.NUMBER\"||s===\"ACCOUNT.XXXX\")if(this.isXXXXForm(W))this.validateXXXX(J)||(H=cnp_j"..., endpos = 0x5a3 <error: Cannot access memory at address 0x5a3>}, r = 0x7f5c48299f24, buff = 0x7f5c30020c68 "s://\"+(r==\"live\"?\"XXXX.XXX\":\"test.XXXX.XXX\")+\"/connectors/XXX/XXXXXXXX.link\",jsonp:\"callback\",dataType:\"jsonp\",success:function(u){var F=cnp_jQuery(\"select.XXXXNameSelectBox\",\"div.otXXXXXXX_XXX\");F.em"...} vrprintf_buf = "\036!\2XX\2XX\\\1XX\000\000$\2XX)H\\\1XX\000\000\2XX\n\002\060\000\000\000\000h\f\002\0XX\\\1XX\000\000\001\000\000\000\000\000\000\000P\3XX~\2XX\\\1XX\000\000\0XX\0XXE", '\000' <repeats 17 times>, "\001\000\000\000\bz\r<\\\1XX\000\000\b\2XX\001(\\\1XX\000\000\b\r\0XX\0XX\\\1XX\000\000\0XX\0XXK\2XX\\\1XX\000\000\0XX\2XX\001(\\\1XX\000\000\2XX\0XX\000\000\000\000\000\000#\2XX)H\\\1XX\000\000\000\000\000\000\000\000\000\000P\3XX~\2XX\\\1XX\000\000\b\r\0XX\0XX\\\1XX\000\000\000\000\000\000\000\000\000\000\0XX^RX\000\000\000\000\001\000\000\000\000\000\000\000\3XX\2XX\001(\\\1XX\000\000\3XX\3XX~\2XX\\\1XX\000\000\2XX\3XX~\2XX\\\1XX\000\000"... #15 0x00007f5c50109160 in ?? () No symbol table info available. #16 0x0000000000aa800e in ?? () No symbol table info available. #17 0x00007f5c897ef5e0 in ?? () No symbol table info available. #18 0x00007f5c9d900ec0 in ?? () No symbol table info available. #19 0x00007f5c897ef5f0 in ?? () No symbol table info available. #20 0x00007f5c501091a8 in ?? () No symbol table info available. #21 0x00007f5c3007ad18 in ?? () No symbol table info available. #22 0x00007f5c50109188 in ?? () No symbol table info available. #23 0x00007f5c897ef610 in ?? () No symbol table info available. #24 0x00007f5c9d9016e0 in ?? () No symbol table info available. #25 0x00007f5c897ef670 in ?? () No symbol table info available. #26 0x00007f5c3007adf8 in ?? () No symbol table info available. #27 0x00007f5c28051f98 in ?? () No symbol table info available. #28 0x00007f5c3007adf8 in ?? () No symbol table info available. #29 0x00007f5c897ef6c0 in ?? () No symbol table info available. #30 0x0000000000453571 in ap_open_piped_log_ex (p=0x0, program=0x7f5c9d4b2240 <error: Cannot access memory at address 0x7f5c9d4b2240>, cmdtype=<optimized out>) at /opt/unpack/httpd-2.4.23/server/log.c:1833 pl = 0x7f5c48006918 #31 0x0000000000000005 in ?? () No symbol table info available. #32 0x0000000000000000 in ?? () No symbol table info available. **** coredump **** **** frame 14 **** (gdb) frame 14 #14 0x0000000000438133 in ap_vrprintf (r=0x7f5c897ef540, fmt=<optimized out>, va=<optimized out>) at /opt/unpack/httpd-2.4.23/server/protocol.c:1658 1658 } (gdb) list 1653 1654 written += n; 1655 } 1656 1657 return written; 1658 } 1659 1660 AP_DECLARE_NONSTD(int) ap_rprintf(request_rec *r, const char *fmt, ...) 1661 { 1662 va_list va; **** frame 30 **** (gdb) frame 30 #30 0x0000000000453571 in ap_open_piped_log_ex (p=0x0, program=0x7f5c9d4b2240 <error: Cannot access memory at address 0x7f5c9d4b2240>, cmdtype=<optimized out>) at /opt/unpack/httpd-2.4.23/server/log.c:1833 1833 apr_pool_cleanup_register(p, pl, piped_log_cleanup, (gdb) list 1828 if (apr_file_pipe_create_ex(&pl->read_fd, 1829 &pl->write_fd, 1830 APR_FULL_BLOCK, p) != APR_SUCCESS) { 1831 return NULL; 1832 } 1833 apr_pool_cleanup_register(p, pl, piped_log_cleanup, 1834 piped_log_cleanup_for_exec); 1835 if (piped_log_spawn(pl) != APR_SUCCESS) { 1836 apr_pool_cleanup_kill(p, pl, piped_log_cleanup); 1837 apr_file_close(pl->read_fd); (gdb)
Hello, We've just got another SEGFAULT, with also a different coredump. Maybe it helps to identify the issue: GNU gdb (Debian 7.7.1+dfsg-5) 7.7.1 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>. Find the GDB manual and other documentation resources online at: <http://www.gnu.org/software/gdb/documentation/>. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /opt/loadbalancer/apache/bin/httpd...done. [New LWP 28271] [New LWP 28274] [New LWP 28277] [New LWP 28278] [New LWP 28279] [New LWP 28280] [New LWP 28275] [New LWP 28281] [New LWP 28286] [New LWP 28273] [New LWP 28287] [New LWP 28289] [New LWP 28291] [New LWP 28272] [New LWP 28299] [New LWP 27530] [New LWP 28300] [New LWP 28301] [New LWP 28284] [New LWP 28283] [New LWP 28303] [New LWP 28282] [New LWP 28304] [New LWP 28276] [New LWP 28309] [New LWP 28310] [New LWP 28290] [New LWP 28311] [New LWP 28288] [New LWP 28319] [New LWP 28285] [New LWP 28321] [New LWP 28294] [New LWP 28293] [New LWP 28322] [New LWP 28292] [New LWP 28325] [New LWP 28326] [New LWP 28327] [New LWP 28329] [New LWP 28302] [New LWP 28336] [New LWP 28298] [New LWP 28297] [New LWP 28337] [New LWP 28296] [New LWP 28338] [New LWP 28295] [New LWP 28343] [New LWP 28347] [New LWP 28308] [New LWP 28350] [New LWP 28307] [New LWP 28351] [New LWP 28254] [New LWP 28306] [New LWP 28255] [New LWP 28305] [New LWP 28257] [New LWP 28320] [New LWP 28258] [New LWP 28259] [New LWP 28318] [New LWP 28261] [New LWP 28266] [New LWP 28268] [New LWP 28317] [New LWP 28270] [New LWP 28315] [New LWP 28323] [New LWP 28316] [New LWP 28324] [New LWP 28328] [New LWP 28330] [New LWP 28314] [New LWP 28331] [New LWP 28313] [New LWP 28332] [New LWP 28312] [New LWP 28333] [New LWP 28269] [New LWP 28334] [New LWP 28335] [New LWP 28267] [New LWP 28339] [New LWP 28265] [New LWP 28340] [New LWP 28341] [New LWP 28264] [New LWP 28342] [New LWP 28344] [New LWP 28345] [New LWP 28346] [New LWP 28348] [New LWP 28349] [New LWP 28352] [New LWP 28252] [New LWP 28253] [New LWP 28256] [New LWP 28260] [New LWP 28262] [New LWP 28263] Core was generated by `/opt/loadbalancer/apache/bin/httpd -k start'. Program terminated with signal SIGSEGV, Segmentation fault. #0 0x0000000000454ac1 in ap_add_common_vars (r=0x0) at /opt/unpack/httpd-2.4.23/server/util_script.c:197 197 if (conf->cgi_pass_auth == AP_CGI_PASS_AUTH_ON) { (gdb) bt full #0 0x0000000000454ac1 in ap_add_common_vars (r=0x0) at /opt/unpack/httpd-2.4.23/server/util_script.c:197 e = 0x0 s = 0x7f5c900e5720 c = 0x7f5c900e3e30 conf = 0x7f5c900e4508 env_temp = <optimized out> hdrs_arr = 0x7f5c877ee700 hdrs = <optimized out> i = -1642676128 rport = <optimized out> q = <optimized out> #1 0x0000000000000006 in ?? () No symbol table info available. #2 0x00007f5c34004928 in ?? () No symbol table info available. #3 0x0000000000000323 in ?? () No symbol table info available. #4 0x00007f5c3400e9a8 in ?? () No symbol table info available. #5 0x0000000000001cdd in ?? () No symbol table info available. #6 0x0000000000499592 in ?? () No symbol table info available. #7 0x0000000000000002 in ?? () No symbol table info available. #8 0x00007f5c34003298 in ?? () No symbol table info available. #9 0x0000000000000006 in ?? () No symbol table info available. #10 0x00007f5c2008edd8 in ?? () No symbol table info available. #11 0x000000000000025b in ?? () No symbol table info available. #12 0x00007f5c20090de8 in ?? () No symbol table info available. #13 0x0000000000001da5 in ?? () No symbol table info available. #14 0x0000000000499592 in ?? () No symbol table info available. #15 0x0000000000000002 in ?? () No symbol table info available. #16 0x00007f5c34003b58 in ?? () No symbol table info available. #17 0x0000000000000004 in ?? () No symbol table info available. #18 0x00007f5c20066c98 in ?? () No symbol table info available. #19 0x00000000000004b3 in ?? () No symbol table info available. #20 0x00007f5c4803cb18 in ?? () No symbol table info available. #21 0x0000000000001b4d in ?? () No symbol table info available. #22 0x0000000000499592 in ?? () No symbol table info available. #23 0x0000000000000002 in ?? () No symbol table info available. #24 0x00007f5c4c054de8 in ?? () No symbol table info available. #25 0x0000000000000006 in ?? () No symbol table info available. #26 0x00007f5c541713e8 in ?? () No symbol table info available. #27 0x00000000000003eb in ?? () No symbol table info available. #28 0x00000001877edc60 in ?? () No symbol table info available. #29 0x00007f5c877edbb0 in ?? () No symbol table info available. #30 0x000000000000000b in ?? () No symbol table info available. #31 0x00000000900e4438 in ?? () No symbol table info available. #32 0x00000000000000c8 in ?? () No symbol table info available. #33 0x0000000000c11e48 in ?? () ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #34 0x00007f5c877edc70 in ?? () No symbol table info available. #35 0x0000000000454d72 in ap_add_common_vars (r=0x1c15) at /opt/unpack/httpd-2.4.23/server/util_script.c:282 e = 0x2 s = 0x7f5c900e5720 c = 0x7f5c900e3e30 conf = 0x7f5c900e4508 env_temp = 0x0 hdrs_arr = 0x7f5c34003478 hdrs = <optimized out> i = <optimized out> rport = <optimized out> q = <optimized out> #36 0x00000000004544c2 in getsfunc_BRIGADE (buf=<optimized out>, len=<optimized out>, arg=<optimized out>) at /opt/unpack/httpd-2.4.23/server/util_script.c:756 bb = <optimized out> dst_end = 0x1 <error: Cannot access memory at address 0x1> dst = <optimized out> e = <optimized out> rv = <optimized out> done = <optimized out> #37 0x00007f5c877edcd0 in ?? () No symbol table info available. #38 0x00007f5c900e43f0 in ?? () No symbol table info available. #39 0x00007f5c900e4048 in ?? () No symbol table info available. #40 0x0000000000000000 in ?? () No symbol table info available. (gdb) frame 2 #2 0x00007f5c34004928 in ?? () (gdb) frame 0 #0 0x0000000000454ac1 in ap_add_common_vars (r=0x0) at /opt/unpack/httpd-2.4.23/server/util_script.c:197 197 if (conf->cgi_pass_auth == AP_CGI_PASS_AUTH_ON) { (gdb) list 192 * in the environment with "ps -e". But, if you must... 193 */ 194 #ifndef SECURITY_HOLE_PASS_AUTHORIZATION 195 else if (!strcasecmp(hdrs[i].key, "Authorization") 196 || !strcasecmp(hdrs[i].key, "Proxy-Authorization")) { 197 if (conf->cgi_pass_auth == AP_CGI_PASS_AUTH_ON) { 198 add_unless_null(e, http2env(r, hdrs[i].key), hdrs[i].val); 199 } 200 } 201 #endif (gdb) frame 35 #35 0x0000000000454d72 in ap_add_common_vars (r=0x1c15) at /opt/unpack/httpd-2.4.23/server/util_script.c:282 282 if (env_temp) { (gdb) list 277 back = back->prev; 278 } 279 } 280 add_unless_null(e, "AUTH_TYPE", r->ap_auth_type); 281 env_temp = ap_get_remote_logname(r); 282 if (env_temp) { 283 apr_table_addn(e, "REMOTE_IDENT", apr_pstrdup(r->pool, env_temp)); 284 } 285 286 /* Apache custom error responses. If we have redirected set two new vars */ (gdb) frame 36 #36 0x00000000004544c2 in getsfunc_BRIGADE (buf=<optimized out>, len=<optimized out>, arg=<optimized out>) at /opt/unpack/httpd-2.4.23/server/util_script.c:756 756 } (gdb) list 751 apr_bucket_delete(e); 752 e = next; 753 } 754 *dst = 0; 755 return done; 756 } 757 758 AP_DECLARE(int) ap_scan_script_header_err_brigade(request_rec *r, 759 apr_bucket_brigade *bb, 760 char *buffer) (gdb)
I don't see how ap_add_common_vars() can be both in frame 0 and frame 35, this is not a recursive function AFAICT. The APR and APR-util symbols seems to be terribly missing here too, any chance to have those (to fill in the backtrace holes)? Finally, which CGI module and its configuration is involved? No third party one?
(In reply to Yann Ylavic from comment #6) > I don't see how ap_add_common_vars() can be both in frame 0 and frame 35, > this is not a recursive function AFAICT. On maybe an ErrorDocument (w/ ProxyErrorOverride) that gets served by a CGI itself?
Hi, Sorry, I've just build the whole thing again having debugging symbols enabled for apr and apr-utils as well. We will deploy the new build tomorrow and update the bug as soon as the next segfault happens. We have no external CGI modules in place. Here is a dump of all modules we have built: Loaded Modules: core_module (static) so_module (static) http_module (static) mpm_worker_module (static) dumpio_module (shared) log_forensic_module (shared) logio_module (shared) proxy_module (shared) proxy_connect_module (shared) proxy_ftp_module (shared) proxy_http_module (shared) proxy_balancer_module (shared) lbmethod_byrequests_module (shared) lbmethod_bybusyness_module (shared) status_module (shared) rewrite_module (shared) unixd_module (shared) auth_basic_module (shared) authz_host_module (shared) authn_core_module (shared) authz_core_module (shared) ldap_module (shared) authnz_ldap_module (shared) expires_module (shared) headers_module (shared) log_config_module (shared) env_module (shared) slotmem_shm_module (shared)
Hello, Finally we have an coredump with all debugging symbols enabled. Sorry for the long delay. We are not able to reproduce the segfault by now, we have to wait for one. They appear not frequently. Here is the backtrace, if more information is needed don't hesitate to ask. GNU gdb (Debian 7.7.1+dfsg-5) 7.7.1 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>. Find the GDB manual and other documentation resources online at: <http://www.gnu.org/software/gdb/documentation/>. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /opt/loadbalancer/apache/bin/httpd...done. [New LWP 126431] [New LWP 125676] [New LWP 126433] [New LWP 126435] [New LWP 126437] [New LWP 126438] [New LWP 126440] [New LWP 126442] [New LWP 126444] [New LWP 126446] [New LWP 126448] [New LWP 126450] [New LWP 126454] [New LWP 126456] [New LWP 126458] [New LWP 126460] [New LWP 126462] [New LWP 126464] [New LWP 126466] [New LWP 126468] [New LWP 126470] [New LWP 126472] [New LWP 126476] [New LWP 126478] [New LWP 126479] [New LWP 126481] [New LWP 126290] [New LWP 126291] [New LWP 126293] [New LWP 126294] [New LWP 126296] [New LWP 126297] [New LWP 126298] [New LWP 126299] [New LWP 126301] [New LWP 126303] [New LWP 126304] [New LWP 126307] [New LWP 126309] [New LWP 126311] [New LWP 126315] [New LWP 126317] [New LWP 126319] [New LWP 126321] [New LWP 126323] [New LWP 126327] [New LWP 126329] [New LWP 126331] [New LWP 126333] [New LWP 126336] [New LWP 126338] [New LWP 126341] [New LWP 126343] [New LWP 126346] [New LWP 126348] [New LWP 126350] [New LWP 126352] [New LWP 126354] [New LWP 126356] [New LWP 126358] [New LWP 126360] [New LWP 126362] [New LWP 126364] [New LWP 126366] [New LWP 126368] [New LWP 126369] [New LWP 126371] [New LWP 126373] [New LWP 126375] [New LWP 126377] [New LWP 126381] [New LWP 126383] [New LWP 126385] [New LWP 126387] [New LWP 126389] [New LWP 126391] [New LWP 126394] [New LWP 126398] [New LWP 126399] [New LWP 126403] [New LWP 126405] [New LWP 126407] [New LWP 126409] [New LWP 126418] [New LWP 126421] [New LWP 126425] [New LWP 126427] [New LWP 126429] [New LWP 126423] [New LWP 126452] [New LWP 126474] [New LWP 126292] [New LWP 126295] [New LWP 126302] [New LWP 126305] [New LWP 126313] [New LWP 126325] [New LWP 126379] [New LWP 126396] [New LWP 126401] [New LWP 126411] [New LWP 126300] warning: Could not load shared library symbols for 2 libraries, e.g. /lib/snoopy.so. Use the "info sharedlibrary" command to see the complete listing. Do you need "set solib-search-path" or "set sysroot"? [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/opt/loadbalancer/apache/bin/httpd -k start'. Program terminated with signal SIGSEGV, Segmentation fault. #0 send_brigade_nonblocking (s=0x7fe19c0e1f08, bb=0x7fe1fc0e4370, bytes_written=0x5, c=0x7fe1a4017168) at /opt/unpack/httpd-2.4.23/server/core_filters.c:664 664 if (!APR_BUCKET_IS_METADATA(bucket)) { (gdb) info sharedlibrary From To Syms Read Shared Object Library No linux-vdso.so.1 No /lib/snoopy.so 0x00007fe20a1720a0 0x00007fe20a187e60 Yes /opt/loadbalancer/pcre/lib/libpcre.so.1 0x00007fe209f53250 0x00007fe209f6774f Yes /opt/loadbalancer/apr-util/lib/libaprutil-1.so.0 0x00007fe209d24b60 0x00007fe209d3cff9 Yes (*) /lib/x86_64-linux-gnu/libexpat.so.1 0x00007fe209afb9b0 0x00007fe209b15f14 Yes /opt/loadbalancer/apr/lib/libapr-1.so.0 0x00007fe2098e9350 0x00007fe2098ec06c Yes /lib/x86_64-linux-gnu/librt.so.1 0x00007fe2096b0cc0 0x00007fe2096b58b4 Yes /lib/x86_64-linux-gnu/libcrypt.so.1 0x00007fe2094989f0 0x00007fe2094a4771 Yes /lib/x86_64-linux-gnu/libpthread.so.0 0x00007fe20928fed0 0x00007fe20929097e Yes /lib/x86_64-linux-gnu/libdl.so.2 0x00007fe208f034a0 0x00007fe20902f943 Yes /lib/x86_64-linux-gnu/libc.so.6 0x00007fe20a595ae0 0x00007fe20a5ae140 Yes /lib64/ld-linux-x86-64.so.2 0x00007fe2038993b0 0x00007fe20389db4e Yes /lib/x86_64-linux-gnu/libnss_compat.so.2 0x00007fe203684160 0x00007fe20368f693 Yes /lib/x86_64-linux-gnu/libnsl.so.1 0x00007fe2034771a0 0x00007fe20347d1bc Yes /lib/x86_64-linux-gnu/libnss_nis.so.2 0x00007fe20326b2a0 0x00007fe203271ba3 Yes /lib/x86_64-linux-gnu/libnss_files.so.2 0x00007fe20681e0e0 0x00007fe20684d474 Yes (*) /usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2 0x00007fe206603540 0x00007fe20660a4d2 Yes (*) /usr/lib/x86_64-linux-gnu/liblber-2.4.so.2 0x00007fe2063eca90 0x00007fe2063f8a76 Yes /lib/x86_64-linux-gnu/libresolv.so.2 0x00007fe2061d0310 0x00007fe2061e1293 Yes (*) /usr/lib/x86_64-linux-gnu/libsasl2.so.2 0x00007fe205ed4400 0x00007fe205f9093a Yes (*) /usr/lib/x86_64-linux-gnu/libgnutls-deb0.so.28 0x00007fe205c95220 0x00007fe205ca6469 Yes (*) /lib/x86_64-linux-gnu/libz.so.1 0x00007fe205a5a4f0 0x00007fe205a76edc Yes (*) /usr/lib/x86_64-linux-gnu/libp11-kit.so.0 0x00007fe20583bc80 0x00007fe205846a0b Yes (*) /usr/lib/x86_64-linux-gnu/libtasn1.so.6 0x00007fe20560d360 0x00007fe2056295ca Yes (*) /usr/lib/x86_64-linux-gnu/libnettle.so.4 0x00007fe2053deb60 0x00007fe2053e9676 Yes (*) /usr/lib/x86_64-linux-gnu/libhogweed.so.2 0x00007fe205161480 0x00007fe2051bf228 Yes (*) /usr/lib/x86_64-linux-gnu/libgmp.so.10 0x00007fe204f4e870 0x00007fe204f5301c Yes (*) /usr/lib/x86_64-linux-gnu/libffi.so.6 0x00007fe203067d70 0x00007fe2030686a0 Yes /opt/loadbalancer/apr-util/lib/apr-util-1/apr_ldap-1.so 0x00007fe208ce2a20 0x00007fe208ce3389 Yes /opt/loadbalancer/apache/modules/mod_dumpio.so 0x00007fe208ae0db0 0x00007fe208ae15d6 Yes /opt/loadbalancer/apache/modules/mod_log_forensic.so 0x00007fe2088dec60 0x00007fe2088df1f5 Yes /opt/loadbalancer/apache/modules/mod_logio.so 0x00007fe2086c70d0 0x00007fe2086d5bae Yes /opt/loadbalancer/apache/modules/mod_proxy.so 0x00007fe2084be1f0 0x00007fe2084bfcfc Yes /opt/loadbalancer/apache/modules/mod_proxy_connect.so 0x00007fe2082b52f0 0x00007fe2082baddc Yes /opt/loadbalancer/apache/modules/mod_proxy_ftp.so 0x00007fe2080ac6b0 0x00007fe2080b0ff8 Yes /opt/loadbalancer/apache/modules/mod_proxy_http.so 0x00007fe207ea0200 0x00007fe207ea607f Yes /opt/loadbalancer/apache/modules/mod_proxy_balancer.so 0x00007fe207c9c740 0x00007fe207c9cbe8 Yes /opt/loadbalancer/apache/modules/mod_lbmethod_byrequests.so 0x00007fe207a9a740 0x00007fe207a9ac1e Yes /opt/loadbalancer/apache/modules/mod_lbmethod_bybusyness.so 0x00007fe2078961c0 0x00007fe2078982db Yes /opt/loadbalancer/apache/modules/mod_status.so 0x00007fe207688350 0x00007fe207690f53 Yes /opt/loadbalancer/apache/modules/mod_rewrite.so 0x00007fe207482fa0 0x00007fe2074839a8 Yes /opt/loadbalancer/apache/modules/mod_unixd.so 0x00007fe20727fe90 0x00007fe207280e78 Yes /opt/loadbalancer/apache/modules/mod_auth_basic.so 0x00007fe20707cdd0 0x00007fe20707d7cc Yes /opt/loadbalancer/apache/modules/mod_authz_host.so 0x00007fe206e79ed0 0x00007fe206e7a652 Yes /opt/loadbalancer/apache/modules/mod_authn_core.so 0x00007fe206c75570 0x00007fe206c7713a Yes /opt/loadbalancer/apache/modules/mod_authz_core.so 0x00007fe206a64620 0x00007fe206a6d29f Yes /opt/loadbalancer/apache/modules/mod_ldap.so 0x00007fe204d41e00 0x00007fe204d48387 Yes /opt/loadbalancer/apache/modules/mod_authnz_ldap.so 0x00007fe204b3df50 0x00007fe204b3e942 Yes /opt/loadbalancer/apache/modules/mod_expires.so 0x00007fe2049397c0 0x00007fe20493b1ed Yes /opt/loadbalancer/apache/modules/mod_headers.so 0x00007fe204732d40 0x00007fe20473589b Yes /opt/loadbalancer/apache/modules/mod_log_config.so 0x00007fe20452faf0 0x00007fe20452fe6b Yes /opt/loadbalancer/apache/modules/mod_env.so 0x00007fe20432c080 0x00007fe20432d7e6 Yes /opt/loadbalancer/apache/modules/mod_slotmem_shm.so 0x00007fe203ff2870 0x00007fe2040d501e Yes (*) /usr/lib/x86_64-linux-gnu/libxml2.so.2 0x00007fe203da3850 0x00007fe203db8082 Yes (*) /lib/x86_64-linux-gnu/liblzma.so.5 0x00007fe203aa5580 0x00007fe203b10d96 Yes /lib/x86_64-linux-gnu/libm.so.6 No /lib/x86_64-linux-gnu/libnss_sss.so.2 0x00007fe200232ab0 0x00007fe2002429a5 Yes (*) /lib/x86_64-linux-gnu/libgcc_s.so.1 0x00007fe20002b100 0x00007fe20002dff0 Yes /lib/x86_64-linux-gnu/libnss_dns.so.2 (*): Shared library is missing debugging information. (gdb) bt full #0 send_brigade_nonblocking (s=0x7fe19c0e1f08, bb=0x7fe1fc0e4370, bytes_written=0x5, c=0x7fe1a4017168) at /opt/unpack/httpd-2.4.23/server/core_filters.c:664 vec = {{iov_base = 0x7fe1c00c8ef8, iov_len = 7482}, {iov_base = 0x7fe1c00caf08, iov_len = 710}, { iov_base = 0x47c2ca, iov_len = 2}, {iov_base = 0x7fe19c0e2d68, iov_len = 5}, { iov_base = 0x7fe194107208, iov_len = 8000}, {iov_base = 0x7fe19c0e18c8, iov_len = 110}, { iov_base = 0x47c2ca, iov_len = 2}, {iov_base = 0x7fe19c0e2548, iov_len = 6}, { iov_base = 0x7fe1c00c0eb8, iov_len = 7882}, {iov_base = 0x7fe1a4006938, iov_len = 310}, { iov_base = 0x47c2ca, iov_len = 2}, {iov_base = 0x7fe19c0e2a48, iov_len = 6}, { iov_base = 0x7fe1c00bae88, iov_len = 7682}, {iov_base = 0x7fe1940f7188, iov_len = 510}, { iov_base = 0x47c2ca, iov_len = 2}, {iov_base = 0x7fe19c0e24a8, iov_len = 6}} nvec = 4 #1 0x00000000004491e1 in send_brigade_blocking (c=<optimized out>, bytes_written=<optimized out>, bb=<optimized out>, s=<optimized out>) at /opt/unpack/httpd-2.4.23/server/core_filters.c:733 No locals. #2 ap_core_output_filter (f=0x7fe19c0e1f08, new_bb=0x7fe1fc0e4370) at /opt/unpack/httpd-2.4.23/server/core_filters.c:542 c = 0x7fe1fc0e3dd8 bytes_in_brigade = 0 non_file_bytes_in_brigade = 140606962540296 eor_buckets_in_brigade = 5 morphing_bucket_in_brigade = -662995824 #3 0x00000000004626b1 in ap_process_request (r=0x7fe194101230) at /opt/unpack/httpd-2.4.23/modules/http/http_request.c:451 bb = 0x7fe1fc0e4370 c = 0x7fe1fc0e3dd8 rv = 48 #4 0x000000000045e9b5 in ap_process_http_sync_connection (c=0x7fe1fc0e3dd8) at /opt/unpack/httpd-2.4.23/modules/http/http_core.c:210 keep_alive_timeout = 5000000 r = 0x7fe194101230 cs = 0x0 csd = 0x0 mpm_state = 0 #5 ap_process_http_connection (c=0x7fe1fc0e3dd8) at /opt/unpack/httpd-2.4.23/modules/http/http_core.c:251 No locals. #6 0x0000000000456d30 in ap_run_process_connection (c=0x7fe1fc0e3dd8) at /opt/unpack/httpd-2.4.23/server/connection.c:42 pHook = <optimized out> n = 0 rv = -1 #7 0x0000000000468f81 in process_socket (bucket_alloc=<optimized out>, my_thread_num=<optimized out>, my_child_num=<optimized out>, sock=<optimized out>, p=<optimized out>, thd=<optimized out>) at /opt/unpack/httpd-2.4.23/server/mpm/worker/worker.c:631 current_conn = 0x7fe1fc0e3dd8 conn_id = 140608573160920 sbh = 0x7fe1fc0e3dd0 #8 worker_thread (thd=0x7fe19c0e1f08, dummy=0x7fe1d87b7c90) at /opt/unpack/httpd-2.4.23/server/mpm/worker/worker.c:992 process_slot = 19 thread_slot = 74 csd = 0x7fe1fc0e3bc0 bucket_alloc = 0x9 last_ptrans = 0x7fe1fc0e3dd8 ptrans = 0x7fe1fc0e3b38 is_idle = -66176064 #9 0x00007fe20949b0a4 in start_thread (arg=0x7fe1d87b8700) at pthread_create.c:309 __res = <optimized out> pd = 0x7fe1d87b8700 now = <optimized out> unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140607976343296, 7960264032231149677, 0, 140608815194208, 152, 140607976343296, -7945694442286690195, -7945304783828627347}, mask_was_saved = 0}}, priv = {pad = { 0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} not_first_call = <optimized out> pagesize_m1 = <optimized out> sp = <optimized out> freesize = <optimized out> __PRETTY_FUNCTION__ = "start_thread" #10 0x00007fe208fcc62d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 No locals. (gdb) best regards, Stefan
(In reply to amd1212 from comment #9) > > Here is the backtrace, if more information is needed don't hesitate to ask. Could you please attach the output of: (gdb) thread apply all bt full ? Thanks!
Hello, Here is the requested output (I've replaced some chars with an equal amount of X'es to obscure some internal data). The comment was to big so I've pasted everything here: http://pastebin.com/X67ptV8R best, Stefan
Created attachment 34566 [details] gdb output As attachment. Thread 1 seems to indeed be the culprit, investigating...
(In reply to Yann Ylavic from comment #12) > > Thread 1 seems to indeed be the culprit, investigating... If you have or could install the .gdbinit provided with httpd source in your path, could you please show (in Thread 1, frame 0) us the output of: (gdb) dump_brigade bb (gdb) dump_bucket bucket (gdb) dump_bucket next (gdb) p *bytes_written Thanks.
Sure thing, only dump_brigade seems to be working: (gdb) thread 1 [Switching to thread 1 (Thread 0x7fe1d87b8700 (LWP 126431))] #0 send_brigade_nonblocking (s=0x7fe19c0e1f08, bb=0x7fe1fc0e4370, bytes_written=0x5, c=0x7fe1a4017168) at /opt/payon/unpack/httpd-2.4.23/server/core_filters.c:664 664 if (!APR_BUCKET_IS_METADATA(bucket)) { (gdb) frame 0 #0 send_brigade_nonblocking (s=0x7fe19c0e1f08, bb=0x7fe1fc0e4370, bytes_written=0x5, c=0x7fe1a4017168) at /opt/payon/unpack/httpd-2.4.23/server/core_filters.c:664 664 if (!APR_BUCKET_IS_METADATA(bucket)) { (gdb) dump_brigade bb dump of brigade 0x7fe1fc0e4370 | type (address) | length | data addr | contents | rc -------------------------------------------------------------------------------- 0 | HEAP (0x7fe19c0e2c28) | 7482 | 0x7fe19c0e27c8 | [44c0d88e5014c13c1...] | 1 1 | HEAP (0x7fe19c0e2228) | 710 | 0x7fe19c0e2908 | [customerId="BUM00...] | 1 2 | IMMORTAL (0x7fe19c0e25e8) | 2 | 0x0047c2ca | [~~] | n/a 3 | HEAP (0x7fe19c0e1f08) | 5 | 0x7fe1a4017168 | [42c~~] | 1 4 | Cannot access memory at address 0x2e512866693b4343 (gdb) dump_bucket bucket value has been optimized out (gdb) dump_bucket next value has been optimized out (gdb) p *bytes_written Cannot access memory at address 0x5 (gdb)
Created attachment 34634 [details] A folder with different core dumps Hello, we update ours httpd to the version 2.4.25 because we hope that resolve the segfault problem. Yesterday we have an segfault again. regards, Claudio
Created attachment 34640 [details] A folder with different core dumps Hi, We just had a segfault again. regards, Claudio
Hi, I did take a closer look at 20 core dumps collected in the meantime with the following observations. The segfault always occurs with one of three backtraces[1][2][3]. Another thing I noticed is that there is always one thread busy in a close() syscall at the time of the crash with a consistent backtrace[4]. Maybe someone that is more familar with the code can tell whether this is suspicious or something to be expected? Any other hints on further debugging appreciated. regards, Andreas Footnotes: [1] --8<---------------cut here---------------start------------->8--- #0 0x0000000000000000 in ?? () #1 0x000000000044a576 in remove_empty_buckets (bb=0x7f81a40ea5f0) at /opt/unpack/httpd-2.4.25/server/core_filters.c:720 apr_bucket_delete(bucket); #2 0x000000000044a852 in send_brigade_nonblocking (s=0x0, bb=0x7f81a40ea5f0, bytes_written=0x7f81480cb620, c=0x7f814c0cc9e8) at /opt/unpack/httpd-2.4.25/server/core_filters.c:625 #3 0x000000000044b5c1 in send_brigade_blocking (c=<optimized out>, bytes_written=<optimized out>, bb=<optimized out>, s=<optimized out>) at /opt/unpack/httpd-2.4.25/server/core_filters.c:733 #4 ap_core_output_filter (f=0x0, new_bb=0x7f81a40ea5f0) at /opt/unpack/httpd-2.4.25/server/core_filters.c:542 #5 0x0000000000464c31 in ap_process_request (r=0x7f8148056c40) at /opt/unpack/httpd-2.4.25/modules/http/http_request.c:477 #6 0x0000000000460df5 in ap_process_http_sync_connection (c=0x7f81a40ea058) at /opt/unpack/httpd-2.4.25/modules/http/http_core.c:210 #7 ap_process_http_connection (c=0x7f81a40ea058) at /opt/unpack/httpd-2.4.25/modules/http/http_core.c:251 #8 0x0000000000459100 in ap_run_process_connection (c=0x7f81a40ea058) at /opt/unpack/httpd-2.4.25/server/connection.c:42 #9 0x000000000046b7c1 in process_socket (bucket_alloc=<optimized out>, my_thread_num=<optimized out>, my_child_num=<optimized out>, sock=<optimized out>, p=<optimized out>, thd=<optimized out>) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:631 #10 worker_thread (thd=0x0, dummy=0x7f81a40ea5f0) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:992 #11 0x00007f81afe500a4 in start_thread (arg=0x7f81897ca700) at pthread_create.c:403 #12 0x00007f81af98162d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 --8<---------------cut here---------------end--------------->8--- [2] --8<---------------cut here---------------start------------->8--- #0 0x00007f81afe52274 in __GI___pthread_mutex_lock (mutex=0x0) at ../nptl/pthread_mutex_lock.c:79 #1 0x00007f81b04bea69 in apr_thread_mutex_lock (mutex=<optimized out>) at /opt/loadbalancer/../unpack/apr-1.5.2/locks/unix/thread_mutex.c:92 #2 0x00007f81b04bf538 in allocator_free (node=0x7f813c0eb050, allocator=0x1bab00e) at /opt/loadbalancer/../unpack/apr-1.5.2/memory/unix/apr_pools.c:370 #3 apr_allocator_free (allocator=0x1bab00e, node=0x7f813c0eb050) at /opt/loadbalancer/../unpack/apr-1.5.2/memory/unix/apr_pools.c:444 #4 0x00007f81b0909d57 in heap_bucket_destroy (data=0x7f81680aaf48) at /opt/loadbalancer/../unpack/apr-util-1.5.4/buckets/apr_buckets_heap.c:36 #5 0x000000000044ada1 in ap_core_input_filter (f=0x7f81a41949d8, b=0x7f813c013190, mode=AP_MODE_GETLINE, block=APR_NONBLOCK_READ, readbytes=0) at /opt/unpack/httpd-2.4.25/server/core_filters.c:132 #6 0x00007f81af292fce in logio_in_filter (f=<optimized out>, bb=0x7f813c013190, mode=<optimized out>, block=<optimized out>, readbytes=<optimized out>) at /opt/unpack/httpd-2.4.25/modules/loggers/mod_logio.c:165 #7 0x0000000000465e9c in ap_http_filter (f=0x7f81500a68b8, b=0x7f813c013190, mode=1745531016, block=(unknown: 2753121024), readbytes=8192) at /opt/unpack/httpd-2.4.25/modules/http/http_filters.c:515 #8 0x00007f81aea61044 in ap_proxy_http_process_response (p=0x1bab1800008, r=0x7f813c090e40, backend_ptr=0x7f81680ab088, server_portstr=0x7f818e7d3c60 "", conf=<optimized out>, conf=<optimized out>, worker=<optimized out>) at /opt/unpack/httpd-2.4.25/modules/proxy/mod_proxy_http.c:1673 #9 0x00007f81aea629a9 in proxy_http_handler (r=0x7f813c090e40, worker=0x1c59560, conf=0x7f81680ab088, url=0x7f813c090dc8 "...", proxyname=0x0, proxyport=31904) at /opt/unpack/httpd-2.4.25/modules/proxy/mod_proxy_http.c:1986 #10 0x00007f81af07f6c3 in proxy_run_scheme_handler (r=0x7f813c090e40, worker=0x1c59560, conf=0x1c1c190, url=0x7f813c007b00 "..."..., proxyhost=0x0, proxyport=0) at /opt/unpack/httpd-2.4.25/modules/proxy/mod_proxy.c:2880 #11 0x00007f81af080631 in proxy_handler (r=0x1bab1800008) at /opt/unpack/httpd-2.4.25/modules/proxy/mod_proxy.c:1230 #12 0x000000000044fbb0 in ap_run_handler (r=r@entry=0x7f813c090e40) at /opt/unpack/httpd-2.4.25/server/config.c:170 #13 0x00000000004500f9 in ap_invoke_handler (r=0x7f813c090e40) at /opt/unpack/httpd-2.4.25/server/config.c:434 #14 0x0000000000464a13 in ap_process_async_request (r=0x7f813c090e40) at /opt/unpack/httpd-2.4.25/modules/http/http_request.c:436 #15 0x0000000000464bb0 in ap_process_request (r=0x7f813c090e40) at /opt/unpack/httpd-2.4.25/modules/http/http_request.c:471 #16 0x0000000000460df5 in ap_process_http_sync_connection (c=0x7f816c015518) at /opt/unpack/httpd-2.4.25/modules/http/http_core.c:210 #17 ap_process_http_connection (c=0x7f816c015518) at /opt/unpack/httpd-2.4.25/modules/http/http_core.c:251 #18 0x0000000000459100 in ap_run_process_connection (c=0x7f816c015518) at /opt/unpack/httpd-2.4.25/server/connection.c:42 #19 0x000000000046b7c1 in process_socket (bucket_alloc=<optimized out>, my_thread_num=<optimized out>, my_child_num=<optimized out>, sock=<optimized out>, p=<optimized out>, thd=<optimized out>) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:631 #20 worker_thread (thd=0x1bab1800008, dummy=0x7f813c0eb050) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:992 #21 0x00007f81afe500a4 in start_thread (arg=0x7f818e7d4700) at pthread_create.c:403 #22 0x00007f81af98162d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 --8<---------------cut here---------------end--------------->8--- [3] --8<---------------cut here---------------start------------->8--- Program terminated with signal SIGSEGV, Segmentation fault. 664 in /opt/unpack/httpd-2.4.25/server/core_filters.c if (!APR_BUCKET_IS_METADATA(bucket)) { #0 send_brigade_nonblocking (s=0x7f81540d77e8, bb=0x7f81a4100d40, bytes_written=0x4, c=0x7f81540d7248) at /opt/unpack/httpd-2.4.25/server/core_filters.c:664 #1 0x000000000044b5c1 in send_brigade_blocking (c=<optimized out>, bytes_written=<optimized out>, bb=<optimized out>, s=<optimized out>) at /opt/unpack/httpd-2.4.25/server/core_filters.c:733 #2 ap_core_output_filter (f=0x7f81540d77e8, new_bb=0x7f81a4100d40) at /opt/unpack/httpd-2.4.25/server/core_filters.c:542 #3 0x0000000000464c31 in ap_process_request (r=0x7f81500bbf20) at /opt/unpack/httpd-2.4.25/modules/http/http_request.c:477 #4 0x0000000000460df5 in ap_process_http_sync_connection (c=0x7f81a41007a8) at /opt/unpack/httpd-2.4.25/modules/http/http_core.c:210 #5 ap_process_http_connection (c=0x7f81a41007a8) at /opt/unpack/httpd-2.4.25/modules/http/http_core.c:251 #6 0x0000000000459100 in ap_run_process_connection (c=0x7f81a41007a8) at /opt/unpack/httpd-2.4.25/server/connection.c:42 #7 0x000000000046b7c1 in process_socket (bucket_alloc=<optimized out>, my_thread_num=<optimized out>, my_child_num=<optimized out>, sock=<optimized out>, p=<optimized out>, thd=<optimized out>) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:631 #8 worker_thread (thd=0x7f81540d77e8, dummy=0x7f8193fdec90) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:992 #9 0x00007f81afe500a4 in start_thread (arg=0x7f8193fdf700) at pthread_create.c:403 #10 0x00007f81af98162d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 --8<---------------cut here---------------end--------------->8--- [4] --8<---------------cut here---------------start------------->8--- #0 0x00007f81afe56add in close () at ../sysdeps/unix/syscall-template.S:81 #1 0x000000000045c324 in ap_mpm_podx_check (pod=<optimized out>) at /opt/unpack/httpd-2.4.25/server/mpm_unix.c:546 #2 0x000000000042bb8d in child_main (child_num_arg=1, child_bucket=29984256) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:1364 #3 0x000000000046c852 in make_child (s=0x1bd85e8, slot=1, bucket=0) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:1456 #4 0x000000000046d185 in perform_idle_server_maintenance (num_buckets=<optimized out>, child_bucket=<optimized out>) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:1672 #5 server_main_loop (num_buckets=<optimized out>, remaining_children_to_start=<optimized out>) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:1805 #6 worker_run (_pconf=0x4, plog=0x1, s=0x0) at /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:1888 #7 0x00000000004336ae in ap_run_mpm (pconf=0x1bb1138, plog=0x1bde378, s=0x1bd85e8) at /opt/unpack/httpd-2.4.25/server/mpm_common.c:94 #8 0x000000000042c5f4 in main (argc=3, argv=0x7ffed664e9f8) at /opt/unpack/httpd-2.4.25/server/main.c:783 --8<---------------cut here---------------end--------------->8---
> --8<---------------cut here---------------start------------->8--- > #0 0x00007f81afe56add in close () at ../sysdeps/unix/syscall-template.S:81 > #1 0x000000000045c324 in ap_mpm_podx_check (pod=<optimized out>) at > /opt/unpack/httpd-2.4.25/server/mpm_unix.c:546 > #2 0x000000000042bb8d in child_main (child_num_arg=1, > child_bucket=29984256) at > /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:1364 > #3 0x000000000046c852 in make_child (s=0x1bd85e8, slot=1, bucket=0) at > /opt/unpack/httpd-2.4.25/server/mpm/worker/worker.c:1456 Unfortunately I think this is no (direct) clue and the debugger is somehow confused as only read() is called in this path. It is a dedicated thread in worker that waits for the parent to tell the worker it's time to exit (due to e.g. MaxSpareThreads).
Created attachment 37288 [details] Patch to add unix: uris to be recognized as absolute paths
Comment on attachment 37288 [details] Patch to add unix: uris to be recognized as absolute paths Um, something weird happened with the bugzilla UI. This was supposed to be attached to bug 57691. Sorry.