Bug 59947 - Crash under mod_cache_socache in run_cleanups
Summary: Crash under mod_cache_socache in run_cleanups
Status: NEW
Alias: None
Product: Apache httpd-2
Classification: Unclassified
Component: mod_cache (show other bugs)
Version: 2.4.23
Hardware: PC FreeBSD
: P2 normal (vote)
Target Milestone: ---
Assignee: Apache HTTPD Bugs Mailing List
URL:
Keywords:
Depends on:
Blocks:
 
Reported: 2016-08-05 06:16 UTC by Alexander Strange
Modified: 2016-11-21 20:35 UTC (History)
0 users



Attachments

Note You need to log in before you can comment on or make changes to this bug.
Description Alexander Strange 2016-08-05 06:16:13 UTC
Found this crash on a web server.

# httpd -V
Server version: Apache/2.4.23 (Unix)
Server built:   Aug  4 2016 02:49:24
Server's Module Magic Number: 20120211:61
Server loaded:  APR 1.5.2, APR-UTIL 1.5.4
Compiled using: APR 1.5.2, APR-UTIL 1.5.4
Architecture:   64-bit
Server MPM:     event
  threaded:     yes (fixed thread count)
    forked:     yes (variable process count)
Server compiled with....
 -D APR_HAS_SENDFILE
 -D APR_HAS_MMAP
 -D APR_HAVE_IPV6 (IPv4-mapped addresses disabled)
 -D APR_USE_FLOCK_SERIALIZE
 -D APR_USE_PTHREAD_SERIALIZE
 -D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
 -D APR_HAS_OTHER_CHILD
 -D AP_HAVE_RELIABLE_PIPED_LOGS
 -D BUFFERED_LOGS
 -D PIPE_BUF=512
 -D DYNAMIC_MODULE_LIMIT=256


(gdb) bt full
#0  0x0000000801c653b4 in run_cleanups (cref=0x70) at memory/unix/apr_pools.c:2348
        c = 0x802dbe238
#1  0x0000000801c642a5 in apr_pool_destroy (pool=0x0) at memory/unix/apr_pools.c:804
        active = 0x4a3d8b <invalidate_entity+155>
        allocator = 0x80cc2e550
#2  0x00000000004a3e5f in commit_entity (r=0x80cc25bb0, h=<optimized out>) at mod_cache_socache.c:1126
        conf = 0x802db28e8
        obj = 0x80cc2e380
        sobj = 0x80cc2e3e8
        rv = 0
#3  invalidate_entity (h=<optimized out>, r=0x80cc25bb0) at mod_cache_socache.c:1165
No locals.
#4  0x0000000000498fb7 in cache_invalidate (cache=cache@entry=0x80cc25828, r=r@entry=0x80cc25bb0) at cache_storage.c:748
        list = 0x80cc25810
        rv = <optimized out>
        status = <optimized out>
        h = 0x80cc2e368
        location_uri = {scheme = 0x7ffff43a1930 "p\031:\364\377\177", 
          hostinfo = 0x8017e6bbd <apr_bucket_heap_make+253> "H\211E\350H\213E\350H\213\025\224]\"", user = 0x43c170 <apr_bucket_free@plt> "\377%\252\276=", 
          password = 0x1f40 <error: Cannot access memory at address 0x1f40>, 
          hostname = 0x80cc2f048 "HTTP/1.1 200 OK\r\nDate: Fri, 05 Aug 2016 03:05:15 GMT\r\nServer: Apache\r\nCache-Control: public, max-age=120, s-maxage=120\r\nVary: Cookie,Accept-Encoding\r\nContent-Encoding: gzip\r\nX-Frame-Options: SAMEORIGI"..., port_str = 0x80cc1e4c8 "\320\340\302\f\b", 
          path = 0x80cc1e548 "\240", query = 0x80cc1e568 "\001", fragment = 0x7ffff43a1970 " \032:\364\377\177", 
          hostent = 0x8017e6c4d <apr_bucket_heap_create+119>, port = 57384, is_initialized = 1, dns_looked_up = 0, dns_resolved = 0}
        content_location_uri = {scheme = 0x1f40 <error: Cannot access memory at address 0x1f40>, 
          hostinfo = 0x80cc2f048 "HTTP/1.1 200 OK\r\nDate: Fri, 05 Aug 2016 03:05:15 GMT\r\nServer: Apache\r\nCache-Control: public, max-age=120, s-maxage=120\r\nVary: Cookie,Accept-Encoding\r\nContent-Encoding: gzip\r\nX-Frame-Options: SAMEORIGI"..., user = 0x80cc2dbf0 "text/html; charset=UTF-8", 
          password = 0x80cc1e4c8 "\320\340\302\f\b", hostname = 0x7ffff43a1a20 "\310\340\302\f\b", 
          port_str = 0x8017e5880 <apr_brigade_writev+805> "H\213E\330H\001E\250H\203E\240\001H\213E\240H;\205h\377\377\377r\236H\213E\270H\213P\030H\213E\230H\001\302H\213E\270H\211P\030\270", path = 0x1000266eafc <error: Cannot access memory at address 0x1000266eafc>, 
          query = 0x20 <error: Cannot access memory at address 0x20>, fragment = 0x80cc2e128 "\250\333\302\f\b", hostent = 0x0, port = 0, 
          is_initialized = 0, dns_looked_up = 0, dns_resolved = 0}
        location = <optimized out>
        location_key = 0x0
        content_location = <optimized out>
        content_location_key = 0x0
---Type <return> to continue, or q <return> to quit---
#5  0x0000000000493d9b in cache_invalidate_filter (f=0x80cc25930, in=0x80cc2e0c8) at mod_cache.c:1682
        r = 0x80cc25bb0
        cache = 0x80cc25828
#6  0x00000000004cd9b3 in ap_http_header_filter (f=<optimized out>, b=0x80cc2dd48) at http_filters.c:1354
        r = 0x80cc25bb0
        c = <optimized out>
        clheader = <optimized out>
        protocol = 0x5c0ae5 "HTTP/1.1"
        e = <optimized out>
        b2 = 0x80cc2e0c8
        h = {pool = 0x80cc20028, bb = 0x80cc2e0c8}
        ctx = 0x0
        ctype = <optimized out>
        eb = <optimized out>
#7  0x0000000000454528 in ap_content_length_filter (f=0x80cc21710, b=0x80cc2dd48) at protocol.c:1443
        r = 0x80cc25bb0
        ctx = 0x80cc2de00
        e = 0x80cc1e248
        eos = <optimized out>
        eblock = <optimized out>
#8  0x00000000004cfceb in ap_byterange_filter (f=0x80cc216e8, bb=<optimized out>) at byterange_filter.c:494
        r = 0x80cc25bb0
        c = 0x80b0e0338
        e = <optimized out>
        bsend = <optimized out>
        tmpbb = <optimized out>
        range_start = <optimized out>
        range_end = <optimized out>
        clength = <optimized out>
        rv = <optimized out>
        found = 0
        bound_head = 0x0
        indexes = <optimized out>
        idx = <optimized out>
        i = <optimized out>
        original_status = <optimized out>
---Type <return> to continue, or q <return> to quit---
        max_ranges = <optimized out>
        max_overlaps = <optimized out>
        max_reversals = 20
        overlaps = <optimized out>
        reversals = <optimized out>
        core_conf = <optimized out>
#9  0x000000000050a957 in session_output_filter (f=0x80cc2a160, in=0x80cc2dd48) at mod_session.c:478
        r = 0x0
#10 0x00000000004c2c97 in deflate_out_filter (f=0x80cc2a138, bb=<optimized out>) at mod_deflate.c:893
        buf = <optimized out>
        b = <optimized out>
        e = 0x80cc1e248
        r = 0x80cc25bb0
        ctx = 0x80cc2dc60
        zRC = <optimized out>
        len = 0
        blen = 34359738368
        data = 0x20 <error: Cannot access memory at address 0x20>
        c = 0x802c99738
#11 0x00000000004b9c1d in filter_harness (f=0x80cc2a138, bb=0x80cc2dda8) at mod_filter.c:323
        ret = <optimized out>
        cachecontrol = <optimized out>
        ctx = 0x80cc2a1c8
        filter = <optimized out>
#12 0x0000000000574196 in action_handler (r=0x80cc200a0) at mod_actions.c:205
        conf = <optimized out>
        t = <optimized out>
        action = <optimized out>
        script = 0x802daf741
        i = <optimized out>
#13 0x000000000046e60a in ap_run_handler (r=0x80cc200a0) at config.c:170
        pHook = <optimized out>
        n = 15
        rv = -1
#14 ap_invoke_handler (r=r@entry=0x80cc200a0) at config.c:434
        handler = <optimized out>
---Type <return> to continue, or q <return> to quit---
        p = <optimized out>
        result = <optimized out>
        old_handler = 0x802daf240 "php-fcgi"
        ignore = <optimized out>
#15 0x00000000004ca8db in ap_process_async_request (r=r@entry=0x80cc200a0) at http_request.c:410
        c = 0x80b0e0338
        access_status = 0
#16 0x00000000004c6181 in ap_process_http_async_connection (c=0x80b0e0338) at http_core.c:154
        r = 0x80cc200a0
        cs = 0x80b0e0310
#17 ap_process_http_connection (c=0x80b0e0338) at http_core.c:248
No locals.
#18 0x0000000000478e6a in ap_run_process_connection (c=c@entry=0x80b0e0338) at connection.c:42
        pHook = <optimized out>
        n = 4
        rv = -1
#19 0x0000000000583b40 in process_socket (my_thread_num=42, my_child_num=2, cs=0x80b0e02a8, sock=0x80b0e00a0, p=0x80b0e0028, thd=0x80b0119d8) at event.c:1102
        c = 0x80b0e0338
        sbh = 0x80b0e0290
        conn_id = <optimized out>
        rc = <optimized out>
#20 worker_thread (thd=0x80b0119d8, dummy=<optimized out>) at event.c:1963
        ti = <optimized out>
        process_slot = 2
        thread_slot = 42
        csd = 0x80b0e00a0
        cs = 0x0
        ptrans = 0x80b0e0028
        rv = <optimized out>
        is_idle = 0
        te = 0x0
#21 0x0000000801c73125 in dummy_worker (opaque=0x80b0119d8) at threadproc/unix/thread.c:142
        thread = 0x80b0119d8
#22 0x00000008020a8585 in thread_start (curthread=0x80b020c00) at /usr/src/lib/libthr/thread/thr_create.c:284
        set = {__bits = {0, 0, 0, 0}}
#23 0x0000000000000000 in ?? ()
---Type <return> to continue, or q <return> to quit---
No symbol table info available.
Backtrace stopped: Cannot access memory at address 0x7ffff43a2000
(gdb)
Comment 1 Alexander Strange 2016-08-08 05:27:54 UTC
The server is using cache/socache/proxy_fcgi/event mpm.

New backtrace:

(gdb) info locals
c = 0x802dbe238
(gdb) info registers
rax            0x70	112
rbx            0x81081bbd8	34636676056
rcx            0x19319	103193
rdx            0x0	0
rsi            0x16	22
rdi            0x70	112
rbp            0x7ffff43a1850	0x7ffff43a1850
rsp            0x7ffff43a1830	0x7ffff43a1830
r8             0x0	0
r9             0xfffffe0000444010	-2199018782704
r10            0xb5c1f	744479
r11            0x202	514
r12            0x810824968	34636712296
r13            0x81081bb70	34636675952
r14            0x802db28e8	34407655656
r15            0x0	0
rip            0x801c63372	0x801c63372 <run_cleanups+16>
eflags         0x10206	[ PF IF RF ]
cs             0x43	67
ss             0x3b	59
ds             <unavailable>
es             <unavailable>
fs             <unavailable>
gs             <unavailable>
(gdb) print *c
$1 = {next = 0x802c21028, data = 0x802c1c330, plain_cleanup_fn = 0x802dbe218, child_cleanup_fn = 0x802dbe238}
(gdb) x/i $pc
=> 0x801c63372 <run_cleanups+16>:	mov    (%rax),%rax
(gdb) bt full
#0  0x0000000801c63372 in run_cleanups (cref=0x70) at memory/unix/apr_pools.c:2348
        c = 0x802dbe238
#1  0x0000000801c6225d in apr_pool_destroy (pool=0x0) at memory/unix/apr_pools.c:804
        active = 0x4a2bae <invalidate_entity+158>
        allocator = 0x81081bd40
#2  0x00000000004a2c7c in commit_entity (r=0x810824968, h=<optimized out>) at mod_cache_socache.c:1155
        conf = <optimized out>
        obj = <optimized out>
        sobj = <optimized out>
        rv = <optimized out>
#3  invalidate_entity (h=<optimized out>, r=0x810824968) at mod_cache_socache.c:1165
No locals.
#4  0x0000000000498503 in cache_invalidate (cache=cache@entry=0x8108245e0, r=r@entry=0x810824968) at cache_storage.c:748
        list = 0x8108245c8
        rv = <optimized out>
        status = <optimized out>
        h = 0x81081bb58
        location_uri = {scheme = 0x7ffff43a1980 "\260\031:\364\377\177", 
          hostinfo = 0x801c7194f <explode_time+274> "\211\302H\213E\250\211P(\220\311\303UH\211\345H\203\354 H\211}\370H\211u\360\211U\354\213U\354H\213u\360H\213E\370\271", user = 0x43c170 <apr_bucket_free@plt> "\377%\252\233=", password = 0x0, 
          hostname = 0x5398268793c1c <error: Cannot access memory at address 0x5398268793c1c>, port_str = 0x7ffff43a19e0 "\340E\202\020\b", 
          path = 0x1f60 <error: Cannot access memory at address 0x1f60>, query = 0x57a7af28 <error: Cannot access memory at address 0x57a7af28>, 
          fragment = 0x3b00000004 <error: Cannot access memory at address 0x3b00000004>, hostent = 0x700000015, port = 7, is_initialized = 0, 
          dns_looked_up = 0, dns_resolved = 0}
        content_location_uri = {scheme = 0x0, hostinfo = 0x0, user = 0x802668710 "UTC", password = 0x81080e4c8 "\300\270\201\020\b", 
          hostname = 0x7ffff43a19b0 "", 
          port_str = 0x8017e388b <apr_brigade_writev+805> "H\213E\330H\001E\250H\203E\240\001H\213E\240H;\205h\377\377\377r\236H\213E\270H\213P\030H\213E\230H\001\302H\213E\270H\211P\030\270", path = 0x1000266cafc <error: Cannot access memory at address 0x1000266cafc>, 
          query = 0x20 <error: Cannot access memory at address 0x20>, fragment = 0x81081b918 "\230\263\201\020\b", hostent = 0x0, port = 0, 
          is_initialized = 0, dns_looked_up = 0, dns_resolved = 0}
        location = <optimized out>
        location_key = 0x0
        content_location = <optimized out>
        content_location_key = 0x0
#5  0x0000000000493c7a in cache_invalidate_filter (f=0x8108246e8, in=0x81081b8b8) at mod_cache.c:1682
        r = 0x810824968
---Type <return> to continue, or q <return> to quit---
        cache = 0x8108245e0
#6  0x00000000004cc857 in ap_http_header_filter (f=<optimized out>, b=0x81081b538) at http_filters.c:1354
        r = 0x810824968
        c = <optimized out>
        clheader = <optimized out>
        protocol = 0x5bf58d "HTTP/1.1"
        e = <optimized out>
        b2 = <optimized out>
        h = {pool = 0x810816028, bb = 0x81081b8b8}
        ctx = 0x0
        ctype = <optimized out>
        eb = <optimized out>
#7  0x0000000000454248 in ap_content_length_filter (f=0x810817710, b=0x81081b538) at protocol.c:1443
        r = 0x810824968
        ctx = 0x81081b5f0
        e = 0x81080e1a8
        eos = <optimized out>
        eblock = <optimized out>
#8  0x00000000004ce936 in ap_byterange_filter (f=0x8108176e8, bb=0x81081b538) at byterange_filter.c:494
        r = 0x810824968
        c = <optimized out>
        e = <optimized out>
        bsend = <optimized out>
        tmpbb = <optimized out>
        range_start = <optimized out>
        range_end = <optimized out>
        clength = <optimized out>
        rv = <optimized out>
        found = 0
        bound_head = 0x0
        indexes = <optimized out>
        idx = <optimized out>
        i = <optimized out>
        original_status = <optimized out>
        max_ranges = <optimized out>
        max_overlaps = <optimized out>
---Type <return> to continue, or q <return> to quit---
        max_reversals = <optimized out>
        overlaps = <optimized out>
        reversals = <optimized out>
        core_conf = <optimized out>
#9  0x0000000000508d04 in session_output_filter (f=0x810810e08, in=0x81081b538) at mod_session.c:478
        r = 0x0
#10 0x00000000004c21ab in deflate_out_filter (f=0x810810de0, bb=<optimized out>) at mod_deflate.c:893
        buf = <optimized out>
        b = <optimized out>
        e = 0x81080e1a8
        r = 0x810824968
        ctx = 0x81081b450
        zRC = <optimized out>
        len = 0
        blen = 5025667
        data = 0x0
        c = 0x802c99738
#11 0x00000000004b8c55 in filter_harness (f=0x810810de0, bb=0x81081b598) at mod_filter.c:323
        ret = <optimized out>
        cachecontrol = <optimized out>
        ctx = 0x810810e70
        filter = <optimized out>
#12 0x0000000000572f52 in action_handler (r=0x8108160a0) at mod_actions.c:205
        conf = <optimized out>
        t = <optimized out>
        action = <optimized out>
        script = 0x802daf741 "/php-www"
        i = <optimized out>
#13 0x000000000046d9d9 in ap_run_handler (r=0x8108160a0) at config.c:170
        pHook = <optimized out>
        n = <optimized out>
        rv = <optimized out>
#14 ap_invoke_handler (r=r@entry=0x8108160a0) at config.c:434
        handler = <optimized out>
        p = <optimized out>
        result = <optimized out>
---Type <return> to continue, or q <return> to quit---
        old_handler = 0x802daf240 "php-fcgi"
        ignore = <optimized out>
#15 0x00000000004c9b23 in ap_process_async_request (r=r@entry=0x8108160a0) at http_request.c:410
        access_status = 0
#16 0x00000000004c52f1 in ap_process_http_async_connection (c=0x80b0b2338) at http_core.c:154
        r = 0x8108160a0
        cs = 0x80b0b2310
#17 ap_process_http_connection (c=0x80b0b2338) at http_core.c:248
No locals.
#18 0x0000000000478ac3 in ap_run_process_connection (c=c@entry=0x80b0b2338) at connection.c:42
        pHook = <optimized out>
        n = 4
        rv = -1
#19 0x0000000000583c9d in process_socket (my_thread_num=42, my_child_num=15, cs=0x80b0b22a8, sock=0x80b0b20a0, p=0x80b0b2028, thd=0x80b0119d8)
    at event.c:1102
        c = 0x80b0b2338
        sbh = 0x80b0b2dc0
        conn_id = <optimized out>
        rc = <optimized out>
#20 worker_thread (thd=0x80b0119d8, dummy=<optimized out>) at event.c:1963
        ti = <optimized out>
        process_slot = 15
        thread_slot = 42
        csd = 0x80b0b20a0
        cs = 0x80b0b22a8
        ptrans = 0x80b0b2028
        rv = <optimized out>
        is_idle = 0
        te = 0x0
#21 0x0000000801c711f8 in dummy_worker (opaque=0x80b0119d8) at threadproc/unix/thread.c:142
        thread = 0x80b0119d8
#22 0x00000008020a6585 in thread_start (curthread=0x80b020c00) at /usr/src/lib/libthr/thread/thr_create.c:284
        set = {__bits = {0, 0, 0, 0}}
#23 0x0000000000000000 in ?? ()
No symbol table info available.
Backtrace stopped: Cannot access memory at address 0x7ffff43a2000
(gdb)
Comment 2 Jim Jagielski 2016-11-21 20:35:19 UTC
What socache provider are you using?