Here is the backtrace: Program ran under gdb with set args -X -f conf/leakd.conf Thread 17 Stack Trace: *** Begin Stack Frame #0 0x403079a7 in memcpy () from /lib/libc.so.6 #1 0x40404661 in shmcb_cyclic_cton_memcpy (buf_size=7190, dest=0xbdbfcd2c "0\201\221\002\001\001\002\002\003\001\004\002", data=0x4048ebea "\0040èË´ëR\222Á3ÿÓ\001àM¯\236ðg\222ë[ù%·ýÆ-f3z ) ÷\023JÌá\233=", src_offset=6402, src_len=10240) at ssl_scache_shmcb.c:915 #2 0x404052cb in shmcb_remove_session_id (s=0x80e2a98, queue=0xbdbff58c, cache=0xbdbff57c, id=0x82708f8 "\177ÉÁvL|\0066=<{w%BQ.øºIÉnÝ7ü\001&\017sI)\224\002 ", idlen=32) at ssl_scache_shmcb.c:1338 #3 0x40404527 in shmcb_remove_session (s=0x80e2a98, shm_segment=0x40452000, id=0x82708f8 "\177ÉÁvL|\0066=<{w%BQ.øºIÉnÝ7ü\001&\017sI)\224\002 ", idlen=32) at ssl_scache_shmcb.c:819 #4 0x40403a2b in ssl_scache_shmcb_remove (s=0x80e2a98, id=0x82708f8 "\177ÉÁvL|\0066=<{w%BQ.øºIÉnÝ7ü\001&\017sI)\224\002 ", idlen=32) at ssl_scache_shmcb.c:477 #5 0x4040291c in ssl_scache_remove (s=0x80e2a98, id=0x82708f8 "\177ÉÁvL|\0066=<{w%BQ.øºIÉnÝ7ü\001&\017sI)\224\002 ", idlen=32) at ssl_scache.c:158 #6 0x403fcfc3 in ssl_callback_DelSessionCacheEntry (ctx=0x80de048, session=0x82708b0) at ssl_engine_kernel.c:1742 #7 0x40042f1b in timeout () from /lib/libssl.so.2 #8 0x400b1d60 in lh_doall_arg () from /lib/libcrypto.so.2 #9 0x40042fa0 in SSL_CTX_flush_sessions () from /lib/libssl.so.2 #10 0x40040691 in ssl_update_cache () from /lib/libssl.so.2 #11 0x4003270f in ssl3_accept () from /lib/libssl.so.2 #12 0x4003f340 in SSL_accept () from /lib/libssl.so.2 #13 0x4003bfe8 in ssl23_get_client_hello () from /lib/libssl.so.2 #14 0x4003b7f5 in ssl23_accept () from /lib/libssl.so.2 #15 0x4003f340 in SSL_accept () from /lib/libssl.so.2 #16 0x403fa2f9 in ssl_io_filter_connect (filter_ctx=0x82313d8) at ssl_engine_io.c:1070 #17 0x403fa664 in ssl_io_filter_input (f=0x82b82d0, bb=0x82a8f28, mode=AP_MODE_GETLINE, block=APR_BLOCK_READ, readbytes=0) at ssl_engine_io.c:1239 #18 0x0807218e in ap_get_brigade (next=0x82b82d0, bb=0x82a8f28, mode=AP_MODE_GETLINE, block=APR_BLOCK_READ, readbytes=0) at util_filter.c:514 #19 0x0807218e in ap_get_brigade (next=0x82a8ec8, bb=0x82a8f28, mode=AP_MODE_GETLINE, block=APR_BLOCK_READ, readbytes=0) at util_filter.c:514 #20 0x08072f93 in ap_rgetline_core (s=0x82a82b8, n=8192, read=0xbdbff9d8, r=0x82a82a0, fold=0, bb=0x82a8f28) at protocol.c:256 #21 0x08073455 in read_request_line (r=0x82a82a0, bb=0x82a8f28) at protocol.c:623 #22 0x080739d7 in ap_read_request (conn=0x8231060) at protocol.c:900 #23 0x080608db in ap_process_http_connection (c=0x8231060) at http_core.c:312 #24 0x0807060a in ap_run_process_connection (c=0x8231060) at connection.c:85 #25 0x08065916 in process_socket (p=0x8230f38, sock=0x8230f70, my_child_num=0, my_thread_num=13, bucket_alloc=0x825d100) at worker.c:632 #26 0x08065f0a in worker_thread (thd=0x80fde88, dummy=0x812bc88) at worker.c:946 #27 0x401f5090 in dummy_worker (opaque=0x80fde88) at thread.c:127 #28 0x40205f77 in pthread_start_thread () from /lib/libpthread.so.0 ***End of Stack Frame Info Threads: 29 Thread 27676 (LWP 1526) 0x40360b60 in poll () from /lib/libc.so.6 18 - 28 in sigsuspend () from /lib/libc.so.6 * 17 Thread 15376 (LWP 1514) 0x403079a7 in memcpy () from /lib/libc.so.6 3 - 16 in sigsuspend () from /lib/libc.so.6 2 Thread 2049 (LWP 1499) 0x40360b60 in poll () from /lib/libc.so.6 1 Thread 1024 (LWP 1492) 0x402b4136 in sigsuspend () from /lib/libc.so.6 CPU Registers: eax 0x24ec 9452 ecx 0x36 54 edx 0xbdbfd040 -1111502784 ebx 0x4041089c 1078003868 esp 0xbdbfcc9c 0xbdbfcc9c ebp 0xbdbfccd4 0xbdbfccd4 esi 0x40490ffe 1078530046 edi 0xbdbff454 -1111493548 eip 0x40404661 0x40404661 eflags 0x202 514 cs 0x23 35 ss 0x2b 43 ds 0x2b 43 es 0x2b 43 fs 0x0 0 gs 0x0 0 fctrl 0x37f 895 fstat 0x20 32 ftag 0xffff 65535 fiseg 0x23 35 fioff 0x400b2af8 1074473720 foseg 0x2b 43 fooff 0x40101950 1074796880 fop 0x5d8 1496 xmm0 {f = {0x0, 0x0, 0x0, 0x0}} {f = {-nan(0x7fffff), -nan(0x7fffff), -nan(0x7fffff), -nan(0x7fffff)}} xmm1-xmm7 the same as xmm0 mxcsr 0x1f80 8064 orig_eax 0xffffffff -1 function where issue is located: 901 static void shmcb_cyclic_cton_memcpy( 902 unsigned int buf_size, 903 unsigned char *dest, 904 unsigned char *data, 905 unsigned int src_offset, 906 unsigned int src_len) 907 { 908 /* Can it be copied all in one go? */ 909 if (src_offset + src_len < buf_size) 910 /* yes */ 911 memcpy(dest, data + src_offset, src_len); 912 else { 913 /* no */ 914 memcpy(dest, data + src_offset, buf_size - src_offset); *915 memcpy(dest + buf_size - src_offset, data, 916 src_len + src_offset - buf_size); (gdb) print dest + buf_size - src_offset $57 = (unsigned char *) 0xfffff4ee <Address 0xfffff4ee out of bounds> (gdb) print src_len + src_offset - buf_size $58 = 2071963774 (gdb) 917 } 918 return; 919 } Frame Information [frame 1]: #1 0x40404661 in shmcb_cyclic_cton_memcpy ( buf_size=7190, dest=0xbdbfcd2c "0\201\221\002\001\001\002\002\003\001\004\002", data=0x4048ebea "\0040èË´ëR\222Á3ÿÓ\001àM¯\236ðg\222ë[ù%·ýÆ-f3z ) ÷\023JÌá\233=", src_offset=6402, src_len=10240 ) at ssl_scache_shmcb.c:915 915 memcpy(dest + buf_size - src_offset, data, 916 src_len + src_offset - buf_size); Variables in the Frame context: (gdb) print buf_size $49 = 7190 (gdb) print dest $51 = (unsigned char *) 0xbdbfcd2c "0\201\221\002\001\001\002\002\003\001\004 \002" (gdb) print data $53 = (unsigned char *) 0x4048ebea "\0040èË´ëR\222Á3ÿÓ\001àM¯\236ðg\222ë[ù%·ýÆ- f3z )÷\023JÌá\233=" (gdb) print src_offset $55 = 3183473748 (gdb) print &src_offset Address requested for identifier "src_offset" which is in register $edi (gdb) print src_len $56 = 3183464512 (gdb) print &src_len Address requested for identifier "src_len" which is in register $edx (gdb) info register edi edx edi 0xbdbff454 -1111493548 edx 0xbdbfd040 -1111502784 These variable values do appear to be valid based on the stack trace? src_offset = 3183473748 location register edi=0xbdbff454 -1111493548 src_len = 3183464512 location register edx=0xbdbfd040 -1111502784 The stack trace shows these are supposed to be: src_offset=6402 src_len=10240 Here is the conf file: # Custom config file for memory leak test ServerRoot "/usr/webserver" PidFile logs/httpd.pid Timeout 300 KeepAlive On MaxKeepAliveRequests 100 KeepAliveTimeout 15 <IfModule worker.c> StartServers 1 MaxClients 25 MinSpareThreads 25 MaxSpareThreads 25 ThreadsPerChild 25 ServerLimit 1 MaxRequestsPerChild 0 </IfModule> <IfModule perchild.c> NumServers 5 StartThreads 5 MinSpareThreads 5 MaxSpareThreads 10 MaxThreadsPerChild 20 MaxRequestsPerChild 0 </IfModule> <IfModule mpm_winnt.c> ThreadsPerChild 250 MaxRequestsPerChild 0 </IfModule> LoadModule access_module modules/mod_access.so LoadModule actions_module modules/mod_actions.so LoadModule alias_module modules/mod_alias.so LoadModule cgi_module modules/mod_cgi.so LoadModule dir_module modules/mod_dir.so LoadModule env_module modules/mod_env.so LoadModule imap_module modules/mod_imap.so LoadModule log_config_module modules/mod_log_config.so LoadModule mime_module modules/mod_mime.so LoadModule proxy_module modules/mod_proxy.so LoadModule proxy_connect_module modules/mod_proxy_connect.so LoadModule proxy_http_module modules/mod_proxy_http.so LoadModule negotiation_module modules/mod_negotiation.so LoadModule rewrite_module modules/mod_rewrite.so LoadModule setenvif_module modules/mod_setenvif.so LoadModule headers_module modules/mod_headers.so LoadModule ssl_module modules/mod_ssl.so LoadModule status_module modules/mod_status.so <IfModule !mpm_winnt.c> # # If you wish httpd to run as a different user or group, you must run # httpd as root initially and it will switch. # User leakd Group leakd </IfModule> UseCanonicalName Off <Directory /> Options FollowSymLinks AllowOverride None #IP_RESTRICTION_BLOCK </Directory> DirectoryIndex index.html index.htm index.php <Files ~ "^\.ht"> Order allow,deny Deny from all </Files> TypesConfig conf/mime.types DefaultType text/plain <IfModule mod_mime_magic.c> MIMEMagicFile conf/magic </IfModule> HostnameLookups Off ErrorLog /usr/webserver/logs/error_log LogLevel error LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined LogFormat "%h %l %u %t \"%r\" %>s %b" common LogFormat "%{Referer}i -> %U" referer LogFormat "%{User-agent}i" agent CustomLog /usr/webserver/logs/access_log common ServerTokens min ServerSignature Off ScriptAlias /cgi-bin/ "/usr/webserver/cgi-bin/" AddEncoding x-compress Z AddEncoding x-gzip gz tgz AddLanguage da .dk AddLanguage nl .nl AddLanguage en .en AddLanguage et .et AddLanguage fr .fr AddLanguage de .de AddLanguage he .he AddLanguage el .el AddLanguage it .it AddLanguage ja .ja AddLanguage pl .po AddLanguage ko .ko AddLanguage pt .pt AddLanguage nn .nn AddLanguage no .no AddLanguage pt-br .pt-br AddLanguage ltz .ltz AddLanguage ca .ca AddLanguage es .es AddLanguage sv .sv AddLanguage cz .cz AddLanguage ru .ru AddLanguage tw .tw AddLanguage zh-tw .tw AddLanguage hr .hr LanguagePriority en da nl et fr de el it ja ko no pl pt pt-br ltz ca es sv tw ForceLanguagePriority Prefer Fallback AddDefaultCharset ISO-8859-1 AddCharset ISO-8859-1 .iso8859-1 .latin1 AddCharset ISO-8859-2 .iso8859-2 .latin2 .cen AddCharset ISO-8859-3 .iso8859-3 .latin3 AddCharset ISO-8859-4 .iso8859-4 .latin4 AddCharset ISO-8859-5 .iso8859-5 .latin5 .cyr .iso-ru AddCharset ISO-8859-6 .iso8859-6 .latin6 .arb AddCharset ISO-8859-7 .iso8859-7 .latin7 .grk AddCharset ISO-8859-8 .iso8859-8 .latin8 .heb AddCharset ISO-8859-9 .iso8859-9 .latin9 .trk AddCharset ISO-2022-JP .iso2022-jp .jis AddCharset ISO-2022-KR .iso2022-kr .kis AddCharset ISO-2022-CN .iso2022-cn .cis AddCharset Big5 .Big5 .big5 # For russian, more than one charset is used (depends on client, mostly): AddCharset WINDOWS-1251 .cp-1251 .win-1251 AddCharset CP866 .cp866 AddCharset KOI8-r .koi8-r .koi8-ru AddCharset KOI8-ru .koi8-uk .ua AddCharset ISO-10646-UCS-2 .ucs2 AddCharset ISO-10646-UCS-4 .ucs4 AddCharset UTF-8 .utf8 AddCharset GB2312 .gb2312 .gb AddCharset utf-7 .utf7 AddCharset utf-8 .utf8 AddCharset big5 .big5 .b5 AddCharset EUC-TW .euc-tw AddCharset EUC-JP .euc-jp AddCharset EUC-KR .euc-kr AddCharset shift_jis .sjis AddType application/x-tar .tgz AddType image/x-icon .ico AddType application/x-httpd-php .php AddType text/html .tpl AddHandler cgi-script cgi exe jpq BrowserMatch "Mozilla/2" nokeepalive BrowserMatch "MSIE 4\.0b2;" nokeepalive downgrade-1.0 force-response-1.0 BrowserMatch "RealPlayer 4\.0" force-response-1.0 BrowserMatch "Java/1\.0" force-response-1.0 BrowserMatch "JDK/1\.0" force-response-1.0 BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect- carefully BrowserMatch "^WebDrive" redirect-carefully BrowserMatch "^WebDAVFS/1.[012]" redirect-carefully BrowserMatch "^gnome-vfs" redirect-carefully <IfModule mod_proxy.c> ProxyRequests Off <Proxy *> Order deny,allow Deny from all Allow from all </Proxy> ProxyVia On </IfModule> <IfModule mod_rewrite.c> RewriteEngine On </IfModule> listen 127.0.0.1:9200 <VirtualHost 127.0.0.1:9200> ServerName 127.0.0.1:9200 DocumentRoot "/usr/webserver/isdocs" <Directory "/usr/webserver/isdocs"> Options MultiViews Options +FollowSymLinks AllowOverride None </Directory> RewriteEngine On RewriteRule ^/login.htm /red9200.html RewriteMap map1 txt:/usr/webserver/conf/musiclist.map RewriteCond %{REQUEST_URI} ^/([^/]+).* RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] RewriteCond %{REQUEST_URI} ^/Music/LookupTag/(.*) RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] RewriteCond %{REQUEST_URI} ^/Music/MusicTag/(.*)RewriteCond ${map1:%1|NONE} ^ (http.*) [NC] RewriteRule ^(/.*) %1$1 [P] ProxyPreserveHost on Header set Server: JKPHTTPServer/9.9 <Location /statusreport> SetHandler server-status </Location> </VirtualHost> listen 172.25.54.114:9200 <VirtualHost 172.25.54.114:9200> ServerName 172.25.54.114:9200 DocumentRoot "/usr/webserver/isdocs" <Directory "/usr/webserver/isdocs"> Options MultiViews Options +FollowSymLinks AllowOverride None </Directory> RewriteEngine On RewriteRule ^/login.htm /red9200.html RewriteMap map1 txt:/usr/webserver/conf/musiclist.map RewriteCond %{REQUEST_URI} ^/([^/]+).* RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] RewriteCond %{REQUEST_URI} ^/Music/LookupTag/(.*) RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] RewriteCond %{REQUEST_URI} ^/Music/MusicTag/(.*) RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] ProxyPreserveHost on Header set Server: HTTPServer/9.9 <Location /statusreport> SetHandler server-status </Location> </VirtualHost> AddType application/x-x509-ca-cert .crt AddType application/x-pkcs7-crl .crl SSLPassPhraseDialog builtin #SSLSessionCache dbm:logs/ssl_scache #SSLSessionCache none SSLSessionCache shmcb:logs/scache(256000) SSLMutex file:logs/ssl_mutex SSLSessionCacheTimeout 300 SSLRandomSeed startup builtin SSLRandomSeed connect builtin listen 127.0.0.1:9201 <VirtualHost 127.0.0.1:9201> ServerName 127.0.0.1:9201 DocumentRoot "/usr/webserver/htdocs" <Directory "/usr/webserver/htdocs"> Options +MultiViews AllowOverride None </Directory> <Directory "/usr/webserver/cgi-bin"> Options +MultiViews AllowOverride None </Directory> <Location /statusreport> SetHandler server-status </Location> RewriteEngine On RewriteMap map1 txt:/usr/webserver/conf/musiclist.map RewriteCond %{REQUEST_URI} ^/([^/]+).* RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] RewriteCond %{REQUEST_URI} ^/Music/LookupTag/(.*) RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] RewriteCond %{REQUEST_URI} ^/Music/MusicTag/(.*) RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] ProxyPreserveHost on SSLEngine on SSLCipherSuite ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP:+eNULL SSLCertificateFile /usr/webserver/conf/cert.pem SSLCertificateKeyFile /usr/webserver/conf/file.pem <Files ~ "\.(jpq|exe|cgi|shtml|phtml|php3?)$"> SSLOptions +StdEnvVars </Files> <Directory "/usr/webserver/cgi-bin"> SSLOptions +StdEnvVars </Directory> Alias /myhelp "/usr/webserver/help" <Directory "/usr/webserver/help"> Options ExecCGI MultiViews AllowOverride None Order allow,deny Allow from all SSLOptions +StdEnvVars </Directory> SetEnvIf User-Agent ".*MSIE.*" \ nokeepalive ssl-unclean-shutdown \ downgrade-1.0 force-response-1.0 </VirtualHost> listen 172.25.54.114:9201 <VirtualHost 172.25.54.114:9201> ServerName 172.25.54.114:9201 DocumentRoot "/usr/webserver/htdocs" <Directory "/usr/webserver/htdocs"> Options +MultiViews AllowOverride None </Directory> <Directory "/usr/webserver/cgi-bin"> Options +MultiViews AllowOverride None </Directory> <Location /statusreport> SetHandler server-status </Location> RewriteEngine On RewriteMap map1 txt:/usr/webserver/conf/musiclist.map RewriteCond %{REQUEST_URI} ^/([^/]+).* RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] RewriteCond %{REQUEST_URI} ^/Music/LookupTag/(.*) RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] RewriteCond %{REQUEST_URI} ^/Music/MusicTag/(.*) RewriteCond ${map1:%1|NONE} ^(http.*) [NC] RewriteRule ^(/.*) %1$1 [P] ProxyPreserveHost on SSLEngine on SSLCipherSuite ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP:+eNULL SSLCertificateFile /usr/webserver/conf/cert.pem SSLCertificateKeyFile /usr/webserver/conf/file.pem <Files ~ "\.(jpq|exe|cgi|shtml|phtml|php3?)$"> SSLOptions +StdEnvVars </Files> <Directory "/usr/webserver/cgi-bin"> SSLOptions +StdEnvVars </Directory> Alias /myhelp "/usr/webserver/help" <Directory "/usr/webserver/help"> Options ExecCGI MultiViews AllowOverride None Order allow,deny Allow from all SSLOptions +StdEnvVars </Directory> SetEnvIf User-Agent ".*MSIE.*" \ nokeepalive ssl-unclean-shutdown \ downgrade-1.0 force-response-1.0 </VirtualHost>
Here is inforamtion from one of our developers: While attempting to locate the cause for what appears to be a memory consumption problem in the SSL code, the server segmentation faults. The first worker child & all of its child threads continue to consume memory while the parent stays the same or gets a little smaller. The child threads never give the memory back unless restarted. Please advise if this is an expected behavior. Running with 'SSLSessionCache none' doesn't consume memory (and doesn't seg fault), but it performs poorly when using 2048 bit keys. I observed the segmentation fault issue in mod_ssl while running the small script listed below. Based on the stack information the issue appears to be in shmcb_cton_memcpy() during an attempt to remove a session id. The server keeps on reponding, but all the child threads die and are restarted. I am not sure what is happening, but the following variables seem to get corrupted: The stack trace shows these are supposed to be: src_offset=6402 src_len=10240 Inside the frame they have these values: (gdb) print src_offset (in edi register) $55 = 3183473748 (gdb) print src_len (in edx register) $56 = 3183464512 The configuration file, and my initial debug session are attached. Apache error_log ... [Mon Mar 15 11:21:33 2004] [notice] Apache/2.0.48 configured -- resuming normal operations [Mon Mar 15 11:25:28 2004] [error] server reached MaxClients setting, consider raising the MaxClients setting [Mon Mar 15 11:38:29 2004] [notice] child pid 1065 exit signal Segmentation fault (11) [Mon Mar 15 12:06:28 2004] [notice] child pid 1154 exit signal Segmentation fault (11) [Mon Mar 15 12:44:49 2004] [notice] child pid 1258 exit signal Segmentation fault (11) [Mon Mar 15 13:04:40 2004] [notice] child pid 1315 exit signal Segmentation fault (11) [Mon Mar 15 13:17:29 2004] [notice] child pid 1363 exit signal Segmentation fault (11) [Mon Mar 15 13:45:12 2004] [notice] child pid 1401 exit signal Segmentation fault (11) ... OS RedHat 7.3 gcc-2.96-113 glibc-2.2.5-43 openssl-0.9.6b-35.7 Apache 2.0.48 Build Script: ./configure --with-program-name=leakd --with-port=9200 --with-mpm=worker -- enable-ssl=shared --enable-maintainer-mode \ --enable-proxy=shared --enable- cgi=shared --enable-setenvif=shared --enable-cgi=shared --enable-access=shared \ --enable-rewrite=shared --enable-dir=shared --enable-actions=shared --enable- mime=shared --enable-proxy_connect=shared \ --enable-proxy_http=shared -- enable-negotiation=shared --enable-alias=shared --enable-env=shared --enable- dir=shared \ --enable-mod-actions=shared --enable-log-config=shared --enable- imap=shared --enable-headers=shared \ --enable-layout=webserver --disable- autoindex --disable-userdir --disable-usertrack --disable-cgid \ --disable- asis --disable-auth --disable-auth_digest --disable-auth_dbm --disable- auth_anon --disable-dav \ --disable-dav_fs --disable-vhost_alias --disable- unique_id --disable-speling --disable-cern_meta --disable-include \ --disable- expires --enable-status=shared --enable-info=shared ldd leakd: libssl.so.2 => /lib/libssl.so.2 (0x40024000) libcrypto.so.2 => /lib/libcrypto.so.2 (0x40052000) libaprutil-0.so.0 => /usr/webserver/lib/libaprutil-0.so.0 (0x40119000) libgdbm.so.2 => /usr/lib/libgdbm.so.2 (0x4012d000) libdb-3.3.so => /lib/libdb-3.3.so (0x40133000) libexpat.so.0 => /usr/lib/libexpat.so.0 (0x401c2000) libapr-0.so.0 => /usr/webserver/lib/libapr-0.so.0 (0x401e1000) libpthread.so.0 => /lib/libpthread.so.0 (0x40200000) librt.so.1 => /lib/librt.so.1 (0x40215000) libm.so.6 => /lib/libm.so.6 (0x40226000) libcrypt.so.1 => /lib/libcrypt.so.1 (0x40247000) libnsl.so.1 => /lib/libnsl.so.1 (0x40274000) libdl.so.2 => /lib/libdl.so.2 (0x40288000) libc.so.6 => /lib/libc.so.6 (0x4028c000) /lib/ld-linux.so.2 => /lib/ld-linux.so.2 (0x40000000) Simple script on external machine downloads copies of the stock Apache index.html.en page under both unsecure & secure sites: #!/bin/sh counter=0 limit=32000 while [ "$counter" -lt "$limit" ] do wget -O - http://myboxaddr:9200 wget -O - https://myboxaddr:9201 counter=`expr $counter + 1` echo "Count=> $counter" done
I added some log messages to the code, and turned on debugging. I attempted to using either SSLMutex file:logs/ssl_mutex or SSLMutex default. It takes longer with SSLMutex default to seg fault, but the stack trace is basically the same. The debug error_log traces are available for both test runs if you want them. Finally the src_offest & src_len variables are not changing. GDB just doesn't reset the registers when you move back in the stack frame.
It seems to me that src_offset and src_len are getting corrupted somehow, but it's not obvious to me where or how this is happening. The versions you're using of redhat, glibc, gcc (etc) are a little dated. and though I'm reluctant to dismiss the issue as being old tools, it would certainly be something to consider - if you're able to build using a different gcc or mess with the optimisation levels, that might hint as to whether this is compiler sensitive or something more macabre. Also, is it possible to insert some debugging lines in the last two frames around the problem area to dump the exact values being passed around? I'm curious how and where those values are getting mangled. As/when you hit a segfault, it would be useful to have something to help pinpoint where the corruption was introduced. (Another possible hint: could those "corrupt" values actually be some unsigned representation of a negative - indicating a possible bug in the "cyclic" logic?) I've added myself to the CC line for this ticket, please let me know how you get on with this.
Logging messages were added into the function to print out the values for src_len and src_offset, and they were actually not changing. The seg fault is in memcpy() frame #0. When you move back to frame #1 to examine things, gdb 5.2- 2 does not reload the registers. Local variables were created inside the function, and assigned the values src_offset & src_len upon entry. The end result was the same (seg fault). It could be the tools, but everything is fine for 15-20 minutes. The function is called 305 times before a failure with the last three calls shown below: CALLER == shmcb_remove_session_id() CALLED == shmcb_cyclic_cton_memcpy() [Wed Mar 17 17:13:20 2004] [info] CALLER: header->cache_data_size=7190 src_offset=3972 src_len=10240 [Wed Mar 17 17:13:20 2004] [info] CALLED: buff_size=7190 src_offset=3972, src_len=10240 [Wed Mar 17 17:13:20 2004] [info] CALLER: header->cache_data_size=7190 src_offset=7166 src_len=10240 [Wed Mar 17 17:13:20 2004] [info] CALLED: buff_size=7190 src_offset=7166, src_len=10240 I have two debug traces.
Ouch, ok - I have this gloomy sense that I'm about to dive back into apache code ... I notice you're on apache 2.0.48 ... I could try to help track the problem in that version and worry about migrating it (if applicable) to cvs after, but to avoid the potential for logjams with other issues already fixed, are you able to move to 2.0.49, or better still, CVS (head or 2.0.**-stable)? At the least, have you diffed the ssl module source against later releases or CVS to check if any fixes have already been made that might cover this? Whatever you do w.r.t. apache versions - please email me a copy of the first few pages of a *trace* log during startup (this should give me all the shmcb geometry settings), and then the last few pages leading up to your first crash. I noticed from the info you've already provided that you are caching sessions around ~10Kb, which would indicate that you're using client-authentication and probably with some biggish certs (or longish cert-chains). My hunch is that this is triggering some wrap-around issue, either in the cyclic logic itself or in the use of variables of insufficient size. Please mail me the details privately, no point drowning the bugzilla database. As/when I have potential suggestions/fixes, how should we handle that? Can I send you diffs to try? Can I shell to a box where this can be reproduced? Thanks again for the detailed report.
Geoff's fix for this is now committed to HEAD and the 2.0 branch - thanks for the report, and thanks to Geoff for tracking it down.