Bug 34589 - continious grow of memory usage when running simple module with never-ending connections
Summary: continious grow of memory usage when running simple module with never-ending ...
Status: RESOLVED DUPLICATE of bug 23567
Alias: None
Product: Apache httpd-2
Classification: Unclassified
Component: All (show other bugs)
Version: 2.0.54
Hardware: PC Linux
: P2 major (vote)
Target Milestone: ---
Assignee: Apache HTTPD Bugs Mailing List
URL:
Keywords:
Depends on:
Blocks:
 
Reported: 2005-04-23 09:12 UTC by Norayr Chilingaryan
Modified: 2005-09-07 09:31 UTC (History)
0 users



Attachments

Note You need to log in before you can comment on or make changes to this bug.
Description Norayr Chilingaryan 2005-04-23 09:12:05 UTC
When running apache versions 2.0.52 2.0.53, 2.0.54 with following simple module
memory usage continuing its grow until all memory and swap is used, after which
system killing processes to free resources. I've tried this module with worker
and prefork mpm compiled httpd's on a 2.4 and 2.6 kernel running machines
(debian and fedora core).
Please, try to establish 500 - 1000 connections to module below and see what
happens with memory usage. Apache just killing the system when occupy all
available memory and swap space. In case of low number of connections, memory
usage still grow but not so fast, so will be noticed after certain time interval

module text:
--------------------------

#include "httpd.h"
#include "http_config.h"

#include <time.h>
#include <sys/time.h>
#include <stdio_ext.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "inc/sp.h"



#define DEFAULT_ENCTYPE "application/x-www-form-urlencoded"


int start_receive(request_rec *r)
{
	int ret;

        while(TRUE)
	{
          ret = ap_rprintf (r, "888839457349857394573495888888888888888\r\n");
	  ret = ap_rflush(r);
	  sleep(1);  
        }
	
	return 0;
}

/*
 * This function is registered as a handler for HTTP methods and will
 * therefore be invoked for all GET requests (and others).  Regardless
 * of the request type, this function simply sends a message to
 * STDERR (which httpd redirects to logs/error_log).  A real module
 * would do *alot* more at this point.
 */

static int receiver_method_handler (request_rec *r)
{
    char* req;
    int	 ret;
    apr_table_t*	table;  

    req = r->the_request;
    if (strstr(req, "messages") == NULL) {
       return DECLINED;
    }
    
    table = apr_table_make(r->pool, 8);
   
    ret = read_post(r, &table);
    
    start_receive(r);

    return OK;
}

/*
 * This function is a callback and it declares what other functions
 * should be called for request processing and configuration requests.
 * This callback function declares the Handlers for other events.
 */
static void register_hooks (apr_pool_t *p)
{
	// I think this is the call to make to register a handler for method calls (GET
PUT et. al.).
	// We will ask to be last so that the comment has a higher tendency to
	// go at the end.
	ap_hook_handler(receiver_method_handler, NULL, NULL, APR_HOOK_LAST);
}

/*
 * Declare and populate the module's data structure.  The
 * name of this structure ('rec_mod') is important - it
 * must match the name of the module.  This structure is the
 * only "glue" between the httpd core and the module.
 */
module AP_MODULE_DECLARE_DATA receiver_module =
{
	// Only one callback function is provided.  Real
	// modules will need to declare callback functions for
	// server/directory configuration, configuration merging
	// and other tasks.
	STANDARD20_MODULE_STUFF,
	NULL,
	NULL,
	NULL,
	NULL,
	NULL,
	register_hooks,			/* callback for registering hooks */
};


static int util_read(request_rec *r, const char **rbuf)
{
   int rc;
   if ((rc = ap_setup_client_block(r, REQUEST_CHUNKED_ERROR)) != OK) {
      return rc;
   }
   if (ap_should_client_block(r)) {
     char argsbuffer[HUGE_STRING_LEN];
     int rsize, len_read, rpos=0;
     long length = r->remaining;
     *rbuf = apr_pcalloc(r->pool, length + 1);
//     ap_hard_timeout("util_read", r);
     while ((len_read = ap_get_client_block(r, argsbuffer, sizeof(argsbuffer)))
> 0) {
//       ap_reset_timeout(r);
       if ((rpos + len_read) > length) {
         rsize = length - rpos;
       }
       else {
         rsize = len_read;
       }
       memcpy((char*)*rbuf + rpos, argsbuffer, rsize);
       rpos += rsize;
    }
//    ap_kill_timeout(r);
  }
  return rc;
}


static int read_post(request_rec *r, apr_table_t **tab)
{
   const char *data;
   const char *key, *val, *type;
   int rc = OK;
   if(r->method_number != M_POST) {
     return rc;
   }
   type = apr_table_get(r->headers_in, "Content-Type");
   if(strcasecmp(type, DEFAULT_ENCTYPE) != 0) {
         return DECLINED;
   }
   if((rc = util_read(r, &data)) != OK) {
     return rc;
   }
   if(*tab) {
     apr_table_clear(*tab);
   }
   else {
     *tab = apr_table_make(r->pool, 8);
   }
   while(*data && (val = ap_getword(r->pool, &data, '&'))) {       
     key = ap_getword(r->pool, &val, '=');
     ap_unescape_url((char*)key);
     ap_unescape_url((char*)val);
     apr_table_merge(*tab, key, val);
   }
   return OK;
}

----------------

Thank You very much!
Comment 1 Paul Querna 2005-04-23 09:48:53 UTC
This is caused by ap_rflush, which calls apr_brigade_create() on every call. 
This could be fixed by re-using a brigade inside ap_rflush.
Comment 2 Norayr Chilingaryan 2005-04-26 09:43:46 UTC
(In reply to comment #1)
> This is caused by ap_rflush, which calls apr_brigade_create() on every call. 
> This could be fixed by re-using a brigade inside ap_rflush.

Thanks... but it seems that your solution doesn't work...
look at this...
original source is:
AP_DECLARE(int) ap_rflush(request_rec *r)
{
    conn_rec *c = r->connection;
    apr_bucket_brigade *bb;
    apr_bucket *b;
    bb = apr_brigade_create(r->pool, c->bucket_alloc);
    b = apr_bucket_flush_create(c->bucket_alloc);
    APR_BRIGADE_INSERT_TAIL(bb, b);

    if (ap_pass_brigade(r->output_filters, bb) != APR_SUCCESS)
        return -1;

    return 0;
}

we added following lines in the protocol.c file:

AP_DECLARE(int) ap_rflush2(request_rec *r, apr_bucket_brigade **bb)
{
conn_rec *c = r->connection;
apr_bucket *b;
if(*bb==NULL){
*bb = apr_brigade_create(r->pool, c->bucket_alloc);
}
b = apr_bucket_flush_create(c->bucket_alloc);
APR_BRIGADE_INSERT_TAIL(*bb, b);

if (ap_pass_brigade(r->output_filters, *bb) != APR_SUCCESS)
return -1;

return 0;
}

and now we are using ap_rflush2 in our module source.

but it didn't help...

memory usage still growing...
Comment 3 Joe Orton 2005-04-26 09:52:54 UTC
It's more complicated than that because other filters higher up the chain will
split and leak a brigade for each FLUSH bucket sent, per bug 23567.
Comment 4 Norayr Chilingaryan 2005-04-29 08:02:45 UTC
(In reply to comment #3)
> It's more complicated than that because other filters higher up the chain will
> split and leak a brigade for each FLUSH bucket sent, per bug 23567.

We had done some other changes in the the code of module and httpd...
But it seems that we can't find out solution.
Could You help us anyway, or could you find out and fix the bug yourself?
Comment 5 Norayr Chilingaryan 2005-04-30 12:25:50 UTC
(In reply to comment #4)
> (In reply to comment #3)
> > It's more complicated than that because other filters higher up the chain will
> > split and leak a brigade for each FLUSH bucket sent, per bug 23567.

----------------quote from bug number 23567 thread------------------

Just brain-dumping, the design issue here is, per Jeff's analysis and subsequent
discussion:

1. brigades are allocated out of pools
2. every call to apr_brigade_split allocates a new brigade
3. every time a FLUSH bucket it sent down the filter stack, it causes at least
one call to apr_brigade_split

fixes for this could be either:

fix (1), allocate brigades out of the bucket-allocator so that they can really
be free'd (very intrusive since many filters presume brigades are never really
destroyed)

fix (3), adjust all filters to ensure that they don't allocate a number of
brigades per request (and hence, memory allocated) which is proportional to
number of FLUSH buckets sent.
----------------------------- end of quote--------------------


Look...

First change we had made is call apr_brigade_create from module code.
And we had played with following functions:

      apr_brigade_cleanup(bb);
      APR_BRIGADE_EMPTY(bb);
      apr_brigade_destroy(bb);

All attempts was unsuccessful...


Then we had attempt to make our module look like this:

int start_receive(request_rec *r, char* user_name, char* group_name)
{
        int ret;
        apr_bucket_brigade *bb = NULL;

        //added
        bb = apr_brigade_create(r->pool, r->connection->bucket_alloc);

        ret = 1;
        ap_rprintf (r, "%s_%s:connected|%d\r\n", user_name, group_name, ret);
        ap_rflush2(r, bb);
             ... ... ...
and made some changes in server/protocol.c:

AP_DECLARE(int) ap_rflush2(request_rec *r, apr_bucket_brigade *bb)
{
conn_rec *c = r->connection;
apr_bucket *b;

//if(*bb==NULL){
//*bb = apr_brigade_create(r->pool, c->bucket_alloc);
//}

b = apr_bucket_flush_create(c->bucket_alloc);
APR_BRIGADE_INSERT_TAIL(bb, b);

//if (ap_pass_brigade(r->output_filters, bb) != APR_SUCCESS)
//return -1;

return 0;
}

So, even when apr_brigade_create function called from module and filters are
commented memory usage still growing.
I can mention, that when following lines
//if (ap_pass_brigade(r->output_filters, bb) != APR_SUCCESS)
//return -1;
are commented then memory usage grow too fast and can hang machine after few
seconds because of lack of resources.

We preferring to use Apache 2 instead of Apache 1.3 because of uncomparable
speed that is very important in our project : 'World Biggest Chat' by Lycos-Europe



Comment 6 Joe Orton 2005-09-07 17:31:44 UTC
This issue is essentially no different from bug 23567, marking as duplicate.

*** This bug has been marked as a duplicate of 23567 ***