Portions of this software were developed by Symantec Corporation at 

Mountain View, California.

# Asynchronous Send File configuration (for WinNT+ only)

#

#   1. Win32 AcceptEx should not be disabled.

#

#   2. EnableSendFile should be on.

#

#   3. AsyncSendFile <anydirectory|givendirectory|off>

#                          |            |          +--->Disable AsyncFileWrite,

#                          |            |               default is off

#                          |            |

#                          |            +--> Force AsyncFileWrite only for folders

#                          |                 specified by <ForceAsyncSendFile>

#                          |                 config item (refer below)

#                          |

#                          +--> Apache does AsyncFileWrite for files greater

#                               than 10KB in size, or greater than the size 

#                               specified by <AsyncSendFileMinSize> config item

#

#   4. AsyncSendFileMinSize <size in Bytes>

#                                  +---> Default is 1MB, any value less than 1MB 

#                                        given will be discarded and set to default.

EnableSendFile on

AsyncSendFile anydirectory

# 1MB limit is put to filter PHP/jscript files so that it gets delivered quickly

AsyncSendFileMinSize	1048576

#

# To force Async File Send only for specific folders

#	AsyncSendFile givendirectory

#

#ForceAsyncSendFile "C:/Program Files/Symantec/Symantec Endpoint Protection Manager/Inetpub/content"

#

# Max number of connection requests that will be queued in the child httpd process

#    ConnectionsToQueuePerChild <number of connections>

#                                  +--> Must be greater than ThreadsPerChild item.

#                                            Default is 5000.

ConnectionsToQueuePerChild 1000

#

# To dedicate worker threads specifically for Async Callback processing

#    DedicatedThreadsForAsyncCallbacks <number of threads>

#					                     +--> Must be greater than half the

#						                      ThreadsPerChild config item,

#                                             Default is 0.

#

#DedicatedThreadsForAsyncCallbacks 1

</IfModule>

<directivesynopsis>

<name>ConnectionsToQueuePerChild</name>

<description>Number of connections to queue</description>  

    <syntax>

      ConnectionsToQueuePerChild <var>number</var>

    </syntax>

    <default>ConnectionsToQueuePerChild 5000</default>

    <contextlist>

      <context>server config</context>

    </contextlist>

    <compatibility>Available in Version 2.2.17 and later</compatibility>

    <modulelist>

      <module>mpmwinnt</module>

    </modulelist>

    <usage>

      <p>

        The <directive>ConnectionsToQueuePerChild</directive> directive sets the 

        limit on the number of simultaneous requests that can be queued if all

        the worker threads are busy. The <directive>ConnectionsToQueuePerChild</directive> 

        cannot not be set to less than <directive>ThreadsPerChild</directive>        

      </p>

      <example>

        ConnectionsToQueuePerChild 1000

    </example>

    </usage>

</directivesynopsis>

<directivesynopsis>

<name>AsyncSendFile</name>

<description>Enable non-blocking asynchronous file transmission</description>  

    <syntax>

      AsyncSendFile Off| <var> AnyDirectory|GivenDirectory </var>    

    </syntax>

    <default>AsyncSendFile Off</default>

    <contextlist>

      <context>server config</context>

    </contextlist>

    <compatibility>Available in mpm_winnt version 2.2.17 and later</compatibility>

    <modulelist>

      <module>mpmwinnt</module>

    </modulelist>

    <usage>

      <p>

        The <directive>AsyncSendFile</directive> directive enables file download 

        requests to be completely handled by kernel asynchronously thereby freeing 

        up Apache worker threads to do other tasks. The <directive>AsyncSendFile</directive> 

        will work only if <directive>EnableSendFile</directive> is enabled, and 

        <directive>Win32DisableAcceptEx</directive> should be disabled. 

      </p>

      <p>

        If this directive is set to <code>AnyDirectory</code>, then file download

        requests will be automatically determined to be transmitted asynchronously or 

        not based on the <directive>AsyncSendFileMinSize</directive> diective.

      </p>

      <p>

        If this directive is set to <code>GivenDirectory</code>, then file download

        requests containting URI paths mentioned by <directive>ForceAsyncSendFile</directive>

        will be transmitted asynchronously.

      </p>

      <example>

        AsyncSendFile AnyDirectory<br />                

    </example>

    </usage>

</directivesynopsis>

<directivesynopsis>

<name>AsyncSendFileMinSize</name>

<description>Denotes when non-blocking asynchronous transmission should be triggered</description>  

    <syntax>

      AsyncSendFileMinSize <var>Size in bytes</var>    

    </syntax>

    <default>AsyncSendFileMinSize 1048576</default>

    <contextlist>

      <context>server config</context>

    </contextlist>

    <compatibility>Available in mpm_winnt version 2.2.17 and later</compatibility>

    <modulelist>

      <module>mpmwinnt</module>

    </modulelist>

    <usage>

      <p>

        The <directive>AsyncSendFileMinSize</directive> directive specifies the minimum

        file size in bytes for it to be transmitted asynchronously. Refer

        <directive>AsyncSendFile</directive> directive for more information.

      </p>

    <example>

      AsyncSendFile AnyDirectory <br />        

      AsyncSendFileMinSize  485760

    </example>

    </usage>

</directivesynopsis>

<directpivesynopsis>

<name>ForceAsyncSendFile</name>

<description>Forces non-blocking asynchronous transmission for a given folder</description>  

    <syntax>

      ForceAsyncSendFile <var>Absoluted folder path</var>    

    </syntax>

    <default>AsyncSendFileMinSize 10485760</default>

    <contextlist>

      <context>server config</context>

    </contextlist>

    <compatibility>Available in mpm_winnt version 2.2.17 and later</compatibility>

    <modulelist>

      <module>mpmwinnt</module>

    </modulelist>

    <usage>

      <p>

        The <directive>AsyncSendFileMinSize</directive> directive specifies the minimum

        required file size in Bytes for it to be transmitted asynchronously, if size

        given is less than the 1MB this directive defaults to 1MB.

        <directive>AsyncSendFile</directive>

      </p>

    <example>

      AsyncSendFile GivenDirectory <br />        

      ForceAsyncSendFile  "C:/ApachePub/Downloads/2012"

    </example>

    </usage>

</directivesynopsis>

/* Handle for Winnt async filter */

#ifdef _WIN32  

extern AP_DECLARE_DATA ap_filter_rec_t *ap_core_async_output_filter_handle;

#endif /* _WIN32 */

/**

 * Log Macros for VC2005 and above compilers for optimized execution

 */

#if _MSC_VER > 1400

extern server_rec* ap_server_conf;

#define  AP_LOG_ERROR(fileline, level, ...)	do{	\

         if ((ap_server_conf != NULL) && ((level) <= ap_server_conf->loglevel))\

		     ap_log_error(fileline, level, __VA_ARGS__);\

    }while(0);

#define  AP_LOG_CERROR(fileline, level, ...) do{ \

         if ((ap_server_conf != NULL) && ((level) <= ap_server_conf->loglevel))\

             ap_log_cerror(fileline, level, __VA_ARGS__);\

    }while(0);

#define  AP_LOG_RERROR(file, line, level, ...)  do{	\

         if ((ap_server_conf != NULL) && ((level) <= ap_server_conf->loglevel))\

		      ap_log_rerror(fileline, level, __VA_ARGS__);\

    }while(0);

#else

#define AP_LOG_ERROR   ap_log_error

#define AP_LOG_CERROR  ap_log_cerror

#define AP_LOG_RERROR  ap_log_rerror

#endif /*_MSC_VER > 1400*/

#define PENDING -3      // Represents request processing is not completed yet

                        // but reliquish the worker thread for now because

                        // an Async I/O may be issued or some background

                        // work is going on that is expected to complete

                        // after few seconds/minutes or more

    /** Store the Isapi Async operational state */

    int nLastAsyncOperation;    

 * @brief Enumeration of connection async state

 */

typedef enum {

    /** Represents there are no Async operations on current connection */

    CONN_OPSTATE_ASYNC_NONE = 0,    

    /* The following states are used for synchronized access of connection

     * context objects between threads*/

    /* Denotes Async request spawned successfully by worker thread but no 

     * other threads should start working on the Async completion notification

     * yet (assuming async operation already completed). This is because the 

     * current worker thread is yet to release the connection context object.     

     */

    CONN_OPSTATE_ASYNC_PRE_READY,   

    /* Denotes the worker thread that spawned Async request has released the

     * connection context, and any thread can start working on the Async 

     * completion notification

     */

    CONN_OPSTATE_ASYNC_READY,

    /* Denotes the Async completion notification is handled, and required 

     * resource cleanup can be done on this connection object

     */

    CONN_OPSTATE_ASYNC_DONE

} conn_operational_state_e;

/** 

    /** Contains the Context object's OVERLAPPED reference */

    void*   pOverlapped;            

    /** Holds IOCP handle, will be null if AcceptEx is disabled */

    HANDLE  hAsyncDispatchIOCP;    

    /** Light weight operational state of connection*/

    volatile conn_operational_state_e opState;

    /** Request record*/

    request_rec*   pRequest;

    /** Async operation state/flags*/

    struct {        

        unsigned char bIsLinkedToIOCP:1;

        unsigned char bAsyncSendFile:1;

        unsigned char bAsyncSockRead:1;        

    } AsyncOperation;

    /* Below items are for debugging purpose, remove for production*/

    void*   pContext;

    long    nUsageCount;

#include "..\..\..\srclib\apr\include\arch\win32\apr_arch_file_io.h"

#include "..\..\..\server\mpm\winnt\mpm_winnt.h"

/* SetNotes

 * Records the last async operation that was invoked

 * LTAC: A function to set async opstate may look as overkill here, but I believe 

 * this will be helpful for debugging and also in future if we may have to 

 * support multiple Async requests on the same connection record? 

 */

int SetNotes(request_rec* r, int nLastAsyncOp)

{

    int rv = ERROR_NOT_CAPABLE;

    if (r){

        /*ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r,

            "Thread(%04d) Connection(%05d,%s) SetNotes(LastAsyncOp,%d) ", 

            GetCurrentThreadId(), r->connection->id, "", nLastAsyncOp);*/

        r->nLastAsyncOperation = nLastAsyncOp;        

    }

    return rv ;

}

/* GetNotes

 * Reads the last async operation that was invoked

 */

int GetNotes(request_rec* r)

{

    int nLastAsyncOp = 0;

    if (r){

        nLastAsyncOp = r->nLastAsyncOperation;

    }

    /*ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r,

        "Thread(%04d) Connection(%05d,%s) GetNotes(LastAsyncOp,%d) ", 

        GetCurrentThreadId(), r->connection->id, "", nLastAsyncOp);*/

    return nLastAsyncOp;    

}

    {   

        if (c->hAsyncDispatchIOCP != NULL){        /* LTAC: using IOCP? */

            int nLastAsyncOp = 0;

            ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r,

                              "Thread(%04d) Connection(%05d,%s) ISAPI: SupportFunc"

                              " HSE_REQ_DONE_WITH_SESSION is called", 

                              GetCurrentThreadId(), c->id, r->uri);

            nLastAsyncOp = GetNotes(r);

            if (nLastAsyncOp == 0){

                ap_log_rerror(APLOG_MARK, APLOG_WARNING, 0, r,

                              "Thread(%04d) Connection(%05d,%s) ISAPI: SupportFunc"

                              " HSE_REQ_DONE_WITH_SESSION is called without prior "

                              "valid HSE_* calls", GetCurrentThreadId(), c->id, 

                              r->uri);

                return 0;

            }

            /* LTAC: For now we support only one Async operation - PUSH */

            else if (nLastAsyncOp == HSE_STATUS_PENDING){

                ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r,

                              "Thread(%04d) Connection(%05d,%s) ISAPI: SupportFunc"

                              " HSE_REQ_DONE_WITH_SESSION invoked for prior "

                              " HSE_STATUS_PENDING call, Posting IOCP packet",

                              GetCurrentThreadId(), c->id, r->uri);

                /* Signal to close - connection and destroy memory associated to it

                 */                

                PostQueuedCompletionStatus(c->hAsyncDispatchIOCP, 0, 

                                 IOCP_CONNECTION_PENDING_DONE, c->pOverlapped);

                /* !!!!CAUTION CAUTION CAUTION!!!!

                 * Do not set HSE_REQ_DONE_WITH_SESSION as Last async op at this

                 * point since request + contexts objects could have been destroyed 

                 * by now due to the above IOCP packet

                 */

            } 

            else{

                SetNotes(r, HSE_REQ_DONE_WITH_SESSION);

            }

            return 1;

        }

    }

        SetNotes(r, HSE_REQ_IO_COMPLETION);

        SetNotes(r, HSE_REQ_TRANSMIT_FILE);

             * Adding send file flag to use Transmit file, however this will 

             * not result in true async mode/callbacks - since EOC bucket

             * is not added to brigade before passing down to output filters.

                        (c->hAsyncDispatchIOCP ? APR_SENDFILE_ENABLED : 0) |

cleanup:

		/* LTAC: By now the Event is used, we can safely clean it 

		 * Note: file_cleanup function closes the event handle, but it gets trigerred only during apr_pool destruction, 

		 * and even though mod_isapi.so creates file handle at Request pool level, the destruction is not happening once request 

		 * processing is done, thereby resulting in event handle leak. 

         */

		if (fd != NULL &&

			fd->pOverlapped != NULL &&

			fd->pOverlapped->hEvent != NULL &&

			fd->pOverlapped->hEvent != INVALID_HANDLE_VALUE){	

				CloseHandle(fd->pOverlapped->hEvent);

				fd->pOverlapped->hEvent = NULL;

		}

        SetNotes(r, HSE_REQ_ASYNC_READ_CLIENT);

			/* LTAC: Just return error, do not advice ISAPI module to unload. Because if ISAPI module is unloaded from

			 * memory, in scenarios where there are outstanding Isapi Requests, mod_isapi would result in access 

			 * violation/crash! at Isapi module entry point, i.e., at (*isa->HttpExtensionProc)(cid->ecb) call. The 

			 * ideal fix would be invoke unload call only if there are no outstanding Isapi requests pending, which is 

			 * hard to deduce in existing pipelined architecture.			

			 *isapi_unload(isa, 0);

             */

    r->connection->nUsageCount++;

    ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r,

                   "Thread(%04d) Connection(%05d, %s, %d) ISAPI: %s: %s", 

                   GetCurrentThreadId(), r->connection->id, "", r->connection

                   ->nUsageCount, r->uri, r->args);

            if (r->connection != NULL &&

                r->connection->hAsyncDispatchIOCP != NULL){

                int nLastAsyncRequest = GetNotes(r);             

                /* Check whether Async request was already emulated and 

                 * completed by Apache (Transmit file or Async read 

                 * operation) on this request?

                 */

                if (nLastAsyncRequest != HSE_REQ_DONE_WITH_SESSION){                                                        

                        SetNotes(r, HSE_STATUS_PENDING);

                        ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r,

                                "Thread(%04d) Connection(%05d,%s) ISAPI: %s: %s ",

                                GetCurrentThreadId(), 0, "ret HSE_STATUS_PENDING", r->uri, r->args);

                        return PENDING;

                }

            }            

        ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r,

                   "Thread(%04d) Connection(%05d,%s) ISAPI: %s: %s", 

                   GetCurrentThreadId(), r->connection->id, "ret OK", r->uri, r->args);

    ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r,

                   "Thread(%04d) Connection(%05d,%s%d) ISAPI: %s: %s", 

                   GetCurrentThreadId(), r->connection->id, "ret ", r->status, r->uri, r->args);

#include "apr_atomic.h"     // For Async framework

#include "http_log.h"

    int rv = 0;

            /* LTAC: In case of PENDING operation return immediately.

             * Each PENDING operation complete can be signalled to 

             * client only by closing the connection, hence further

             * processing the other requests on same connection can 

             * again result in yet another PENDING operation 

             * notifying such operations can only be done by closing the

             * socket.

             */

            if (rv == PENDING){

                apr_atomic_set32((volatile apr_uint32_t*) &c->opState, 

                                 CONN_OPSTATE_ASYNC_PRE_READY);

                break;

            }

    int rv = 0;

    int nTotalReqInConnection = 0;

        AP_LOG_CERROR(APLOG_MARK, APLOG_DEBUG, 0, c,

            "Thread(%04d) --%d-- Connection(%05d,%s) context: %08x", 

            GetCurrentThreadId(), ++nTotalReqInConnection, c->id, r->uri,

            c->pContext);

        /* LTAC: In case of PENDING operation return immediately.

         * Each PENDING operation completion can be signalled to 

         * client only by closing the connection, hence further

         * processing the other requests on same connection can 

         * again result in yet another PENDING operation which

         * is not supported atleast for now.

         */

        if (rv == PENDING){

            /* LTAC: Update score board is not required since a connection is 

             * no longer tied to a thread, so we don't need to update the 

             * score board now

             */

             /* ap_update_child_status(c->sbh, SERVER_BUSY_KEEPALIVE, r); */

            /* LTAC: Update connection operational state 

             */

            apr_atomic_set32((volatile apr_uint32_t*) &r->connection->opState,

                             CONN_OPSTATE_ASYNC_PRE_READY);

            break;

        }

#ifdef _WIN32

        /* LTAC: Hook async filter only if IOCP is present */

        if (r->connection->hAsyncDispatchIOCP){

            ap_add_output_filter_handle(ap_core_async_output_filter_handle,

                                        NULL, r, r->connection);

        }

#endif /*_WIN32*/

    int status = APR_SUCCESS;

    if (access_status == PENDING)

    {        

        /* LTAC: For Async framework, this state represents a long time consuming 

         * operation 

         */

        r->connection->pRequest = r;

        return PENDING;

    }

	return access_status;

    /* LTAC: For Async framework - make PENDING to be an acceptable return

     * value as well, PENDING - represents either Async I/O was initiated

     * or a Long duration operation (push connection) was invoked

     */

    AP_LOG_CERROR(APLOG_MARK, APLOG_DEBUG, 0, c,

                   "Thread(%04d) Connection(%05d,%s, %d) Socket: %d",

                   GetCurrentThreadId(), c->id, "CLOSING", c->nUsageCount, 

                   *(((BYTE*)csd) + sizeof(apr_pool_t*))); /*get socket handle*/

        /* Async ? */        

        if (status == PENDING && c->aborted == 0){            

            return PENDING;

        }

/* For Async Send file feature */

#ifdef _WIN32

#include "mpm\winnt\mpm_winnt.h"

#include "..\srclib\apr\include\arch\win32\apr_arch_networkio.h"

#endif /*_WIN32*/

#ifdef _WIN32

__inline apr_status_t do_sendfile_ex(apr_socket_t* client_socket,

									apr_file_t* fd,

									apr_hdtr_t* hdtr,

									apr_off_t  file_offset,

									apr_size_t  file_bytes_left,

									apr_int32_t flags,

									conn_rec* pConnRec)

{

	apr_status_t rv = APR_BADARG;

	apr_size_t tmplen = file_bytes_left;

	ap_assert(pConnRec != NULL);

	ap_assert(pConnRec->AsyncOperation.bIsLinkedToIOCP == 0);

	ap_assert(pConnRec->AsyncOperation.bAsyncSendFile);

	if (client_socket == NULL){

		return rv;

	}

	/* Associate the context's overlapped info to tcp socket, since 

     * connection is accepted, it is kept unused */

    client_socket->overlapped = &((PCOMP_CONTEXT)pConnRec->pContext)

                                    ->Overlapped;

    /* We use IOCP and not events any more */

    ap_assert(client_socket->overlapped->hEvent == NULL);    

    rv = apr_socket_sendfileEx(client_socket, fd, hdtr, &file_offset,

							   &tmplen, flags, pConnRec->hAsyncDispatchIOCP);

    switch (rv)

    {

    case ERROR_IO_PENDING:                

        pConnRec->AsyncOperation.bIsLinkedToIOCP = 1;                

        AP_LOG_CERROR(APLOG_MARK, APLOG_DEBUG, 0, pConnRec, 

                      "Thread(%04d) Invoked TransmitFile(%08x) ",

                      GetCurrentThreadId(), client_socket->socketdes);                   

        rv = PENDING; 

		break;

    case WSAEWOULDBLOCK:

        AP_LOG_CERROR(APLOG_MARK, APLOG_ERR, 0, pConnRec, 

                     "Thread(%04d) Cannot invoke TransmitFile(%08x) "

                     "without overlapped structure falling back to "

                     "Sync sendfile", GetCurrentThreadId(), 

                     client_socket->socketdes);            

        break;

    case ERROR_UNKNOWN_PORT:                

        rv = apr_get_os_error();

        AP_LOG_CERROR(APLOG_MARK, APLOG_ERR, 0, pConnRec, 

                     "Thread(%04d) Cannot assoc CompletionPort for "

					 "TransmitFile(%08x) to socket(), falling back to "

					 "Sync sendfile", GetCurrentThreadId(), 

					 pConnRec->hAsyncDispatchIOCP, client_socket);

        break;

    case APR_SUCCESS:

		AP_LOG_CERROR(APLOG_MARK, APLOG_WARNING, 0, pConnRec, 

					  "Thread(%04d) Async IO completed upon request "

					  "TransmitFile(%08x) ", GetCurrentThreadId(), 

					  pConnRec->hAsyncDispatchIOCP, client_socket);

        break;

    default:

        AP_LOG_CERROR(APLOG_MARK, APLOG_ERR, rv, pConnRec, 

                      "Thread(%04d) Error while invoking "

                      "apr_socket_sendfileEx for socket %08x"

                      "falling back to sync sendfile()",

                      GetCurrentThreadId(),client_socket->socketdes);

    }

	return rv;

}

#endif /*_WIN32*/

#if _WIN32        

        /* Check whether Async file send operation is possible:

        *  1. Connection record should not be NULL

        *  2. Socket should not be already associated to IOCP - error check

        *  3. Connection should be set to use Async Write

        */

        if (pConnRec != NULL  &&        

            pConnRec->AsyncOperation.bIsLinkedToIOCP == 0 &&    

            pConnRec->AsyncOperation.bAsyncSendFile){

			/* Call SendFileEx */

			rv = do_sendfile_ex(c->client_socket, fd, hdtr, file_offset, 

								tmplen, flags, pConnRec);			           

			/* Async File write initiated, return immediately */

			if (rv == PENDING){

				return rv;

			} 

            /* Fall back to Stock Sendfile in case of TransmitFile 

			 * initialization errors, errors during transmission will be caught 

			 * by the code path handling Completion packet deqeueing method -

			 * mpm_winnt_get_connection() and HandlePendingContext()

             */

        }

#endif /* _WIN32 */

void HandlePendingContext(PCOMP_CONTEXT pContext, ap_sb_handle_t *pSbh);

/* LTAC: Store the actual handle of the IO completion port that the worker

 * thread is supposed to work on. Stored in Thread local storage area so 

 * that each thread can be associated to differnt IOCP Queue efficiently.

 * This way we make specific thread to work on specific completion ports 

 * (Connection acceptence or Async Callback IOCP port)   

 */

__declspec(thread) static HANDLE tls_hIOCP = 0;

/* If "DedicatedThreadsForAsyncCallbacks" experimental httpd configuration

 * item is set, AsyncDispatchIOCP will store the IOCP handle specifically 

 * meant for Async callback notifications.

 */

static HANDLE AsyncDispatchIOCP = NULL;

/* Holds the count of dedicated threads initialized for Async 

 * completion notification processing.

 */

static int s_nAsyncHandlingThreadsInit = 0;    

void mpm_recycle_iocp_completion_context(PCOMP_CONTEXT context)

{

    /* LTAC: We use Async framework in AcceptEx mode - where one thread

     * is not always associated to one context/connection - since a 

     * context can result in a Async context/connection which has to be 

     * queued seperately for later usage and cleanup. 

     * Therefore in Async / AcceptEx mode - we cleanup the context once

     * we are done with the sync/async connection 

     * LTAC TODO OPTIMIZATION: 

     * We can try to optimize by truly recycling the context rather than 

	 * freeing and creating contexts every time - this saves us some CPU 

	 * cycles, however - when we recycle, memory used within the context 

	 * (transaction pool to be precise) doesn't get cleaned up until the 

	 * child restarts. This is something we may need to fix it in core 

	 * Apache if we really want to recycle completion contexts for performance

	 * reasons

     *

     * Note:

     * context->accept_socket may be in a disconnected but reusable

     * state so -don't- close it.

     */        

    ap_assert(use_acceptex);

    ap_assert(ThreadDispatchIOCP != NULL);

    if (context) {                    

            if(context->pAsyncConRec != NULL){

                if (context->pAsyncConRec->opState != CONN_OPSTATE_ASYNC_DONE ){

                    /* LTAC: This is an Async context don't clean up 

                     */

                    AP_LOG_CERROR(APLOG_MARK,APLOG_WARNING, 0, context->pAsyncConRec,

                        "Thread(%04d) Connection(%05d,%s) Not Destroying async context since state is pending: %08x",

                     GetCurrentThreadId(), context->pAsyncConRec->id, "", context);

                    return;

                }

                else{

                    AP_LOG_CERROR(APLOG_MARK,APLOG_DEBUG, 0, context->pAsyncConRec,

                        "Thread(%04d) Connection(%05d,%s) Destroying async context: %08x",

                     GetCurrentThreadId(), context->pAsyncConRec->id, "", context);

                }

            }

            else{

                AP_LOG_ERROR(APLOG_MARK,APLOG_DEBUG, 0, ap_server_conf,

                  "Thread(%04d) Connection(%05d,%s) Destroying context: %08x",

                     GetCurrentThreadId(), 0, "", context);         

            }

            /* Acquire child lock

             */

            apr_thread_mutex_lock(child_lock);

            /* By now this event should not exist but do a sanity cleanup

             * TODO: remove this later

             */

            if (context->Overlapped.hEvent != NULL){

                CloseHandle(context->Overlapped.hEvent);

                context->Overlapped.hEvent = NULL;

            }

            apr_pool_destroy(context->ptrans);

            apr_atomic_dec32(&num_completion_contexts);

            free(context);                

            /* Release lock */

            apr_thread_mutex_unlock(child_lock);

            context = NULL;   

    }

}

PCOMP_CONTEXT mpm_get_Iocp_completion_context(void)

{

    apr_status_t rv;

    PCOMP_CONTEXT context = NULL;

    int max_num_contexts = g_MaxConnectionsToQueuePerChild;

    ap_assert(use_acceptex);

    ap_assert(ThreadDispatchIOCP != NULL);

    /* LTAC: Throttle connection accept based on number of completion/

     * connection contexts    

     * We account only for Sync completion context - since these are the 

     * connections that are actually eating httpd.exe - CPU time slice,

     * hence we deduct the Async contexts

     */    

    if (num_completion_contexts - g_AsyncContextCount >= max_num_contexts ){

        static DWORD dwLastTime = 0;

        /* Log about this resource contention very first time or 

         * every 10 minutes 

         */

        if (dwLastTime == 0 ||

            GetTickCount() - dwLastTime > 10*60*1000 /*10 minutes*/){      

                AP_LOG_ERROR(APLOG_MARK,APLOG_WARNING, 0, ap_server_conf,

                    "Thread(%04d) Contexts(Total: %d Sync: %d Async: %d "

                    "Max: %d) returning null context", GetCurrentThreadId(), 

                    num_completion_contexts,  num_completion_contexts - 

                    g_AsyncContextCount, g_AsyncContextCount, max_num_contexts);

                dwLastTime = GetTickCount();

        }

        /* Wait for a second for accept thread to free CPU

         * TODO: Make the sleep interval configurable if accept thread is 

         * eating more CPU?

         */

        Sleep(1000);

        return NULL;

    }

    while (1) {

        apr_allocator_t *allocator;       

        apr_thread_mutex_lock(child_lock);

        /* LTAC: Dont use APR to allocate contexts, doing so will spike memory

         * usage, this is because garbage collection of free'd APR memory 

         * happens only when child httpd process die's

         */

        context = (PCOMP_CONTEXT) malloc(sizeof(COMP_CONTEXT));

        memset(context, 0, sizeof(COMP_CONTEXT));

        /* This event is not really required if we are using Apache single 

         * process mode - refer win_acceptIocp() function for more info

         */

        context->Overlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);

        if (context->Overlapped.hEvent == NULL) {

            /* Hopefully this is a temporary condition ... */

            ap_log_error(APLOG_MARK,APLOG_WARNING, apr_get_os_error(), ap_server_conf,

                "mpm_get_Iocp_completion_context: CreateEvent failed.");

            free(context);

            apr_thread_mutex_unlock(child_lock);

            return NULL;

        }

        /* Create the tranaction pool */

        apr_allocator_create(&allocator);

        apr_allocator_max_free_set(allocator, ap_max_mem_free);

        rv = apr_pool_create_ex(&context->ptrans, pchild, NULL, allocator);

        if (rv != APR_SUCCESS) {

            ap_log_error(APLOG_MARK,APLOG_WARNING, rv, ap_server_conf,

                "mpm_get_Iocp_completion_context: Failed to create the transaction pool.");

            CloseHandle(context->Overlapped.hEvent);

            free(context);            

            apr_thread_mutex_unlock(child_lock);

            return NULL;

        }

        apr_allocator_owner_set(allocator, context->ptrans);

        apr_pool_tag(context->ptrans, "IocpTransaction");

        context->accept_socket = INVALID_SOCKET;

        context->ba = apr_bucket_alloc_create(context->ptrans);

        /* LTAC: Setup new Async entities within context

         */

        context->pAsyncConRec = NULL;

        context->next = NULL;                

        apr_atomic_inc32(&num_completion_contexts);

        apr_thread_mutex_unlock(child_lock);

        break;    

    }

    return context;

}

/* Windows NT/2000 specific code...

 * This function is modified version of of winnt_accept(). 

 * Accept processing for on Windows NT uses a producer/consumer queue

 * An accept thread accepts connections off the network using IO compeletion

 * ports then issues PostQueuedCompletionStatus() to awake a thread blocked 

 * on the ThreadDispatch IOCompletionPort (IOCP)

 *

 * winnt_acceptIocp()

 *    One or more accept threads run in this function, each of which accepts

 *    connections off the network using Completion ports and calls 

 *    PostQueuedCompletionStatus() to queue an io completion packet to the 

 *    ThreadDispatch IOCompletionPort.

 * winnt_get_connection()

 *    Worker threads block on the ThreadDispatch IOCompletionPort awaiting

 *    connections to service.

 */

static unsigned int __stdcall winnt_acceptIocp(void *lr_)

{

    ap_listen_rec *lr = (ap_listen_rec *)lr_;

    apr_os_sock_info_t sockinfo;

    PCOMP_CONTEXT context = NULL;

    DWORD BytesRead = 0;

    SOCKET nlsd;

    int rv, err_count = 0;    

    DWORD dwCompKey = IOCP_CONNECTION_ACCEPTED;

    LPOVERLAPPED pOvl = 0;    

    HANDLE hTemp = 0;

    int nRetry = 0;

#if APR_HAVE_IPV6

    SOCKADDR_STORAGE ss_listen;

    int namelen = sizeof(ss_listen);

#endif

    apr_os_sock_get(&nlsd, lr->sd);

#if APR_HAVE_IPV6

    if (getsockname(nlsd, (struct sockaddr *)&ss_listen, &namelen) == SOCKET_ERROR) {

        ap_log_error(APLOG_MARK,APLOG_ERR, apr_get_netos_error(), ap_server_conf,

                    "winnt_accept: getsockname error on listening socket, is IPv6 available?");

        return 1;

   }

#endif

    ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, ap_server_conf,

                 "Child %d Thread %d: Starting thread to listen on port %d.",

                 my_pid, GetCurrentThreadId(),lr->bind_addr->port);

    while (!shutdown_in_progress) {

        /* LTAC: Clean up the old contexts before using a new one

         * For Async framework - the number of contexts are limited to the new

         * config item - "MaxConnections", and we do not queue all contexts

         * (a.k.a connections) for reuse as in Stock apache since each 

         * connection is holding socket + file resources (handles) and this 

         * would result in instability of apache during long runs

         */

        if (context){

            mpm_recycle_iocp_completion_context(context);

            context = NULL;            

        }

        context = mpm_get_Iocp_completion_context();

        if (!context) {

            /* Temporary resource constraint? */

            /* LTAC: Max connections could have been hit?

             */

            Sleep(0);

            continue;

        }

        /* Create and initialize the accept socket */

#if APR_HAVE_IPV6

        if (context->accept_socket == INVALID_SOCKET) {

            context->accept_socket = socket(ss_listen.ss_family, SOCK_STREAM, IPPROTO_TCP);

            context->socket_family = ss_listen.ss_family;

        }

        else if (context->socket_family != ss_listen.ss_family) {

            closesocket(context->accept_socket);

            context->accept_socket = socket(ss_listen.ss_family, SOCK_STREAM, IPPROTO_TCP);

            context->socket_family = ss_listen.ss_family;

        }

        if (context->accept_socket == INVALID_SOCKET) {

            ap_log_error(APLOG_MARK,APLOG_WARNING, apr_get_netos_error(), ap_server_conf,

                         "winnt_accept: Failed to allocate an accept socket. "

                         "Temporary resource constraint? Try again.");

            Sleep(100);

            continue;

        }

#else

        if (context->accept_socket == INVALID_SOCKET) {

            context->accept_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);

            if (context->accept_socket == INVALID_SOCKET) {

                /* Another temporary condition? */

                ap_log_error(APLOG_MARK,APLOG_WARNING, apr_get_netos_error(), ap_server_conf,

                             "winnt_acceptIocp: Failed to allocate an accept socket. "

                             "Temporary resource constraint? Try again.");

                Sleep(100);

                continue;

            }

        }

#endif

        /* AcceptEx on the completion context. The completion context will be

         * signaled when a connection is accepted.

         */                                

        if (!AcceptEx(nlsd, context->accept_socket,

                      context->buff,

                      0,

                      PADDED_ADDR_SIZE,

                      PADDED_ADDR_SIZE,

                      &BytesRead,

                      &context->Overlapped)) {

            rv = apr_get_netos_error();

            if ((rv == APR_FROM_OS_ERROR(WSAEINVAL)) ||

                (rv == APR_FROM_OS_ERROR(WSAENOTSOCK))) {

                /* We can get here when:

                 * 1) the client disconnects early

                 * 2) TransmitFile does not properly recycle the accept socket (typically

                 *    because the client disconnected)

                 * 3) there is VPN or Firewall software installed with buggy AcceptEx implementation

                 * 4) the webserver is using a dynamic address that has changed

                 */

                ++err_count;

                closesocket(context->accept_socket);

                context->accept_socket = INVALID_SOCKET;

                if (err_count > MAX_ACCEPTEX_ERR_COUNT) {

                    ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf,

                                 "Child %d: Encountered too many errors accepting client connections. "

                                 "Possible causes: dynamic address renewal, or incompatible VPN or firewall software. "

                                 "Try using the Win32DisableAcceptEx directive.", my_pid);

                    err_count = 0;

                }

                continue;

            }

            else if ((rv != APR_FROM_OS_ERROR(ERROR_IO_PENDING)) &&

                     (rv != APR_FROM_OS_ERROR(WSA_IO_PENDING))) {

                ++err_count;

                if (err_count > MAX_ACCEPTEX_ERR_COUNT) {

                    ap_log_error(APLOG_MARK,APLOG_ERR, rv, ap_server_conf,

                                 "Child %d: Encountered too many errors accepting client connections. "

                                 "Possible causes: Unknown. "

                                 "Try using the Win32DisableAcceptEx directive.", my_pid);

                    err_count = 0;

                }

                closesocket(context->accept_socket);

                context->accept_socket = INVALID_SOCKET;

                continue;

            }

            err_count = 0;

            /* Wait for pending i/o.

             * Wake up once per second to check for shutdown .

             * XXX: We should be waiting on exit_event instead of polling

             */

            while (1) {

                    rv = WaitForSingleObject(context->Overlapped.hEvent, 1000);                

                    if (rv == WAIT_OBJECT_0) {                    

                        if (context->accept_socket == INVALID_SOCKET) {

                            /* socket already closed */

                            break;

                        }

                        if (!GetOverlappedResult((HANDLE)context->accept_socket,

                                                 &context->Overlapped,

                                                 &BytesRead, FALSE)) {

                            ap_log_error(APLOG_MARK, APLOG_WARNING,

                                         apr_get_os_error(), ap_server_conf,

                                 "winnt_acceptIocp: Asynchronous AcceptEx failed.");

                            closesocket(context->accept_socket);

                            context->accept_socket = INVALID_SOCKET;

                        }

                        break;

                    }

                /* WAIT_TIMEOUT */

                if (shutdown_in_progress) {

                    closesocket(context->accept_socket);

                    context->accept_socket = INVALID_SOCKET;

                    break;

                }                

            }

            if (context->accept_socket == INVALID_SOCKET) {

                continue;

            }

        }        

        err_count = 0;

        /* Inherit the listen socket settings. Required for

         * shutdown() to work

         */

        /* LTAC: LEAK! LEAK! LEAK!

         * Setsockopt - results in handle leaks, internally WinSock module

         * sets up a timerhandle when the following call is made to inherit 

         * lisenting socket properties (running in parent httpd.exe process

         * or service context)to associate this socket with the service 

         * shutdown event. I don't see this essential since we close the 

         * the open sockets during shutdown gracefully in at end of 

         * woker_main() 

         * 

         *if (setsockopt(context->accept_socket, SOL_SOCKET,

         *               SO_UPDATE_ACCEPT_CONTEXT, (char *)&nlsd,

         *               sizeof(nlsd))) {

         *    ap_log_error(APLOG_MARK, APLOG_WARNING, apr_get_netos_error(), ap_server_conf,

         *                 "setsockopt(SO_UPDATE_ACCEPT_CONTEXT) failed.");

         *     * Not a failure condition. Keep running. *

         *}

         */

        /* Get the local & remote address */

        GetAcceptExSockaddrs(context->buff,

                             0,

                             PADDED_ADDR_SIZE,

                             PADDED_ADDR_SIZE,

                             &context->sa_server,

                             &context->sa_server_len,

                             &context->sa_client,

                             &context->sa_client_len);

        sockinfo.os_sock = &context->accept_socket;

        sockinfo.local   = context->sa_server;

        sockinfo.remote  = context->sa_client;

        sockinfo.family  = context->sa_server->sa_family;

        sockinfo.type    = SOCK_STREAM;

        apr_os_sock_make(&context->sock, &sockinfo, context->ptrans);

        /* LTAC: The event handle is no longer required since we will utilize

         * IOCP to fire events - this way we reduce the number of handles used 

         * by Apache

         */

        CloseHandle(context->Overlapped.hEvent);

        context->Overlapped.hEvent=NULL;

        /* When a connection is received, send an io completion notification to

         * the ThreadDispatchIOCP. This function could be replaced by

         * mpm_post_completion_context(), but why do an extra function call...

         */

        PostQueuedCompletionStatus(ThreadDispatchIOCP, 0, IOCP_CONNECTION_ACCEPTED,

                                   &context->Overlapped);

        context = NULL;

    }

    if (!shutdown_in_progress) {

        /* Yow, hit an irrecoverable error! Tell the child to die. */

        SetEvent(exit_event);

    }

    AP_LOG_ERROR(APLOG_MARK, APLOG_INFO, APR_SUCCESS, ap_server_conf,

                 "Child %d: Thread: %d Accept thread exiting.", my_pid, 

                 GetCurrentThreadId());

    return 0;

}

        case IOCP_CONNECTION_PENDING_DONE:

        {

            // LTAC: Move Async processing to a dedicated thread if child 

            // threads are too busy ?

            // Once we get real data from PQA tests we can make this call

            //

            context = CONTAINING_RECORD(pol, COMP_CONTEXT, Overlapped);                           

            break;

        }

        case IOCP_TRANSMITFILE_DONE:

        {         

            /* LTAC: TF I/O operation is done or have error'd  */

            context = CONTAINING_RECORD(pol, COMP_CONTEXT, Overlapped); 

            if (!context && !context->pAsyncConRec){

                ap_log_error(APLOG_MARK, APLOG_ERR, 0, ap_server_conf,

                             "TransmitFile operation done - context/connection"

                             "is invalid");

                /* Cleanup the memory associated to context */

                mpm_recycle_iocp_completion_context(context);

                context = NULL;    

            }

            break;

        }

            AP_LOG_ERROR(APLOG_MARK,APLOG_ERR, 0, ap_server_conf,

                         "Thread(%04d) Unknown completion key: %d", 

                         GetCurrentThreadId(), CompKey);  

    /* LTAC: Initialize the TLS IOCP handle varibale during thread startup */

    if (use_acceptex && tls_hIOCP == 0)

    {

        /* using child lock for thread synchronization */

        apr_thread_mutex_lock(child_lock);