#Stuck thread detection Valve

stuckThreadDetectionValve.notifyStuckThreadDetected=Thread "{0}" has been active for {1} milliseconds (started {2}) and may be stuck. There is/are {3} thread(s) in total that are monitored by this Valve and may be stuck.

stuckThreadDetectionValve.notifyStuckThreadCompleted=Thread "{0}" was previously reported to be stuck but has completed. It was active for approximately {1} milliseconds. There is/are {2} thread(s) in total that are monitored by this Valve and still may be stuck.

/*

 * Licensed to the Apache Software Foundation (ASF) under one or more

 * contributor license agreements.  See the NOTICE file distributed with

 * this work for additional information regarding copyright ownership.

 * The ASF licenses this file to You under the Apache License, Version 2.0

 * (the "License"); you may not use this file except in compliance with

 * the License.  You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package org.apache.catalina.valves;

import java.io.IOException;

import java.util.Date;

import java.util.Queue;

import java.util.concurrent.ConcurrentHashMap;

import java.util.concurrent.ConcurrentLinkedQueue;

import java.util.concurrent.atomic.AtomicInteger;

import javax.servlet.ServletException;

import org.apache.catalina.Container;

import org.apache.catalina.Engine;

import org.apache.catalina.LifecycleException;

import org.apache.catalina.connector.Request;

import org.apache.catalina.connector.Response;

import org.apache.juli.logging.Log;

import org.apache.juli.logging.LogFactory;

import org.apache.tomcat.util.res.StringManager;

public class StuckThreadDetectionValve extends ValveBase {

    /**

     * The descriptive information related to this implementation.

     */

    private static final String info =

            "org.apache.catalina.valves.StuckThreadDetectionValve/1.0";

    /**

     * Logger

     */

    private static final Log log = LogFactory.getLog(StuckThreadDetectionValve.class);

    /**

     * Keeps count of the number of stuck threads detected

     */

    private final AtomicInteger stuckCount = new AtomicInteger(0);

    /**

     * In seconds. Default 600 (10 minutes).

     */

    private int threshold = 600;

    /**

     * 'SO_TIMEOUT' is 20 seconds in Tomcat.

     * The threshold must at least be larger than that.

     */

    private static final int THRESHOLD_MIN_VALUE = 120;

    /**

     * The only references we keep to actual running Thread objects are in

     * this Map (which is automatically cleaned in invoke()s finally clause).

     * That way, Threads can be GC'ed, eventhough the Valve still thinks they

     * are stuck (caused by a long monitor interval)

     */

    private ConcurrentHashMap<Long, MonitoredThread> activeThreads =

            new ConcurrentHashMap<Long, MonitoredThread>();

    /**

     *

     */

    private Queue<CompletedStuckThread> completedStuckThreadsQueue =

            new ConcurrentLinkedQueue<CompletedStuckThread>();

    /**

     * The string manager for this package.

     */

    private static final StringManager sm =

            StringManager.getManager(Constants.Package);

    /**

     * Specify the threshold (in seconds) used when checking for stuck threads.

     * Minimum value is 120 seconds. The default is 600 seconds.

     *

     * @param threshold   The new threshold in seconds

     */

    public void setThreshold(int threshold) {

        if (threshold >= THRESHOLD_MIN_VALUE) {

            this.threshold = threshold;

            if (log.isDebugEnabled()) {

                log.debug("Threshold set to " + threshold + " seconds");

            }

        } else {

            log.warn("Threshold too low. Must be "

                    + THRESHOLD_MIN_VALUE

                    + " or larger. Actual: " + threshold);

        }

    }

    /**

     * @see #setThreshold(int)

     * @return  The current threshold in seconds

     */

    public int getThreshold() {

        return threshold;

    }

    @Override

    protected void initInternal() throws LifecycleException {

        super.initInternal();

        if (log.isDebugEnabled()) {

            log.debug("Monitoring stuck threads with threshold = "

                    + threshold

                    + " sec");

        }

    }

    /**

     * Return descriptive information about this Valve implementation.

     */

    @Override

    public String getInfo() {

        return info;

    }

    private void notifyStuckThreadDetected(String threadName,

            StackTraceElement[] trace, long activeTime,

            Date startTime, int numStuckThreads) {

        String msg = sm.getString(

                "stuckThreadDetectionValve.notifyStuckThreadDetected",

                threadName, activeTime, startTime, numStuckThreads);

        msg += "\n" + getStackTraceAsString(trace);

        log.warn(msg);

    }

    private void notifyStuckThreadCompleted(String threadName,

            long activeTime, int numStuckThreads) {

        String msg = sm.getString(

                "stuckThreadDetectionValve.notifyStuckThreadCompleted",

                threadName, activeTime, numStuckThreads);

        //Since the "stuck thread notification" is warn, this should also be warn

        log.warn(msg);

    }

    //TODO Utility method, should be moved to utility class

    public static String getStackTraceAsString(StackTraceElement[] trace) {

        StringBuilder buf = new StringBuilder();

        for (int i = 0; i < trace.length; i++) {

            buf.append("\tat ").append(trace[i]).append("\n");

        }

        return buf.toString();

    }

    //FIXME where does it make sense to add this Valve?

    /**

     * Set the Container to which this Valve is attached.

     *

     * @param container The container to which we are attached

     */

    @Override

    public void setContainer(Container container) {

        if (container == null){

            throw new IllegalArgumentException("Configuration error:  "

                    + "Container cannot be null. Valve must be attached to an Engine");

        }

        if(!(container instanceof Engine)){

            throw new IllegalArgumentException("Configuration error:  "

                    + "Valve must be attached to an Engine. Actual: "

                    +container.getClass().getName());

        }

        super.setContainer(container);

//        this.engine = (Engine) container;

    }

    /**

     * {@inheritDoc}

     */

    @Override

    public void invoke(Request request, Response response)

            throws IOException, ServletException {

        //Save the thread/runnable

        //Keeping a reference to the thread object here does not prevent GC'ing,

        //as the reference is removed from the Map in the finally clause

        Long key = new Long(Thread.currentThread().getId());

        MonitoredThread previous = activeThreads.put(key,

                new MonitoredThread(Thread.currentThread()));

        //In theory we might get collisions when using "new Long(thread.getId()).hashCode()".

        //We can prevent this by using new Integer(AtomicInt.getAndIncrement()) for uniqueness

        if (previous != null) {

            log.error("Key already added to map. Previous value has been removed ("

                    + previous.getThread().getName() + ")");

        }

        if (log.isTraceEnabled()) {

            log.trace("Monitoring execution time for Request "

                    + request.getRequestURI() + " with originalRemoteAddr '"

                    + request.getRemoteAddr() + ":" + request.getRemotePort()

                    + "'");

        }

        try {

            getNext().invoke(request, response);

        } finally {

            MonitoredThread monitoredThread = activeThreads.remove(key);

            if (monitoredThread.isMarkedAsStuck()) {

                completedStuckThreadsQueue.add(

                        new CompletedStuckThread(monitoredThread.getThread().getName(),

                        monitoredThread.getActiveTimeInMillis()));

            }

        }

    }

    @Override

    public void backgroundProcess() {

        super.backgroundProcess();

        long thresholdInMillis = threshold * 1000;

        //Check monitored threads

        for (MonitoredThread monitoredThread : activeThreads.values()) {

            long activeTime = monitoredThread.getActiveTimeInMillis();

            if (!monitoredThread.isMarkedAsStuck() && activeTime >= thresholdInMillis) {

                int numStuckThreads = stuckCount.incrementAndGet();

                monitoredThread.markAsStuck();

                notifyStuckThreadDetected(monitoredThread.getThread().getName(),

                        monitoredThread.getThread().getStackTrace(),

                        activeTime, monitoredThread.getStartTime(), numStuckThreads);

            }

        }

        //Check if any threads previously reported as stuck, have finished.

        CompletedStuckThread completedStuckThread = completedStuckThreadsQueue.poll();

        while (completedStuckThread != null) {

            int numStuckThreads = stuckCount.decrementAndGet();

            notifyStuckThreadCompleted(completedStuckThread.getName(),

                    completedStuckThread.getTotalActiveTime(), numStuckThreads);

            completedStuckThread = completedStuckThreadsQueue.poll();

        }

    }

    private class MonitoredThread {

        /**

         * Reference to the thread to get a stack trace from background task

         */

        private Thread thread;

        private long start = System.currentTimeMillis();

        private volatile boolean isStuck = false;

        public MonitoredThread(Thread thread) {

            this.thread = thread;

        }

        public Thread getThread() {

            return this.thread;

        }

        public long getActiveTimeInMillis() {

            return System.currentTimeMillis() - start;

        }

        public Date getStartTime() {

            return new Date(start);

        }

        public void markAsStuck() {

            this.isStuck = true;

        }

        public boolean isMarkedAsStuck() {

            return this.isStuck;

        }

    }

    private class CompletedStuckThread {

        private String threadName;

        private long totalActiveTime;

        public CompletedStuckThread(String threadName, long totalActiveTime) {

            this.threadName = threadName;

            this.totalActiveTime = totalActiveTime;

        }

        public String getName() {

            return this.threadName;

        }

        public long getTotalActiveTime() {

            return this.totalActiveTime;

        }

    }

}