Attachment 26769 Details for Bug 50306 – New Valve: StuckThreadDetectionValve

[patch] New Valve: StuckThreadDetectionValve

stuckthreadvalve.patch (text/plain), 12.26 KB, created by TomLu on 2011-03-14 10:30:23 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: TomLu

Created: 2011-03-14 10:30:23 UTC

Size: 12.26 KB

patch

obsolete

># This patch can be applied using context Tools: Patch action on respective folder.
># It uses platform neutral UTF-8 encoding and \n newlines.
># Above lines and this line are ignored by the patching process.
>Index: java/org/apache/catalina/valves/LocalStrings.properties
>--- java/org/apache/catalina/valves/LocalStrings.properties Base (BASE)
>+++ java/org/apache/catalina/valves/LocalStrings.properties Locally Modified (Based On LOCAL)
>@@ -41,6 +41,10 @@
> # Remote IP valve
> remoteIpValve.syntax=Invalid regular expressions [{0}] provided.
> 
>+#Stuck thread detection Valve
>+stuckThreadDetectionValve.notifyStuckThreadDetected=Thread "{0}" has been active for {1} milliseconds (started {2}) and may be stuck. There is/are {3} thread(s) in total that are monitored by this Valve and may be stuck.
>+stuckThreadDetectionValve.notifyStuckThreadCompleted=Thread "{0}" was previously reported to be stuck but has completed. It was active for approximately {1} milliseconds. There is/are {2} thread(s) in total that are monitored by this Valve and still may be stuck.
>+
> # HTTP status reports
> http.100=The client may continue ({0}).
> http.101=The server is switching protocols according to the "Upgrade" header ({0}).
>Index: java/org/apache/catalina/valves/StuckThreadDetectionValve.java
>--- java/org/apache/catalina/valves/StuckThreadDetectionValve.java Locally New
>+++ java/org/apache/catalina/valves/StuckThreadDetectionValve.java Locally New
>@@ -0,0 +1,300 @@
>+/*
>+ * Licensed to the Apache Software Foundation (ASF) under one or more
>+ * contributor license agreements.  See the NOTICE file distributed with
>+ * this work for additional information regarding copyright ownership.
>+ * The ASF licenses this file to You under the Apache License, Version 2.0
>+ * (the "License"); you may not use this file except in compliance with
>+ * the License.  You may obtain a copy of the License at
>+ *
>+ *      http://www.apache.org/licenses/LICENSE-2.0
>+ *
>+ * Unless required by applicable law or agreed to in writing, software
>+ * distributed under the License is distributed on an "AS IS" BASIS,
>+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>+ * See the License for the specific language governing permissions and
>+ * limitations under the License.
>+ */
>+package org.apache.catalina.valves;
>+
>+import java.io.IOException;
>+import java.util.Date;
>+import java.util.Queue;
>+import java.util.concurrent.ConcurrentHashMap;
>+import java.util.concurrent.ConcurrentLinkedQueue;
>+import java.util.concurrent.atomic.AtomicInteger;
>+
>+import javax.servlet.ServletException;
>+import org.apache.catalina.Container;
>+import org.apache.catalina.Engine;
>+import org.apache.catalina.LifecycleException;
>+
>+import org.apache.catalina.connector.Request;
>+import org.apache.catalina.connector.Response;
>+import org.apache.juli.logging.Log;
>+import org.apache.juli.logging.LogFactory;
>+import org.apache.tomcat.util.res.StringManager;
>+
>+public class StuckThreadDetectionValve extends ValveBase {
>+
>+    /**
>+     * The descriptive information related to this implementation.
>+     */
>+    private static final String info =
>+            "org.apache.catalina.valves.StuckThreadDetectionValve/1.0";
>+    /**
>+     * Logger
>+     */
>+    private static final Log log = LogFactory.getLog(StuckThreadDetectionValve.class);
>+    /**
>+     * Keeps count of the number of stuck threads detected
>+     */
>+    private final AtomicInteger stuckCount = new AtomicInteger(0);
>+    /**
>+     * In seconds. Default 600 (10 minutes).
>+     */
>+    private int threshold = 600;
>+    /**
>+     * 'SO_TIMEOUT' is 20 seconds in Tomcat.
>+     * The threshold must at least be larger than that.
>+     */
>+    private static final int THRESHOLD_MIN_VALUE = 120;
>+    /**
>+     * The only references we keep to actual running Thread objects are in
>+     * this Map (which is automatically cleaned in invoke()s finally clause).
>+     * That way, Threads can be GC'ed, eventhough the Valve still thinks they
>+     * are stuck (caused by a long monitor interval)
>+     */
>+    private ConcurrentHashMap<Long, MonitoredThread> activeThreads =
>+            new ConcurrentHashMap<Long, MonitoredThread>();
>+    /**
>+     *
>+     */
>+    private Queue<CompletedStuckThread> completedStuckThreadsQueue =
>+            new ConcurrentLinkedQueue<CompletedStuckThread>();
>+    /**
>+     * The string manager for this package.
>+     */
>+    private static final StringManager sm =
>+            StringManager.getManager(Constants.Package);
>+
>+    /**
>+     * Specify the threshold (in seconds) used when checking for stuck threads.
>+     * Minimum value is 120 seconds. The default is 600 seconds.
>+     *
>+     * @param threshold   The new threshold in seconds
>+     */
>+    public void setThreshold(int threshold) {
>+        if (threshold >= THRESHOLD_MIN_VALUE) {
>+            this.threshold = threshold;
>+            if (log.isDebugEnabled()) {
>+                log.debug("Threshold set to " + threshold + " seconds");
>+            }
>+        } else {
>+            log.warn("Threshold too low. Must be "
>+                    + THRESHOLD_MIN_VALUE
>+                    + " or larger. Actual: " + threshold);
>+        }
>+    }
>+
>+    /**
>+     * @see #setThreshold(int)
>+     * @return  The current threshold in seconds
>+     */
>+    public int getThreshold() {
>+        return threshold;
>+    }
>+
>+    @Override
>+    protected void initInternal() throws LifecycleException {
>+        super.initInternal();
>+
>+        if (log.isDebugEnabled()) {
>+            log.debug("Monitoring stuck threads with threshold = "
>+                    + threshold
>+                    + " sec");
>+        }
>+    }
>+
>+    /**
>+     * Return descriptive information about this Valve implementation.
>+     */
>+    @Override
>+    public String getInfo() {
>+        return info;
>+    }
>+
>+    private void notifyStuckThreadDetected(String threadName,
>+            StackTraceElement[] trace, long activeTime,
>+            Date startTime, int numStuckThreads) {
>+
>+        String msg = sm.getString(
>+                "stuckThreadDetectionValve.notifyStuckThreadDetected",
>+                threadName, activeTime, startTime, numStuckThreads);
>+        msg += "\n" + getStackTraceAsString(trace);
>+        log.warn(msg);
>+    }
>+
>+    private void notifyStuckThreadCompleted(String threadName,
>+            long activeTime, int numStuckThreads) {
>+
>+        String msg = sm.getString(
>+                "stuckThreadDetectionValve.notifyStuckThreadCompleted",
>+                threadName, activeTime, numStuckThreads);
>+        //Since the "stuck thread notification" is warn, this should also be warn
>+        log.warn(msg);
>+    }
>+
>+    //TODO Utility method, should be moved to utility class
>+    public static String getStackTraceAsString(StackTraceElement[] trace) {
>+        StringBuilder buf = new StringBuilder();
>+        for (int i = 0; i < trace.length; i++) {
>+            buf.append("\tat ").append(trace[i]).append("\n");
>+        }
>+        return buf.toString();
>+    }
>+
>+    //FIXME where does it make sense to add this Valve?
>+    /**
>+     * Set the Container to which this Valve is attached.
>+     *
>+     * @param container The container to which we are attached
>+     */
>+    @Override
>+    public void setContainer(Container container) {
>+
>+        if (container == null){
>+            throw new IllegalArgumentException("Configuration error:  "
>+                    + "Container cannot be null. Valve must be attached to an Engine");
>+        }
>+        if(!(container instanceof Engine)){
>+            throw new IllegalArgumentException("Configuration error:  "
>+                    + "Valve must be attached to an Engine. Actual: "
>+                    +container.getClass().getName());
>+        }
>+
>+        super.setContainer(container);
>+//        this.engine = (Engine) container;
>+    }
>+
>+    /**
>+     * {@inheritDoc}
>+     */
>+    @Override
>+    public void invoke(Request request, Response response)
>+            throws IOException, ServletException {
>+        //Save the thread/runnable
>+        //Keeping a reference to the thread object here does not prevent GC'ing,
>+        //as the reference is removed from the Map in the finally clause
>+
>+        Long key = new Long(Thread.currentThread().getId());
>+
>+        MonitoredThread previous = activeThreads.put(key,
>+                new MonitoredThread(Thread.currentThread()));
>+
>+        //In theory we might get collisions when using "new Long(thread.getId()).hashCode()".
>+        //We can prevent this by using new Integer(AtomicInt.getAndIncrement()) for uniqueness
>+        if (previous != null) {
>+            log.error("Key already added to map. Previous value has been removed ("
>+                    + previous.getThread().getName() + ")");
>+        }
>+
>+        if (log.isTraceEnabled()) {
>+            log.trace("Monitoring execution time for Request "
>+                    + request.getRequestURI() + " with originalRemoteAddr '"
>+                    + request.getRemoteAddr() + ":" + request.getRemotePort()
>+                    + "'");
>+        }
>+
>+        try {
>+            getNext().invoke(request, response);
>+        } finally {
>+            MonitoredThread monitoredThread = activeThreads.remove(key);
>+            if (monitoredThread.isMarkedAsStuck()) {
>+                completedStuckThreadsQueue.add(
>+                        new CompletedStuckThread(monitoredThread.getThread().getName(),
>+                        monitoredThread.getActiveTimeInMillis()));
>+            }
>+        }
>+    }
>+
>+    @Override
>+    public void backgroundProcess() {
>+        super.backgroundProcess();
>+
>+        long thresholdInMillis = threshold * 1000;
>+
>+        //Check monitored threads
>+        for (MonitoredThread monitoredThread : activeThreads.values()) {
>+            long activeTime = monitoredThread.getActiveTimeInMillis();
>+
>+            if (!monitoredThread.isMarkedAsStuck() && activeTime >= thresholdInMillis) {
>+                int numStuckThreads = stuckCount.incrementAndGet();
>+                monitoredThread.markAsStuck();
>+                notifyStuckThreadDetected(monitoredThread.getThread().getName(),
>+                        monitoredThread.getThread().getStackTrace(),
>+                        activeTime, monitoredThread.getStartTime(), numStuckThreads);
>+            }
>+        }
>+        //Check if any threads previously reported as stuck, have finished.
>+        CompletedStuckThread completedStuckThread = completedStuckThreadsQueue.poll();
>+        while (completedStuckThread != null) {
>+            int numStuckThreads = stuckCount.decrementAndGet();
>+            notifyStuckThreadCompleted(completedStuckThread.getName(),
>+                    completedStuckThread.getTotalActiveTime(), numStuckThreads);
>+            completedStuckThread = completedStuckThreadsQueue.poll();
>+        }
>+    }
>+
>+    private class MonitoredThread {
>+
>+        /**
>+         * Reference to the thread to get a stack trace from background task
>+         */
>+        private Thread thread;
>+        private long start = System.currentTimeMillis();
>+        private volatile boolean isStuck = false;
>+
>+        public MonitoredThread(Thread thread) {
>+            this.thread = thread;
>+        }
>+
>+        public Thread getThread() {
>+            return this.thread;
>+        }
>+
>+        public long getActiveTimeInMillis() {
>+            return System.currentTimeMillis() - start;
>+        }
>+
>+        public Date getStartTime() {
>+            return new Date(start);
>+        }
>+
>+        public void markAsStuck() {
>+            this.isStuck = true;
>+        }
>+
>+        public boolean isMarkedAsStuck() {
>+            return this.isStuck;
>+        }
>+    }
>+
>+    private class CompletedStuckThread {
>+
>+        private String threadName;
>+        private long totalActiveTime;
>+
>+        public CompletedStuckThread(String threadName, long totalActiveTime) {
>+            this.threadName = threadName;
>+            this.totalActiveTime = totalActiveTime;
>+        }
>+
>+        public String getName() {
>+            return this.threadName;
>+        }
>+
>+        public long getTotalActiveTime() {
>+            return this.totalActiveTime;
>+        }
>+    }
>+}
>#END

Actions: View | Diff

Attachments on bug 50306: 26451 | 26769 | 28961 | 28966