ASF Bugzilla – Attachment 26769 Details for
Bug 50306
Detect stuck threads
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
Remember
[x]
|
Forgot Password
Login:
[x]
[patch]
New Valve: StuckThreadDetectionValve
stuckthreadvalve.patch (text/plain), 12.26 KB, created by
TomLu
on 2011-03-14 10:30:23 UTC
(
hide
)
Description:
New Valve: StuckThreadDetectionValve
Filename:
MIME Type:
Creator:
TomLu
Created:
2011-03-14 10:30:23 UTC
Size:
12.26 KB
patch
obsolete
># This patch can be applied using context Tools: Patch action on respective folder. ># It uses platform neutral UTF-8 encoding and \n newlines. ># Above lines and this line are ignored by the patching process. >Index: java/org/apache/catalina/valves/LocalStrings.properties >--- java/org/apache/catalina/valves/LocalStrings.properties Base (BASE) >+++ java/org/apache/catalina/valves/LocalStrings.properties Locally Modified (Based On LOCAL) >@@ -41,6 +41,10 @@ > # Remote IP valve > remoteIpValve.syntax=Invalid regular expressions [{0}] provided. > >+#Stuck thread detection Valve >+stuckThreadDetectionValve.notifyStuckThreadDetected=Thread "{0}" has been active for {1} milliseconds (started {2}) and may be stuck. There is/are {3} thread(s) in total that are monitored by this Valve and may be stuck. >+stuckThreadDetectionValve.notifyStuckThreadCompleted=Thread "{0}" was previously reported to be stuck but has completed. It was active for approximately {1} milliseconds. There is/are {2} thread(s) in total that are monitored by this Valve and still may be stuck. >+ > # HTTP status reports > http.100=The client may continue ({0}). > http.101=The server is switching protocols according to the "Upgrade" header ({0}). >Index: java/org/apache/catalina/valves/StuckThreadDetectionValve.java >--- java/org/apache/catalina/valves/StuckThreadDetectionValve.java Locally New >+++ java/org/apache/catalina/valves/StuckThreadDetectionValve.java Locally New >@@ -0,0 +1,300 @@ >+/* >+ * Licensed to the Apache Software Foundation (ASF) under one or more >+ * contributor license agreements. See the NOTICE file distributed with >+ * this work for additional information regarding copyright ownership. >+ * The ASF licenses this file to You under the Apache License, Version 2.0 >+ * (the "License"); you may not use this file except in compliance with >+ * the License. You may obtain a copy of the License at >+ * >+ * http://www.apache.org/licenses/LICENSE-2.0 >+ * >+ * Unless required by applicable law or agreed to in writing, software >+ * distributed under the License is distributed on an "AS IS" BASIS, >+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. >+ * See the License for the specific language governing permissions and >+ * limitations under the License. >+ */ >+package org.apache.catalina.valves; >+ >+import java.io.IOException; >+import java.util.Date; >+import java.util.Queue; >+import java.util.concurrent.ConcurrentHashMap; >+import java.util.concurrent.ConcurrentLinkedQueue; >+import java.util.concurrent.atomic.AtomicInteger; >+ >+import javax.servlet.ServletException; >+import org.apache.catalina.Container; >+import org.apache.catalina.Engine; >+import org.apache.catalina.LifecycleException; >+ >+import org.apache.catalina.connector.Request; >+import org.apache.catalina.connector.Response; >+import org.apache.juli.logging.Log; >+import org.apache.juli.logging.LogFactory; >+import org.apache.tomcat.util.res.StringManager; >+ >+public class StuckThreadDetectionValve extends ValveBase { >+ >+ /** >+ * The descriptive information related to this implementation. >+ */ >+ private static final String info = >+ "org.apache.catalina.valves.StuckThreadDetectionValve/1.0"; >+ /** >+ * Logger >+ */ >+ private static final Log log = LogFactory.getLog(StuckThreadDetectionValve.class); >+ /** >+ * Keeps count of the number of stuck threads detected >+ */ >+ private final AtomicInteger stuckCount = new AtomicInteger(0); >+ /** >+ * In seconds. Default 600 (10 minutes). >+ */ >+ private int threshold = 600; >+ /** >+ * 'SO_TIMEOUT' is 20 seconds in Tomcat. >+ * The threshold must at least be larger than that. >+ */ >+ private static final int THRESHOLD_MIN_VALUE = 120; >+ /** >+ * The only references we keep to actual running Thread objects are in >+ * this Map (which is automatically cleaned in invoke()s finally clause). >+ * That way, Threads can be GC'ed, eventhough the Valve still thinks they >+ * are stuck (caused by a long monitor interval) >+ */ >+ private ConcurrentHashMap<Long, MonitoredThread> activeThreads = >+ new ConcurrentHashMap<Long, MonitoredThread>(); >+ /** >+ * >+ */ >+ private Queue<CompletedStuckThread> completedStuckThreadsQueue = >+ new ConcurrentLinkedQueue<CompletedStuckThread>(); >+ /** >+ * The string manager for this package. >+ */ >+ private static final StringManager sm = >+ StringManager.getManager(Constants.Package); >+ >+ /** >+ * Specify the threshold (in seconds) used when checking for stuck threads. >+ * Minimum value is 120 seconds. The default is 600 seconds. >+ * >+ * @param threshold The new threshold in seconds >+ */ >+ public void setThreshold(int threshold) { >+ if (threshold >= THRESHOLD_MIN_VALUE) { >+ this.threshold = threshold; >+ if (log.isDebugEnabled()) { >+ log.debug("Threshold set to " + threshold + " seconds"); >+ } >+ } else { >+ log.warn("Threshold too low. Must be " >+ + THRESHOLD_MIN_VALUE >+ + " or larger. Actual: " + threshold); >+ } >+ } >+ >+ /** >+ * @see #setThreshold(int) >+ * @return The current threshold in seconds >+ */ >+ public int getThreshold() { >+ return threshold; >+ } >+ >+ @Override >+ protected void initInternal() throws LifecycleException { >+ super.initInternal(); >+ >+ if (log.isDebugEnabled()) { >+ log.debug("Monitoring stuck threads with threshold = " >+ + threshold >+ + " sec"); >+ } >+ } >+ >+ /** >+ * Return descriptive information about this Valve implementation. >+ */ >+ @Override >+ public String getInfo() { >+ return info; >+ } >+ >+ private void notifyStuckThreadDetected(String threadName, >+ StackTraceElement[] trace, long activeTime, >+ Date startTime, int numStuckThreads) { >+ >+ String msg = sm.getString( >+ "stuckThreadDetectionValve.notifyStuckThreadDetected", >+ threadName, activeTime, startTime, numStuckThreads); >+ msg += "\n" + getStackTraceAsString(trace); >+ log.warn(msg); >+ } >+ >+ private void notifyStuckThreadCompleted(String threadName, >+ long activeTime, int numStuckThreads) { >+ >+ String msg = sm.getString( >+ "stuckThreadDetectionValve.notifyStuckThreadCompleted", >+ threadName, activeTime, numStuckThreads); >+ //Since the "stuck thread notification" is warn, this should also be warn >+ log.warn(msg); >+ } >+ >+ //TODO Utility method, should be moved to utility class >+ public static String getStackTraceAsString(StackTraceElement[] trace) { >+ StringBuilder buf = new StringBuilder(); >+ for (int i = 0; i < trace.length; i++) { >+ buf.append("\tat ").append(trace[i]).append("\n"); >+ } >+ return buf.toString(); >+ } >+ >+ //FIXME where does it make sense to add this Valve? >+ /** >+ * Set the Container to which this Valve is attached. >+ * >+ * @param container The container to which we are attached >+ */ >+ @Override >+ public void setContainer(Container container) { >+ >+ if (container == null){ >+ throw new IllegalArgumentException("Configuration error: " >+ + "Container cannot be null. Valve must be attached to an Engine"); >+ } >+ if(!(container instanceof Engine)){ >+ throw new IllegalArgumentException("Configuration error: " >+ + "Valve must be attached to an Engine. Actual: " >+ +container.getClass().getName()); >+ } >+ >+ super.setContainer(container); >+// this.engine = (Engine) container; >+ } >+ >+ /** >+ * {@inheritDoc} >+ */ >+ @Override >+ public void invoke(Request request, Response response) >+ throws IOException, ServletException { >+ //Save the thread/runnable >+ //Keeping a reference to the thread object here does not prevent GC'ing, >+ //as the reference is removed from the Map in the finally clause >+ >+ Long key = new Long(Thread.currentThread().getId()); >+ >+ MonitoredThread previous = activeThreads.put(key, >+ new MonitoredThread(Thread.currentThread())); >+ >+ //In theory we might get collisions when using "new Long(thread.getId()).hashCode()". >+ //We can prevent this by using new Integer(AtomicInt.getAndIncrement()) for uniqueness >+ if (previous != null) { >+ log.error("Key already added to map. Previous value has been removed (" >+ + previous.getThread().getName() + ")"); >+ } >+ >+ if (log.isTraceEnabled()) { >+ log.trace("Monitoring execution time for Request " >+ + request.getRequestURI() + " with originalRemoteAddr '" >+ + request.getRemoteAddr() + ":" + request.getRemotePort() >+ + "'"); >+ } >+ >+ try { >+ getNext().invoke(request, response); >+ } finally { >+ MonitoredThread monitoredThread = activeThreads.remove(key); >+ if (monitoredThread.isMarkedAsStuck()) { >+ completedStuckThreadsQueue.add( >+ new CompletedStuckThread(monitoredThread.getThread().getName(), >+ monitoredThread.getActiveTimeInMillis())); >+ } >+ } >+ } >+ >+ @Override >+ public void backgroundProcess() { >+ super.backgroundProcess(); >+ >+ long thresholdInMillis = threshold * 1000; >+ >+ //Check monitored threads >+ for (MonitoredThread monitoredThread : activeThreads.values()) { >+ long activeTime = monitoredThread.getActiveTimeInMillis(); >+ >+ if (!monitoredThread.isMarkedAsStuck() && activeTime >= thresholdInMillis) { >+ int numStuckThreads = stuckCount.incrementAndGet(); >+ monitoredThread.markAsStuck(); >+ notifyStuckThreadDetected(monitoredThread.getThread().getName(), >+ monitoredThread.getThread().getStackTrace(), >+ activeTime, monitoredThread.getStartTime(), numStuckThreads); >+ } >+ } >+ //Check if any threads previously reported as stuck, have finished. >+ CompletedStuckThread completedStuckThread = completedStuckThreadsQueue.poll(); >+ while (completedStuckThread != null) { >+ int numStuckThreads = stuckCount.decrementAndGet(); >+ notifyStuckThreadCompleted(completedStuckThread.getName(), >+ completedStuckThread.getTotalActiveTime(), numStuckThreads); >+ completedStuckThread = completedStuckThreadsQueue.poll(); >+ } >+ } >+ >+ private class MonitoredThread { >+ >+ /** >+ * Reference to the thread to get a stack trace from background task >+ */ >+ private Thread thread; >+ private long start = System.currentTimeMillis(); >+ private volatile boolean isStuck = false; >+ >+ public MonitoredThread(Thread thread) { >+ this.thread = thread; >+ } >+ >+ public Thread getThread() { >+ return this.thread; >+ } >+ >+ public long getActiveTimeInMillis() { >+ return System.currentTimeMillis() - start; >+ } >+ >+ public Date getStartTime() { >+ return new Date(start); >+ } >+ >+ public void markAsStuck() { >+ this.isStuck = true; >+ } >+ >+ public boolean isMarkedAsStuck() { >+ return this.isStuck; >+ } >+ } >+ >+ private class CompletedStuckThread { >+ >+ private String threadName; >+ private long totalActiveTime; >+ >+ public CompletedStuckThread(String threadName, long totalActiveTime) { >+ this.threadName = threadName; >+ this.totalActiveTime = totalActiveTime; >+ } >+ >+ public String getName() { >+ return this.threadName; >+ } >+ >+ public long getTotalActiveTime() { >+ return this.totalActiveTime; >+ } >+ } >+} >#END
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
Attachments on
bug 50306
:
26451
| 26769 |
28961
|
28966