View | Details | Raw Unified | Return to bug 50306
Collapse All | Expand All

(-)java/org/apache/catalina/valves/LocalStrings.properties (+4 lines)
Lines 56-61 Link Here
56
56
57
sslValve.invalidProvider=The SSL provider specified on the connector associated with this request of [{0}] is invalid. The certificate data could not be processed.
57
sslValve.invalidProvider=The SSL provider specified on the connector associated with this request of [{0}] is invalid. The certificate data could not be processed.
58
58
59
#Stuck thread detection Valve
60
stuckThreadDetectionValve.notifyStuckThreadDetected=Thread "{0}" (id={6}) has been active for {1} milliseconds (since {2}) to serve the same request for {4} and may be stuck (configured threshold for this StuckThreadDetectionValve is {5} seconds). There is/are {3} thread(s) in total that are monitored by this Valve and may be stuck.
61
stuckThreadDetectionValve.notifyStuckThreadCompleted=Thread "{0}" (id={3}) was previously reported to be stuck but has completed. It was active for approximately {1} milliseconds.{2,choice,0#|0< There is/are still {2} thread(s) that are monitored by this Valve and may be stuck.}
62
59
# HTTP status reports
63
# HTTP status reports
60
http.100=The client may continue ({0}).
64
http.100=The client may continue ({0}).
61
http.101=The server is switching protocols according to the "Upgrade" header ({0}).
65
http.101=The server is switching protocols according to the "Upgrade" header ({0}).
(-)java/org/apache/catalina/valves/mbeans-descriptors.xml (+27 lines)
Lines 459-462 Link Here
459
               writeable="false" />
459
               writeable="false" />
460
               
460
               
461
  </mbean>
461
  </mbean>
462
463
  <mbean name="StuckThreadDetectionValve"
464
         description="Detect long requests for which their thread might be stuck"
465
         domain="Catalina"
466
         group="Valve"
467
         type="org.apache.catalina.valves.StuckThreadDetectionValve">
468
469
    <attribute name="className"
470
               description="Fully qualified class name of the managed object"
471
               type="java.lang.String"
472
               writeable="false"/>
473
474
    <attribute name="stuckThreadIds"
475
               description="IDs of the threads currently considered stuck. Each ID can then be used with the Threading MBean to retrieve data about it."
476
               type="long[]"
477
               writeable="false"/>
478
479
    <attribute name="stuckThreadNames"
480
               description="Names of the threads currently considered stuck."
481
               type="java.lang.String[]"
482
               writeable="false"/>
483
484
    <attribute name="threshold"
485
               description="Duration in seconds after which a request is considered as stuck"
486
               type="int"/>
487
488
  </mbean>
462
</mbeans-descriptors>
489
</mbeans-descriptors>
(-)java/org/apache/catalina/valves/StuckThreadDetectionValve.java (+297 lines)
Line 0 Link Here
1
/*
2
 * Licensed to the Apache Software Foundation (ASF) under one or more
3
 * contributor license agreements.  See the NOTICE file distributed with
4
 * this work for additional information regarding copyright ownership.
5
 * The ASF licenses this file to You under the Apache License, Version 2.0
6
 * (the "License"); you may not use this file except in compliance with
7
 * the License.  You may obtain a copy of the License at
8
 *
9
 *      http://www.apache.org/licenses/LICENSE-2.0
10
 *
11
 * Unless required by applicable law or agreed to in writing, software
12
 * distributed under the License is distributed on an "AS IS" BASIS,
13
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
 * See the License for the specific language governing permissions and
15
 * limitations under the License.
16
 */
17
package org.apache.catalina.valves;
18
19
import java.io.IOException;
20
import java.util.ArrayList;
21
import java.util.Date;
22
import java.util.List;
23
import java.util.Queue;
24
import java.util.concurrent.ConcurrentHashMap;
25
import java.util.concurrent.ConcurrentLinkedQueue;
26
import java.util.concurrent.atomic.AtomicInteger;
27
28
import javax.servlet.ServletException;
29
30
import org.apache.catalina.connector.Request;
31
import org.apache.catalina.connector.Response;
32
import org.apache.juli.logging.Log;
33
import org.apache.juli.logging.LogFactory;
34
import org.apache.tomcat.util.res.StringManager;
35
36
/**
37
 * This valve allows to detect requests that take a long time to process, which
38
 * might indicate that the thread that is processing it is stuck.
39
 */
40
public class StuckThreadDetectionValve extends ValveBase {
41
42
    /**
43
     * Logger
44
     */
45
    private static final Log log = LogFactory.getLog(StuckThreadDetectionValve.class);
46
47
    /**
48
     * The string manager for this package.
49
     */
50
    private static final StringManager sm =
51
        StringManager.getManager(Constants.Package);
52
53
    /**
54
     * Keeps count of the number of stuck threads detected
55
     */
56
    private final AtomicInteger stuckCount = new AtomicInteger(0);
57
58
    /**
59
     * In seconds. Default 600 (10 minutes).
60
     */
61
    private int threshold = 600;
62
63
    /**
64
     * The only references we keep to actual running Thread objects are in
65
     * this Map (which is automatically cleaned in invoke()s finally clause).
66
     * That way, Threads can be GC'ed, eventhough the Valve still thinks they
67
     * are stuck (caused by a long monitor interval)
68
     */
69
    private final ConcurrentHashMap<Long, MonitoredThread> activeThreads =
70
            new ConcurrentHashMap<Long, MonitoredThread>();
71
    /**
72
     *
73
     */
74
    private final Queue<CompletedStuckThread> completedStuckThreadsQueue =
75
            new ConcurrentLinkedQueue<CompletedStuckThread>();
76
77
    /**
78
     * Specify the threshold (in seconds) used when checking for stuck threads.
79
     * If &lt;=0, the detection is disabled. The default is 600 seconds.
80
     *
81
     * @param threshold
82
     *            The new threshold in seconds
83
     */
84
    public void setThreshold(int threshold) {
85
        this.threshold = threshold;
86
    }
87
88
    /**
89
     * @see #setThreshold(int)
90
     * @return The current threshold in seconds
91
     */
92
    public int getThreshold() {
93
        return threshold;
94
    }
95
96
97
    private void notifyStuckThreadDetected(MonitoredThread monitoredThread,
98
        long activeTime, int numStuckThreads) {
99
        if (log.isWarnEnabled()) {
100
            String msg = sm.getString(
101
                "stuckThreadDetectionValve.notifyStuckThreadDetected",
102
                monitoredThread.getThread().getName(),
103
                Long.valueOf(activeTime),
104
                monitoredThread.getStartTime(),
105
                Integer.valueOf(numStuckThreads),
106
                monitoredThread.getRequestUri(),
107
                Integer.valueOf(threshold),
108
                String.valueOf(monitoredThread.getThread().getId())
109
                );
110
            // msg += "\n" + getStackTraceAsString(trace);
111
            Throwable th = new Throwable();
112
            th.setStackTrace(monitoredThread.getThread().getStackTrace());
113
            log.warn(msg, th);
114
        }
115
    }
116
117
    private void notifyStuckThreadCompleted(CompletedStuckThread thread,
118
            int numStuckThreads) {
119
        if (log.isWarnEnabled()) {
120
            String msg = sm.getString(
121
                "stuckThreadDetectionValve.notifyStuckThreadCompleted",
122
                thread.getName(),
123
                Long.valueOf(thread.getTotalActiveTime()),
124
                Integer.valueOf(numStuckThreads),
125
                String.valueOf(thread.getId()));
126
            // Since the "stuck thread notification" is warn, this should also
127
            // be warn
128
            log.warn(msg);
129
        }
130
    }
131
132
    /**
133
     * {@inheritDoc}
134
     */
135
    @Override
136
    public void invoke(Request request, Response response)
137
            throws IOException, ServletException {
138
139
        if (threshold <= 0) {
140
            // short-circuit if not monitoring stuck threads
141
            getNext().invoke(request, response);
142
            return;
143
        }
144
145
        // Save the thread/runnable
146
        // Keeping a reference to the thread object here does not prevent
147
        // GC'ing, as the reference is removed from the Map in the finally clause
148
149
        Long key = Long.valueOf(Thread.currentThread().getId());
150
        StringBuffer requestUrl = request.getRequestURL();
151
        if(request.getQueryString()!=null) {
152
            requestUrl.append("?");
153
            requestUrl.append(request.getQueryString());
154
        }
155
        MonitoredThread monitoredThread = new MonitoredThread(Thread.currentThread(),
156
            requestUrl.toString());
157
        activeThreads.put(key, monitoredThread);
158
159
        try {
160
            getNext().invoke(request, response);
161
        } finally {
162
            activeThreads.remove(key);
163
            if (monitoredThread.markAsDone() == MonitoredThreadState.STUCK) {
164
                completedStuckThreadsQueue.add(
165
                        new CompletedStuckThread(monitoredThread.getThread(),
166
                            monitoredThread.getActiveTimeInMillis()));
167
            }
168
        }
169
    }
170
171
    @Override
172
    public void backgroundProcess() {
173
        super.backgroundProcess();
174
175
        long thresholdInMillis = threshold * 1000;
176
177
        // Check monitored threads, being careful that the request might have
178
        // completed by the time we examine it
179
        for (MonitoredThread monitoredThread : activeThreads.values()) {
180
            long activeTime = monitoredThread.getActiveTimeInMillis();
181
182
            if (activeTime >= thresholdInMillis && monitoredThread.markAsStuckIfStillRunning()) {
183
                int numStuckThreads = stuckCount.incrementAndGet();
184
                notifyStuckThreadDetected(monitoredThread, activeTime, numStuckThreads);
185
            }
186
        }
187
        // Check if any threads previously reported as stuck, have finished.
188
        for (CompletedStuckThread completedStuckThread = completedStuckThreadsQueue.poll();
189
            completedStuckThread != null; completedStuckThread = completedStuckThreadsQueue.poll()) {
190
191
            int numStuckThreads = stuckCount.decrementAndGet();
192
            notifyStuckThreadCompleted(completedStuckThread, numStuckThreads);
193
        }
194
    }
195
196
    public long[] getStuckThreadIds() {
197
        List<Long> idList = new ArrayList<Long>();
198
        for (MonitoredThread monitoredThread : activeThreads.values()) {
199
            if (monitoredThread.isMarkedAsStuck()) {
200
                idList.add(Long.valueOf(monitoredThread.getThread().getId()));
201
            }
202
        }
203
204
        long[] result = new long[idList.size()];
205
        for (int i = 0; i < result.length; i++) {
206
            result[i] = idList.get(i).longValue();
207
        }
208
        return result;
209
    }
210
211
    public String[] getStuckThreadNames() {
212
        List<String> nameList = new ArrayList<String>();
213
        for (MonitoredThread monitoredThread : activeThreads.values()) {
214
            if (monitoredThread.isMarkedAsStuck()) {
215
                nameList.add(monitoredThread.getThread().getName());
216
            }
217
        }
218
        return nameList.toArray(new String[nameList.size()]);
219
    }
220
221
    private static class MonitoredThread {
222
223
        /**
224
         * Reference to the thread to get a stack trace from background task
225
         */
226
        private final Thread thread;
227
        private final String requestUri;
228
        private final long start;
229
        private final AtomicInteger state = new AtomicInteger(
230
            MonitoredThreadState.RUNNING.ordinal());
231
232
        public MonitoredThread(Thread thread, String requestUri) {
233
            this.thread = thread;
234
            this.requestUri = requestUri;
235
            this.start = System.currentTimeMillis();
236
        }
237
238
        public Thread getThread() {
239
            return this.thread;
240
        }
241
242
        public String getRequestUri() {
243
            return requestUri;
244
        }
245
246
        public long getActiveTimeInMillis() {
247
            return System.currentTimeMillis() - start;
248
        }
249
250
        public Date getStartTime() {
251
            return new Date(start);
252
        }
253
254
        public boolean markAsStuckIfStillRunning() {
255
            return this.state.compareAndSet(MonitoredThreadState.RUNNING.ordinal(),
256
                MonitoredThreadState.STUCK.ordinal());
257
        }
258
259
        public MonitoredThreadState markAsDone() {
260
            int val = this.state.getAndSet(MonitoredThreadState.DONE.ordinal());
261
            return MonitoredThreadState.values()[val];
262
        }
263
264
        boolean isMarkedAsStuck() {
265
            return this.state.get() == MonitoredThreadState.STUCK.ordinal();
266
        }
267
    }
268
269
    private static class CompletedStuckThread {
270
271
        private final String threadName;
272
        private final long threadId;
273
        private final long totalActiveTime;
274
275
        public CompletedStuckThread(Thread thread, long totalActiveTime) {
276
            this.threadName = thread.getName();
277
            this.threadId = thread.getId();
278
            this.totalActiveTime = totalActiveTime;
279
        }
280
281
        public String getName() {
282
            return this.threadName;
283
        }
284
285
        public long getId() {
286
            return this.threadId;
287
        }
288
289
        public long getTotalActiveTime() {
290
            return this.totalActiveTime;
291
        }
292
    }
293
294
    private enum MonitoredThreadState {
295
        RUNNING, STUCK, DONE;
296
    }
297
}
(-)webapps/docs/config/valve.xml (+44 lines)
Lines 873-878 Link Here
873
</section>
873
</section>
874
874
875
875
876
<section name="Stuck Thread Detection Valve">
877
878
  <subsection name="Introduction">
879
880
    <p>This valve allows to detect requests that take a long time to process, which might
881
    indicate that the thread that is processing it is stuck.</p>
882
    <p>When such a request is detected, the current stack trace of its thread is written
883
    to Tomcat log with a WARN level.</p>
884
    <p>The IDs and names of the stuck threads are available through JMX in the
885
    <code>stuckThreadIds</code> and <code>stuckThreadNames</code> attributes.
886
    The IDs can be used with the standard Threading JVM MBean
887
    (<code>java.lang:type=Threading</code>) to retrieve other information
888
    about each stuck thread.</p>
889
890
  </subsection>
891
892
  <subsection name="Attributes">
893
894
    <p>The <strong>Stuck Thread Detection Valve</strong> supports the
895
    following configuration attributes:</p>
896
897
    <attributes>
898
899
      <attribute name="className" required="true">
900
        <p>Java class name of the implementation to use.  This MUST be set to
901
        <strong>org.apache.catalina.valves.StuckThreadDetectionValve</strong>.
902
        </p>
903
      </attribute>
904
905
      <attribute name="threshold" required="false">
906
        <p>Minimum duration in seconds after which a thread is considered stuck.
907
        Default is 600 seconds. If set to 0, the detection is disabled.</p>
908
        <p>Note: since the detection is done in the background thread of the Container
909
        (Engine, Host or Context) declaring this Valve, the threshold should be higher
910
        than the <code>backgroundProcessorDelay</code> of this Container.</p>
911
      </attribute>
912
913
    </attributes>
914
915
  </subsection>
916
917
</section>
918
919
876
</body>
920
</body>
877
921
878
922

Return to bug 50306