Line 0
Link Here
|
|
|
1 |
/* |
2 |
* Licensed to the Apache Software Foundation (ASF) under one or more |
3 |
* contributor license agreements. See the NOTICE file distributed with |
4 |
* this work for additional information regarding copyright ownership. |
5 |
* The ASF licenses this file to You under the Apache License, Version 2.0 |
6 |
* (the "License"); you may not use this file except in compliance with |
7 |
* the License. You may obtain a copy of the License at |
8 |
* |
9 |
* http://www.apache.org/licenses/LICENSE-2.0 |
10 |
* |
11 |
* Unless required by applicable law or agreed to in writing, software |
12 |
* distributed under the License is distributed on an "AS IS" BASIS, |
13 |
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
14 |
* See the License for the specific language governing permissions and |
15 |
* limitations under the License. |
16 |
* |
17 |
*/ |
18 |
|
19 |
package org.apache.jmeter.protocol.http.util.accesslog; |
20 |
|
21 |
import java.io.File; |
22 |
import java.io.FileNotFoundException; |
23 |
import java.io.IOException; |
24 |
import java.io.InputStream; |
25 |
import java.net.URISyntaxException; |
26 |
import java.text.ParseException; |
27 |
import java.util.ArrayList; |
28 |
import java.util.HashMap; |
29 |
import java.util.HashSet; |
30 |
import java.util.List; |
31 |
import java.util.Map; |
32 |
import java.util.Set; |
33 |
import java.util.regex.Matcher; |
34 |
import java.util.regex.Pattern; |
35 |
|
36 |
import org.apache.commons.io.FileUtils; |
37 |
import org.apache.commons.io.FilenameUtils; |
38 |
import org.apache.commons.io.IOUtils; |
39 |
import org.apache.commons.io.LineIterator; |
40 |
import org.apache.commons.lang3.StringUtils; |
41 |
import org.apache.jmeter.timers.AccessLogTimer; |
42 |
import org.apache.jmeter.util.AccessLogUtils; |
43 |
|
44 |
public class TestPlanGenerator { |
45 |
|
46 |
Map<String, List<String>> entriesByIp; |
47 |
Map<String, Long> offsetsByIp; |
48 |
Set<String> ips; |
49 |
String groupedLogName; |
50 |
String planFileName; |
51 |
String resultFileName; |
52 |
|
53 |
public TestPlanGenerator() { |
54 |
super(); |
55 |
} |
56 |
|
57 |
public static class DateRange { |
58 |
public String startTime; |
59 |
public String endTime; |
60 |
|
61 |
public DateRange(String startTime, String endTime) { |
62 |
this.startTime = startTime; |
63 |
this.endTime = endTime; |
64 |
} |
65 |
} |
66 |
|
67 |
public static class PlanRequest implements Cloneable { |
68 |
public String server; |
69 |
public String port; |
70 |
public String protocol; |
71 |
public DateRange range; |
72 |
public String logFile; |
73 |
public String outputDir; |
74 |
public String virtualhost; |
75 |
public boolean printProgress; |
76 |
public int progressInterval; |
77 |
public String logFormat; |
78 |
} |
79 |
|
80 |
public String generateTestPlan(PlanRequest request) throws IOException, |
81 |
ParseException, FileNotFoundException { |
82 |
generateGroupedAccessLog(request); |
83 |
ips = findIPSetInAccessLog(new File(groupedLogName)); |
84 |
return generateTestPlanForIPs(request); |
85 |
} |
86 |
|
87 |
protected void generateGroupedAccessLog(PlanRequest request) |
88 |
throws IOException, ParseException { |
89 |
|
90 |
long start = parseTime(request.range.startTime); |
91 |
long end = parseTime(request.range.endTime); |
92 |
|
93 |
entriesByIp = filterEntries(request, start, end); |
94 |
|
95 |
File groupedLog = createGroupedLog(request.logFile, request.outputDir); |
96 |
writeGroupedAccessLog(groupedLog); |
97 |
|
98 |
offsetsByIp = calculateOffsets(); |
99 |
|
100 |
} |
101 |
|
102 |
private long parseTime(String time) throws ParseException { |
103 |
return AccessLogUtils.parseDateAsLong(time, |
104 |
AccessLogTimer.DEFAULT_DATE_FORMAT); |
105 |
} |
106 |
|
107 |
protected Set<String> findIPSetInAccessLog(File accessLog) |
108 |
throws IOException { |
109 |
Set<String> ips = new HashSet<String>(); |
110 |
LineIterator it = getLineIteratorForFilename(accessLog); |
111 |
|
112 |
while (it.hasNext()) { |
113 |
String ip = AccessLogUtils.getIpAddress(it.next()); |
114 |
ips.add(ip); |
115 |
} |
116 |
it.close(); |
117 |
return ips; |
118 |
} |
119 |
|
120 |
private String generateTestPlanForIPs(PlanRequest userPlan) |
121 |
throws ParseException { |
122 |
|
123 |
// the thread groups are created, one for each different ip |
124 |
StringBuffer threadGroups = new StringBuffer(); |
125 |
for (String ip : ips) { |
126 |
String threadGroup = generateThreadGroupForTheIP(ip, userPlan.range); |
127 |
threadGroups.append(threadGroup); |
128 |
} |
129 |
|
130 |
// if requested start or end time is empty, the first or end date of the |
131 |
// access log is really used as plan time |
132 |
PlanRequest realPlan = userPlan; |
133 |
|
134 |
if (StringUtils.isEmpty(realPlan.range.startTime)) { |
135 |
realPlan.range.startTime = AccessLogUtils |
136 |
.getDateFromEntry(firstEntry()); |
137 |
} |
138 |
|
139 |
if (StringUtils.isEmpty(realPlan.range.endTime)) { |
140 |
realPlan.range.endTime = AccessLogUtils |
141 |
.getDateFromEntry(lastEntry()); |
142 |
} |
143 |
|
144 |
// the test plan is built and written in the jmx file |
145 |
planFileName = realPlan.outputDir |
146 |
+ constructPlanFileName(realPlan.range); |
147 |
String plan = buildPlan(realPlan, threadGroups); |
148 |
writePlan(planFileName, plan); |
149 |
|
150 |
return planFileName; |
151 |
} |
152 |
|
153 |
String constructPlanFileName(DateRange dateRange) { |
154 |
|
155 |
String startDateName = contructDateNameFromDate(dateRange.startTime); |
156 |
String endDateName = contructDateNameFromDate(dateRange.endTime); |
157 |
return startDateName + "_" + endDateName + "-plan.jmx"; |
158 |
} |
159 |
|
160 |
String contructDateNameFromDate(String date) { |
161 |
return removeOffsetFromDate(date).replaceAll("[:/]", ""); |
162 |
} |
163 |
|
164 |
String removeOffsetFromDate(String date) { |
165 |
if (StringUtils.isEmpty(date)) { |
166 |
return null; |
167 |
} |
168 |
|
169 |
String pattern = "([\\w:/]+)(\\s[+\\-]\\d{4})"; |
170 |
Pattern p = Pattern.compile(pattern); |
171 |
Matcher matcher = p.matcher(date); |
172 |
matcher.matches(); |
173 |
return matcher.group(1); |
174 |
} |
175 |
|
176 |
private String buildPlan(PlanRequest request, StringBuffer threadGroups) { |
177 |
String plan = null; |
178 |
try { |
179 |
String planName = constructPlanName(request.range); |
180 |
resultFileName = contructResultFileName(request.outputDir); |
181 |
plan = readTemplate("template-testPlan.jmx"); |
182 |
plan = plan.replace("@PLAN_NAME@", planName); |
183 |
plan = plan.replace("@SERVER_NAME@", request.server); |
184 |
plan = plan.replace("@SERVER_PORT@", request.port); |
185 |
plan = plan.replace("@SERVER_PROTOCOL@", request.protocol); |
186 |
plan = plan.replace("@THREAD_GROUPS@", threadGroups.toString()); |
187 |
plan = plan.replace("@LOG_FILE@", groupedLogName); |
188 |
plan = plan.replace("@DATE_FORMAT@", |
189 |
AccessLogTimer.DEFAULT_DATE_FORMAT); |
190 |
plan = plan.replace("@DATE_PATTERN@", |
191 |
AccessLogTimer.DEFAULT_DATE_PATTERN); |
192 |
plan = plan.replace("@RESULT_FILE@", resultFileName); |
193 |
|
194 |
} catch (IOException e) { |
195 |
// TODO Auto-generated catch block |
196 |
e.printStackTrace(); |
197 |
} catch (URISyntaxException e) { |
198 |
// TODO Auto-generated catch block |
199 |
e.printStackTrace(); |
200 |
} |
201 |
return plan; |
202 |
} |
203 |
|
204 |
private String contructResultFileName(String outputDir) { |
205 |
return outputDir + FilenameUtils.getBaseName(planFileName) + ".csv"; |
206 |
} |
207 |
|
208 |
private String constructPlanName(DateRange range) { |
209 |
return "Accesses from " + removeOffsetFromDate(range.startTime) |
210 |
+ " to " + removeOffsetFromDate(range.endTime); |
211 |
} |
212 |
|
213 |
private void writePlan(String fileName, String plan) { |
214 |
|
215 |
try { |
216 |
FileUtils.write(new File(fileName), plan); |
217 |
} catch (IOException e) { |
218 |
// TODO Auto-generated catch block |
219 |
e.printStackTrace(); |
220 |
} |
221 |
} |
222 |
|
223 |
private void writeGroupedAccessLog(File groupedLog) throws IOException { |
224 |
for (String ip : entriesByIp.keySet()) { |
225 |
FileUtils.writeLines(groupedLog, entriesByIp.get(ip), true); |
226 |
} |
227 |
} |
228 |
|
229 |
private Map<String, Long> calculateOffsets() throws IOException { |
230 |
Map<String, Long> offsetByIp = new HashMap<String, Long>(); |
231 |
long offset = 0; |
232 |
for (String ip : entriesByIp.keySet()) { |
233 |
offsetByIp.put(ip, offset); |
234 |
offset += entriesByIp.get(ip).size(); |
235 |
} |
236 |
return offsetByIp; |
237 |
} |
238 |
|
239 |
private File createGroupedLog(String accessLog, String outputDir) |
240 |
throws IOException { |
241 |
groupedLogName = outputDir + FilenameUtils.getBaseName(accessLog) |
242 |
+ ".jmeter.log"; |
243 |
File groupedLog = new File(groupedLogName); |
244 |
groupedLog.createNewFile(); |
245 |
return groupedLog; |
246 |
} |
247 |
|
248 |
private Map<String, List<String>> filterEntries(PlanRequest request, |
249 |
long start, long end) throws IOException, ParseException { |
250 |
Map<String, List<String>> entriesByIp = new HashMap<String, List<String>>(); |
251 |
|
252 |
LineIterator it = getLineIteratorForFilename(new File(request.logFile)); |
253 |
|
254 |
int entryNumber = 1; |
255 |
|
256 |
while (it.hasNext()) { |
257 |
|
258 |
if (request.printProgress) { |
259 |
if (0 == entryNumber % request.progressInterval) { |
260 |
System.out.println(entryNumber); |
261 |
} |
262 |
entryNumber++; |
263 |
} |
264 |
|
265 |
String entry = it.next(); |
266 |
// the entry date is queried. |
267 |
long entryDate = AccessLogUtils.parseDateAsLongFromEntry(entry, |
268 |
AccessLogTimer.DEFAULT_DATE_PATTERN, |
269 |
AccessLogTimer.DEFAULT_DATE_FORMAT); |
270 |
|
271 |
// if entry date is outside range, the entry is discarded. |
272 |
if (!isEntryDateInsideRequestedRange(entryDate, start, end)) { |
273 |
continue; |
274 |
} |
275 |
|
276 |
if (StringUtils.isNotEmpty(request.virtualhost)) { |
277 |
if (!isEntryTargetedToRequestedVirtualHost(entry, |
278 |
request.logFormat, request.virtualhost)) { |
279 |
continue; |
280 |
} |
281 |
} |
282 |
|
283 |
// otherwise the entry ip is included in the grouped log. |
284 |
String ip = AccessLogUtils.getIpAddress(entry); |
285 |
if (!entriesByIp.containsKey(ip)) { |
286 |
entriesByIp.put(ip, new ArrayList<String>()); |
287 |
} |
288 |
entriesByIp.get(ip).add(entry); |
289 |
} |
290 |
|
291 |
it.close(); |
292 |
|
293 |
return entriesByIp; |
294 |
} |
295 |
|
296 |
private boolean isEntryTargetedToRequestedVirtualHost(String entry, |
297 |
String logFormat, String vhost) { |
298 |
String addressedTo = AccessLogUtils.getFieldFromEntry( |
299 |
AccessLogUtils.LOGFORMAT_VHOST, entry, logFormat); |
300 |
return vhost.equals(addressedTo); |
301 |
} |
302 |
|
303 |
public boolean isEntryDateInsideRequestedRange(long entryDate, long start, |
304 |
long end) { |
305 |
|
306 |
if (unlimitedRange(start, end)) { |
307 |
return true; |
308 |
} |
309 |
|
310 |
if (startTimeIsUnlimited(start, end)) { |
311 |
return entryDate <= end; |
312 |
} |
313 |
|
314 |
if (endTimeIsUnlimited(start, end)) { |
315 |
return entryDate >= start; |
316 |
} |
317 |
|
318 |
return entryDate >= start && entryDate <= end; |
319 |
} |
320 |
|
321 |
private boolean startTimeIsUnlimited(long start, long end) { |
322 |
return AccessLogUtils.DATE_EMPTY == start |
323 |
&& AccessLogUtils.DATE_EMPTY != end; |
324 |
} |
325 |
|
326 |
private boolean endTimeIsUnlimited(long start, long end) { |
327 |
return AccessLogUtils.DATE_EMPTY != start |
328 |
&& AccessLogUtils.DATE_EMPTY == end; |
329 |
} |
330 |
|
331 |
private boolean unlimitedRange(long start, long end) { |
332 |
return AccessLogUtils.DATE_EMPTY == start |
333 |
&& AccessLogUtils.DATE_EMPTY == end; |
334 |
} |
335 |
|
336 |
private LineIterator getLineIteratorForFilename(File accessLog) |
337 |
throws IOException { |
338 |
return IOUtils.lineIterator(TCLogParser.getReader(accessLog, 0)); |
339 |
} |
340 |
|
341 |
protected String generateThreadGroupForTheIP(String ip, DateRange range) { |
342 |
|
343 |
String threadGroup = ""; |
344 |
try { |
345 |
|
346 |
// the thread group delay is calculated accordingly the first access |
347 |
List<String> entries = entriesByIp.get(ip); |
348 |
long firstAccess = AccessLogUtils.parseDateAsLongFromEntry(entries |
349 |
.get(0)); // first access |
350 |
long start = AccessLogUtils.parseDateAsLong(range.startTime); |
351 |
long groupDelay = Math.abs(firstAccess - start) / 1000; |
352 |
|
353 |
// the thread group duration is calculated |
354 |
// the duration is not really important, as what matters is the loop |
355 |
// counter |
356 |
// given by the number of entries. We make twice the range so is big |
357 |
// enough to |
358 |
// loop all the samples. |
359 |
|
360 |
long end = parseTime(range.endTime); |
361 |
long duration = (end - start) * 2 / 1000; |
362 |
|
363 |
// the thread group is written |
364 |
threadGroup = readTemplate("template-threadGroup.jmx"); |
365 |
threadGroup = threadGroup.replaceAll("@IP@", ip); |
366 |
threadGroup = threadGroup.replace("@DELAY@", |
367 |
Long.toString(groupDelay)); |
368 |
threadGroup = threadGroup.replace("@DURATION@", |
369 |
Long.toString(duration)); |
370 |
threadGroup = threadGroup.replace("@NUM_ENTRIES@", |
371 |
Long.toString(entries.size())); |
372 |
threadGroup = threadGroup.replace("@OFFSET@", |
373 |
Long.toString(offsetsByIp.get(ip))); |
374 |
|
375 |
} catch (IOException e) { |
376 |
// TODO Auto-generated catch block |
377 |
e.printStackTrace(); |
378 |
} catch (ParseException e) { |
379 |
// TODO Auto-generated catch block |
380 |
e.printStackTrace(); |
381 |
} catch (URISyntaxException e) { |
382 |
// TODO Auto-generated catch block |
383 |
e.printStackTrace(); |
384 |
} |
385 |
|
386 |
return threadGroup; |
387 |
|
388 |
} |
389 |
|
390 |
private String readTemplate(String templateName) throws IOException, |
391 |
URISyntaxException { |
392 |
InputStream is = this.getClass().getResourceAsStream(templateName); |
393 |
return IOUtils.toString(is); |
394 |
} |
395 |
|
396 |
public int ipCount() { |
397 |
return entriesByIp.keySet().size(); |
398 |
} |
399 |
|
400 |
public int entryCount() { |
401 |
int count = 0; |
402 |
for (String ip : entriesByIp.keySet()) { |
403 |
count += entriesByIp.get(ip).size(); |
404 |
} |
405 |
return count; |
406 |
} |
407 |
|
408 |
// TODO refactor firstDate and LastEntry |
409 |
|
410 |
public String firstEntry() { |
411 |
long firstAccess = Long.MAX_VALUE; |
412 |
String firstEntry = ""; |
413 |
for (String ip : entriesByIp.keySet()) { |
414 |
for (String entry : entriesByIp.get(ip)) { |
415 |
try { |
416 |
long entryDate = AccessLogUtils |
417 |
.parseDateAsLongFromEntry(entry); |
418 |
if (entryDate < firstAccess) { |
419 |
firstAccess = entryDate; |
420 |
firstEntry = entry; |
421 |
} |
422 |
} catch (ParseException e) { |
423 |
// TODO Auto-generated catch block |
424 |
e.printStackTrace(); |
425 |
} |
426 |
} |
427 |
} |
428 |
return firstEntry; |
429 |
} |
430 |
|
431 |
public String lastEntry() { |
432 |
long lastAccess = 0; |
433 |
String lastEntry = ""; |
434 |
for (String ip : entriesByIp.keySet()) { |
435 |
for (String entry : entriesByIp.get(ip)) { |
436 |
try { |
437 |
long entryDate = AccessLogUtils |
438 |
.parseDateAsLongFromEntry(entry); |
439 |
if (entryDate > lastAccess) { |
440 |
lastAccess = entryDate; |
441 |
lastEntry = entry; |
442 |
} |
443 |
} catch (ParseException e) { |
444 |
// TODO Auto-generated catch block |
445 |
e.printStackTrace(); |
446 |
} |
447 |
} |
448 |
} |
449 |
return lastEntry; |
450 |
} |
451 |
|
452 |
} |