Attachment 33181 Details for Bug 57045 – Program to decode output of fixed mod_dumpio

Program to decode output of fixed mod_dumpio

error_log_data_extract (text/plain), 4.62 KB, created by Kurtis Rader on 2015-10-12 01:04:40 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Kurtis Rader

Created: 2015-10-12 01:04:40 UTC

Size: 4.62 KB

patch

obsolete

>#!/Users/krader/symlinks/python3
>"""Extract data captured by Apache mod_dumpio.
>
>Scan the input looking for lines from mod_dumpio representing the input
>received from the remote system. Capture the data. When the remote client ID
>changes dump the captured data.
>
>Note that we do absolutely everything using binary I/O and byte-arrays rather
>than text I/O and strings. This is because we don't actually care what encoding
>is used in the mod_dumpio output or the Apache HTTP error log. We're
>potentially dealing with non-ASCII and non-UTF-8 data (e.g., binary data such
>as a Zip archive).
>
>Usage:
>    error_log_data_extract < error_log > decoded_data
>
>Note that you do not have to pipe the entire error log through this program.
>You can preprocess the log by, for example, extracting with grep just the lines
>associated with a given IP address and sending that subset of lines through
>this program.
>
>This program assumes you're running the corrected mod_dumpio module I posted to
>Apache PR57045 (https://bz.apache.org/bugzilla/show_bug.cgi?id=57045). It can
>be used on output from the broken mod_dumpio module but it won't, obviously,
>correctly handle 0x00 or 0xFF bytes.
>"""
>import re
>import sys
>
>data_sep_in = (b'=#= input ' + b'=#=' * (70 // 3)) + b'\n'
>data_sep_out = (b'=#= output ' + b'=#=' * (70 // 3)) + b'\n'
>dumpio_new_req_re = re.compile(rb' mod_dumpio: new request$')
>dumpio_re = re.compile(rb'^.*? \[client (?P<client_id>.+?)\]\s'
>                       rb'mod_dumpio:\s+(?P<which>dumpio_(?:in|out))\s+'
>                       rb'$data-[^)]+$:\s(?P<data>.*)$')
>escape_re = re.compile(rb'(\\\\|\\x[fF][fF]\\x0[12]|'
>                       rb'\\x[0-9a-fA-F][0-9a-fA-F]|\\[abfnrtv"])')
>
>hex_escape_to_bin = {'\\x{:02x}'.format(i).encode('ascii'): bytes([i])
>                     for i in range(256)}
>escape_to_char = {b'\\"': b'"', b'\\\\': b'\\', b'\\a': b'\a', b'\\b': b'\b',
>                  b'\\f': b'\f', b'\\n': b'\n', b'\\r': b'\r', b'\\t': b'\t',
>                  b'\\v': b'\v', b'\\xff\\x02': b'\xff', b'\\xff\\x01': b'\x00'}
>escape_to_char.update(hex_escape_to_bin)
>
>
>class ReconstructCapturedData(object):
>    """Capture the data from mod_dumpio and emit it on demand."""
>
>    def __init__(self, out_fh):
>        self.out_fh = out_fh
>        self.captured_input = []
>        self.captured_output = []
>
>    def _SubEscapes(self, match):
>        """Convert text escaped by the mod_dumpio module to their original chars."""
>        _ = self
>        return escape_to_char.get(match.group(0).lower())
>
>    def CaptureData(self, which, data):
>        """Capture the data logged by mod_dumpio."""
>        data = escape_re.sub(self._SubEscapes, data)
>        if which == b'dumpio_in':
>            self.captured_input.append(data)
>        else:
>            self.captured_output.append(data)
>
>    def _DumpCapturedData(self, data, sep):
>        """Write the unescaped data we captured from the mod_dumpio module."""
>        if not data:
>            return
>        self.out_fh.write(sep)
>        eol = ''
>        for data_in in data:
>            eol = data_in[-1] if data_in else ''
>            self.out_fh.write(data_in)
>        if eol != b'\n':
>            self.out_fh.write(b'\n')
>        self.out_fh.write(sep)
>
>    def DumpCapturedData(self):
>        """Write the unescaped data we captured from the mod_dumpio module."""
>        self._DumpCapturedData(self.captured_input, data_sep_in)
>        self._DumpCapturedData(self.captured_output, data_sep_out)
>        self.captured_input = []
>        self.captured_output = []
>
>
>def main():
>    """Filter an Apache error log containing mod_dumpio data."""
>    in_fh = open(sys.stdin.fileno(), 'rb', closefd=False)
>    out_fh = open(sys.stdout.fileno(), 'wb', closefd=False)
>    captured_data = ReconstructCapturedData(out_fh)
>    prev_client_id = None
>    for line in in_fh:
>        if line.endswith(b' mod_dumpio: new request\n'):
>            captured_data.DumpCapturedData()
>            out_fh.write(line)
>            continue
>
>        match = dumpio_re.match(line)
>        if match:
>            client_id = match.group('client_id')
>            which = match.group('which')
>            data = match.group('data')
>            if client_id != prev_client_id and prev_client_id is not None:
>                # This should be a "can't happen" situation if we're parsing
>                # mod_dumpio output that includes the "new request" message.
>                # This will occur for output from older versions of mod_dumpio.
>                captured_data.DumpCapturedData()
>                prev_client_id = client_id
>
>            captured_data.CaptureData(which, data)
>
>        out_fh.write(line)
>
>    captured_data.DumpCapturedData()
>
>
>if __name__ == '__main__':
>    main()

Actions: View

Attachments on bug 57045: 33162 | 33163 | 33164 | 33179 | 33180 | 33181