--- src/examples/src/org/apache/poi/poifs/poibrowser/DocumentDescriptor.java (revision 1753737) +++ src/examples/src/org/apache/poi/poifs/poibrowser/DocumentDescriptor.java (working copy) @@ -17,8 +17,11 @@ package org.apache.poi.poifs.poibrowser; -import java.io.*; -import org.apache.poi.poifs.filesystem.*; +import java.io.IOException; + +import org.apache.poi.poifs.filesystem.DocumentInputStream; +import org.apache.poi.poifs.filesystem.POIFSDocumentPath; +import org.apache.poi.util.IOUtils; /** *

Describes the most important (whatever that is) features of a @@ -49,26 +52,20 @@ public DocumentDescriptor(final String name, final POIFSDocumentPath path, final DocumentInputStream stream, - final int nrOfBytes) - { + final int nrOfBytes) { this.name = name; this.path = path; this.stream = stream; - try - { - size = stream.available(); - if (stream.markSupported()) - { + try { + if (stream.markSupported()) { stream.mark(nrOfBytes); - final byte[] b = new byte[nrOfBytes]; - final int read = stream.read(b, 0, Math.min(size, b.length)); - bytes = new byte[read]; - System.arraycopy(b, 0, bytes, 0, read); + bytes = IOUtils.toByteArray(stream, nrOfBytes); stream.reset(); + } else { + bytes = new byte[0]; } - } - catch (IOException ex) - { + size = bytes.length + stream.available(); + } catch (IOException ex) { System.out.println(ex); } } --- src/java/org/apache/poi/hpsf/PropertySet.java (revision 1753737) +++ src/java/org/apache/poi/hpsf/PropertySet.java (working copy) @@ -235,15 +236,11 @@ throws NoPropertySetStreamException, MarkUnsupportedException, IOException, UnsupportedEncodingException { - if (isPropertySetStream(stream)) - { - final int avail = stream.available(); - final byte[] buffer = new byte[avail]; - IOUtils.readFully(stream, buffer); - init(buffer, 0, buffer.length); - } - else + if (!isPropertySetStream(stream)) { throw new NoPropertySetStreamException(); + } + final byte[] buffer = IOUtils.toByteArray(stream); + init(buffer, 0, buffer.length); } @@ -322,25 +320,19 @@ * reset to this position if the stream does not contain a * property set. */ - if (!stream.markSupported()) + if (!stream.markSupported()) { throw new MarkUnsupportedException(stream.getClass().getName()); + } stream.mark(BUFFER_SIZE); /* * Read a couple of bytes from the stream. */ - final byte[] buffer = new byte[BUFFER_SIZE]; - final int bytes = - stream.read(buffer, 0, - Math.min(buffer.length, stream.available())); - final boolean isPropertySetStream = - isPropertySetStream(buffer, 0, bytes); + final byte[] buffer = IOUtils.toByteArray(stream, -BUFFER_SIZE); stream.reset(); - return isPropertySetStream; + return (buffer.length > 0 && isPropertySetStream(buffer, 0, buffer.length)); } - - /** *

Checks whether a byte array is in the Horrible Property Set * Format.

--- src/java/org/apache/poi/hssf/dev/BiffViewer.java (revision 1753737) +++ src/java/org/apache/poi/hssf/dev/BiffViewer.java (working copy) @@ -194,10 +194,12 @@ import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; import org.apache.poi.util.HexDump; +import org.apache.poi.util.IOUtils; import org.apache.poi.util.LittleEndian; import org.apache.poi.util.POILogFactory; import org.apache.poi.util.POILogger; import org.apache.poi.util.StringUtil; +import org.apache.poi.util.SuppressForbidden; /** * Utility for reading in BIFF8 records and displaying data from them. @@ -567,10 +569,7 @@ try { if (cmdArgs.shouldOutputRawHexOnly()) { - int size = is.available(); - byte[] data = new byte[size]; - - is.read(data); + byte[] data = IOUtils.toByteArray(is); HexDump.dump(data, 0, System.out, 0); } else { boolean dumpInterpretedRecords = cmdArgs.shouldDumpRecordInterpretations(); --- src/java/org/apache/poi/hssf/record/ObjRecord.java (revision 1753737) +++ src/java/org/apache/poi/hssf/record/ObjRecord.java (working copy) @@ -18,10 +18,13 @@ package org.apache.poi.hssf.record; import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import org.apache.poi.util.HexDump; +import org.apache.poi.util.IOUtils; import org.apache.poi.util.LittleEndian; import org.apache.poi.util.LittleEndianByteArrayOutputStream; import org.apache.poi.util.LittleEndianInputStream; @@ -88,57 +89,69 @@ subrecords = new ArrayList(); ByteArrayInputStream bais = new ByteArrayInputStream(subRecordData); LittleEndianInputStream subRecStream = new LittleEndianInputStream(bais); - CommonObjectDataSubRecord cmo = (CommonObjectDataSubRecord)SubRecord.createSubRecord(subRecStream, 0); - subrecords.add(cmo); - while (true) { - SubRecord subRecord = SubRecord.createSubRecord(subRecStream, cmo.getObjectType()); - subrecords.add(subRecord); - if (subRecord.isTerminating()) { - break; - } - } - int nRemainingBytes = bais.available(); - if (nRemainingBytes > 0) { - // At present (Oct-2008), most unit test samples have (subRecordData.length % 2 == 0) - _isPaddedToQuadByteMultiple = subRecordData.length % MAX_PAD_ALIGNMENT == 0; - if (nRemainingBytes >= (_isPaddedToQuadByteMultiple ? MAX_PAD_ALIGNMENT : NORMAL_PAD_ALIGNMENT)) { - if (!canPaddingBeDiscarded(subRecordData, nRemainingBytes)) { - String msg = "Leftover " + nRemainingBytes - + " bytes in subrecord data " + HexDump.toHex(subRecordData); - throw new RecordFormatException(msg); - } - _isPaddedToQuadByteMultiple = false; - } - - } else { - _isPaddedToQuadByteMultiple = false; + try { + CommonObjectDataSubRecord cmo = (CommonObjectDataSubRecord)SubRecord.createSubRecord(subRecStream, 0); + subrecords.add(cmo); + SubRecord subRecord; + do { + subRecord = SubRecord.createSubRecord(subRecStream, cmo.getObjectType()); + subrecords.add(subRecord); + } while (!subRecord.isTerminating()); + + _isPaddedToQuadByteMultiple = isPaddedToQuadByteMultiple(bais, subRecordData.length); + _uninterpretedData = null; + } finally { + try { + subRecStream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } } - _uninterpretedData = null; } - /** - * Some XLS files have ObjRecords with nearly 8Kb of excessive padding. These were probably - * written by a version of POI (around 3.1) which incorrectly interpreted the second short of - * the ftLbs subrecord (0x1FEE) as a length, and read that many bytes as padding (other bugs - * helped allow this to occur). - * - * Excel reads files with this excessive padding OK, truncating the over-sized ObjRecord back - * to the its proper size. POI does the same. - */ - private static boolean canPaddingBeDiscarded(byte[] data, int nRemainingBytes) { - // make sure none of the padding looks important - for(int i=data.length-nRemainingBytes; i= padAlignment) { + if (dataInPadding) { + String msg = "Leftover bytes in subrecord data " + HexDump.toHex(nRemainingBytes); + throw new RecordFormatException(msg); + } + isPaddedToQuad = false; + } + + return isPaddedToQuad; } @Override public String toString() { --- src/java/org/apache/poi/hssf/record/SubRecord.java (revision 1753737) +++ src/java/org/apache/poi/hssf/record/SubRecord.java (working copy) @@ -23,6 +23,7 @@ import org.apache.poi.util.LittleEndianOutputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; /** * Subrecords are part of the OBJ class. @@ -76,15 +77,26 @@ public byte[] serialize() { int size = getDataSize() + 4; ByteArrayOutputStream baos = new ByteArrayOutputStream(size); - serialize(new LittleEndianOutputStream(baos)); - if (baos.size() != size) { - throw new RuntimeException("write size mismatch"); + LittleEndianOutputStream leos = new LittleEndianOutputStream(baos); + try { + serialize(leos); + if (baos.size() != size) { + throw new RuntimeException("write size mismatch"); + } + return baos.toByteArray(); + } finally { + try { + leos.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } } - return baos.toByteArray(); } public abstract void serialize(LittleEndianOutput out); --- src/java/org/apache/poi/poifs/crypt/ChunkedCipherInputStream.java (revision 1753737) +++ src/java/org/apache/poi/poifs/crypt/ChunkedCipherInputStream.java (working copy) @@ -57,27 +44,29 @@ public int read(byte[] b, int off, int len) throws IOException { int total = 0; - if (available() <= 0) return -1; + if (remainingBytes() <= 0) { + return -1; + } while (len > 0) { - if (_chunk == null) { - try { - _chunk = nextChunk(); - } catch (GeneralSecurityException e) { - throw new EncryptedDocumentException(e.getMessage(), e); - } + int tmp = -1; + if (!_chunkValid) { + tmp = nextChunk(); + _chunkValid = true; } int count = (int)(chunkSize - (_pos & chunkMask)); - int avail = available(); + int avail = remainingBytes(); + assert(tmp == chunkSize || tmp == -1 || avail <= tmp); if (avail == 0) { return total; } @@ -86,8 +89,9 @@ off += count; len -= count; _pos += count; - if ((_pos & chunkMask) == 0) - _chunk = null; + if ((_pos & chunkMask) == 0) { + _chunkValid = false; + } total += count; } @@ -97,16 +101,22 @@ @Override public long skip(long n) throws IOException { long start = _pos; - long skip = Math.min(available(), n); + long skip = Math.min(remainingBytes(), n); - if ((((_pos + skip) ^ start) & ~chunkMask) != 0) - _chunk = null; + if ((((_pos + skip) ^ start) & ~chunkMask) != 0) { + _chunkValid = false; + } _pos += skip; return skip; } @Override public int available() { + return remainingBytes(); + } + + // helper method (... for forbidden-apis-check :) ...) + private int remainingBytes() { return (int)(_size - _pos); } @@ -125,17 +135,35 @@ throw new UnsupportedOperationException(); } - private byte[] nextChunk() throws GeneralSecurityException, IOException { - int index = (int)(_pos >> chunkBits); - initCipherForBlock(_cipher, index); + /** + * Read the next chunk from the super InputStream. + * This also handles non-continued reads, because of {@link #skip(long)} calls. + * + * @return the actual chunk size, which might be less than {@link #chunkSize} + * @throws IOException if the underlying stream can't be read + */ + private int nextChunk() throws IOException { + final int index = (int)(_pos >> chunkBits); if (_lastIndex != index) { super.skip((index - _lastIndex) << chunkBits); } - - byte[] block = new byte[Math.min(super.available(), chunkSize)]; - super.read(block, 0, block.length); _lastIndex = index + 1; - return _cipher.doFinal(block); + + + int totalBytes = 0; + for (int readBytes; totalBytes < chunkSize; totalBytes += readBytes) { + readBytes = super.read(_chunk, totalBytes, chunkSize-totalBytes); + if (readBytes == -1) { + break; + } + } + + try { + initCipherForBlock(_cipher, index); + return _cipher.doFinal(_chunk, 0, totalBytes, _chunk); + } catch (GeneralSecurityException e) { + throw new EncryptedDocumentException(e.getMessage(), e); + } } } --- src/java/org/apache/poi/poifs/eventfilesystem/POIFSReader.java (revision 1753737) +++ src/java/org/apache/poi/poifs/eventfilesystem/POIFSReader.java (working copy) @@ -15,13 +15,14 @@ See the License for the specific language governing permissions and limitations under the License. ==================================================================== */ - -package org.apache.poi.poifs.eventfilesystem; -import java.io.*; +package org.apache.poi.poifs.eventfilesystem; -import java.util.*; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; import org.apache.poi.poifs.filesystem.DocumentInputStream; import org.apache.poi.poifs.filesystem.OPOIFSDocument; @@ -34,6 +35,7 @@ import org.apache.poi.poifs.storage.HeaderBlock; import org.apache.poi.poifs.storage.RawDataBlockList; import org.apache.poi.poifs.storage.SmallBlockTableReader; +import org.apache.poi.util.IOUtils; /** * An event-driven reader for POIFS file systems. Users of this class @@ -258,11 +257,12 @@ while (listeners.hasNext()) { POIFSReaderListener listener = listeners.next(); - - listener.processPOIFSReaderEvent( - new POIFSReaderEvent( - new DocumentInputStream(document), path, - name)); + DocumentInputStream dis = new DocumentInputStream(document); + try { + listener.processPOIFSReaderEvent(new POIFSReaderEvent(dis, path, name)); + } finally { + dis.close(); + } } } else @@ -309,9 +310,7 @@ try { - byte[] data = new byte[ istream.available() ]; - - istream.read(data); + byte[] data = IOUtils.toByteArray(istream); int pathLength = path.length(); for (int k = 0; k < pathLength; k++) --- src/java/org/apache/poi/poifs/filesystem/ODocumentInputStream.java (revision 1753737) +++ src/java/org/apache/poi/poifs/filesystem/ODocumentInputStream.java (working copy) @@ -134,7 +134,7 @@ if (atEOD()) { return EOF; } - int limit = Math.min(available(), len); + int limit = Math.min(_document_size - _current_offset, len); readFully(b, off, limit); return limit; } --- src/java/org/apache/poi/util/IOUtils.java (revision 1753737) +++ src/java/org/apache/poi/util/IOUtils.java (working copy) @@ -74,21 +74,30 @@ /** * Reads up to {@code length} bytes from the input stream, and returns the bytes read. + * + * @param stream the stream to read from + * @param length the amount of bytes to be read, if {@code length} is {@value Integer.MAX_VALUE} + * or negative, the byte length is not checked + * + * @return the read bytes + * + * @throws IOException if {@code length} can't be read. the check is omitted, if {@code length} is negative */ - public static byte[] toByteArray(InputStream stream, int length) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(length == Integer.MAX_VALUE ? 4096 : length); + public static byte[] toByteArray(InputStream stream, final int length) throws IOException { + final int len = Math.abs(length); + ByteArrayOutputStream baos = new ByteArrayOutputStream(len == Integer.MAX_VALUE ? 4096 : len); byte[] buffer = new byte[4096]; int totalBytes = 0, readBytes; do { - readBytes = stream.read(buffer, 0, Math.min(buffer.length, length-totalBytes)); + readBytes = stream.read(buffer, 0, Math.min(buffer.length, len-totalBytes)); totalBytes += Math.max(readBytes,0); if (readBytes > 0) { baos.write(buffer, 0, readBytes); } - } while (totalBytes < length && readBytes > -1); + } while (totalBytes < len && readBytes > -1); - if (length != Integer.MAX_VALUE && totalBytes < length) { + if (length >= 0 && len != Integer.MAX_VALUE && totalBytes < len) { throw new IOException("unexpected EOF"); } --- src/ooxml/java/org/apache/poi/openxml4j/opc/internal/marshallers/ZipPartMarshaller.java (revision 1753737) +++ src/ooxml/java/org/apache/poi/openxml4j/opc/internal/marshallers/ZipPartMarshaller.java (working copy) @@ -36,6 +36,7 @@ import org.apache.poi.openxml4j.opc.internal.PartMarshaller; import org.apache.poi.openxml4j.opc.internal.ZipHelper; import org.apache.poi.util.DocumentHelper; +import org.apache.poi.util.IOUtils; import org.apache.poi.util.POILogFactory; import org.apache.poi.util.POILogger; import org.apache.poi.xssf.usermodel.XSSFRelation; @@ -81,19 +81,12 @@ // Saving data in the ZIP file InputStream ins = part.getInputStream(); - byte[] buff = new byte[ZipHelper.READ_WRITE_FILE_BUFFER_SIZE]; - while (ins.available() > 0) { - int resultRead = ins.read(buff); - if (resultRead == -1) { - // End of file reached - break; - } - zos.write(buff, 0, resultRead); - } + IOUtils.copy(ins, zos); + ins.close(); + zos.closeEntry(); } catch (IOException ioe) { - logger.log(POILogger.ERROR,"Cannot write: " + part.getPartName() + ": in ZIP", - ioe); + logger.log(POILogger.ERROR,"Cannot write: " + part.getPartName() + ": in ZIP", ioe); return false; } --- src/ooxml/testcases/org/apache/poi/TestPOIXMLProperties.java (revision 1753737) +++ src/ooxml/testcases/org/apache/poi/TestPOIXMLProperties.java (working copy) @@ -30,6 +30,7 @@ import org.apache.poi.POIXMLProperties.CoreProperties; import org.apache.poi.openxml4j.util.Nullable; +import org.apache.poi.util.IOUtils; import org.apache.poi.util.LocaleUtil; import org.apache.poi.xssf.XSSFTestDataSamples; import org.apache.poi.xssf.usermodel.XSSFWorkbook; @@ -249,13 +250,13 @@ assertNotNull(noThumbProps.getThumbnailPart()); assertEquals("Testing.png", noThumbProps.getThumbnailFilename()); assertNotNull(noThumbProps.getThumbnailImage()); - assertEquals(1, noThumbProps.getThumbnailImage().available()); + assertEquals(1, IOUtils.toByteArray(noThumbProps.getThumbnailImage()).length); noThumbProps.setThumbnail("Testing2.png", new ByteArrayInputStream(new byte[2])); assertNotNull(noThumbProps.getThumbnailPart()); assertEquals("Testing.png", noThumbProps.getThumbnailFilename()); assertNotNull(noThumbProps.getThumbnailImage()); - assertEquals(2, noThumbProps.getThumbnailImage().available()); + assertEquals(2, IOUtils.toByteArray(noThumbProps.getThumbnailImage()).length); } private static String zeroPad(long i) { --- src/ooxml/testcases/org/apache/poi/poifs/crypt/TestDecryptor.java (revision 1753737) +++ src/ooxml/testcases/org/apache/poi/poifs/crypt/TestDecryptor.java (working copy) @@ -90,10 +91,14 @@ // crc32 is checked within zip-stream - zin.skip(entry.getSize()); + zin.skip(entry.getSize()-1); byte buf[] = new byte[10]; int readBytes = zin.read(buf); // zin.available() doesn't work for entries - assertEquals("size failed for "+entry.getName(), -1, readBytes); + assertEquals("size failed for "+entry.getName(), 1, readBytes); } zin.close(); @@ -126,10 +131,11 @@ break; } - while (zin.available()>0) { - zin.skip(zin.available()); - } + IOUtils.toByteArray(zin); } + + is.close(); + fs.close(); } @Test --- src/ooxml/testcases/org/apache/poi/poifs/crypt/TestEncryptor.java (revision 1753737) +++ src/ooxml/testcases/org/apache/poi/poifs/crypt/TestEncryptor.java (working copy) @@ -42,10 +42,10 @@ import org.apache.poi.poifs.filesystem.Entry; import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; import org.apache.poi.poifs.filesystem.POIFSFileSystem; -import org.apache.poi.util.BoundedInputStream; import org.apache.poi.util.IOUtils; import org.apache.poi.util.TempFile; import org.apache.poi.xwpf.usermodel.XWPFDocument; +import org.bouncycastle.util.Arrays; import org.junit.Assume; import org.junit.Ignore; import org.junit.Test; @@ -112,9 +112,10 @@ assertEquals(decPackLenExpected, payloadExpected.length); is = nfs.getRoot().createDocumentInputStream(Decryptor.DEFAULT_POIFS_ENTRY); - is = new BoundedInputStream(is, is.available()-16); // ignore padding block byte encPackExpected[] = IOUtils.toByteArray(is); is.close(); + // ignore padding block + encPackExpected = Arrays.copyOf(encPackExpected, encPackExpected.length-16); // listDir(nfs.getRoot(), "orig", ""); @@ -164,8 +166,9 @@ long decPackLenActual = decActual.getLength(); is = nfs.getRoot().createDocumentInputStream(Decryptor.DEFAULT_POIFS_ENTRY); - is = new BoundedInputStream(is, is.available()-16); // ignore padding block byte encPackActual[] = IOUtils.toByteArray(is); + // ignore padding block + encPackActual = Arrays.copyOf(encPackActual, encPackActual.length-16); is.close(); // listDir(nfs.getRoot(), "copy", ""); @@ -245,14 +248,10 @@ bos.reset(); fs.writeFilesystem(bos); + fs.close(); ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); - // FileOutputStream fos = new FileOutputStream("encrypted.docx"); - // IOUtils.copy(bis, fos); - // fos.close(); - // bis.reset(); - nfs = new NPOIFSFileSystem(bis); infoExpected = new EncryptionInfo(nfs); d = Decryptor.getInstance(infoExpected); @@ -279,6 +278,7 @@ public void encryptPackageWithoutCoreProperties() throws Exception { // Open our file without core properties File inp = POIDataSamples.getOpenXML4JInstance().getFile("OPCCompliance_NoCoreProperties.xlsx"); + @SuppressWarnings("resource") OPCPackage pkg = OPCPackage.open(inp.getPath()); // It doesn't have any core properties yet --- src/resources/devtools/forbidden-signatures.txt (revision 1753737) +++ src/resources/devtools/forbidden-signatures.txt (working copy) @@ -109,3 +109,6 @@ @defaultMessage Don't interrupt threads use FutureUtils#cancel(Future) instead java.util.concurrent.Future#cancel(boolean) + +@defaultMessage Don't use ...InputStream.available() as it gives wrong result for certain streams - use IOUtils.toByteArray to read the stream fully and then count the available bytes +java.io.InputStream#available() --- src/scratchpad/src/org/apache/poi/hwmf/usermodel/HwmfPicture.java (revision 1753737) +++ src/scratchpad/src/org/apache/poi/hwmf/usermodel/HwmfPicture.java (working copy) @@ -53,40 +53,50 @@ placeableHeader = HwmfPlaceableHeader.readHeader(leis); header = new HwmfHeader(leis); - for (;;) { - if (leis.available() < 6) { - logger.log(POILogger.ERROR, "unexpected eof - wmf file was truncated"); - break; - } - // recordSize in DWORDs - long recordSize = leis.readUInt()*2; - int recordFunction = leis.readShort(); - // 4 bytes (recordSize) + 2 bytes (recordFunction) - int consumedSize = 6; - HwmfRecordType wrt = HwmfRecordType.getById(recordFunction); - if (wrt == null) { - throw new IOException("unexpected record type: "+recordFunction); - } - if (wrt == HwmfRecordType.eof) break; - if (wrt.clazz == null) { - throw new IOException("unsupported record type: "+recordFunction); - } - - HwmfRecord wr; - try { - wr = wrt.clazz.newInstance(); - records.add(wr); - } catch (Exception e) { - throw (IOException)new IOException("can't create wmf record").initCause(e); - } - - consumedSize += wr.init(leis, recordSize, recordFunction); - int remainingSize = (int)(recordSize - consumedSize); - assert(remainingSize >= 0); - if (remainingSize > 0) { - // skip size in loops, because not always all bytes are skipped in one call - for (int i=remainingSize; i>0; i-=leis.skip(i)); + try { + for (;;) { + long recordSize; + int recordFunction; + try { + // recordSize in DWORDs + recordSize = leis.readUInt()*2; + recordFunction = leis.readShort(); + } catch (Exception e) { + logger.log(POILogger.ERROR, "unexpected eof - wmf file was truncated"); + break; + } + // 4 bytes (recordSize) + 2 bytes (recordFunction) + int consumedSize = 6; + HwmfRecordType wrt = HwmfRecordType.getById(recordFunction); + if (wrt == null) { + throw new IOException("unexpected record type: "+recordFunction); + } + if (wrt == HwmfRecordType.eof) { + break; + } + if (wrt.clazz == null) { + throw new IOException("unsupported record type: "+recordFunction); + } + + HwmfRecord wr; + try { + wr = wrt.clazz.newInstance(); + records.add(wr); + } catch (Exception e) { + throw (IOException)new IOException("can't create wmf record").initCause(e); + } + + consumedSize += wr.init(leis, recordSize, recordFunction); + int remainingSize = (int)(recordSize - consumedSize); + assert(remainingSize >= 0); + if (remainingSize > 0) { + // skip size in loops, because not always all bytes are skipped in one call + for (int i=remainingSize; i>0; i-=leis.skip(i)); + } } + } finally { + leis.close(); + bis.close(); } } --- src/testcases/org/apache/poi/poifs/filesystem/ReaderWriter.java (revision 1753737) +++ src/testcases/org/apache/poi/poifs/filesystem/ReaderWriter.java (working copy) @@ -29,6 +29,7 @@ import org.apache.poi.poifs.eventfilesystem.POIFSReader; import org.apache.poi.poifs.eventfilesystem.POIFSReaderEvent; import org.apache.poi.poifs.eventfilesystem.POIFSReaderListener; +import org.apache.poi.util.IOUtils; /** * Test (Proof of concept) program that employs the @@ -105,18 +108,18 @@ * @param event the POIFSReaderEvent */ + @Override public void processPOIFSReaderEvent(final POIFSReaderEvent event) { + @SuppressWarnings("resource") DocumentInputStream istream = event.getStream(); POIFSDocumentPath path = event.getPath(); String name = event.getName(); - try - { - int size = istream.available(); - byte[] data = new byte[ istream.available() ]; + try { + byte[] data = IOUtils.toByteArray(istream); + int size = data.length; - istream.read(data); DocumentDescriptor descriptor = new DocumentDescriptor(path, name); --- src/testcases/org/apache/poi/poifs/filesystem/TestDocumentInputStream.java (revision 1753737) +++ src/testcases/org/apache/poi/poifs/filesystem/TestDocumentInputStream.java (working copy) @@ -17,22 +17,28 @@ package org.apache.poi.poifs.filesystem; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; -import junit.framework.TestCase; - import org.apache.poi.POIDataSamples; import org.apache.poi.poifs.property.DirectoryProperty; import org.apache.poi.poifs.storage.RawDataBlock; +import org.apache.poi.util.SuppressForbidden; +import org.junit.Before; +import org.junit.Test; /** * Class to test DocumentInputStream functionality */ -public final class TestDocumentInputStream extends TestCase { +public final class TestDocumentInputStream { private DocumentNode _workbook_n; private DocumentNode _workbook_o; private byte[] _workbook_data; @@ -42,7 +48,8 @@ // any block size private static final int _buffer_size = 6; - protected void setUp() throws Exception { + @Before + public void setUp() throws Exception { int blocks = (_workbook_size + 511) / 512; _workbook_data = new byte[ 512 * blocks ]; @@ -91,6 +98,7 @@ /** * test constructor */ + @Test public void testConstructor() throws IOException { DocumentInputStream ostream = new ODocumentInputStream(_workbook_o); DocumentInputStream nstream = new NDocumentInputStream(_workbook_n); @@ -98,8 +106,8 @@ assertEquals(_workbook_size, _workbook_o.getSize()); assertEquals(_workbook_size, _workbook_n.getSize()); - assertEquals(_workbook_size, ostream.available()); - assertEquals(_workbook_size, nstream.available()); + assertEquals(_workbook_size, available(ostream)); + assertEquals(_workbook_size, available(nstream)); ostream.close(); nstream.close(); @@ -108,23 +116,24 @@ /** * test available() behavior */ + @Test public void testAvailable() throws IOException { DocumentInputStream ostream = new DocumentInputStream(_workbook_o); DocumentInputStream nstream = new NDocumentInputStream(_workbook_n); - assertEquals(_workbook_size, ostream.available()); - assertEquals(_workbook_size, nstream.available()); + assertEquals(_workbook_size, available(ostream)); + assertEquals(_workbook_size, available(nstream)); ostream.close(); nstream.close(); try { - ostream.available(); + available(ostream); fail("Should have caught IOException"); } catch (IllegalStateException ignored) { // as expected } try { - nstream.available(); + available(nstream); fail("Should have caught IOException"); } catch (IllegalStateException ignored) { // as expected @@ -134,6 +143,7 @@ /** * test mark/reset/markSupported. */ + @Test public void testMarkFunctions() throws IOException { byte[] buffer = new byte[ _workbook_size / 5 ]; byte[] small_buffer = new byte[212]; @@ -151,12 +161,12 @@ _workbook_data[ j ], buffer[ j ] ); } - assertEquals(_workbook_size - buffer.length, stream.available()); + assertEquals(_workbook_size - buffer.length, available(stream)); // Reset, and check the available goes back to being the // whole of the stream stream.reset(); - assertEquals(_workbook_size, stream.available()); + assertEquals(_workbook_size, available(stream)); // Read part of a block @@ -167,7 +177,7 @@ _workbook_data[ j ], small_buffer[ j ] ); } - assertEquals(_workbook_size - small_buffer.length, stream.available()); + assertEquals(_workbook_size - small_buffer.length, available(stream)); stream.mark(0); // Read the next part @@ -178,11 +188,11 @@ _workbook_data[ j+small_buffer.length ], small_buffer[ j ] ); } - assertEquals(_workbook_size - 2*small_buffer.length, stream.available()); + assertEquals(_workbook_size - 2*small_buffer.length, available(stream)); // Reset, check it goes back to where it was stream.reset(); - assertEquals(_workbook_size - small_buffer.length, stream.available()); + assertEquals(_workbook_size - small_buffer.length, available(stream)); // Read stream.read(small_buffer); @@ -192,7 +202,7 @@ _workbook_data[ j+small_buffer.length ], small_buffer[ j ] ); } - assertEquals(_workbook_size - 2*small_buffer.length, stream.available()); + assertEquals(_workbook_size - 2*small_buffer.length, available(stream)); // Now read at various points @@ -235,11 +245,11 @@ _workbook_data[ j ], buffer[ j ] ); } - assertEquals(_workbook_size - buffer.length, stream.available()); + assertEquals(_workbook_size - buffer.length, available(stream)); // Read all of it again, check it began at the start again stream.reset(); - assertEquals(_workbook_size, stream.available()); + assertEquals(_workbook_size, available(stream)); stream.read(buffer); for (int j = 0; j < buffer.length; j++) { @@ -253,7 +263,7 @@ stream.mark(12); stream.read(buffer); assertEquals(_workbook_size - (2 * buffer.length), - stream.available()); + available(stream)); for (int j = buffer.length; j < (2 * buffer.length); j++) { assertEquals("checking byte " + j, _workbook_data[ j ], @@ -262,12 +272,12 @@ // Reset, should go back to only one buffer full read stream.reset(); - assertEquals(_workbook_size - buffer.length, stream.available()); + assertEquals(_workbook_size - buffer.length, available(stream)); // Read the buffer again stream.read(buffer); assertEquals(_workbook_size - (2 * buffer.length), - stream.available()); + available(stream)); for (int j = buffer.length; j < (2 * buffer.length); j++) { assertEquals("checking byte " + j, _workbook_data[ j ], @@ -280,6 +290,7 @@ /** * test simple read method */ + @Test public void testReadSingleByte() throws IOException { DocumentInputStream[] streams = new DocumentInputStream[] { new DocumentInputStream(_workbook_o), @@ -296,7 +307,7 @@ ( byte ) b); remaining--; assertEquals("checking remaining after reading byte " + j, - remaining, stream.available()); + remaining, available(stream)); } // Ensure we fell off the end @@ -316,6 +327,7 @@ /** * Test buffered read */ + @Test public void testBufferRead() throws IOException { DocumentInputStream[] streams = new DocumentInputStream[] { new DocumentInputStream(_workbook_o), @@ -332,23 +344,22 @@ // test reading zero length buffer assertEquals(0, stream.read(new byte[ 0 ])); - assertEquals(_workbook_size, stream.available()); + assertEquals(_workbook_size, available(stream)); byte[] buffer = new byte[ _buffer_size ]; int offset = 0; - while (stream.available() >= buffer.length) + while (available(stream) >= buffer.length) { assertEquals(_buffer_size, stream.read(buffer)); - for (int j = 0; j < buffer.length; j++) - { + for (byte element : buffer) { assertEquals("in main loop, byte " + offset, - _workbook_data[ offset ], buffer[ j ]); + _workbook_data[ offset ], element); offset++; } assertEquals("offset " + offset, _workbook_size - offset, - stream.available()); + available(stream)); } - assertEquals(_workbook_size % _buffer_size, stream.available()); + assertEquals(_workbook_size % _buffer_size, available(stream)); Arrays.fill(buffer, ( byte ) 0); int count = stream.read(buffer); @@ -378,6 +389,7 @@ /** * Test complex buffered read */ + @Test public void testComplexBufferRead() throws IOException { DocumentInputStream[] streams = new DocumentInputStream[] { new DocumentInputStream(_workbook_o), @@ -413,11 +425,11 @@ // test reading zero assertEquals(0, stream.read(new byte[ 5 ], 0, 0)); - assertEquals(_workbook_size, stream.available()); + assertEquals(_workbook_size, available(stream)); byte[] buffer = new byte[ _workbook_size ]; int offset = 0; - while (stream.available() >= _buffer_size) + while (available(stream) >= _buffer_size) { Arrays.fill(buffer, ( byte ) 0); assertEquals(_buffer_size, @@ -437,9 +449,9 @@ } offset += _buffer_size; assertEquals("offset " + offset, _workbook_size - offset, - stream.available()); + available(stream)); } - assertEquals(_workbook_size % _buffer_size, stream.available()); + assertEquals(_workbook_size % _buffer_size, available(stream)); Arrays.fill(buffer, ( byte ) 0); int count = stream.read(buffer, offset, _workbook_size % _buffer_size); @@ -474,38 +486,40 @@ /** * Tests that we can skip within the stream */ + @Test public void testSkip() throws IOException { DocumentInputStream[] streams = new DocumentInputStream[] { new DocumentInputStream(_workbook_o), new NDocumentInputStream(_workbook_n) }; for(DocumentInputStream stream : streams) { - assertEquals(_workbook_size, stream.available()); - int count = stream.available(); + assertEquals(_workbook_size, available(stream)); + int count = available(stream); - while (stream.available() >= _buffer_size) { + while (available(stream) >= _buffer_size) { assertEquals(_buffer_size, stream.skip(_buffer_size)); count -= _buffer_size; - assertEquals(count, stream.available()); + assertEquals(count, available(stream)); } assertEquals(_workbook_size % _buffer_size, stream.skip(_buffer_size)); - assertEquals(0, stream.available()); + assertEquals(0, available(stream)); stream.reset(); - assertEquals(_workbook_size, stream.available()); + assertEquals(_workbook_size, available(stream)); assertEquals(_workbook_size, stream.skip(_workbook_size * 2)); - assertEquals(0, stream.available()); + assertEquals(0, available(stream)); stream.reset(); - assertEquals(_workbook_size, stream.available()); + assertEquals(_workbook_size, available(stream)); assertEquals(_workbook_size, stream.skip(2 + ( long ) Integer.MAX_VALUE)); - assertEquals(0, stream.available()); + assertEquals(0, available(stream)); } } /** * Test that we can read files at multiple levels down the tree */ + @Test public void testReadMultipleTreeLevels() throws Exception { final POIDataSamples _samples = POIDataSamples.getPublisherInstance(); File sample = _samples.getFile("Sample.pub"); @@ -551,4 +565,9 @@ npoifs.close(); } } + + @SuppressForbidden("just for testing") + private static int available(InputStream is) throws IOException { + return is.available(); + } } --- src/testcases/org/apache/poi/poifs/filesystem/TestPOIFSFileSystem.java (revision 1753737) +++ src/testcases/org/apache/poi/poifs/filesystem/TestPOIFSFileSystem.java (working copy) @@ -33,12 +31,15 @@ import org.apache.poi.poifs.storage.BlockAllocationTableReader; import org.apache.poi.poifs.storage.HeaderBlock; import org.apache.poi.poifs.storage.RawDataBlockList; +import org.apache.poi.util.IOUtils; @@ -284,9 +288,7 @@ DocumentNode doc = (DocumentNode) entry; DocumentInputStream dis = new DocumentInputStream(doc); try { - int numBytes = dis.available(); - byte[] data = new byte [numBytes]; - dis.read(data); + IOUtils.toByteArray(dis); } finally { dis.close(); } --- src/testcases/org/apache/poi/util/TestLittleEndian.java (revision 1753737) +++ src/testcases/org/apache/poi/util/TestLittleEndian.java (working copy) @@ -241,15 +252,20 @@ /** * test the readShort method */ public void testReadShort() throws IOException { short expected_value = 0x0201; InputStream stream = new ByteArrayInputStream(_good_array); int count = 0; - while (stream.available() > 0) { - short value = LittleEndian.readShort(stream); - assertEquals(value, expected_value); - count++; + while (true) { + try { + short value = LittleEndian.readShort(stream); + assertEquals(value, expected_value); + count++; + } catch (BufferUnderrunException e) { + break; + } } assertEquals(count, _good_array.length / LittleEndianConsts.SHORT_SIZE); @@ -265,15 +281,20 @@ /** * test the readInt method */ public void testReadInt() throws IOException { int expected_value = 0x02010201; InputStream stream = new ByteArrayInputStream(_good_array); int count = 0; - while (stream.available() > 0) { - int value = LittleEndian.readInt(stream); - assertEquals(value, expected_value); - count++; + while (true) { + try { + int value = LittleEndian.readInt(stream); + assertEquals(value, expected_value); + count++; + } catch (BufferUnderrunException e) { + break; + } } assertEquals(count, _good_array.length / LittleEndianConsts.INT_SIZE); stream = new ByteArrayInputStream(_bad_array); @@ -289,15 +310,20 @@ /** * test the readLong method */ public void testReadLong() throws IOException { long expected_value = 0x0201020102010201L; InputStream stream = new ByteArrayInputStream(_good_array); int count = 0; - while (stream.available() > 0) { - long value = LittleEndian.readLong(stream); - assertEquals(value, expected_value); - count++; + while (true) { + try { + long value = LittleEndian.readLong(stream); + assertEquals(value, expected_value); + count++; + } catch (BufferUnderrunException e) { + break; + } } assertEquals(count, _good_array.length / LittleEndianConsts.LONG_SIZE);