diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ExtentsByteProvider.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RangeMappedByteProvider.java similarity index 65% rename from Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ExtentsByteProvider.java rename to Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RangeMappedByteProvider.java index 4c18e4651e..e30f49129c 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/ExtentsByteProvider.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/RangeMappedByteProvider.java @@ -26,17 +26,17 @@ import ghidra.formats.gfilesystem.FSRL; * A {@link ByteProvider} that is a concatenation of sub-ranges of another ByteProvider, also * allowing for non-initialized (sparse) regions. *

- * Not thread-safe when extents are being added. + * Not thread-safe when ranges are being added. *

* Does not assume ownership of wrapped ByteProvider. */ -public class ExtentsByteProvider implements ByteProvider { +public class RangeMappedByteProvider implements ByteProvider { private ByteProvider delegate; /** * TreeMap of this-provider offsets to the delegate-provider's offsets. *

- * Each extent/region in the delegate provider is defined by the gap between + * Each range in the delegate provider is defined by the gap between * adjacent offsetMap entries. The last entry is bounded by the total * length of the provider as specified by the length field. */ @@ -45,37 +45,67 @@ public class ExtentsByteProvider implements ByteProvider { private FSRL fsrl; /** - * Creates a new {@link ExtentsByteProvider}. + * Creates a new {@link RangeMappedByteProvider}. * * @param provider {@link ByteProvider} to wrap * @param fsrl {@link FSRL} of this new byte provider */ - public ExtentsByteProvider(ByteProvider provider, FSRL fsrl) { + public RangeMappedByteProvider(ByteProvider provider, FSRL fsrl) { this.delegate = provider; this.fsrl = fsrl; } /** - * Adds an extent to the current end of this instance. + * Adds a range to the current end of this instance. + *

+ * If the new range immediately follows the previous range, the new range is merged + * into the previous entry. * - * @param offset long byte offset in the delegate ByteProvider - * @param extentLen long length of the extent region in the delegate ByteProvider + * @param offset long byte offset in the delegate ByteProvider, -1 indicates a sparse + * range with no storage + * @param rangeLen long length of the range in the delegate ByteProvider */ - public void addExtent(long offset, long extentLen) { - if (extentLen <= 0) { + public void addRange(long offset, long rangeLen) { + if (rangeLen <= 0) { throw new IllegalArgumentException(); } + Entry lastEntry = offsetMap.lastEntry(); + if (lastEntry != null) { + // try to merge sparse ranges + long lastRangeOffset = lastEntry.getValue(); + if (offset == -1 && lastRangeOffset == -1) { + length += rangeLen; + return; + } + + // try to merge this new range into the previous range + long lastRangeLen = length - lastEntry.getKey(); + if (lastRangeOffset + lastRangeLen == offset) { + length += rangeLen; + return; + } + } offsetMap.put(length, offset); - length += extentLen; + length += rangeLen; } /** - * Adds a sparse extent to the current end of this instance. + * Adds a sparse range to the current end of this instance. * - * @param extentLen long length of the sparse extent region + * @param rangeLen long length of the sparse range */ - public void addSparseExtent(long extentLen) { - addExtent(-1, extentLen); + public void addSparseRange(long rangeLen) { + addRange(-1, rangeLen); + } + + /** + * Return the number of ranges. Adjacent ranges that were merged + * will count as a single range. + * + * @return number of ranges + */ + public int getRangeCount() { + return offsetMap.size(); } @Override @@ -118,12 +148,12 @@ public class ExtentsByteProvider implements ByteProvider { ensureBounds(index, 1); Entry entry = offsetMap.floorEntry(index); - long extentStart = entry.getKey(); - long extentOffset = index - extentStart; - long delegateExtentStart = entry.getValue(); + long rangeStart = entry.getKey(); + long rangeOffset = index - rangeStart; + long delegateRangeStart = entry.getValue(); - return (delegateExtentStart != -1) - ? delegate.readByte(delegateExtentStart + extentOffset) + return (delegateRangeStart != -1) + ? delegate.readByte(delegateRangeStart + rangeOffset) : 0; } @@ -166,20 +196,20 @@ public class ExtentsByteProvider implements ByteProvider { Entry entry = offsetMap.floorEntry(currentIndex); Entry nextEntry = offsetMap.higherEntry(entry.getKey()); - long extentStart = entry.getKey(); - long extentOffset = currentIndex - extentStart; - long extentEnd = (nextEntry != null) ? nextEntry.getKey() : length; - long delegateExtentStart = entry.getValue(); + long rangeStart = entry.getKey(); + long rangeOffset = currentIndex - rangeStart; + long rangeEnd = (nextEntry != null) ? nextEntry.getKey() : length; + long delegateRangeStart = entry.getValue(); int bytesToRead = - (int) Math.min(len - totalBytesRead, extentEnd - extentStart - extentOffset); - if (delegateExtentStart != -1) { - long delegateOffsetToRead = delegateExtentStart + extentOffset; + (int) Math.min(len - totalBytesRead, rangeEnd - rangeStart - rangeOffset); + if (delegateRangeStart != -1) { + long delegateOffsetToRead = delegateRangeStart + rangeOffset; // TODO: when ByteProvider interface has better readBytes() method, use it here - byte[] extentBytes = delegate.readBytes(delegateOffsetToRead, bytesToRead); - System.arraycopy(extentBytes, 0, buffer, bufferDest, bytesToRead); + byte[] rangeBytes = delegate.readBytes(delegateOffsetToRead, bytesToRead); + System.arraycopy(rangeBytes, 0, buffer, bufferDest, bytesToRead); } else { - // the extent was not present, result will be 0's + // the range was not present, result will be 0's Arrays.fill(buffer, bufferDest, bufferDest + bytesToRead, (byte) 0 /* fill value */); } diff --git a/Ghidra/Features/Base/src/test/java/ghidra/app/util/bin/ExtentsByteProviderTest.java b/Ghidra/Features/Base/src/test/java/ghidra/app/util/bin/ExtentsByteProviderTest.java deleted file mode 100644 index 8bb53f67f0..0000000000 --- a/Ghidra/Features/Base/src/test/java/ghidra/app/util/bin/ExtentsByteProviderTest.java +++ /dev/null @@ -1,214 +0,0 @@ -/* ### - * IP: GHIDRA - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ghidra.app.util.bin; - -import static org.junit.Assert.*; - -import java.io.IOException; -import java.util.Arrays; - -import org.junit.Test; - -public class ExtentsByteProviderTest { - private ByteArrayProvider bap(int... values) { - byte[] bytes = new byte[values.length]; - for (int i = 0; i < values.length; i++) { - bytes[i] = (byte) values[i]; - } - return new ByteArrayProvider(bytes); - } - - /* - * "NN 01 NN 03 NN 05 NN 07 NN 09"... (NN = blockNumber, 00-FF = offset in block) - */ - private ByteArrayProvider patternedBAP(int bs, int count) { - byte[] bytes = new byte[bs * count]; - for (int blockNum = 0; blockNum < count; blockNum++) { - int blockStart = blockNum * bs; - Arrays.fill(bytes, blockStart, blockStart + bs, (byte) blockNum); - for (int i = 1; i < bs; i += 2) { - bytes[i + blockStart] = (byte) (i % 256); - } - } - return new ByteArrayProvider(bytes); - } - - @Test(expected = IOException.class) - public void testEmptyExtentBP() throws IOException { - try (ExtentsByteProvider ebp = new ExtentsByteProvider(bap(55), null)) { - ebp.readByte(0); - } - } - - @Test - public void testExtentBP_SingleByteRead() throws IOException { - try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) { - ebp.addExtent(10, 10); - ebp.addExtent(0, 1); - ebp.addExtent(0, 10); - - assertEquals(21, ebp.length()); - - assertEquals(0x01, ebp.readByte(0)); - assertEquals(0x01, ebp.readByte(1)); - assertEquals(0x01, ebp.readByte(2)); - assertEquals(0x03, ebp.readByte(3)); - assertEquals(0x09, ebp.readByte(9)); - - assertEquals(0x00, ebp.readByte(11)); - assertEquals(0x01, ebp.readByte(12)); - assertEquals(0x00, ebp.readByte(13)); - assertEquals(0x03, ebp.readByte(14)); - assertEquals(0x09, ebp.readByte(20)); - - try { - ebp.readByte(21); - fail(); - } - catch (IOException e) { - // good - } - } - } - - @Test - public void testExtentBP_MultiByteRead() throws IOException { - try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) { - ebp.addExtent(10, 10); - ebp.addExtent(0, 1); - ebp.addExtent(0, 10); - - assertEquals(21, ebp.length()); - - byte[] bytes = ebp.readBytes(0, 21); - assertEquals(0x01, bytes[0]); - assertEquals(0x01, bytes[1]); - assertEquals(0x01, bytes[2]); - assertEquals(0x03, bytes[3]); - assertEquals(0x09, bytes[9]); - - assertEquals(0x00, bytes[11]); - assertEquals(0x01, bytes[12]); - assertEquals(0x00, bytes[13]); - assertEquals(0x03, bytes[14]); - assertEquals(0x09, bytes[20]); - } - } - - @Test - public void testExtentBP_MisalignedMultiByteRead() throws IOException { - try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) { - ebp.addExtent(10, 10); - ebp.addExtent(0, 10); - - assertEquals(20, ebp.length()); - - byte[] bytes = ebp.readBytes(5, 10); - assertEquals(0x05, bytes[0]); - assertEquals(0x01, bytes[1]); - assertEquals(0x07, bytes[2]); - assertEquals(0x01, bytes[3]); - assertEquals(0x09, bytes[4]); - assertEquals(0x00, bytes[5]); - assertEquals(0x01, bytes[6]); - assertEquals(0x00, bytes[7]); - assertEquals(0x03, bytes[8]); - assertEquals(0x00, bytes[9]); - } - } - - @Test - public void testSmallExtentBP() throws IOException { - try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) { - ebp.addExtent(10, 1); - ebp.addExtent(0, 1); - - assertEquals(2, ebp.length()); - - assertEquals(0x01, ebp.readByte(0)); - assertEquals(0x00, ebp.readByte(1)); - - try { - ebp.readByte(3); - fail(); - } - catch (IOException e) { - // good - } - } - } - - @Test - public void testExtentBP_SparseMultiByteRead() throws IOException { - try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) { - ebp.addExtent(-1, 5); - - assertEquals(5, ebp.length()); - - byte[] bytes = ebp.readBytes(0, 5); - assertEquals(0x00, bytes[0]); - assertEquals(0x00, bytes[1]); - assertEquals(0x00, bytes[2]); - assertEquals(0x00, bytes[3]); - assertEquals(0x00, bytes[4]); - } - } - - @Test - public void testExtentBP_SparseSingleByteRead() throws IOException { - try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) { - ebp.addExtent(-1, 5); - - assertEquals(5, ebp.length()); - - assertEquals(0x00, ebp.readByte(0)); - assertEquals(0x00, ebp.readByte(1)); - assertEquals(0x00, ebp.readByte(2)); - assertEquals(0x00, ebp.readByte(3)); - assertEquals(0x00, ebp.readByte(4)); - } - } - - @Test - public void testExtentBP_MixedSparseMultiByteRead() throws IOException { - try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) { - ebp.addExtent(10, 10); - ebp.addExtent(-1, 5); - ebp.addExtent(0, 10); - - assertEquals(25, ebp.length()); - - byte[] bytes = ebp.readBytes(0, 25); - assertEquals(0x01, bytes[0]); - assertEquals(0x01, bytes[1]); - assertEquals(0x01, bytes[2]); - assertEquals(0x03, bytes[3]); - assertEquals(0x09, bytes[9]); - - assertEquals(0x00, bytes[10]); - assertEquals(0x00, bytes[11]); - assertEquals(0x00, bytes[12]); - assertEquals(0x00, bytes[13]); - assertEquals(0x00, bytes[14]); - - assertEquals(0x00, bytes[15]); - assertEquals(0x01, bytes[16]); - assertEquals(0x00, bytes[17]); - assertEquals(0x03, bytes[18]); - assertEquals(0x09, bytes[24]); - } - } -} diff --git a/Ghidra/Features/Base/src/test/java/ghidra/app/util/bin/RangeMappedByteProviderTest.java b/Ghidra/Features/Base/src/test/java/ghidra/app/util/bin/RangeMappedByteProviderTest.java new file mode 100644 index 0000000000..609803c3d6 --- /dev/null +++ b/Ghidra/Features/Base/src/test/java/ghidra/app/util/bin/RangeMappedByteProviderTest.java @@ -0,0 +1,271 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.app.util.bin; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.Arrays; + +import org.junit.Test; + +public class RangeMappedByteProviderTest { + private ByteArrayProvider bap(int... values) { + byte[] bytes = new byte[values.length]; + for (int i = 0; i < values.length; i++) { + bytes[i] = (byte) values[i]; + } + return new ByteArrayProvider(bytes); + } + + /* + * "NN 01 NN 03 NN 05 NN 07 NN 09"... (NN = blockNumber, 00-FF = offset in block) + */ + private ByteArrayProvider patternedBAP(int bs, int count) { + byte[] bytes = new byte[bs * count]; + for (int blockNum = 0; blockNum < count; blockNum++) { + int blockStart = blockNum * bs; + Arrays.fill(bytes, blockStart, blockStart + bs, (byte) blockNum); + for (int i = 1; i < bs; i += 2) { + bytes[i + blockStart] = (byte) (i % 256); + } + } + return new ByteArrayProvider(bytes); + } + + @Test(expected = IOException.class) + public void testEmptyRangeMappedBP() throws IOException { + try (RangeMappedByteProvider rmbp = new RangeMappedByteProvider(bap(55), null)) { + rmbp.readByte(0); + } + } + + @Test + public void testRangeMapppedBP_SingleByteRead() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addRange(10, 10); + rmbp.addRange(0, 1); + rmbp.addRange(0, 10); + + assertEquals(21, rmbp.length()); + + assertEquals(0x01, rmbp.readByte(0)); + assertEquals(0x01, rmbp.readByte(1)); + assertEquals(0x01, rmbp.readByte(2)); + assertEquals(0x03, rmbp.readByte(3)); + assertEquals(0x09, rmbp.readByte(9)); + + assertEquals(0x00, rmbp.readByte(11)); + assertEquals(0x01, rmbp.readByte(12)); + assertEquals(0x00, rmbp.readByte(13)); + assertEquals(0x03, rmbp.readByte(14)); + assertEquals(0x09, rmbp.readByte(20)); + + try { + rmbp.readByte(21); + fail(); + } + catch (IOException e) { + // good + } + } + } + + @Test + public void testRangeMapppedBP_MultiByteRead() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addRange(10, 10); + rmbp.addRange(0, 1); + rmbp.addRange(0, 10); + + assertEquals(21, rmbp.length()); + + byte[] bytes = rmbp.readBytes(0, 21); + assertEquals(0x01, bytes[0]); + assertEquals(0x01, bytes[1]); + assertEquals(0x01, bytes[2]); + assertEquals(0x03, bytes[3]); + assertEquals(0x09, bytes[9]); + + assertEquals(0x00, bytes[11]); + assertEquals(0x01, bytes[12]); + assertEquals(0x00, bytes[13]); + assertEquals(0x03, bytes[14]); + assertEquals(0x09, bytes[20]); + } + } + + @Test + public void testRangeMapppedBP_MisalignedMultiByteRead() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addRange(10, 10); + rmbp.addRange(0, 10); + + assertEquals(20, rmbp.length()); + + byte[] bytes = rmbp.readBytes(5, 10); + assertEquals(0x05, bytes[0]); + assertEquals(0x01, bytes[1]); + assertEquals(0x07, bytes[2]); + assertEquals(0x01, bytes[3]); + assertEquals(0x09, bytes[4]); + assertEquals(0x00, bytes[5]); + assertEquals(0x01, bytes[6]); + assertEquals(0x00, bytes[7]); + assertEquals(0x03, bytes[8]); + assertEquals(0x00, bytes[9]); + } + } + + @Test + public void testSmallRangeMapppedBP() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addRange(10, 1); + rmbp.addRange(0, 1); + + assertEquals(2, rmbp.length()); + + assertEquals(0x01, rmbp.readByte(0)); + assertEquals(0x00, rmbp.readByte(1)); + + try { + rmbp.readByte(3); + fail(); + } + catch (IOException e) { + // good + } + } + } + + @Test + public void testRangeMapppedBP_SparseMultiByteRead() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addSparseRange(5); + + assertEquals(5, rmbp.length()); + + byte[] bytes = rmbp.readBytes(0, 5); + assertEquals(0x00, bytes[0]); + assertEquals(0x00, bytes[1]); + assertEquals(0x00, bytes[2]); + assertEquals(0x00, bytes[3]); + assertEquals(0x00, bytes[4]); + } + } + + @Test + public void testRangeMapppedBP_SparseSingleByteRead() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addSparseRange(5); + + assertEquals(5, rmbp.length()); + + assertEquals(0x00, rmbp.readByte(0)); + assertEquals(0x00, rmbp.readByte(1)); + assertEquals(0x00, rmbp.readByte(2)); + assertEquals(0x00, rmbp.readByte(3)); + assertEquals(0x00, rmbp.readByte(4)); + } + } + + @Test + public void testRangeMapppedBP_MixedSparseMultiByteRead() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addRange(10, 10); + rmbp.addSparseRange(5); + rmbp.addRange(0, 10); + + assertEquals(25, rmbp.length()); + + byte[] bytes = rmbp.readBytes(0, 25); + assertEquals(0x01, bytes[0]); + assertEquals(0x01, bytes[1]); + assertEquals(0x01, bytes[2]); + assertEquals(0x03, bytes[3]); + assertEquals(0x09, bytes[9]); + + assertEquals(0x00, bytes[10]); + assertEquals(0x00, bytes[11]); + assertEquals(0x00, bytes[12]); + assertEquals(0x00, bytes[13]); + assertEquals(0x00, bytes[14]); + + assertEquals(0x00, bytes[15]); + assertEquals(0x01, bytes[16]); + assertEquals(0x00, bytes[17]); + assertEquals(0x03, bytes[18]); + assertEquals(0x09, bytes[24]); + } + } + + @Test + public void testMergeAdjacentRanges() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addRange(10, 10); + rmbp.addRange(20, 5); + rmbp.addRange(25, 5); + + assertEquals(20, rmbp.length()); + assertEquals(1, rmbp.getRangeCount()); + } + } + + @Test + public void testDontMergeAlmostAdjacentRanges() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addRange(10, 10); + rmbp.addRange(21, 5); + + assertEquals(15, rmbp.length()); + assertEquals(2, rmbp.getRangeCount()); + } + } + + @Test + public void testDontMergeAlmostAdjacentRanges2() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addRange(10, 10); + rmbp.addRange(19, 5);// creates a weird overlapped result, but good boundary cond test + + assertEquals(15, rmbp.length()); + assertEquals(2, rmbp.getRangeCount()); + } + } + + @Test + public void testMergeAdjacentSparseRanges() throws IOException { + try (RangeMappedByteProvider rmbp = + new RangeMappedByteProvider(patternedBAP(10, 10), null)) { + rmbp.addRange(10, 10); + rmbp.addSparseRange(5); + rmbp.addSparseRange(5); + + assertEquals(20, rmbp.length()); + assertEquals(2, rmbp.getRangeCount()); + } + } +} diff --git a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4Analyzer.java b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4Analyzer.java index 21efefa456..b8ba92b2a2 100644 --- a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4Analyzer.java +++ b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4Analyzer.java @@ -16,7 +16,6 @@ package ghidra.file.formats.ext4; import java.io.IOException; -import java.util.List; import ghidra.app.util.bin.*; import ghidra.app.util.importer.MessageLog; @@ -191,27 +190,27 @@ public class Ext4Analyzer extends FileFormatAnalyzer { boolean isDirEntry2 = (superBlock.getS_feature_incompat() & Ext4Constants.INCOMPAT_FILETYPE) != 0; // if uses extents if( (inode.getI_flags() & Ext4Constants.EXT4_EXTENTS_FL) != 0 ) { - Ext4IBlock i_block = inode.getI_block(); - Ext4ExtentHeader header = i_block.getHeader(); - if( header.getEh_depth() == 0 ) { - short numEntries = header.getEh_entries(); - List entries = i_block.getExtentEntries(); - for( int i = 0; i < numEntries; i++ ) { - Ext4Extent extent = entries.get(i); - long offset = extent.getExtentStartBlockNumber() * blockSize; - reader.setPointerIndex(offset); - Address address = toAddr(program, offset); - if( isDirEntry2 ) { - while( (reader.getPointerIndex() - offset) < (extent.getEe_len() * blockSize)) { - Ext4DirEntry2 dirEnt2 = Ext4DirEntry2.read(reader); - DataType dataType = dirEnt2.toDataType(); - createData(program, address, dataType); - address = address.add(dataType.getLength()); - } - } - } - - } +// Ext4IBlock i_block = inode.getI_block(); +// Ext4ExtentHeader header = i_block.getHeader(); +// if( header.getEh_depth() == 0 ) { +// short numEntries = header.getEh_entries(); +// List entries = i_block.getExtentEntries(); +// for( int i = 0; i < numEntries; i++ ) { +// Ext4Extent extent = entries.get(i); +// long offset = extent.getExtentStartBlockNumber() * blockSize; +// reader.setPointerIndex(offset); +// Address address = toAddr(program, offset); +// if( isDirEntry2 ) { +// while( (reader.getPointerIndex() - offset) < (extent.getEe_len() * blockSize)) { +// Ext4DirEntry2 dirEnt2 = Ext4DirEntry2.read(reader); +// DataType dataType = dirEnt2.toDataType(); +// createData(program, address, dataType); +// address = address.add(dataType.getLength()); +// } +// } +// } +// +// } } } diff --git a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4BlockMapHelper.java b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4BlockMapHelper.java new file mode 100644 index 0000000000..273a2153a1 --- /dev/null +++ b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4BlockMapHelper.java @@ -0,0 +1,93 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.file.formats.ext4; + +import java.io.IOException; + +import ghidra.app.util.bin.*; +import ghidra.formats.gfilesystem.FSRL; + +/** + * Helper class that handles the blockmap data stored in an inode's i_block[] array + */ +public class Ext4BlockMapHelper { + private static final int INDIRECT_BLOCK_INDEX = 12; + private static final int DOUBLE_INDIRECT_BLOCK_INDEX = 13; + private static final int TRIPLE_INDIRECT_BLOCK_INDEX = 14; + private static final int INODE_BLOCKMAP_COUNT = 15; + + /** + * Creates a {@link RangeMappedByteProvider} from the old-style block map data found in the + * inode's i_block field. + * + * @param rawIBlockBytes raw bytes from the inode's i_block + * @param provider the file system volume provider + * @param expectedLength the length the file should be (from the inode) + * @param blockSize file system blockSize + * @param fsrl {@link FSRL} to assign to the new ByteProvider + * @return new {@link ByteProvider} containing the blocks of the volume that were specified + * by the blockmap data + * @throws IOException if error + */ + public static ByteProvider getByteProvider(byte[] rawIBlockBytes, ByteProvider provider, + long expectedLength, int blockSize, FSRL fsrl) throws IOException { + BinaryReader iBlockReader = + new BinaryReader(new ByteArrayProvider(rawIBlockBytes), true /* LE */); + int[] blockNumbers = iBlockReader.readNextIntArray(INODE_BLOCKMAP_COUNT); + + RangeMappedByteProvider bp = new RangeMappedByteProvider(provider, fsrl); + + // the location of the first 12 blocks of the file are held in [0..11] + addFromArray(blockNumbers, 0, INDIRECT_BLOCK_INDEX, 0, bp, blockSize, expectedLength, + provider); + + // the location of the next blockSize/4 (ie. 4096/4=1024) blocks of the file are + // held in an array that is located in the block pointed to by [12] + addFromArray(blockNumbers, INDIRECT_BLOCK_INDEX, DOUBLE_INDIRECT_BLOCK_INDEX, 1, bp, + blockSize, expectedLength, provider); + + // the location of the next blocks of the file are held in an array that is + // double-ly indirectly pointed to by [13] + addFromArray(blockNumbers, DOUBLE_INDIRECT_BLOCK_INDEX, TRIPLE_INDIRECT_BLOCK_INDEX, 2, bp, + blockSize, expectedLength, provider); + + // the location of the next blocks of the file are held in an array that is + // triply indirectly pointed to by [14] + addFromArray(blockNumbers, TRIPLE_INDIRECT_BLOCK_INDEX, INODE_BLOCKMAP_COUNT, 3, bp, + blockSize, expectedLength, provider); + + return bp; + } + + private static void addFromArray(int[] blockNums, int start, int end, int indirectLevel, + RangeMappedByteProvider ebp, int blockSize, long expectedLength, ByteProvider provider) + throws IOException { + BinaryReader reader = new BinaryReader(provider, true /* LE */ ); + for (int i = start; i < end && ebp.length() < expectedLength; i++) { + if ( indirectLevel > 0 ) { + int[] subBlockNumbers = reader.readIntArray(blockNums[i] * blockSize, + blockSize / BinaryReader.SIZEOF_INT); + addFromArray(subBlockNumbers, 0, subBlockNumbers.length, indirectLevel - 1, ebp, + blockSize, expectedLength, provider); + } + else { + int bytesFromBlock = (int) Math.min(blockSize, expectedLength - ebp.length()); + long blockNum = Integer.toUnsignedLong(blockNums[i]); + ebp.addRange(blockNum == 0 ? -1 : blockNum * blockSize, bytesFromBlock); + } + } + } +} diff --git a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4ExtentHeader.java b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4ExtentHeader.java index 38e45352eb..febe74cbe5 100644 --- a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4ExtentHeader.java +++ b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4ExtentHeader.java @@ -15,17 +15,14 @@ */ package ghidra.file.formats.ext4; -import ghidra.app.util.bin.BinaryReader; -import ghidra.app.util.bin.ByteProvider; -import ghidra.app.util.bin.StructConverter; -import ghidra.program.model.data.DataType; -import ghidra.program.model.data.Structure; -import ghidra.program.model.data.StructureDataType; -import ghidra.util.exception.DuplicateNameException; - import java.io.IOException; +import ghidra.app.util.bin.*; +import ghidra.program.model.data.*; +import ghidra.util.exception.DuplicateNameException; + public class Ext4ExtentHeader implements StructConverter { + private static final int SIZEOF = 12; private short eh_magic; private short eh_entries; @@ -33,6 +30,21 @@ public class Ext4ExtentHeader implements StructConverter { private short eh_depth; private int eh_generation; + /** + * Read a Ext4ExtentHeader from the stream. + * + * @param reader BinaryReader to read from + * @return new Ext4ExtentHeader instance, or null if eof or no magic value + * @throws IOException if error + */ + public static Ext4ExtentHeader read(BinaryReader reader) throws IOException { + if (reader.getPointerIndex() + SIZEOF >= reader.length() || + Short.toUnsignedInt(reader.peekNextShort()) != Ext4Constants.EXTENT_HEADER_MAGIC) { + return null; + } + return new Ext4ExtentHeader(reader); + } + public Ext4ExtentHeader( ByteProvider provider ) throws IOException { this( new BinaryReader( provider, true ) ); } diff --git a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4ExtentsHelper.java b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4ExtentsHelper.java new file mode 100644 index 0000000000..3c5972b3fc --- /dev/null +++ b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4ExtentsHelper.java @@ -0,0 +1,95 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.file.formats.ext4; + +import java.io.IOException; + +import ghidra.app.util.bin.*; +import ghidra.formats.gfilesystem.FSRL; + +/** + * Helper class that handles the extent data stored in an inode's i_block[] array + */ +public class Ext4ExtentsHelper { + + /** + * Creates a {@link RangeMappedByteProvider} from the extents data found in the + * inode's i_block field. + * + * @param rawIBlockBytes raw bytes from the inode's i_block + * @param provider the file system volume provider + * @param expectedLength the length the file should be (from the inode) + * @param blockSize file system blockSize + * @param fsrl {@link FSRL} to assign to the new ByteProvider + * @return new {@link ByteProvider} containing the blocks of the volume that were specified + * by the extent data + * @throws IOException if error + */ + public static ByteProvider getByteProvider(byte[] rawIBlockBytes, ByteProvider provider, + long expectedLength, int blockSize, FSRL fsrl) throws IOException { + BinaryReader iBlockReader = + new BinaryReader(new ByteArrayProvider(rawIBlockBytes), true /* LE */); + + RangeMappedByteProvider ebp = new RangeMappedByteProvider(provider, fsrl); + processExtents(iBlockReader, provider, ebp, blockSize, expectedLength); + if (ebp.length() < expectedLength) { + // trailing sparse. not sure if possible. + ebp.addSparseRange(expectedLength - ebp.length()); + } + + return ebp; + } + + private static void processExtents(BinaryReader reader, ByteProvider provider, + RangeMappedByteProvider ebp, int blockSize, long expectedLength) throws IOException { + Ext4ExtentHeader header = Ext4ExtentHeader.read(reader); + if ( header == null ) { + throw new IOException("Bad/missing extents header"); + } + if (header.getEh_depth() == 0) { + for (int i = 0; i < header.getEh_entries() && ebp.length() < expectedLength; i++) { + Ext4Extent extent = new Ext4Extent(reader); + + long startPos = extent.getStreamBlockNumber() * blockSize; + long providerOfs = extent.getExtentStartBlockNumber() * blockSize; + long extentLen = extent.getExtentBlockCount() * blockSize; + if (ebp.length() < startPos) { + ebp.addSparseRange(startPos - ebp.length()); + } + if (ebp.length() + extentLen > expectedLength) { + // the last extent may have a trailing partial block + extentLen = expectedLength - ebp.length(); + } + + ebp.addRange(providerOfs, extentLen); + } + } + else { + for (int i = 0; i < header.getEh_entries(); i++) { + Ext4ExtentIdx idx = new Ext4ExtentIdx(reader); + long offset = idx.getEi_leaf() * blockSize; + try (ByteProviderWrapper bpw = + new ByteProviderWrapper(provider, offset, blockSize)) { + BinaryReader subReader = new BinaryReader(bpw, true /* LE */); + processExtents(subReader, provider, ebp, blockSize, expectedLength); + } + } + } + + } + + +} diff --git a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4FileSystem.java b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4FileSystem.java index 795904a401..886f49ebad 100644 --- a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4FileSystem.java +++ b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4FileSystem.java @@ -70,7 +70,7 @@ public class Ext4FileSystem implements GFileSystem { numGroups++; } - int groupDescriptorOffset = blockSize; + int groupDescriptorOffset = blockSize + (superBlock.getS_first_data_block() * blockSize); reader.setPointerIndex(groupDescriptorOffset); monitor.initialize(numGroups); monitor.setMessage("Reading inode tables"); @@ -112,41 +112,6 @@ public class Ext4FileSystem implements GFileSystem { } } - interface Checked2Consumer { - void accept(T t) throws E1, E2; - } - - interface ExtentConsumer extends Checked2Consumer { - // no additional def - } - - private void forEachExtentEntry(Ext4IBlock i_block, ExtentConsumer extentConsumer, - TaskMonitor monitor) throws CancelledException, IOException { - Ext4ExtentHeader header = i_block.getHeader(); - if (header.getEh_depth() == 0) { - short numEntries = header.getEh_entries(); - List entries = i_block.getExtentEntries(); - for (int i = 0; i < numEntries; i++) { - monitor.checkCanceled(); - Ext4Extent extent = entries.get(i); - extentConsumer.accept(extent); - } - } - else { - short numEntries = header.getEh_entries(); - List entries = i_block.getIndexEntries(); - for (int i = 0; i < numEntries; i++) { - monitor.checkCanceled(); - - Ext4ExtentIdx extentIndex = entries.get(i); - long offset = extentIndex.getEi_leaf() * blockSize; - - forEachExtentEntry(Ext4IBlock.readIBlockWithExtents(provider, offset), - extentConsumer, monitor); - } - } - } - private void processDirectory(Ext4Inode inode, GFile dirFile, Ext4Inode[] inodes, BitSet processedInodes, TaskMonitor monitor) throws IOException, CancelledException { try (ByteProvider bp = getInodeByteProvider(inode, dirFile.getFSRL(), monitor)) { @@ -186,6 +151,7 @@ public class Ext4FileSystem implements GFileSystem { throw new IOException("Inode " + inode + " has unhandled file type: " + Integer.toHexString(inode.getFileType())); } + String name = dirEntry.getName(); if (".".equals(name) || "..".equals(name)) { // skip the ".", and ".." self-reference directories @@ -331,55 +297,19 @@ public class Ext4FileSystem implements GFileSystem { private ByteProvider getInodeByteProvider(Ext4Inode inode, FSRL inodeFSRL, TaskMonitor monitor) throws IOException { if (inode.isFlagExtents()) { - return getExtentsByteProvider(inodeFSRL, inode, monitor); + return Ext4ExtentsHelper.getByteProvider(inode.getI_block(), provider, inode.getSize(), + blockSize, inodeFSRL); } else if (inode.isFlagInlineData() || inode.isSymLink()) { - return getInlineDataProvider(inodeFSRL, inode, monitor); + byte[] data = inode.getInlineDataValue(); + return new ByteArrayProvider(data, inodeFSRL); } else { - throw new IOException("Unsupported file storage: " + inodeFSRL.getPath()); + return Ext4BlockMapHelper.getByteProvider(inode.getI_block(), provider, inode.getSize(), + blockSize, inodeFSRL); } } - private ByteProvider getInlineDataProvider(FSRL fileFSRL, Ext4Inode inode, TaskMonitor monitor) - throws IOException { - byte[] data = inode.getInlineDataValue(); - return new ByteArrayProvider(data, fileFSRL); - } - - private ByteProvider getExtentsByteProvider(FSRL fileFSRL, Ext4Inode inode, TaskMonitor monitor) - throws IOException { - try { - long fileSize = inode.getSize(); - ExtentsByteProvider result = new ExtentsByteProvider(provider, fileFSRL); - - Ext4IBlock i_block = inode.getI_block(); - forEachExtentEntry(i_block, extent -> { - long startPos = extent.getStreamBlockNumber() * blockSize; - long providerOfs = extent.getExtentStartBlockNumber() * blockSize; - long extentLen = extent.getExtentBlockCount() * blockSize; - if (result.length() < startPos) { - result.addSparseExtent(startPos - result.length()); - } - if (result.length() + extentLen > fileSize) { - // the last extent may have a trailing partial block - extentLen = fileSize - result.length(); - } - - result.addExtent(providerOfs, extentLen); - }, monitor); - if (result.length() < fileSize) { - // trailing sparse. not sure if possible. - result.addSparseExtent(fileSize - result.length()); - } - return result; - } - catch (CancelledException e) { - throw new IOException(e); - } - - } - private Ext4Inode[] getInodes(BinaryReader reader, Ext4GroupDescriptor[] groupDescriptors, TaskMonitor monitor) throws IOException, CancelledException { diff --git a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4Inode.java b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4Inode.java index 071dd1241e..b02edb2ee3 100644 --- a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4Inode.java +++ b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/Ext4Inode.java @@ -40,7 +40,7 @@ public class Ext4Inode implements StructConverter { private int i_blocks_lo; // 1C 4 private int i_flags; // 20 4 see Ext4Constants.EXT4_SECRM_FL...EXT4_RESERVED_FL private int i_osd1; // 24 4 - private Ext4IBlock i_block; // 28 60 + private byte[] i_block; // 28 60 private int i_generation; // 64 4 private int i_file_acl_lo; // 68 4 private int i_size_high; // 6C 4 @@ -82,7 +82,7 @@ public class Ext4Inode implements StructConverter { i_blocks_lo = reader.readNextInt(); i_flags = reader.readNextInt(); i_osd1 = reader.readNextInt(); - i_block = new Ext4IBlock(reader, isFlagExtents()); + i_block = reader.readNextByteArray(60); i_generation = reader.readNextInt(); i_file_acl_lo = reader.readNextInt(); i_size_high = reader.readNextInt(); @@ -154,7 +154,7 @@ public class Ext4Inode implements StructConverter { return i_osd1; } - public Ext4IBlock getI_block() { + public byte[] getI_block() { return i_block; } @@ -257,7 +257,7 @@ public class Ext4Inode implements StructConverter { } /** - * Returns the bytes in this inode's iblock.extra and the system.data + * Returns the bytes in this inode's i_block and the "system.data" * extended attribute. * * @return bytes of this file that were stored inline in the inode @@ -267,14 +267,13 @@ public class Ext4Inode implements StructConverter { public byte[] getInlineDataValue() throws IOException { int bytesRemaining = (int) getSize(); byte[] result = new byte[bytesRemaining]; - byte[] iblockExtra = i_block.getExtra(); byte[] eaSystemData = getEAValue("system.data"); if (eaSystemData == null) { eaSystemData = new byte[0]; } int bytesCopied = 0; - int copyLen = Math.min(bytesRemaining, iblockExtra.length); - System.arraycopy(i_block.getExtra(), 0, result, 0, copyLen); + int copyLen = Math.min(bytesRemaining, i_block.length); + System.arraycopy(i_block, 0, result, 0, copyLen); bytesCopied += copyLen; bytesRemaining -= copyLen; if (bytesRemaining > 0) { @@ -296,8 +295,7 @@ public class Ext4Inode implements StructConverter { @Override public DataType toDataType() throws DuplicateNameException, IOException { - DataType iBlockDataType = i_block.toDataType(); - Structure structure = new StructureDataType("ext4_inode_"+iBlockDataType.getName( ), 0); + Structure structure = new StructureDataType("ext4_inode", 0); structure.add(WORD, "i_mode", null); structure.add(WORD, "i_uid", null); structure.add(DWORD, "i_size_lo", null); @@ -310,7 +308,7 @@ public class Ext4Inode implements StructConverter { structure.add(DWORD, "i_blocks_lo", null); structure.add(DWORD, "i_flags", null); structure.add(DWORD, "i_osd1", null); - structure.add(iBlockDataType, "i_block", null); + structure.add(new ArrayDataType(BYTE, 60, BYTE.getLength()), "i_block", null); structure.add(DWORD, "i_generation", null); structure.add(DWORD, "i_file_acl_lo", null); structure.add(DWORD, "i_size_high", null); diff --git a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/NewExt4Analyzer.java b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/NewExt4Analyzer.java index 9ecabc45f2..ef8658b84a 100644 --- a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/NewExt4Analyzer.java +++ b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/formats/ext4/NewExt4Analyzer.java @@ -252,14 +252,14 @@ public class NewExt4Analyzer extends FileFormatAnalyzer { comment += "Group Descriptor ID: 0x" + Integer.toHexString( i ) + "\n"; comment += "Inode Offset Into Group: 0x" + Integer.toHexString( j ) + "\n"; - Ext4IBlock iBlock = inode.getI_block( ); - if ( iBlock != null ) { - for ( Ext4Extent extent : iBlock.getExtentEntries( ) ) { - monitor.checkCanceled( ); - long destination = extent.getExtentStartBlockNumber() * blockSize; - comment += "Extent: 0x" + Long.toHexString( destination ) + "\n"; - } - } +// Ext4IBlock iBlock = inode.getI_block( ); +// if ( iBlock != null ) { +// for ( Ext4Extent extent : iBlock.getExtentEntries( ) ) { +// monitor.checkCanceled( ); +// long destination = extent.getExtentStartBlockNumber() * blockSize; +// comment += "Extent: 0x" + Long.toHexString( destination ) + "\n"; +// } +// } setPlateComment( program, address, comment ); createLabel( program, address, "INODE_" + "0x" + Integer.toHexString( inodeIndex + 1 ) ); @@ -322,8 +322,8 @@ public class NewExt4Analyzer extends FileFormatAnalyzer { boolean isDirEntry2 = (superBlock.getS_feature_incompat() & Ext4Constants.INCOMPAT_FILETYPE) != 0; // if uses extents if ( (inode.getI_flags() & Ext4Constants.EXT4_EXTENTS_FL) != 0 ) { - Ext4IBlock i_block = inode.getI_block(); - processIBlock( program, reader, isDirEntry2, i_block, monitor ); +// Ext4IBlock i_block = inode.getI_block(); +// processIBlock( program, reader, isDirEntry2, i_block, monitor ); } }