Merge remote-tracking branch

'origin/GP-1094_dev747368_ext4_bs1024_blockmaps--SQUASHED' (Closes 1877)
This commit is contained in:
ghidra1
2021-07-15 18:11:36 -04:00
10 changed files with 586 additions and 372 deletions
@@ -26,17 +26,17 @@ import ghidra.formats.gfilesystem.FSRL;
* A {@link ByteProvider} that is a concatenation of sub-ranges of another ByteProvider, also
* allowing for non-initialized (sparse) regions.
* <p>
* Not thread-safe when extents are being added.
* Not thread-safe when ranges are being added.
* <p>
* Does not assume ownership of wrapped ByteProvider.
*/
public class ExtentsByteProvider implements ByteProvider {
public class RangeMappedByteProvider implements ByteProvider {
private ByteProvider delegate;
/**
* TreeMap of this-provider offsets to the delegate-provider's offsets.
* <p>
* Each extent/region in the delegate provider is defined by the gap between
* Each range in the delegate provider is defined by the gap between
* adjacent offsetMap entries. The last entry is bounded by the total
* length of the provider as specified by the length field.
*/
@@ -45,37 +45,67 @@ public class ExtentsByteProvider implements ByteProvider {
private FSRL fsrl;
/**
* Creates a new {@link ExtentsByteProvider}.
* Creates a new {@link RangeMappedByteProvider}.
*
* @param provider {@link ByteProvider} to wrap
* @param fsrl {@link FSRL} of this new byte provider
*/
public ExtentsByteProvider(ByteProvider provider, FSRL fsrl) {
public RangeMappedByteProvider(ByteProvider provider, FSRL fsrl) {
this.delegate = provider;
this.fsrl = fsrl;
}
/**
* Adds an extent to the current end of this instance.
* Adds a range to the current end of this instance.
* <p>
* If the new range immediately follows the previous range, the new range is merged
* into the previous entry.
*
* @param offset long byte offset in the delegate ByteProvider
* @param extentLen long length of the extent region in the delegate ByteProvider
* @param offset long byte offset in the delegate ByteProvider, -1 indicates a sparse
* range with no storage
* @param rangeLen long length of the range in the delegate ByteProvider
*/
public void addExtent(long offset, long extentLen) {
if (extentLen <= 0) {
public void addRange(long offset, long rangeLen) {
if (rangeLen <= 0) {
throw new IllegalArgumentException();
}
Entry<Long, Long> lastEntry = offsetMap.lastEntry();
if (lastEntry != null) {
// try to merge sparse ranges
long lastRangeOffset = lastEntry.getValue();
if (offset == -1 && lastRangeOffset == -1) {
length += rangeLen;
return;
}
// try to merge this new range into the previous range
long lastRangeLen = length - lastEntry.getKey();
if (lastRangeOffset + lastRangeLen == offset) {
length += rangeLen;
return;
}
}
offsetMap.put(length, offset);
length += extentLen;
length += rangeLen;
}
/**
* Adds a sparse extent to the current end of this instance.
* Adds a sparse range to the current end of this instance.
*
* @param extentLen long length of the sparse extent region
* @param rangeLen long length of the sparse range
*/
public void addSparseExtent(long extentLen) {
addExtent(-1, extentLen);
public void addSparseRange(long rangeLen) {
addRange(-1, rangeLen);
}
/**
* Return the number of ranges. Adjacent ranges that were merged
* will count as a single range.
*
* @return number of ranges
*/
public int getRangeCount() {
return offsetMap.size();
}
@Override
@@ -118,12 +148,12 @@ public class ExtentsByteProvider implements ByteProvider {
ensureBounds(index, 1);
Entry<Long, Long> entry = offsetMap.floorEntry(index);
long extentStart = entry.getKey();
long extentOffset = index - extentStart;
long delegateExtentStart = entry.getValue();
long rangeStart = entry.getKey();
long rangeOffset = index - rangeStart;
long delegateRangeStart = entry.getValue();
return (delegateExtentStart != -1)
? delegate.readByte(delegateExtentStart + extentOffset)
return (delegateRangeStart != -1)
? delegate.readByte(delegateRangeStart + rangeOffset)
: 0;
}
@@ -166,20 +196,20 @@ public class ExtentsByteProvider implements ByteProvider {
Entry<Long, Long> entry = offsetMap.floorEntry(currentIndex);
Entry<Long, Long> nextEntry = offsetMap.higherEntry(entry.getKey());
long extentStart = entry.getKey();
long extentOffset = currentIndex - extentStart;
long extentEnd = (nextEntry != null) ? nextEntry.getKey() : length;
long delegateExtentStart = entry.getValue();
long rangeStart = entry.getKey();
long rangeOffset = currentIndex - rangeStart;
long rangeEnd = (nextEntry != null) ? nextEntry.getKey() : length;
long delegateRangeStart = entry.getValue();
int bytesToRead =
(int) Math.min(len - totalBytesRead, extentEnd - extentStart - extentOffset);
if (delegateExtentStart != -1) {
long delegateOffsetToRead = delegateExtentStart + extentOffset;
(int) Math.min(len - totalBytesRead, rangeEnd - rangeStart - rangeOffset);
if (delegateRangeStart != -1) {
long delegateOffsetToRead = delegateRangeStart + rangeOffset;
// TODO: when ByteProvider interface has better readBytes() method, use it here
byte[] extentBytes = delegate.readBytes(delegateOffsetToRead, bytesToRead);
System.arraycopy(extentBytes, 0, buffer, bufferDest, bytesToRead);
byte[] rangeBytes = delegate.readBytes(delegateOffsetToRead, bytesToRead);
System.arraycopy(rangeBytes, 0, buffer, bufferDest, bytesToRead);
}
else {
// the extent was not present, result will be 0's
// the range was not present, result will be 0's
Arrays.fill(buffer, bufferDest, bufferDest + bytesToRead,
(byte) 0 /* fill value */);
}
@@ -1,214 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.app.util.bin;
import static org.junit.Assert.*;
import java.io.IOException;
import java.util.Arrays;
import org.junit.Test;
public class ExtentsByteProviderTest {
private ByteArrayProvider bap(int... values) {
byte[] bytes = new byte[values.length];
for (int i = 0; i < values.length; i++) {
bytes[i] = (byte) values[i];
}
return new ByteArrayProvider(bytes);
}
/*
* "NN 01 NN 03 NN 05 NN 07 NN 09"... (NN = blockNumber, 00-FF = offset in block)
*/
private ByteArrayProvider patternedBAP(int bs, int count) {
byte[] bytes = new byte[bs * count];
for (int blockNum = 0; blockNum < count; blockNum++) {
int blockStart = blockNum * bs;
Arrays.fill(bytes, blockStart, blockStart + bs, (byte) blockNum);
for (int i = 1; i < bs; i += 2) {
bytes[i + blockStart] = (byte) (i % 256);
}
}
return new ByteArrayProvider(bytes);
}
@Test(expected = IOException.class)
public void testEmptyExtentBP() throws IOException {
try (ExtentsByteProvider ebp = new ExtentsByteProvider(bap(55), null)) {
ebp.readByte(0);
}
}
@Test
public void testExtentBP_SingleByteRead() throws IOException {
try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) {
ebp.addExtent(10, 10);
ebp.addExtent(0, 1);
ebp.addExtent(0, 10);
assertEquals(21, ebp.length());
assertEquals(0x01, ebp.readByte(0));
assertEquals(0x01, ebp.readByte(1));
assertEquals(0x01, ebp.readByte(2));
assertEquals(0x03, ebp.readByte(3));
assertEquals(0x09, ebp.readByte(9));
assertEquals(0x00, ebp.readByte(11));
assertEquals(0x01, ebp.readByte(12));
assertEquals(0x00, ebp.readByte(13));
assertEquals(0x03, ebp.readByte(14));
assertEquals(0x09, ebp.readByte(20));
try {
ebp.readByte(21);
fail();
}
catch (IOException e) {
// good
}
}
}
@Test
public void testExtentBP_MultiByteRead() throws IOException {
try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) {
ebp.addExtent(10, 10);
ebp.addExtent(0, 1);
ebp.addExtent(0, 10);
assertEquals(21, ebp.length());
byte[] bytes = ebp.readBytes(0, 21);
assertEquals(0x01, bytes[0]);
assertEquals(0x01, bytes[1]);
assertEquals(0x01, bytes[2]);
assertEquals(0x03, bytes[3]);
assertEquals(0x09, bytes[9]);
assertEquals(0x00, bytes[11]);
assertEquals(0x01, bytes[12]);
assertEquals(0x00, bytes[13]);
assertEquals(0x03, bytes[14]);
assertEquals(0x09, bytes[20]);
}
}
@Test
public void testExtentBP_MisalignedMultiByteRead() throws IOException {
try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) {
ebp.addExtent(10, 10);
ebp.addExtent(0, 10);
assertEquals(20, ebp.length());
byte[] bytes = ebp.readBytes(5, 10);
assertEquals(0x05, bytes[0]);
assertEquals(0x01, bytes[1]);
assertEquals(0x07, bytes[2]);
assertEquals(0x01, bytes[3]);
assertEquals(0x09, bytes[4]);
assertEquals(0x00, bytes[5]);
assertEquals(0x01, bytes[6]);
assertEquals(0x00, bytes[7]);
assertEquals(0x03, bytes[8]);
assertEquals(0x00, bytes[9]);
}
}
@Test
public void testSmallExtentBP() throws IOException {
try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) {
ebp.addExtent(10, 1);
ebp.addExtent(0, 1);
assertEquals(2, ebp.length());
assertEquals(0x01, ebp.readByte(0));
assertEquals(0x00, ebp.readByte(1));
try {
ebp.readByte(3);
fail();
}
catch (IOException e) {
// good
}
}
}
@Test
public void testExtentBP_SparseMultiByteRead() throws IOException {
try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) {
ebp.addExtent(-1, 5);
assertEquals(5, ebp.length());
byte[] bytes = ebp.readBytes(0, 5);
assertEquals(0x00, bytes[0]);
assertEquals(0x00, bytes[1]);
assertEquals(0x00, bytes[2]);
assertEquals(0x00, bytes[3]);
assertEquals(0x00, bytes[4]);
}
}
@Test
public void testExtentBP_SparseSingleByteRead() throws IOException {
try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) {
ebp.addExtent(-1, 5);
assertEquals(5, ebp.length());
assertEquals(0x00, ebp.readByte(0));
assertEquals(0x00, ebp.readByte(1));
assertEquals(0x00, ebp.readByte(2));
assertEquals(0x00, ebp.readByte(3));
assertEquals(0x00, ebp.readByte(4));
}
}
@Test
public void testExtentBP_MixedSparseMultiByteRead() throws IOException {
try (ExtentsByteProvider ebp = new ExtentsByteProvider(patternedBAP(10, 10), null)) {
ebp.addExtent(10, 10);
ebp.addExtent(-1, 5);
ebp.addExtent(0, 10);
assertEquals(25, ebp.length());
byte[] bytes = ebp.readBytes(0, 25);
assertEquals(0x01, bytes[0]);
assertEquals(0x01, bytes[1]);
assertEquals(0x01, bytes[2]);
assertEquals(0x03, bytes[3]);
assertEquals(0x09, bytes[9]);
assertEquals(0x00, bytes[10]);
assertEquals(0x00, bytes[11]);
assertEquals(0x00, bytes[12]);
assertEquals(0x00, bytes[13]);
assertEquals(0x00, bytes[14]);
assertEquals(0x00, bytes[15]);
assertEquals(0x01, bytes[16]);
assertEquals(0x00, bytes[17]);
assertEquals(0x03, bytes[18]);
assertEquals(0x09, bytes[24]);
}
}
}
@@ -0,0 +1,271 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.app.util.bin;
import static org.junit.Assert.*;
import java.io.IOException;
import java.util.Arrays;
import org.junit.Test;
public class RangeMappedByteProviderTest {
private ByteArrayProvider bap(int... values) {
byte[] bytes = new byte[values.length];
for (int i = 0; i < values.length; i++) {
bytes[i] = (byte) values[i];
}
return new ByteArrayProvider(bytes);
}
/*
* "NN 01 NN 03 NN 05 NN 07 NN 09"... (NN = blockNumber, 00-FF = offset in block)
*/
private ByteArrayProvider patternedBAP(int bs, int count) {
byte[] bytes = new byte[bs * count];
for (int blockNum = 0; blockNum < count; blockNum++) {
int blockStart = blockNum * bs;
Arrays.fill(bytes, blockStart, blockStart + bs, (byte) blockNum);
for (int i = 1; i < bs; i += 2) {
bytes[i + blockStart] = (byte) (i % 256);
}
}
return new ByteArrayProvider(bytes);
}
@Test(expected = IOException.class)
public void testEmptyRangeMappedBP() throws IOException {
try (RangeMappedByteProvider rmbp = new RangeMappedByteProvider(bap(55), null)) {
rmbp.readByte(0);
}
}
@Test
public void testRangeMapppedBP_SingleByteRead() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addRange(10, 10);
rmbp.addRange(0, 1);
rmbp.addRange(0, 10);
assertEquals(21, rmbp.length());
assertEquals(0x01, rmbp.readByte(0));
assertEquals(0x01, rmbp.readByte(1));
assertEquals(0x01, rmbp.readByte(2));
assertEquals(0x03, rmbp.readByte(3));
assertEquals(0x09, rmbp.readByte(9));
assertEquals(0x00, rmbp.readByte(11));
assertEquals(0x01, rmbp.readByte(12));
assertEquals(0x00, rmbp.readByte(13));
assertEquals(0x03, rmbp.readByte(14));
assertEquals(0x09, rmbp.readByte(20));
try {
rmbp.readByte(21);
fail();
}
catch (IOException e) {
// good
}
}
}
@Test
public void testRangeMapppedBP_MultiByteRead() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addRange(10, 10);
rmbp.addRange(0, 1);
rmbp.addRange(0, 10);
assertEquals(21, rmbp.length());
byte[] bytes = rmbp.readBytes(0, 21);
assertEquals(0x01, bytes[0]);
assertEquals(0x01, bytes[1]);
assertEquals(0x01, bytes[2]);
assertEquals(0x03, bytes[3]);
assertEquals(0x09, bytes[9]);
assertEquals(0x00, bytes[11]);
assertEquals(0x01, bytes[12]);
assertEquals(0x00, bytes[13]);
assertEquals(0x03, bytes[14]);
assertEquals(0x09, bytes[20]);
}
}
@Test
public void testRangeMapppedBP_MisalignedMultiByteRead() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addRange(10, 10);
rmbp.addRange(0, 10);
assertEquals(20, rmbp.length());
byte[] bytes = rmbp.readBytes(5, 10);
assertEquals(0x05, bytes[0]);
assertEquals(0x01, bytes[1]);
assertEquals(0x07, bytes[2]);
assertEquals(0x01, bytes[3]);
assertEquals(0x09, bytes[4]);
assertEquals(0x00, bytes[5]);
assertEquals(0x01, bytes[6]);
assertEquals(0x00, bytes[7]);
assertEquals(0x03, bytes[8]);
assertEquals(0x00, bytes[9]);
}
}
@Test
public void testSmallRangeMapppedBP() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addRange(10, 1);
rmbp.addRange(0, 1);
assertEquals(2, rmbp.length());
assertEquals(0x01, rmbp.readByte(0));
assertEquals(0x00, rmbp.readByte(1));
try {
rmbp.readByte(3);
fail();
}
catch (IOException e) {
// good
}
}
}
@Test
public void testRangeMapppedBP_SparseMultiByteRead() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addSparseRange(5);
assertEquals(5, rmbp.length());
byte[] bytes = rmbp.readBytes(0, 5);
assertEquals(0x00, bytes[0]);
assertEquals(0x00, bytes[1]);
assertEquals(0x00, bytes[2]);
assertEquals(0x00, bytes[3]);
assertEquals(0x00, bytes[4]);
}
}
@Test
public void testRangeMapppedBP_SparseSingleByteRead() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addSparseRange(5);
assertEquals(5, rmbp.length());
assertEquals(0x00, rmbp.readByte(0));
assertEquals(0x00, rmbp.readByte(1));
assertEquals(0x00, rmbp.readByte(2));
assertEquals(0x00, rmbp.readByte(3));
assertEquals(0x00, rmbp.readByte(4));
}
}
@Test
public void testRangeMapppedBP_MixedSparseMultiByteRead() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addRange(10, 10);
rmbp.addSparseRange(5);
rmbp.addRange(0, 10);
assertEquals(25, rmbp.length());
byte[] bytes = rmbp.readBytes(0, 25);
assertEquals(0x01, bytes[0]);
assertEquals(0x01, bytes[1]);
assertEquals(0x01, bytes[2]);
assertEquals(0x03, bytes[3]);
assertEquals(0x09, bytes[9]);
assertEquals(0x00, bytes[10]);
assertEquals(0x00, bytes[11]);
assertEquals(0x00, bytes[12]);
assertEquals(0x00, bytes[13]);
assertEquals(0x00, bytes[14]);
assertEquals(0x00, bytes[15]);
assertEquals(0x01, bytes[16]);
assertEquals(0x00, bytes[17]);
assertEquals(0x03, bytes[18]);
assertEquals(0x09, bytes[24]);
}
}
@Test
public void testMergeAdjacentRanges() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addRange(10, 10);
rmbp.addRange(20, 5);
rmbp.addRange(25, 5);
assertEquals(20, rmbp.length());
assertEquals(1, rmbp.getRangeCount());
}
}
@Test
public void testDontMergeAlmostAdjacentRanges() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addRange(10, 10);
rmbp.addRange(21, 5);
assertEquals(15, rmbp.length());
assertEquals(2, rmbp.getRangeCount());
}
}
@Test
public void testDontMergeAlmostAdjacentRanges2() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addRange(10, 10);
rmbp.addRange(19, 5);// creates a weird overlapped result, but good boundary cond test
assertEquals(15, rmbp.length());
assertEquals(2, rmbp.getRangeCount());
}
}
@Test
public void testMergeAdjacentSparseRanges() throws IOException {
try (RangeMappedByteProvider rmbp =
new RangeMappedByteProvider(patternedBAP(10, 10), null)) {
rmbp.addRange(10, 10);
rmbp.addSparseRange(5);
rmbp.addSparseRange(5);
assertEquals(20, rmbp.length());
assertEquals(2, rmbp.getRangeCount());
}
}
}
@@ -16,7 +16,6 @@
package ghidra.file.formats.ext4;
import java.io.IOException;
import java.util.List;
import ghidra.app.util.bin.*;
import ghidra.app.util.importer.MessageLog;
@@ -191,27 +190,27 @@ public class Ext4Analyzer extends FileFormatAnalyzer {
boolean isDirEntry2 = (superBlock.getS_feature_incompat() & Ext4Constants.INCOMPAT_FILETYPE) != 0;
// if uses extents
if( (inode.getI_flags() & Ext4Constants.EXT4_EXTENTS_FL) != 0 ) {
Ext4IBlock i_block = inode.getI_block();
Ext4ExtentHeader header = i_block.getHeader();
if( header.getEh_depth() == 0 ) {
short numEntries = header.getEh_entries();
List<Ext4Extent> entries = i_block.getExtentEntries();
for( int i = 0; i < numEntries; i++ ) {
Ext4Extent extent = entries.get(i);
long offset = extent.getExtentStartBlockNumber() * blockSize;
reader.setPointerIndex(offset);
Address address = toAddr(program, offset);
if( isDirEntry2 ) {
while( (reader.getPointerIndex() - offset) < (extent.getEe_len() * blockSize)) {
Ext4DirEntry2 dirEnt2 = Ext4DirEntry2.read(reader);
DataType dataType = dirEnt2.toDataType();
createData(program, address, dataType);
address = address.add(dataType.getLength());
}
}
}
}
// Ext4IBlock i_block = inode.getI_block();
// Ext4ExtentHeader header = i_block.getHeader();
// if( header.getEh_depth() == 0 ) {
// short numEntries = header.getEh_entries();
// List<Ext4Extent> entries = i_block.getExtentEntries();
// for( int i = 0; i < numEntries; i++ ) {
// Ext4Extent extent = entries.get(i);
// long offset = extent.getExtentStartBlockNumber() * blockSize;
// reader.setPointerIndex(offset);
// Address address = toAddr(program, offset);
// if( isDirEntry2 ) {
// while( (reader.getPointerIndex() - offset) < (extent.getEe_len() * blockSize)) {
// Ext4DirEntry2 dirEnt2 = Ext4DirEntry2.read(reader);
// DataType dataType = dirEnt2.toDataType();
// createData(program, address, dataType);
// address = address.add(dataType.getLength());
// }
// }
// }
//
// }
}
}
@@ -0,0 +1,93 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.file.formats.ext4;
import java.io.IOException;
import ghidra.app.util.bin.*;
import ghidra.formats.gfilesystem.FSRL;
/**
* Helper class that handles the blockmap data stored in an inode's i_block[] array
*/
public class Ext4BlockMapHelper {
private static final int INDIRECT_BLOCK_INDEX = 12;
private static final int DOUBLE_INDIRECT_BLOCK_INDEX = 13;
private static final int TRIPLE_INDIRECT_BLOCK_INDEX = 14;
private static final int INODE_BLOCKMAP_COUNT = 15;
/**
* Creates a {@link RangeMappedByteProvider} from the old-style block map data found in the
* inode's i_block field.
*
* @param rawIBlockBytes raw bytes from the inode's i_block
* @param provider the file system volume provider
* @param expectedLength the length the file should be (from the inode)
* @param blockSize file system blockSize
* @param fsrl {@link FSRL} to assign to the new ByteProvider
* @return new {@link ByteProvider} containing the blocks of the volume that were specified
* by the blockmap data
* @throws IOException if error
*/
public static ByteProvider getByteProvider(byte[] rawIBlockBytes, ByteProvider provider,
long expectedLength, int blockSize, FSRL fsrl) throws IOException {
BinaryReader iBlockReader =
new BinaryReader(new ByteArrayProvider(rawIBlockBytes), true /* LE */);
int[] blockNumbers = iBlockReader.readNextIntArray(INODE_BLOCKMAP_COUNT);
RangeMappedByteProvider bp = new RangeMappedByteProvider(provider, fsrl);
// the location of the first 12 blocks of the file are held in [0..11]
addFromArray(blockNumbers, 0, INDIRECT_BLOCK_INDEX, 0, bp, blockSize, expectedLength,
provider);
// the location of the next blockSize/4 (ie. 4096/4=1024) blocks of the file are
// held in an array that is located in the block pointed to by [12]
addFromArray(blockNumbers, INDIRECT_BLOCK_INDEX, DOUBLE_INDIRECT_BLOCK_INDEX, 1, bp,
blockSize, expectedLength, provider);
// the location of the next blocks of the file are held in an array that is
// double-ly indirectly pointed to by [13]
addFromArray(blockNumbers, DOUBLE_INDIRECT_BLOCK_INDEX, TRIPLE_INDIRECT_BLOCK_INDEX, 2, bp,
blockSize, expectedLength, provider);
// the location of the next blocks of the file are held in an array that is
// triply indirectly pointed to by [14]
addFromArray(blockNumbers, TRIPLE_INDIRECT_BLOCK_INDEX, INODE_BLOCKMAP_COUNT, 3, bp,
blockSize, expectedLength, provider);
return bp;
}
private static void addFromArray(int[] blockNums, int start, int end, int indirectLevel,
RangeMappedByteProvider ebp, int blockSize, long expectedLength, ByteProvider provider)
throws IOException {
BinaryReader reader = new BinaryReader(provider, true /* LE */ );
for (int i = start; i < end && ebp.length() < expectedLength; i++) {
if ( indirectLevel > 0 ) {
int[] subBlockNumbers = reader.readIntArray(blockNums[i] * blockSize,
blockSize / BinaryReader.SIZEOF_INT);
addFromArray(subBlockNumbers, 0, subBlockNumbers.length, indirectLevel - 1, ebp,
blockSize, expectedLength, provider);
}
else {
int bytesFromBlock = (int) Math.min(blockSize, expectedLength - ebp.length());
long blockNum = Integer.toUnsignedLong(blockNums[i]);
ebp.addRange(blockNum == 0 ? -1 : blockNum * blockSize, bytesFromBlock);
}
}
}
}
@@ -15,17 +15,14 @@
*/
package ghidra.file.formats.ext4;
import ghidra.app.util.bin.BinaryReader;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.bin.StructConverter;
import ghidra.program.model.data.DataType;
import ghidra.program.model.data.Structure;
import ghidra.program.model.data.StructureDataType;
import ghidra.util.exception.DuplicateNameException;
import java.io.IOException;
import ghidra.app.util.bin.*;
import ghidra.program.model.data.*;
import ghidra.util.exception.DuplicateNameException;
public class Ext4ExtentHeader implements StructConverter {
private static final int SIZEOF = 12;
private short eh_magic;
private short eh_entries;
@@ -33,6 +30,21 @@ public class Ext4ExtentHeader implements StructConverter {
private short eh_depth;
private int eh_generation;
/**
* Read a Ext4ExtentHeader from the stream.
*
* @param reader BinaryReader to read from
* @return new Ext4ExtentHeader instance, or null if eof or no magic value
* @throws IOException if error
*/
public static Ext4ExtentHeader read(BinaryReader reader) throws IOException {
if (reader.getPointerIndex() + SIZEOF >= reader.length() ||
Short.toUnsignedInt(reader.peekNextShort()) != Ext4Constants.EXTENT_HEADER_MAGIC) {
return null;
}
return new Ext4ExtentHeader(reader);
}
public Ext4ExtentHeader( ByteProvider provider ) throws IOException {
this( new BinaryReader( provider, true ) );
}
@@ -0,0 +1,95 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.file.formats.ext4;
import java.io.IOException;
import ghidra.app.util.bin.*;
import ghidra.formats.gfilesystem.FSRL;
/**
* Helper class that handles the extent data stored in an inode's i_block[] array
*/
public class Ext4ExtentsHelper {
/**
* Creates a {@link RangeMappedByteProvider} from the extents data found in the
* inode's i_block field.
*
* @param rawIBlockBytes raw bytes from the inode's i_block
* @param provider the file system volume provider
* @param expectedLength the length the file should be (from the inode)
* @param blockSize file system blockSize
* @param fsrl {@link FSRL} to assign to the new ByteProvider
* @return new {@link ByteProvider} containing the blocks of the volume that were specified
* by the extent data
* @throws IOException if error
*/
public static ByteProvider getByteProvider(byte[] rawIBlockBytes, ByteProvider provider,
long expectedLength, int blockSize, FSRL fsrl) throws IOException {
BinaryReader iBlockReader =
new BinaryReader(new ByteArrayProvider(rawIBlockBytes), true /* LE */);
RangeMappedByteProvider ebp = new RangeMappedByteProvider(provider, fsrl);
processExtents(iBlockReader, provider, ebp, blockSize, expectedLength);
if (ebp.length() < expectedLength) {
// trailing sparse. not sure if possible.
ebp.addSparseRange(expectedLength - ebp.length());
}
return ebp;
}
private static void processExtents(BinaryReader reader, ByteProvider provider,
RangeMappedByteProvider ebp, int blockSize, long expectedLength) throws IOException {
Ext4ExtentHeader header = Ext4ExtentHeader.read(reader);
if ( header == null ) {
throw new IOException("Bad/missing extents header");
}
if (header.getEh_depth() == 0) {
for (int i = 0; i < header.getEh_entries() && ebp.length() < expectedLength; i++) {
Ext4Extent extent = new Ext4Extent(reader);
long startPos = extent.getStreamBlockNumber() * blockSize;
long providerOfs = extent.getExtentStartBlockNumber() * blockSize;
long extentLen = extent.getExtentBlockCount() * blockSize;
if (ebp.length() < startPos) {
ebp.addSparseRange(startPos - ebp.length());
}
if (ebp.length() + extentLen > expectedLength) {
// the last extent may have a trailing partial block
extentLen = expectedLength - ebp.length();
}
ebp.addRange(providerOfs, extentLen);
}
}
else {
for (int i = 0; i < header.getEh_entries(); i++) {
Ext4ExtentIdx idx = new Ext4ExtentIdx(reader);
long offset = idx.getEi_leaf() * blockSize;
try (ByteProviderWrapper bpw =
new ByteProviderWrapper(provider, offset, blockSize)) {
BinaryReader subReader = new BinaryReader(bpw, true /* LE */);
processExtents(subReader, provider, ebp, blockSize, expectedLength);
}
}
}
}
}
@@ -70,7 +70,7 @@ public class Ext4FileSystem implements GFileSystem {
numGroups++;
}
int groupDescriptorOffset = blockSize;
int groupDescriptorOffset = blockSize + (superBlock.getS_first_data_block() * blockSize);
reader.setPointerIndex(groupDescriptorOffset);
monitor.initialize(numGroups);
monitor.setMessage("Reading inode tables");
@@ -112,41 +112,6 @@ public class Ext4FileSystem implements GFileSystem {
}
}
interface Checked2Consumer<T, E1 extends Throwable, E2 extends Throwable> {
void accept(T t) throws E1, E2;
}
interface ExtentConsumer extends Checked2Consumer<Ext4Extent, IOException, CancelledException> {
// no additional def
}
private void forEachExtentEntry(Ext4IBlock i_block, ExtentConsumer extentConsumer,
TaskMonitor monitor) throws CancelledException, IOException {
Ext4ExtentHeader header = i_block.getHeader();
if (header.getEh_depth() == 0) {
short numEntries = header.getEh_entries();
List<Ext4Extent> entries = i_block.getExtentEntries();
for (int i = 0; i < numEntries; i++) {
monitor.checkCanceled();
Ext4Extent extent = entries.get(i);
extentConsumer.accept(extent);
}
}
else {
short numEntries = header.getEh_entries();
List<Ext4ExtentIdx> entries = i_block.getIndexEntries();
for (int i = 0; i < numEntries; i++) {
monitor.checkCanceled();
Ext4ExtentIdx extentIndex = entries.get(i);
long offset = extentIndex.getEi_leaf() * blockSize;
forEachExtentEntry(Ext4IBlock.readIBlockWithExtents(provider, offset),
extentConsumer, monitor);
}
}
}
private void processDirectory(Ext4Inode inode, GFile dirFile, Ext4Inode[] inodes,
BitSet processedInodes, TaskMonitor monitor) throws IOException, CancelledException {
try (ByteProvider bp = getInodeByteProvider(inode, dirFile.getFSRL(), monitor)) {
@@ -186,6 +151,7 @@ public class Ext4FileSystem implements GFileSystem {
throw new IOException("Inode " + inode + " has unhandled file type: " +
Integer.toHexString(inode.getFileType()));
}
String name = dirEntry.getName();
if (".".equals(name) || "..".equals(name)) {
// skip the ".", and ".." self-reference directories
@@ -331,55 +297,19 @@ public class Ext4FileSystem implements GFileSystem {
private ByteProvider getInodeByteProvider(Ext4Inode inode, FSRL inodeFSRL, TaskMonitor monitor)
throws IOException {
if (inode.isFlagExtents()) {
return getExtentsByteProvider(inodeFSRL, inode, monitor);
return Ext4ExtentsHelper.getByteProvider(inode.getI_block(), provider, inode.getSize(),
blockSize, inodeFSRL);
}
else if (inode.isFlagInlineData() || inode.isSymLink()) {
return getInlineDataProvider(inodeFSRL, inode, monitor);
byte[] data = inode.getInlineDataValue();
return new ByteArrayProvider(data, inodeFSRL);
}
else {
throw new IOException("Unsupported file storage: " + inodeFSRL.getPath());
return Ext4BlockMapHelper.getByteProvider(inode.getI_block(), provider, inode.getSize(),
blockSize, inodeFSRL);
}
}
private ByteProvider getInlineDataProvider(FSRL fileFSRL, Ext4Inode inode, TaskMonitor monitor)
throws IOException {
byte[] data = inode.getInlineDataValue();
return new ByteArrayProvider(data, fileFSRL);
}
private ByteProvider getExtentsByteProvider(FSRL fileFSRL, Ext4Inode inode, TaskMonitor monitor)
throws IOException {
try {
long fileSize = inode.getSize();
ExtentsByteProvider result = new ExtentsByteProvider(provider, fileFSRL);
Ext4IBlock i_block = inode.getI_block();
forEachExtentEntry(i_block, extent -> {
long startPos = extent.getStreamBlockNumber() * blockSize;
long providerOfs = extent.getExtentStartBlockNumber() * blockSize;
long extentLen = extent.getExtentBlockCount() * blockSize;
if (result.length() < startPos) {
result.addSparseExtent(startPos - result.length());
}
if (result.length() + extentLen > fileSize) {
// the last extent may have a trailing partial block
extentLen = fileSize - result.length();
}
result.addExtent(providerOfs, extentLen);
}, monitor);
if (result.length() < fileSize) {
// trailing sparse. not sure if possible.
result.addSparseExtent(fileSize - result.length());
}
return result;
}
catch (CancelledException e) {
throw new IOException(e);
}
}
private Ext4Inode[] getInodes(BinaryReader reader, Ext4GroupDescriptor[] groupDescriptors,
TaskMonitor monitor) throws IOException, CancelledException {
@@ -40,7 +40,7 @@ public class Ext4Inode implements StructConverter {
private int i_blocks_lo; // 1C 4
private int i_flags; // 20 4 see Ext4Constants.EXT4_SECRM_FL...EXT4_RESERVED_FL
private int i_osd1; // 24 4
private Ext4IBlock i_block; // 28 60
private byte[] i_block; // 28 60
private int i_generation; // 64 4
private int i_file_acl_lo; // 68 4
private int i_size_high; // 6C 4
@@ -82,7 +82,7 @@ public class Ext4Inode implements StructConverter {
i_blocks_lo = reader.readNextInt();
i_flags = reader.readNextInt();
i_osd1 = reader.readNextInt();
i_block = new Ext4IBlock(reader, isFlagExtents());
i_block = reader.readNextByteArray(60);
i_generation = reader.readNextInt();
i_file_acl_lo = reader.readNextInt();
i_size_high = reader.readNextInt();
@@ -154,7 +154,7 @@ public class Ext4Inode implements StructConverter {
return i_osd1;
}
public Ext4IBlock getI_block() {
public byte[] getI_block() {
return i_block;
}
@@ -257,7 +257,7 @@ public class Ext4Inode implements StructConverter {
}
/**
* Returns the bytes in this inode's iblock.extra and the system.data
* Returns the bytes in this inode's i_block and the "system.data"
* extended attribute.
*
* @return bytes of this file that were stored inline in the inode
@@ -267,14 +267,13 @@ public class Ext4Inode implements StructConverter {
public byte[] getInlineDataValue() throws IOException {
int bytesRemaining = (int) getSize();
byte[] result = new byte[bytesRemaining];
byte[] iblockExtra = i_block.getExtra();
byte[] eaSystemData = getEAValue("system.data");
if (eaSystemData == null) {
eaSystemData = new byte[0];
}
int bytesCopied = 0;
int copyLen = Math.min(bytesRemaining, iblockExtra.length);
System.arraycopy(i_block.getExtra(), 0, result, 0, copyLen);
int copyLen = Math.min(bytesRemaining, i_block.length);
System.arraycopy(i_block, 0, result, 0, copyLen);
bytesCopied += copyLen;
bytesRemaining -= copyLen;
if (bytesRemaining > 0) {
@@ -296,8 +295,7 @@ public class Ext4Inode implements StructConverter {
@Override
public DataType toDataType() throws DuplicateNameException, IOException {
DataType iBlockDataType = i_block.toDataType();
Structure structure = new StructureDataType("ext4_inode_"+iBlockDataType.getName( ), 0);
Structure structure = new StructureDataType("ext4_inode", 0);
structure.add(WORD, "i_mode", null);
structure.add(WORD, "i_uid", null);
structure.add(DWORD, "i_size_lo", null);
@@ -310,7 +308,7 @@ public class Ext4Inode implements StructConverter {
structure.add(DWORD, "i_blocks_lo", null);
structure.add(DWORD, "i_flags", null);
structure.add(DWORD, "i_osd1", null);
structure.add(iBlockDataType, "i_block", null);
structure.add(new ArrayDataType(BYTE, 60, BYTE.getLength()), "i_block", null);
structure.add(DWORD, "i_generation", null);
structure.add(DWORD, "i_file_acl_lo", null);
structure.add(DWORD, "i_size_high", null);
@@ -252,14 +252,14 @@ public class NewExt4Analyzer extends FileFormatAnalyzer {
comment += "Group Descriptor ID: 0x" + Integer.toHexString( i ) + "\n";
comment += "Inode Offset Into Group: 0x" + Integer.toHexString( j ) + "\n";
Ext4IBlock iBlock = inode.getI_block( );
if ( iBlock != null ) {
for ( Ext4Extent extent : iBlock.getExtentEntries( ) ) {
monitor.checkCanceled( );
long destination = extent.getExtentStartBlockNumber() * blockSize;
comment += "Extent: 0x" + Long.toHexString( destination ) + "\n";
}
}
// Ext4IBlock iBlock = inode.getI_block( );
// if ( iBlock != null ) {
// for ( Ext4Extent extent : iBlock.getExtentEntries( ) ) {
// monitor.checkCanceled( );
// long destination = extent.getExtentStartBlockNumber() * blockSize;
// comment += "Extent: 0x" + Long.toHexString( destination ) + "\n";
// }
// }
setPlateComment( program, address, comment );
createLabel( program, address, "INODE_" + "0x" + Integer.toHexString( inodeIndex + 1 ) );
@@ -322,8 +322,8 @@ public class NewExt4Analyzer extends FileFormatAnalyzer {
boolean isDirEntry2 = (superBlock.getS_feature_incompat() & Ext4Constants.INCOMPAT_FILETYPE) != 0;
// if uses extents
if ( (inode.getI_flags() & Ext4Constants.EXT4_EXTENTS_FL) != 0 ) {
Ext4IBlock i_block = inode.getI_block();
processIBlock( program, reader, isDirEntry2, i_block, monitor );
// Ext4IBlock i_block = inode.getI_block();
// processIBlock( program, reader, isDirEntry2, i_block, monitor );
}
}