GT-2845: updated Mach-O, DYLD, Prelink loaders to utilize FilesBytes.

This commit is contained in:
Ryan Kurtz
2019-07-02 14:59:41 -04:00
parent 12af9291c9
commit 7e5252f6c7
7 changed files with 108 additions and 118 deletions
@@ -330,7 +330,7 @@ public class BinaryLoader extends AbstractProgramLoader {
block.setWrite(isOverlay ? false : true);
block.setExecute(isOverlay ? false : true);
block.setSourceName("Binary Loader");
MemoryBlockUtil.adjustFragment(prog.getListing(), block.getStart(), blockName);
MemoryBlockUtils.adjustFragment(prog, block.getStart(), blockName);
}
catch (LockException | MemoryConflictException e) {
Msg.error(this, "Unexpected exception creating memory block", e);
@@ -18,6 +18,7 @@ package ghidra.app.util.opinion;
import java.io.IOException;
import java.util.*;
import ghidra.app.util.MemoryBlockUtils;
import ghidra.app.util.Option;
import ghidra.app.util.bin.BinaryReader;
import ghidra.app.util.bin.ByteProvider;
@@ -44,8 +45,7 @@ public class DyldCacheLoader extends AbstractLibrarySupportLoader {
static final boolean PROCESS_SYMBOLS_OPTION_DEFAULT = true;
/** Loader option to create memory blocks for DYLIB sections */
static final String CREATE_DYLIB_SECTIONS_OPTION_NAME =
"Create DYLIB section memory blocks (slow!)";
static final String CREATE_DYLIB_SECTIONS_OPTION_NAME = "Create DYLIB section memory blocks";
/** Default value for loader option to create memory blocks for DYLIB sections */
static final boolean CREATE_DYLIB_SECTIONS_OPTION_DEFAULT = false;
@@ -85,8 +85,9 @@ public class DyldCacheLoader extends AbstractLibrarySupportLoader {
throws IOException {
try {
DyldCacheProgramBuilder.buildProgram(program, provider, shouldProcessSymbols(options),
shouldCreateDylibSections(options), log, handler, monitor);
DyldCacheProgramBuilder.buildProgram(program, provider,
MemoryBlockUtils.createFileBytes(program, provider), shouldProcessSymbols(options),
shouldCreateDylibSections(options), log, monitor);
}
catch (CancelledException e) {
return;
@@ -19,13 +19,16 @@ import java.io.File;
import java.io.IOException;
import java.util.*;
import ghidra.app.util.MemoryBlockUtils;
import ghidra.app.util.bin.BinaryReader;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.bin.format.macho.MachException;
import ghidra.app.util.bin.format.macho.MachHeader;
import ghidra.app.util.bin.format.macho.commands.NList;
import ghidra.app.util.bin.format.macho.dyld.*;
import ghidra.app.util.importer.*;
import ghidra.app.util.importer.MessageLog;
import ghidra.app.util.importer.MessageLogContinuesFactory;
import ghidra.program.database.mem.FileBytes;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressSpace;
import ghidra.program.model.listing.*;
@@ -47,17 +50,17 @@ public class DyldCacheProgramBuilder extends MachoProgramBuilder {
*
* @param program The {@link Program} to build up
* @param provider The {@link ByteProvider} that contains the DYLD Cache bytes
* @param fileBytes Where the Mach-O's bytes came from
* @param shouldProcessSymbols True if symbols should be processed; otherwise, false
* @param shouldCreateDylibSections True if memory blocks should be created for DYLIB sections;
* otherwise, false
* @param log The log
* @param memoryConflictHandler How to handle memory conflicts that may occur
* @param monitor A cancelable task monitor
*/
protected DyldCacheProgramBuilder(Program program, ByteProvider provider,
protected DyldCacheProgramBuilder(Program program, ByteProvider provider, FileBytes fileBytes,
boolean shouldProcessSymbols, boolean shouldCreateDylibSections, MessageLog log,
MemoryConflictHandler memoryConflictHandler, TaskMonitor monitor) {
super(program, provider, log, memoryConflictHandler, monitor);
TaskMonitor monitor) {
super(program, provider, fileBytes, log, monitor);
this.shouldProcessSymbols = shouldProcessSymbols;
this.shouldCreateDylibSections = shouldCreateDylibSections;
}
@@ -67,20 +70,19 @@ public class DyldCacheProgramBuilder extends MachoProgramBuilder {
*
* @param program The {@link Program} to build up
* @param provider The {@link ByteProvider} that contains the DYLD Cache's bytes
* @param fileBytes Where the Mach-O's bytes came from
* @param shouldProcessSymbols True if symbols should be processed; otherwise, false
* @param shouldCreateDylibSections True if memory blocks should be created for DYLIB sections;
* otherwise, false
* @param log The log
* @param memoryConflictHandler How to handle memory conflicts that may occur
* @param monitor A cancelable task monitor
* @throws Exception if a problem occurs
*/
public static void buildProgram(Program program, ByteProvider provider,
public static void buildProgram(Program program, ByteProvider provider, FileBytes fileBytes,
boolean shouldProcessSymbols, boolean shouldCreateDylibSections, MessageLog log,
MemoryConflictHandler memoryConflictHandler, TaskMonitor monitor) throws Exception {
TaskMonitor monitor) throws Exception {
DyldCacheProgramBuilder dyldCacheProgramBuilder = new DyldCacheProgramBuilder(program,
provider, shouldProcessSymbols, shouldCreateDylibSections, log, memoryConflictHandler,
monitor);
provider, fileBytes, shouldProcessSymbols, shouldCreateDylibSections, log, monitor);
dyldCacheProgramBuilder.build();
}
@@ -93,20 +95,12 @@ public class DyldCacheProgramBuilder extends MachoProgramBuilder {
dyldCacheHeader.parseFromFile(shouldProcessSymbols, log, monitor);
monitor.incrementProgress(1);
try {
setDyldCacheImageBase();
processDyldCacheMemoryBlocks();
markupHeaders();
markupBranchIslands();
createSymbols();
processDylibs();
}
finally {
if (mbu != null) {
mbu.dispose();
mbu = null;
}
}
setDyldCacheImageBase();
processDyldCacheMemoryBlocks();
markupHeaders();
markupBranchIslands();
createSymbols();
processDylibs();
}
/**
@@ -135,9 +129,9 @@ public class DyldCacheProgramBuilder extends MachoProgramBuilder {
for (DyldCacheMappingInfo mappingInfo : mappingInfos) {
long offset = mappingInfo.getFileOffset();
long size = mappingInfo.getSize();
mbu.createInitializedBlock("DYLD", space.getAddress(mappingInfo.getAddress()),
provider.getInputStream(offset), size, "", "", mappingInfo.isRead(),
mappingInfo.isWrite(), mappingInfo.isExecute(), monitor);
MemoryBlockUtils.createInitializedBlock(program, false, "DYLD",
space.getAddress(mappingInfo.getAddress()), fileBytes, offset, size, "", "",
mappingInfo.isRead(), mappingInfo.isWrite(), mappingInfo.isExecute(), log);
if (offset + size > endOfMappedOffset) {
endOfMappedOffset = offset + size;
}
@@ -147,9 +141,10 @@ public class DyldCacheProgramBuilder extends MachoProgramBuilder {
if (endOfMappedOffset < provider.length()) {
monitor.setMessage("Processing DYLD unmapped memory block...");
mbu.createOverlayBlock("FILE", AddressSpace.OTHER_SPACE.getAddress(endOfMappedOffset),
provider.getInputStream(endOfMappedOffset), provider.length() - endOfMappedOffset,
"Useful bytes that don't get mapped into memory", "", false, false, false, monitor);
MemoryBlockUtils.createInitializedBlock(program, true, "FILE",
AddressSpace.OTHER_SPACE.getAddress(endOfMappedOffset), fileBytes,
endOfMappedOffset, provider.length() - endOfMappedOffset,
"Useful bytes that don't get mapped into memory", "", false, false, false, log);
}
}
@@ -228,10 +223,9 @@ public class DyldCacheProgramBuilder extends MachoProgramBuilder {
// easier
monitor.setMessage("Parsing DYLIB's...");
monitor.initialize(dyldCacheHeader.getImageInfos().size());
TreeSet<DyldCacheMachoInfo> dyldCacheMachoInfoSet =
new TreeSet<>((a, b) -> a.headerAddr.compareTo(b.headerAddr));
List<DyldCacheMachoInfo> infoList = new ArrayList<>(dyldCacheHeader.getImageInfos().size());
for (DyldCacheImageInfo dyldCacheImageInfo : dyldCacheHeader.getImageInfos()) {
dyldCacheMachoInfoSet.add(new DyldCacheMachoInfo(provider,
infoList.add(new DyldCacheMachoInfo(provider,
dyldCacheImageInfo.getAddress() - dyldCacheHeader.getBaseAddress(),
space.getAddress(dyldCacheImageInfo.getAddress()), dyldCacheImageInfo.getPath()));
monitor.checkCanceled();
@@ -240,8 +234,8 @@ public class DyldCacheProgramBuilder extends MachoProgramBuilder {
// Markup DyldCache Mach-O headers
monitor.setMessage("Marking up DYLIB headers...");
monitor.initialize(dyldCacheMachoInfoSet.size());
for (DyldCacheMachoInfo info : dyldCacheMachoInfoSet) {
monitor.initialize(infoList.size());
for (DyldCacheMachoInfo info : infoList) {
info.markupHeaders();
monitor.checkCanceled();
monitor.incrementProgress(1);
@@ -249,8 +243,8 @@ public class DyldCacheProgramBuilder extends MachoProgramBuilder {
// Add DyldCache Mach-O's to program tree
monitor.setMessage("Adding DYLIB's to program tree...");
monitor.initialize(dyldCacheMachoInfoSet.size());
Iterator<DyldCacheMachoInfo> iter = dyldCacheMachoInfoSet.iterator();
monitor.initialize(infoList.size());
Iterator<DyldCacheMachoInfo> iter = infoList.iterator();
if (iter.hasNext()) {
DyldCacheMachoInfo curr = iter.next();
do {
@@ -263,13 +257,11 @@ public class DyldCacheProgramBuilder extends MachoProgramBuilder {
while (iter.hasNext());
}
// Process DyldCache DYLIB memory blocks. Need to do it in descending (reverse) order or
// the memory block splitting will be way too slow.
// Process DyldCache DYLIB memory blocks.
monitor.setMessage("Processing DYLIB memory blocks...");
monitor.initialize(dyldCacheMachoInfoSet.size());
Iterator<DyldCacheMachoInfo> descendingIter = dyldCacheMachoInfoSet.descendingIterator();
while (descendingIter.hasNext()) {
descendingIter.next().processMemoryBlocks();
monitor.initialize(infoList.size());
for (DyldCacheMachoInfo info : infoList) {
info.processMemoryBlocks();
monitor.checkCanceled();
monitor.incrementProgress(1);
}
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.*;
import generic.continues.RethrowContinuesFactory;
import ghidra.app.util.MemoryBlockUtils;
import ghidra.app.util.Option;
import ghidra.app.util.bin.*;
import ghidra.app.util.bin.format.macho.*;
@@ -28,6 +29,7 @@ import ghidra.app.util.bin.format.ubi.*;
import ghidra.app.util.importer.MemoryConflictHandler;
import ghidra.app.util.importer.MessageLog;
import ghidra.framework.model.DomainFolder;
import ghidra.program.database.mem.FileBytes;
import ghidra.program.model.listing.Program;
import ghidra.util.LittleEndianDataConverter;
import ghidra.util.exception.CancelledException;
@@ -81,15 +83,18 @@ public class MachoLoader extends AbstractLibrarySupportLoader {
throws IOException {
try {
FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider);
// A Mach-O file may contain PRELINK information. If so, we use a special
// program builder that knows how to deal with it.
List<PrelinkMap> prelinkList = MachoPrelinkUtils.parsePrelinkXml(provider, monitor);
if (!prelinkList.isEmpty()) {
MachoPrelinkProgramBuilder.buildProgram(program, provider, prelinkList, log, handler,
monitor);
MachoPrelinkProgramBuilder.buildProgram(program, provider, fileBytes, prelinkList,
log, monitor);
}
else {
MachoProgramBuilder.buildProgram(program, provider, log, handler, monitor);
MachoProgramBuilder.buildProgram(program, provider, fileBytes, log, handler,
monitor);
}
}
catch (Exception e) {
@@ -25,7 +25,9 @@ import ghidra.app.util.bin.format.macho.MachHeader;
import ghidra.app.util.bin.format.macho.Section;
import ghidra.app.util.bin.format.macho.commands.SegmentNames;
import ghidra.app.util.bin.format.macho.prelink.PrelinkMap;
import ghidra.app.util.importer.*;
import ghidra.app.util.importer.MessageLog;
import ghidra.app.util.importer.MessageLogContinuesFactory;
import ghidra.program.database.mem.FileBytes;
import ghidra.program.model.address.Address;
import ghidra.program.model.data.DataUtilities;
import ghidra.program.model.data.Pointer64DataType;
@@ -48,15 +50,15 @@ public class MachoPrelinkProgramBuilder extends MachoProgramBuilder {
*
* @param program The {@link Program} to build up.
* @param provider The {@link ByteProvider} that contains the Mach-O's bytes.
* @param fileBytes Where the Mach-O's bytes came from.
* @param prelinkList Parsed {@link PrelinkMap PRELINK} information.
* @param log The log.
* @param memoryConflictHandler How to handle memory conflicts that may occur.
* @param monitor A cancelable task monitor.
*/
protected MachoPrelinkProgramBuilder(Program program, ByteProvider provider,
List<PrelinkMap> prelinkList, MessageLog log,
MemoryConflictHandler memoryConflictHandler, TaskMonitor monitor) {
super(program, provider, log, memoryConflictHandler, monitor);
FileBytes fileBytes, List<PrelinkMap> prelinkList, MessageLog log,
TaskMonitor monitor) {
super(program, provider, fileBytes, log, monitor);
this.prelinkList = prelinkList;
}
@@ -65,17 +67,16 @@ public class MachoPrelinkProgramBuilder extends MachoProgramBuilder {
*
* @param program The {@link Program} to build up.
* @param provider The {@link ByteProvider} that contains the Mach-O's bytes.
* @param fileBytes Where the Mach-O's bytes came from.
* @param prelinkList Parsed {@link PrelinkMap PRELINK} information.
* @param log The log.
* @param memoryConflictHandler How to handle memory conflicts that may occur.
* @param monitor A cancelable task monitor.
* @throws Exception if a problem occurs.
*/
public static void buildProgram(Program program, ByteProvider provider,
List<PrelinkMap> prelinkList, MessageLog log,
MemoryConflictHandler memoryConflictHandler, TaskMonitor monitor) throws Exception {
public static void buildProgram(Program program, ByteProvider provider, FileBytes fileBytes,
List<PrelinkMap> prelinkList, MessageLog log, TaskMonitor monitor) throws Exception {
MachoPrelinkProgramBuilder machoPrelinkProgramBuilder = new MachoPrelinkProgramBuilder(
program, provider, prelinkList, log, memoryConflictHandler, monitor);
program, provider, fileBytes, prelinkList, log, monitor);
machoPrelinkProgramBuilder.build();
}
@@ -19,7 +19,7 @@ import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import ghidra.app.util.MemoryBlockUtil;
import ghidra.app.util.MemoryBlockUtils;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.bin.StructConverter;
import ghidra.app.util.bin.format.macho.*;
@@ -30,6 +30,7 @@ import ghidra.app.util.bin.format.objectiveC.ObjectiveC1_Constants;
import ghidra.app.util.importer.*;
import ghidra.framework.options.Options;
import ghidra.program.database.function.OverlappingFunctionException;
import ghidra.program.database.mem.FileBytes;
import ghidra.program.model.address.*;
import ghidra.program.model.data.*;
import ghidra.program.model.lang.Processor;
@@ -55,32 +56,32 @@ public class MachoProgramBuilder {
protected Program program;
protected ByteProvider provider;
protected FileBytes fileBytes;
protected MessageLog log;
protected TaskMonitor monitor;
protected Memory memory;
protected Listing listing;
protected AddressSpace space;
protected MemoryBlockUtil mbu;
/**
* Creates a new {@link MachoProgramBuilder} based on the given information.
*
* @param program The {@link Program} to build up.
* @param provider The {@link ByteProvider} that contains the Mach-O's bytes.
* @param fileBytes Where the Mach-O's bytes came from.
* @param log The log.
* @param memoryConflictHandler How to handle memory conflicts that may occur.
* @param monitor A cancelable task monitor.
*/
protected MachoProgramBuilder(Program program, ByteProvider provider, MessageLog log,
MemoryConflictHandler memoryConflictHandler, TaskMonitor monitor) {
protected MachoProgramBuilder(Program program, ByteProvider provider, FileBytes fileBytes,
MessageLog log, TaskMonitor monitor) {
this.program = program;
this.provider = provider;
this.fileBytes = fileBytes;
this.log = log;
this.monitor = monitor;
this.memory = program.getMemory();
this.listing = program.getListing();
this.space = program.getAddressFactory().getDefaultAddressSpace();
this.mbu = new MemoryBlockUtil(program, memoryConflictHandler);
}
/**
@@ -88,15 +89,17 @@ public class MachoProgramBuilder {
*
* @param program The {@link Program} to build up.
* @param provider The {@link ByteProvider} that contains the Mach-O's bytes.
* @param fileBytes Where the Mach-O's bytes came from.
* @param log The log.
* @param memoryConflictHandler How to handle memory conflicts that may occur.
* @param monitor A cancelable task monitor.
* @throws Exception if a problem occurs.
*/
public static void buildProgram(Program program, ByteProvider provider, MessageLog log,
MemoryConflictHandler memoryConflictHandler, TaskMonitor monitor) throws Exception {
public static void buildProgram(Program program, ByteProvider provider, FileBytes fileBytes,
MessageLog log, MemoryConflictHandler memoryConflictHandler, TaskMonitor monitor)
throws Exception {
MachoProgramBuilder machoProgramBuilder =
new MachoProgramBuilder(program, provider, log, memoryConflictHandler, monitor);
new MachoProgramBuilder(program, provider, fileBytes, log, monitor);
machoProgramBuilder.build();
}
@@ -115,33 +118,25 @@ public class MachoProgramBuilder {
}
monitor.setCancelEnabled(true);
try {
setImageBase();
processEntryPoint();
processMemoryBlocks(machoHeader, provider.getName(), true, true);
processUnsupportedLoadCommands();
processSymbolTables();
processIndirectSymbols();
setRelocatableProperty();
processLibraries();
processProgramDescription();
renameObjMsgSendRtpSymbol();
processUndefinedSymbols();
processAbsoluteSymbols();
processDyldInfo();
markupHeaders(machoHeader, headerAddr);
markupSections();
processProgramVars();
loadSectionRelocations();
loadExternalRelocations();
loadLocalRelocations();
}
finally {
if (mbu != null) {
mbu.dispose();
mbu = null;
}
}
setImageBase();
processEntryPoint();
processMemoryBlocks(machoHeader, provider.getName(), true, true);
processUnsupportedLoadCommands();
processSymbolTables();
processIndirectSymbols();
setRelocatableProperty();
processLibraries();
processProgramDescription();
renameObjMsgSendRtpSymbol();
processUndefinedSymbols();
processAbsoluteSymbols();
processDyldInfo();
markupHeaders(machoHeader, headerAddr);
markupSections();
processProgramVars();
loadSectionRelocations();
loadExternalRelocations();
loadLocalRelocations();
}
private void setImageBase() throws Exception {
@@ -219,12 +214,8 @@ public class MachoProgramBuilder {
return;
}
// Create memory blocks for segments. Create them in reverse order so the splitting
// is more efficient.
List<SegmentCommand> segments = header.getAllSegments();
segments.sort((SegmentCommand a, SegmentCommand b) -> Long.compare(b.getVMaddress(),
a.getVMaddress()));
for (SegmentCommand segment : segments) {
// Create memory blocks for segments.
for (SegmentCommand segment : header.getAllSegments()) {
if (monitor.isCancelled()) {
break;
}
@@ -256,12 +247,9 @@ public class MachoProgramBuilder {
}
// Create memory blocks for sections. They will be in the segments we just created, so the
// segment blocks will be split and possibly replaced. Create them in reverse order so
// the splitting is more efficient.
// segment blocks will be split and possibly replaced.
if (processSections) {
List<Section> sections = header.getAllSections();
sections.sort((Section a, Section b) -> Long.compare(b.getAddress(), a.getAddress()));
for (Section section : sections) {
for (Section section : header.getAllSections()) {
if (monitor.isCancelled()) {
break;
}
@@ -331,12 +319,12 @@ public class MachoProgramBuilder {
if (intersectingBlocks.isEmpty()) {
if (zeroFill) {
// Treat zero-fill blocks as uninitialized to save space
return mbu.createUninitializedBlock(false, name, start, dataLength, comment, source,
r, w, x);
return MemoryBlockUtils.createUninitializedBlock(program, false, name, start,
dataLength, comment, source, r, w, x, log);
}
return mbu.createInitializedBlock(name, start, provider.getInputStream(dataOffset),
dataLength, comment, source, r, w, x, monitor);
return MemoryBlockUtils.createInitializedBlock(program, false, name, start, fileBytes,
dataOffset, dataLength, comment, source, r, w, x, log);
}
// Split the starting block (if necessary). Splitting is not necessary if the start of our
@@ -363,9 +351,7 @@ public class MachoProgramBuilder {
for (MemoryBlock block : memory.getBlocks()) {
if (range.intersects(block.getStart(), block.getEnd())) {
block.setName(name);
block.setRead(r);
block.setWrite(w);
block.setExecute(x);
block.setPermissions(r, w, x);
block.setSourceName(source);
block.setComment(comment);
}
@@ -23,6 +23,7 @@ import org.apache.commons.collections4.BidiMap;
import org.jdom.JDOMException;
import generic.continues.RethrowContinuesFactory;
import ghidra.app.util.MemoryBlockUtils;
import ghidra.app.util.bin.*;
import ghidra.app.util.bin.format.macho.*;
import ghidra.app.util.bin.format.macho.commands.*;
@@ -37,6 +38,7 @@ import ghidra.formats.gfilesystem.factory.GFileSystemBaseFactory;
import ghidra.framework.store.local.LocalFileSystem;
import ghidra.macosx.MacosxLanguageHelper;
import ghidra.program.database.ProgramDB;
import ghidra.program.database.mem.FileBytes;
import ghidra.program.model.address.Address;
import ghidra.program.model.lang.LanguageCompilerSpecPair;
import ghidra.program.model.lang.LanguageService;
@@ -185,9 +187,12 @@ public class PrelinkFileSystem extends GFileSystemBase implements GFileSystemPro
int id = program.startTransaction(getName());
boolean success = false;
try {
MachoProgramBuilder.buildProgram(program,
new ByteProviderWrapper(provider, offset, provider.length() - offset),
new MessageLog(), MemoryConflictHandler.NEVER_OVERWRITE, monitor);
FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider, offset,
provider.length() - offset);
ByteProvider providerWrapper =
new ByteProviderWrapper(provider, offset, provider.length() - offset);
MachoProgramBuilder.buildProgram(program, providerWrapper, fileBytes, new MessageLog(),
MemoryConflictHandler.NEVER_OVERWRITE, monitor);
program.setExecutableFormat(MachoLoader.MACH_O_NAME);
program.setExecutablePath(file.getPath());