diff --git a/Ghidra/Debug/Debugger-rmi-trace/src/main/java/ghidra/app/plugin/core/debug/service/tracermi/TraceRmiHandler.java b/Ghidra/Debug/Debugger-rmi-trace/src/main/java/ghidra/app/plugin/core/debug/service/tracermi/TraceRmiHandler.java index 81122cd771..4c4ad6a843 100644 --- a/Ghidra/Debug/Debugger-rmi-trace/src/main/java/ghidra/app/plugin/core/debug/service/tracermi/TraceRmiHandler.java +++ b/Ghidra/Debug/Debugger-rmi-trace/src/main/java/ghidra/app/plugin/core/debug/service/tracermi/TraceRmiHandler.java @@ -56,6 +56,7 @@ import ghidra.trace.model.Lifespan; import ghidra.trace.model.Trace; import ghidra.trace.model.guest.TracePlatform; import ghidra.trace.model.memory.*; +import ghidra.trace.model.memory.TraceMemoryOperations.StatePredicate; import ghidra.trace.model.target.*; import ghidra.trace.model.target.TraceObject.ConflictResolution; import ghidra.trace.model.target.path.*; @@ -926,11 +927,10 @@ public class TraceRmiHandler extends AbstractTraceRmiConnection { TraceMemoryManager memoryManager = open.trace.getMemoryManager(); AddressSetView readOnly = memoryManager.getRegionsAddressSetWith(snap, r -> !r.isWrite(snap)); - AddressSetView everKnown = memoryManager.getAddressesWithState(Lifespan.since(snap), - s -> s == TraceMemoryState.KNOWN); + AddressSetView everKnown = + memoryManager.getAddressesWithState(Lifespan.since(snap), StatePredicate.IS_KNOWN); AddressSetView roEverKnown = new IntersectionAddressSetView(readOnly, everKnown); - AddressSetView known = - memoryManager.getAddressesWithState(snap, s -> s == TraceMemoryState.KNOWN); + AddressSetView known = memoryManager.getAddressesWithState(snap, StatePredicate.IS_KNOWN); AddressSetView disassemblable = new AddressSet(new UnionAddressSetView(known, roEverKnown)); Address start = open.toAddress(req.getStart(), true); diff --git a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java index e69bd9883e..e3f04a5a2a 100644 --- a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java +++ b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java @@ -39,6 +39,7 @@ import ghidra.trace.model.*; import ghidra.trace.model.guest.TraceGuestPlatform; import ghidra.trace.model.guest.TracePlatform; import ghidra.trace.model.memory.*; +import ghidra.trace.model.memory.TraceMemoryOperations.StatePredicate; import ghidra.trace.model.program.TraceProgramView; import ghidra.util.IntersectionAddressSetView; import ghidra.util.UnionAddressSetView; @@ -159,11 +160,10 @@ public class DebuggerDisassemblerPlugin extends Plugin implements PopupActionPro TraceMemoryManager memoryManager = trace.getMemoryManager(); AddressSetView readOnly = memoryManager.getRegionsAddressSetWith(ks, r -> !r.isWrite(ks)); - AddressSetView everKnown = memoryManager.getAddressesWithState(Lifespan.since(ks), - s -> s == TraceMemoryState.KNOWN); + AddressSetView everKnown = + memoryManager.getAddressesWithState(Lifespan.since(ks), StatePredicate.IS_KNOWN); AddressSetView roEverKnown = new IntersectionAddressSetView(readOnly, everKnown); - AddressSetView known = - memoryManager.getAddressesWithState(ks, s -> s == TraceMemoryState.KNOWN); + AddressSetView known = memoryManager.getAddressesWithState(ks, StatePredicate.IS_KNOWN); AddressSetView disassemblable = new UnionAddressSetView(known, roEverKnown); return disassemblable; } diff --git a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/action/BasicAutoReadMemorySpec.java b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/action/BasicAutoReadMemorySpec.java index 13211c35c5..4deced1017 100644 --- a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/action/BasicAutoReadMemorySpec.java +++ b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/action/BasicAutoReadMemorySpec.java @@ -36,6 +36,7 @@ import ghidra.program.model.address.*; import ghidra.program.model.mem.MemoryAccessException; import ghidra.trace.model.*; import ghidra.trace.model.memory.*; +import ghidra.trace.model.memory.TraceMemoryOperations.StatePredicate; import ghidra.util.MathUtilities; import ghidra.util.task.TaskMonitor; @@ -65,7 +66,7 @@ public enum BasicAutoReadMemorySpec implements AutoReadMemorySpec { Target target = coordinates.getTarget(); TraceMemoryManager mm = coordinates.getTrace().getMemoryManager(); AddressSetView alreadyKnown = mm.getAddressesWithState(coordinates.getSnap(), visible, - s -> s == TraceMemoryState.KNOWN || s == TraceMemoryState.ERROR); + StatePredicate.IS_KNOWN_OR_ERROR); AddressSet toRead = visible.subtract(alreadyKnown); if (toRead.isEmpty()) { @@ -91,8 +92,8 @@ public enum BasicAutoReadMemorySpec implements AutoReadMemorySpec { Target target = coordinates.getTarget(); TraceMemoryManager mm = coordinates.getTrace().getMemoryManager(); long snap = coordinates.getSnap(); - AddressSetView alreadyKnown = mm.getAddressesWithState(snap, visible, - s -> s == TraceMemoryState.KNOWN || s == TraceMemoryState.ERROR); + AddressSetView alreadyKnown = + mm.getAddressesWithState(snap, visible, StatePredicate.IS_KNOWN_OR_ERROR); AddressSet toRead = visible.subtract(alreadyKnown); if (toRead.isEmpty()) { @@ -171,8 +172,7 @@ public enum BasicAutoReadMemorySpec implements AutoReadMemorySpec { AddressSet toRead = new AddressSet(quantize(12, visible)); for (Lifespan span : coordinates.getView().getViewport().getOrderedSpans()) { AddressSetView alreadyKnown = - mm.getAddressesWithState(span.lmin(), visible, - s -> s == TraceMemoryState.KNOWN); + mm.getAddressesWithState(span.lmin(), visible, StatePredicate.IS_KNOWN); toRead.delete(alreadyKnown); if (span.lmax() != span.lmin() || toRead.isEmpty()) { break; diff --git a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/copying/DebuggerCopyPlan.java b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/copying/DebuggerCopyPlan.java index 2d08d29041..d9123498c0 100644 --- a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/copying/DebuggerCopyPlan.java +++ b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/copying/DebuggerCopyPlan.java @@ -31,6 +31,7 @@ import ghidra.trace.model.Lifespan; import ghidra.trace.model.breakpoint.TraceBreakpointLocation; import ghidra.trace.model.memory.TraceMemoryManager; import ghidra.trace.model.memory.TraceMemoryState; +import ghidra.trace.model.memory.TraceMemoryOperations.StatePredicate; import ghidra.trace.model.program.TraceProgramView; import ghidra.util.exception.InvalidInputException; import ghidra.util.task.TaskMonitor; @@ -91,9 +92,9 @@ public class DebuggerCopyPlan { AddressSet rngAsSet = new AddressSet(fromRange); TraceMemoryManager mm = from.getTrace().getMemoryManager(); AddressSetView knownSet = mm.getAddressesWithState(from.getSnap(), rngAsSet, - s -> s == TraceMemoryState.KNOWN); + StatePredicate.IS_KNOWN); AddressSetView errorSet = mm.getAddressesWithState(from.getSnap(), rngAsSet, - s -> s == TraceMemoryState.ERROR); + StatePredicate.IS_ERROR); AddressSetView staleSet = rngAsSet.subtract(knownSet).subtract(errorSet); setShifted(map, fromRange.getMinAddress(), intoAddress, errorSet, DebuggerResources.COLOR_BACKGROUND_ERROR.getRGB()); diff --git a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/diff/DebuggerTraceViewDiffPlugin.java b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/diff/DebuggerTraceViewDiffPlugin.java index 7a2af04404..bb0213e3bd 100644 --- a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/diff/DebuggerTraceViewDiffPlugin.java +++ b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/diff/DebuggerTraceViewDiffPlugin.java @@ -55,6 +55,7 @@ import ghidra.program.util.ProgramLocation; import ghidra.trace.model.Trace; import ghidra.trace.model.memory.TraceMemoryManager; import ghidra.trace.model.memory.TraceMemoryState; +import ghidra.trace.model.memory.TraceMemoryOperations.StatePredicate; import ghidra.trace.model.program.TraceProgramView; import ghidra.trace.model.time.schedule.TraceSchedule; import ghidra.util.Msg; @@ -539,8 +540,8 @@ public class DebuggerTraceViewDiffPlugin extends AbstractDebuggerPlugin { TraceMemoryManager mm = trace.getMemoryManager(); - AddressSetView known1 = mm.getAddressesWithState(snap1, s -> s == TraceMemoryState.KNOWN); - AddressSetView known2 = mm.getAddressesWithState(snap2, s -> s == TraceMemoryState.KNOWN); + AddressSetView known1 = mm.getAddressesWithState(snap1, StatePredicate.IS_KNOWN); + AddressSetView known2 = mm.getAddressesWithState(snap2, StatePredicate.IS_KNOWN); //AddressSet knownEither = known1.union(known2); AddressSet knownBoth = known1.intersect(known2); // Will need byte-by-byte examination diff --git a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/register/DebuggerRegistersProvider.java b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/register/DebuggerRegistersProvider.java index cc54e35d71..6c3b5ecf0b 100644 --- a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/register/DebuggerRegistersProvider.java +++ b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/gui/register/DebuggerRegistersProvider.java @@ -69,6 +69,7 @@ import ghidra.trace.model.*; import ghidra.trace.model.guest.TracePlatform; import ghidra.trace.model.listing.*; import ghidra.trace.model.memory.*; +import ghidra.trace.model.memory.TraceMemoryOperations.StatePredicate; import ghidra.trace.model.program.TraceProgramView; import ghidra.trace.model.thread.TraceThread; import ghidra.trace.util.TraceEvents; @@ -895,7 +896,8 @@ public class DebuggerRegistersProvider extends ComponentProviderAdapter } BigInteger getRegisterValue(Register register) { - TraceMemorySpace regs = getRegisterMemorySpace(register.getAddressSpace(), false); + Address hostReg = current.getPlatform().mapGuestToHost(register.getAddress()); + TraceMemorySpace regs = getRegisterMemorySpace(hostReg.getAddressSpace(), false); if (regs == null) { return BigInteger.ZERO; } @@ -971,7 +973,8 @@ public class DebuggerRegistersProvider extends ComponentProviderAdapter } TraceData getRegisterData(Register register) { - TraceCodeSpace space = getRegisterCodeSpace(register.getAddressSpace(), false); + Address hostReg = current.getPlatform().mapGuestToHost(register.getAddress()); + TraceCodeSpace space = getRegisterCodeSpace(hostReg.getAddressSpace(), false); if (space == null) { return null; } @@ -1058,8 +1061,8 @@ public class DebuggerRegistersProvider extends ComponentProviderAdapter AddressSetView guestRegs = platform.getLanguage().getRegisterAddresses(); AddressSetView hostRegs = platform.mapGuestToHost(guestRegs); AddressSetView viewKnownMem = view.getViewport() - .unionedAddresses(snap -> mem.getAddressesWithState(snap, hostRegs, - state -> state == TraceMemoryState.KNOWN)); + .unionedAddresses( + snap -> mem.getAddressesWithState(snap, hostRegs, StatePredicate.IS_KNOWN)); AddressSpace regSpace = platform.getAddressFactory().getRegisterSpace(); if (regSpace == null) { viewKnown = new AddressSet(viewKnownMem); @@ -1073,8 +1076,8 @@ public class DebuggerRegistersProvider extends ComponentProviderAdapter AddressSetView overlayRegs = TraceRegisterUtils.getOverlaySet(regs.getAddressSpace(), hostRegs); AddressSetView viewKnownRegs = view.getViewport() - .unionedAddresses(snap -> regs.getAddressesWithState(snap, overlayRegs, - state -> state == TraceMemoryState.KNOWN)); + .unionedAddresses( + snap -> regs.getAddressesWithState(snap, overlayRegs, StatePredicate.IS_KNOWN)); viewKnown = viewKnownRegs.union(viewKnownMem); } @@ -1082,8 +1085,9 @@ public class DebuggerRegistersProvider extends ComponentProviderAdapter if (viewKnown == null) { return false; } - TraceMemorySpace regs = getRegisterMemorySpace(current, register.getAddressSpace(), false); - if (regs == null && register.getAddressSpace().isRegisterSpace()) { + Address hostReg = current.getPlatform().mapGuestToHost(register.getAddress()); + TraceMemorySpace regs = getRegisterMemorySpace(current, hostReg.getAddressSpace(), false); + if (regs == null && hostReg.getAddressSpace().isRegisterSpace()) { return false; } AddressRange range = current.getPlatform() @@ -1102,10 +1106,11 @@ public class DebuggerRegistersProvider extends ComponentProviderAdapter if (!isRegisterKnown(register)) { return false; } + Address hostReg = current.getPlatform().mapGuestToHost(register.getAddress()); TraceMemorySpace curSpace = - getRegisterMemorySpace(current, register.getAddressSpace(), false); + getRegisterMemorySpace(current, hostReg.getAddressSpace(), false); TraceMemorySpace prevSpace = - getRegisterMemorySpace(previous, register.getAddressSpace(), false); + getRegisterMemorySpace(previous, hostReg.getAddressSpace(), false); if (prevSpace == null) { return false; } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/pcode/exec/trace/TraceEmulationIntegration.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/pcode/exec/trace/TraceEmulationIntegration.java index 7c9c4dc53a..484f6c4da4 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/pcode/exec/trace/TraceEmulationIntegration.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/pcode/exec/trace/TraceEmulationIntegration.java @@ -385,10 +385,13 @@ public enum TraceEmulationIntegration { knownButUninit.getMinAddress(), knownButUninit.getMaxAddress()); ByteBuffer buf = ByteBuffer.allocate((int) knownBound.getLength()); - acc.getBytes(knownBound.getMinAddress(), buf); + Address knownMin = knownBound.getMinAddress(); + acc.getBytes(knownMin, buf); for (AddressRange range : knownButUninit) { - piece.setVarInternal(range.getAddressSpace(), range.getMinAddress().getOffset(), - (int) range.getLength(), buf.array()); + byte[] sub = new byte[(int) range.getLength()]; + Address rngMin = range.getMinAddress(); + buf.get((int) rngMin.subtract(knownMin), sub); + piece.setVarInternal(range.getAddressSpace(), rngMin.getOffset(), sub.length, sub); remains.delete(range); } return remains; diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/pcode/exec/trace/data/AbstractPcodeTraceDataAccess.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/pcode/exec/trace/data/AbstractPcodeTraceDataAccess.java index a0cb53020a..91604854f5 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/pcode/exec/trace/data/AbstractPcodeTraceDataAccess.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/pcode/exec/trace/data/AbstractPcodeTraceDataAccess.java @@ -23,6 +23,7 @@ import ghidra.trace.model.Lifespan; import ghidra.trace.model.TraceTimeViewport; import ghidra.trace.model.guest.TracePlatform; import ghidra.trace.model.memory.*; +import ghidra.trace.model.memory.TraceMemoryOperations.StatePredicate; import ghidra.trace.util.TraceRegisterUtils; /** @@ -130,8 +131,7 @@ public abstract class AbstractPcodeTraceDataAccess implements InternalPcodeTrace AddressSet hostSet = new AddressSet(toOverlay(hostRange)); for (long snap : viewport.getOrderedSnaps()) { - hostSet.delete( - ops.getAddressesWithState(snap, hostSet, s -> s == TraceMemoryState.KNOWN)); + hostSet.delete(ops.getAddressesWithState(snap, hostSet, StatePredicate.IS_KNOWN)); } return hostSet.isEmpty() ? TraceMemoryState.KNOWN : TraceMemoryState.UNKNOWN; } @@ -147,14 +147,14 @@ public abstract class AbstractPcodeTraceDataAccess implements InternalPcodeTrace AddressSet hostKnown = new AddressSet(); if (useFullSpans) { for (Lifespan span : viewport.getOrderedSpans()) { - hostKnown.add(ops.getAddressesWithState(span, hostView, - st -> st != null && st != TraceMemoryState.UNKNOWN)); + hostKnown.add( + ops.getAddressesWithState(span, hostView, StatePredicate.IS_KNOWN_OR_ERROR)); } } else { for (long snap : viewport.getOrderedSnaps()) { - hostKnown.add(ops.getAddressesWithState(snap, hostView, - st -> st != null && st != TraceMemoryState.UNKNOWN)); + hostKnown.add( + ops.getAddressesWithState(snap, hostView, StatePredicate.IS_KNOWN_OR_ERROR)); } } AddressSetView hostResult = diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/DBTrace.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/DBTrace.java index ac062d3d81..c41c5528b1 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/DBTrace.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/DBTrace.java @@ -74,6 +74,9 @@ import ghidra.util.task.TaskMonitor; // Applies to creation, and to setting end snap // Also to deleting a thread altogether. public class DBTrace extends DBCachedDomainObjectAdapter implements Trace, TraceChangeManager { + + public final static int CHUNK_SIZE = 4096; + protected static final String TRACE_INFO = "Trace Information"; protected static final String NAME = "Name"; protected static final String DATE_CREATED = "Date Created"; diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/context/DBTraceRegisterContextSpace.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/context/DBTraceRegisterContextSpace.java index e5580944b5..7e3bc30242 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/context/DBTraceRegisterContextSpace.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/context/DBTraceRegisterContextSpace.java @@ -84,8 +84,10 @@ public class DBTraceRegisterContextSpace implements TraceRegisterContextSpace, D protected final AddressRange all; protected final DBCachedObjectStore registerStore; - protected final Map, DBTraceAddressSnapRangePropertyMapSpace> registerValueMaps = - new HashMap<>(); + protected final Map, + DBTraceAddressSnapRangePropertyMapSpace> registerValueMaps = + new HashMap<>(); public DBTraceRegisterContextSpace(DBTraceRegisterContextManager manager, DBHandle dbh, AddressSpace space, DBTraceSpaceEntry ent) throws VersionException, IOException { @@ -133,8 +135,9 @@ public class DBTraceRegisterContextSpace implements TraceRegisterContextSpace, D language.getLanguageID().getIdAsString() + "_" + register.getName(); } - protected DBTraceAddressSnapRangePropertyMapSpace createRegisterValueMap( - Pair lr) throws VersionException { + protected DBTraceAddressSnapRangePropertyMapSpace + createRegisterValueMap( + Pair lr) throws VersionException { String name = tableName(lr.getLeft(), lr.getRight()); try { return new DBTraceAddressSnapRangePropertyMapSpace<>(name, trace, @@ -147,8 +150,9 @@ public class DBTraceRegisterContextSpace implements TraceRegisterContextSpace, D } } - protected DBTraceAddressSnapRangePropertyMapSpace getRegisterValueMap( - Language language, Register register, boolean createIfAbsent) { + protected DBTraceAddressSnapRangePropertyMapSpace + getRegisterValueMap( + Language language, Register register, boolean createIfAbsent) { ImmutablePair pair = new ImmutablePair<>(language, register); DBTraceGuestLanguage guest = manager.languageManager.getLanguageByLanguage(language); if (createIfAbsent) { @@ -209,11 +213,15 @@ public class DBTraceRegisterContextSpace implements TraceRegisterContextSpace, D protected void doRemove(DBTraceAddressSnapRangePropertyMapSpace valueMap, TraceAddressSnapRange range) { Map toPutBack = new HashMap<>(); + List> toRemove = new ArrayList<>(); for (Entry entry : valueMap.reduce( TraceAddressSnapRangeQuery.intersecting(range)).entries()) { for (TraceAddressSnapRange diff : doSubtract(entry.getKey(), range)) { toPutBack.put(diff, entry.getValue()); } + toRemove.add(entry); + } + for (Entry entry : toRemove) { valueMap.remove(entry); } for (Entry entry : toPutBack.entrySet()) { @@ -430,8 +438,9 @@ public class DBTraceRegisterContextSpace implements TraceRegisterContextSpace, D @Override public void clear(Lifespan span, AddressRange range) { try (LockHold hold = LockHold.lock(lock.writeLock())) { - for (DBTraceAddressSnapRangePropertyMapSpace valueMap : registerValueMaps - .values()) { + for (DBTraceAddressSnapRangePropertyMapSpace valueMap : registerValueMaps + .values()) { for (Entry entry : valueMap.reduce( TraceAddressSnapRangeQuery.intersecting(range, span)).entries()) { DBTraceRegisterContextEntry record = @@ -448,8 +457,9 @@ public class DBTraceRegisterContextSpace implements TraceRegisterContextSpace, D try (LockHold hold = LockHold.lock(lock.writeLock())) { registerStore.invalidateCache(); loadRegisterValueMaps(); - for (DBTraceAddressSnapRangePropertyMapSpace map : registerValueMaps - .values()) { + for (DBTraceAddressSnapRangePropertyMapSpace map : registerValueMaps + .values()) { map.invalidateCache(); } } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/data/DBTraceDataSettingsOperations.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/data/DBTraceDataSettingsOperations.java index 8797ad8b38..f347eca4ee 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/data/DBTraceDataSettingsOperations.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/data/DBTraceDataSettingsOperations.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -62,8 +62,9 @@ public interface DBTraceDataSettingsOperations } default void doMakeWay(Lifespan span, Address address, String name) { - for (DBTraceSettingsEntry entry : reduce(TraceAddressSnapRangeQuery.intersecting( - new AddressRangeImpl(address, address), span)).values()) { + for (DBTraceSettingsEntry entry : List.copyOf(reduce( + TraceAddressSnapRangeQuery.intersecting(new AddressRangeImpl(address, address), span)) + .values())) { if (name == null || name.equals(entry.name)) { makeWay(entry, span); } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/AbstractBaseDBTraceDefinedUnitsView.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/AbstractBaseDBTraceDefinedUnitsView.java index ecc867d8f9..b96826fc0c 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/AbstractBaseDBTraceDefinedUnitsView.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/AbstractBaseDBTraceDefinedUnitsView.java @@ -20,9 +20,8 @@ import java.util.Map.Entry; import ghidra.program.model.address.*; import ghidra.program.model.util.CodeUnitInsertionException; -import ghidra.trace.database.DBTraceCacheForContainingQueries; +import ghidra.trace.database.*; import ghidra.trace.database.DBTraceCacheForContainingQueries.GetKey; -import ghidra.trace.database.DBTraceCacheForSequenceQueries; import ghidra.trace.database.context.DBTraceRegisterContextSpace; import ghidra.trace.database.map.DBTraceAddressSnapRangePropertyMapAddressSetView; import ghidra.trace.database.map.DBTraceAddressSnapRangePropertyMapSpace; @@ -50,6 +49,7 @@ import ghidra.util.task.TaskMonitor; public abstract class AbstractBaseDBTraceDefinedUnitsView> extends AbstractSingleDBTraceCodeUnitsView { + protected final static int CHUNK_SIZE = DBTrace.CHUNK_SIZE; protected final static int CACHE_MAX_REGIONS = 1000; protected final static int CACHE_ADDRESS_BREADTH = 10000; protected final static int CACHE_MAX_POINTS = 10000; @@ -306,27 +306,35 @@ public abstract class AbstractBaseDBTraceDefinedUnitsView chunk = submap.values().stream().limit(CHUNK_SIZE).toList(); + for (T unit : chunk) { + monitor.checkCancelled(); + if (unit.getStartSnap() < startSnap) { + Lifespan oldSpan = unit.getLifespan(); + if (clearContext) { + clearContext(Lifespan.span(span.lmin(), oldSpan.lmax()), + unit.getRange()); + } + unit.setEndSnap(startSnap - 1); + } + else { + if (clearContext) { + clearContext(unit.getLifespan(), unit.getRange()); + } + unit.delete(); } - unit.setEndSnap(startSnap - 1); } - else { - if (clearContext) { - clearContext(unit.getLifespan(), unit.getRange()); - } - unit.delete(); + if (chunk.size() < CHUNK_SIZE) { + break; } } } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/DBTraceCodeSpace.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/DBTraceCodeSpace.java index 348bbd1773..7ae80025e8 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/DBTraceCodeSpace.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/DBTraceCodeSpace.java @@ -16,8 +16,7 @@ package ghidra.trace.database.listing; import java.io.IOException; -import java.util.HashSet; -import java.util.Set; +import java.util.*; import java.util.concurrent.locks.ReadWriteLock; import db.DBHandle; @@ -55,6 +54,8 @@ import ghidra.util.task.TaskMonitor; * {@link TraceCodeManager#getCodeRegisterSpace(TraceThread, int, boolean)}. */ public class DBTraceCodeSpace implements TraceCodeSpace, DBTraceSpaceBased { + protected final static int CHUNK_SIZE = DBTrace.CHUNK_SIZE; + protected final DBTraceCodeManager manager; protected final DBHandle dbh; protected final AddressSpace space; @@ -65,7 +66,8 @@ public class DBTraceCodeSpace implements TraceCodeSpace, DBTraceSpaceBased { protected final DBTraceReferenceManager referenceManager; protected final AddressRange all; - protected final DBTraceAddressSnapRangePropertyMapSpace instructionMapSpace; + protected final DBTraceAddressSnapRangePropertyMapSpace instructionMapSpace; protected final DBTraceAddressSnapRangePropertyMapSpace dataMapSpace; // NOTE: All combinations except () and (INSTRUCTIONS,UNDEFINED) @@ -178,28 +180,42 @@ public class DBTraceCodeSpace implements TraceCodeSpace, DBTraceSpaceBased { definedData.invalidateCache(); undefinedData.invalidateCache(); - for (DBTraceInstruction instruction : instructionMapSpace.reduce( - TraceAddressSnapRangeQuery.intersecting(range, span)).values()) { - monitor.checkCancelled(); - monitor.incrementProgress(1); - if (instruction.platform != guest) { - continue; + TraceAddressSnapRangeQuery query = TraceAddressSnapRangeQuery.intersecting(range, span); + var instructionSubmap = instructionMapSpace.reduce(query); + while (true) { + List chunk = + instructionSubmap.values().stream().limit(CHUNK_SIZE).toList(); + for (DBTraceInstruction instruction : chunk) { + monitor.checkCancelled(); + monitor.incrementProgress(1); + if (instruction.platform != guest) { + continue; + } + instructionMapSpace.deleteData(instruction); + instructions.unitRemoved(instruction); + } + if (chunk.size() < CHUNK_SIZE) { + break; } - instructionMapSpace.deleteData(instruction); - instructions.unitRemoved(instruction); } + monitor.setMessage("Clearing data"); monitor.setMaximum(dataMapSpace.size()); // This is OK - for (DBTraceData dataUnit : dataMapSpace.reduce( - TraceAddressSnapRangeQuery.intersecting(range, span)).values()) { - monitor.checkCancelled(); - monitor.incrementProgress(1); - if (dataUnit.platform != guest) { - continue; + var dataSubmap = dataMapSpace.reduce(query); + while (true) { + List chunk = dataSubmap.values().stream().limit(CHUNK_SIZE).toList(); + for (DBTraceData dataUnit : chunk) { + monitor.checkCancelled(); + monitor.incrementProgress(1); + if (dataUnit.platform != guest) { + continue; + } + dataMapSpace.deleteData(dataUnit); + definedData.unitRemoved(dataUnit); + } + if (chunk.size() < CHUNK_SIZE) { + break; } - // TODO: I don't yet have guest-language data units. - dataMapSpace.deleteData(dataUnit); - definedData.unitRemoved(dataUnit); } } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/DBTraceCommentAdapter.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/DBTraceCommentAdapter.java index ee373953b0..416d7bf65a 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/DBTraceCommentAdapter.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/listing/DBTraceCommentAdapter.java @@ -16,6 +16,7 @@ package ghidra.trace.database.listing; import java.io.IOException; +import java.util.List; import java.util.concurrent.locks.ReadWriteLock; import org.apache.commons.lang3.StringUtils; @@ -138,8 +139,8 @@ public class DBTraceCommentAdapter } String oldValue = null; try (LockHold hold = LockHold.lock(lock.writeLock())) { - for (DBTraceCommentEntry entry : reduce(TraceAddressSnapRangeQuery - .intersecting(new AddressRangeImpl(address, address), lifespan)).values()) { + for (DBTraceCommentEntry entry : List.copyOf(reduce(TraceAddressSnapRangeQuery + .intersecting(new AddressRangeImpl(address, address), lifespan)).values())) { if (entry.type == commentType.ordinal()) { if (entry.getLifespan().contains(lifespan.lmin())) { oldValue = entry.comment; @@ -207,8 +208,8 @@ public class DBTraceCommentAdapter */ public void clearComments(Lifespan span, AddressRange range, CommentType commentType) { try (LockHold hold = LockHold.lock(lock.writeLock())) { - for (DBTraceCommentEntry entry : reduce( - TraceAddressSnapRangeQuery.intersecting(range, span)).values()) { + for (DBTraceCommentEntry entry : List.copyOf(reduce( + TraceAddressSnapRangeQuery.intersecting(range, span)).values())) { if (commentType == null || entry.type == commentType.ordinal()) { makeWay(entry, span); } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/map/AbstractDBTracePropertyMap.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/map/AbstractDBTracePropertyMap.java index f108a5603c..43610b11be 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/map/AbstractDBTracePropertyMap.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/map/AbstractDBTracePropertyMap.java @@ -21,6 +21,7 @@ import java.lang.reflect.InvocationTargetException; import java.util.Collection; import java.util.Map.Entry; import java.util.concurrent.locks.ReadWriteLock; +import java.util.function.BiConsumer; import db.*; import ghidra.framework.data.OpenMode; @@ -39,13 +40,17 @@ import ghidra.util.*; import ghidra.util.database.*; import ghidra.util.database.DBCachedObjectStoreFactory.AbstractDBFieldCodec; import ghidra.util.database.annot.*; +import ghidra.util.database.spatial.SpatialMap; import ghidra.util.exception.NotYetImplementedException; import ghidra.util.exception.VersionException; import ghidra.util.task.TaskMonitor; -public abstract class AbstractDBTracePropertyMap> +public abstract class AbstractDBTracePropertyMap> extends DBTraceAddressSnapRangePropertyMap implements TracePropertyMap { + protected static final int CHUNK_SIZE = DBTrace.CHUNK_SIZE; + public AbstractDBTracePropertyMap(String name, DBHandle dbh, OpenMode openMode, ReadWriteLock lock, TaskMonitor monitor, Language baseLanguage, DBTrace trace, DBTraceThreadManager threadManager, Class dataType, @@ -96,26 +101,36 @@ public abstract class AbstractDBTracePropertyMap> boolean + doClear(Lifespan span, SpatialMap sub, + BiConsumer, Lifespan> makeWay) { + boolean result = false; + while (true) { + var chunk = sub.entries().stream().limit(CHUNK_SIZE).toList(); + for (Entry entry : chunk) { + makeWay.accept(entry, span); + result = true; + } + if (chunk.size() < CHUNK_SIZE) { + break; + } + } + return result; + } + @Override public boolean clear(Lifespan span, AddressRange range) { try (LockHold hold = LockHold.lock(lock.writeLock())) { - boolean result = false; - for (Entry entry : reduce( - TraceAddressSnapRangeQuery.intersecting(range, span)).entries()) { - makeWay(entry, span); - result = true; - } - return result; + return doClear(span, reduce(TraceAddressSnapRangeQuery.intersecting(range, span)), + this::makeWay); } } @Override public T put(TraceAddressSnapRange shape, T value) { try (LockHold hold = LockHold.lock(lock.writeLock())) { - for (Entry entry : reduce( - TraceAddressSnapRangeQuery.intersecting(shape)).entries()) { - makeWay(entry, shape.getLifespan()); - } + doClear(shape.getLifespan(), reduce(TraceAddressSnapRangeQuery.intersecting(shape)), + this::makeWay); return super.put(shape, value); } } @@ -206,23 +221,16 @@ public abstract class AbstractDBTracePropertyMap entry : reduce( - TraceAddressSnapRangeQuery.intersecting(range, span)).entries()) { - makeWay(entry, span); - result = true; - } - return result; + return doClear(span, reduce(TraceAddressSnapRangeQuery.intersecting(range, span)), + this::makeWay); } } @Override public T put(TraceAddressSnapRange shape, T value) { try (LockHold hold = LockHold.lock(lock.writeLock())) { - for (Entry entry : reduce( - TraceAddressSnapRangeQuery.intersecting(shape)).entries()) { - makeWay(entry, shape.getLifespan()); - } + doClear(shape.getLifespan(), reduce(TraceAddressSnapRangeQuery.intersecting(shape)), + this::makeWay); return super.put(shape, value); } } @@ -324,8 +332,9 @@ public abstract class AbstractDBTracePropertyMap valueClass; @SuppressWarnings({ "rawtypes", "unchecked" }) - protected static Class> getEntryClass( - Class valueClass) { + protected static Class> + getEntryClass( + Class valueClass) { return (Class) DBTraceSaveablePropertyMapEntry.class; } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/map/DBTraceAddressSnapRangePropertyMapTree.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/map/DBTraceAddressSnapRangePropertyMapTree.java index 0411587e7f..8ab5dec911 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/map/DBTraceAddressSnapRangePropertyMapTree.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/map/DBTraceAddressSnapRangePropertyMapTree.java @@ -496,9 +496,28 @@ public class DBTraceAddressSnapRangePropertyMapTree> result = new ArrayList<>(); for (DBTraceMemorySpace space : spaces.values()) { - AddressRange rng = - new AddressRangeImpl(space.space.getMinAddress(), space.space.getMaxAddress()); - result.addAll( - space.stateMapSpace.reduce(TraceAddressSnapRangeQuery.enclosed(rng, between)) - .entries()); + result.addAll(space.stateMapSpace + .reduce(TraceAddressSnapRangeQuery.minWithin(between, space.space)) + .entries()); } return result; } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemorySpace.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemorySpace.java index c109191ad5..831a22f06f 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemorySpace.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemorySpace.java @@ -29,10 +29,10 @@ import ghidra.program.model.address.*; import ghidra.program.model.mem.MemBuffer; import ghidra.trace.database.DBTrace; import ghidra.trace.database.DBTraceTimeViewport; -import ghidra.trace.database.DBTraceUtils.AddressRangeMapSetter; import ghidra.trace.database.DBTraceUtils.OffsetSnap; import ghidra.trace.database.listing.DBTraceCodeSpace; -import ghidra.trace.database.map.*; +import ghidra.trace.database.map.DBTraceAddressSnapRangePropertyMapAddressSetView; +import ghidra.trace.database.map.DBTraceAddressSnapRangePropertyMapSpace; import ghidra.trace.database.map.DBTraceAddressSnapRangePropertyMapTree.Painter; import ghidra.trace.database.map.DBTraceAddressSnapRangePropertyMapTree.TraceAddressSnapRangeQuery; import ghidra.trace.database.space.AbstractDBTraceSpaceBasedManager.DBTraceSpaceEntry; @@ -61,9 +61,21 @@ public class DBTraceMemorySpace public static final int BLOCK_SIZE = 1 << BLOCK_SHIFT; public static final int BLOCK_MASK = -1 << BLOCK_SHIFT; public static final int DEPENDENT_COMPRESSED_SIZE_TOLERANCE = BLOCK_SIZE >>> 2; + public static final int BLOCK_CACHE_SIZE = 10; public static final int BLOCKS_PER_BUFFER = 256; // Must be a power of 2 and >= 8; + public static final int STATE_BLOCK_SHIFT = 8; + public static final int STATE_BLOCK_SIZE = 1 << STATE_BLOCK_SHIFT; + public static final int STATE_BLOCK_MASK = -1 << STATE_BLOCK_SHIFT; + public static final int STATE_CACHE_SIZE = 500; + + public static final int STATE_MR_CACHE_SIZE = 100; + + record AddressSetStateCacheKey(Lifespan span, long offset, StatePredicate predicate) {} + + record MostRecentStateCacheKey(long snap, Address address) {} + protected final DBTraceMemoryManager manager; protected final DBHandle dbh; protected final AddressSpace space; @@ -77,7 +89,12 @@ public class DBTraceMemorySpace protected final DBCachedObjectStore blockStore; protected final DBCachedObjectIndex blocksByOffset; protected final Map blockCacheMostRecent = - new FixedSizeHashMap<>(10); + Collections.synchronizedMap(new FixedSizeHashMap<>(BLOCK_CACHE_SIZE)); + protected final Map addressSetStateCache = + Collections.synchronizedMap(new FixedSizeHashMap<>(STATE_CACHE_SIZE)); + protected final Map> mostRecentStateEntryCache = + Collections.synchronizedMap(new FixedSizeHashMap<>(STATE_MR_CACHE_SIZE)); protected final DBTraceTimeViewport viewport; @@ -91,6 +108,7 @@ public class DBTraceMemorySpace DBCachedObjectStoreFactory factory = trace.getStoreFactory(); + // TODO: Remove the Exp thing from this and related this.stateMapSpace = new DBTraceAddressSnapRangePropertyMapSpace<>( DBTraceMemoryStateEntry.tableName(space), trace, factory, lock, space, DBTraceMemoryStateEntry.class, DBTraceMemoryStateEntry::new); @@ -133,56 +151,169 @@ public class DBTraceMemorySpace return space; } - protected void doSetState(long snap, Address start, Address end, TraceMemoryState state) { - if (state == null) { - throw new NullPointerException(); + protected void doPutState(AddressRange range, Lifespan span, TraceMemoryState state) { + if (!range.getMinAddress().equals(range.getAddressSpace().getMinAddress())) { + Entry candidateLeft = stateMapSpace + .reduce(TraceAddressSnapRangeQuery.at(range.getMinAddress().previous(), + span.lmin())) + .firstEntry(); + if (candidateLeft != null && candidateLeft.getValue() == state && + candidateLeft.getKey().getLifespan().equals(span)) { + range = new AddressRangeImpl(candidateLeft.getKey().getRange().getMinAddress(), + range.getMaxAddress()); + stateMapSpace.remove(candidateLeft); + } } - var l = new Object() { - boolean changed; - }; - new AddressRangeMapSetter, - TraceMemoryState>() { - - @Override - protected AddressRange getRange(Entry entry) { - return entry.getKey().getRange(); + if (!range.getMaxAddress().equals(range.getAddressSpace().getMaxAddress())) { + Entry candidateRight = stateMapSpace + .reduce(TraceAddressSnapRangeQuery.at(range.getMaxAddress().next(), + span.lmin())) + .firstEntry(); + if (candidateRight != null && candidateRight.getValue() == state && + candidateRight.getKey().getLifespan().equals(span)) { + range = new AddressRangeImpl(range.getMinAddress(), + candidateRight.getKey().getRange().getMaxAddress()); + stateMapSpace.remove(candidateRight); } + } + stateMapSpace.put(range, span, state); + } - @Override - protected TraceMemoryState getValue( - Entry entry) { - return entry.getValue(); + protected void doTruncateAndPut(long snap, AddressSet remains, TraceMemoryState state) { + if (remains.isEmpty()) { + return; + } + Lifespan nowOn = Lifespan.nowOnMaybeScratch(snap); + // If there can't be anything ahead, just add it and be done + if (nowOn.lmin() == nowOn.lmax()) { + for (AddressRange rng : remains) { + doPutState(rng, nowOn, state); } - - @Override - protected void remove(Entry entry) { - stateMapSpace.remove(entry); + return; + } + // Scan ahead to see how the new entry should be split, if truncation is needed + Lifespan future = Lifespan.nowOnMaybeScratch(snap + 1); + Iterator> it = stateMapSpace + .reduce(TraceAddressSnapRangeQuery.intersectingEnclosed( + new AddressRangeImpl(remains.getMinAddress(), remains.getMaxAddress()), future) + .starting(Rectangle2DDirection.BOTTOMMOST)) + .orderedEntries() + .iterator(); + // Avoid concurrent modification, lest new entries confuse our splitting + List toAdd = new ArrayList<>(); + while (!remains.isEmpty() && it.hasNext()) { + Entry next = it.next(); + if (next.getValue() != TraceMemoryState.KNOWN) { + continue; } - - @Override - protected Iterable> getIntersecting( - Address lower, Address upper) { - return stateMapSpace - .reduce(TraceAddressSnapRangeQuery.intersecting(lower, upper, snap, snap)) - .entries(); - } - - @Override - protected Entry put(AddressRange range, - TraceMemoryState value) { - // This should not get called if the range is already the desired state - l.changed = true; - if (value != TraceMemoryState.UNKNOWN) { - stateMapSpace.put(new ImmutableTraceAddressSnapRange(range, snap), value); + TraceAddressSnapRange key = next.getKey(); + AddressSet intersection = remains.intersectRange(key.getRange()); + if (!intersection.isEmpty()) { + Lifespan portion = Lifespan.span(snap, key.getLifespan().lmin() - 1); + for (AddressRange rng : intersection) { + toAdd.add(new ImmutableTraceAddressSnapRange(rng, portion)); } - return null; // Don't need to return it } - }.set(start, end, state); - - if (l.changed) { - trace.setChanged(new TraceChangeRecord<>(TraceEvents.BYTES_STATE_CHANGED, space, - new ImmutableTraceAddressSnapRange(start, end, snap, snap), state)); + remains.delete(key.getRange()); } + for (AddressRange rng : remains) { // may be empty, anyway + toAdd.add(new ImmutableTraceAddressSnapRange(rng, nowOn)); + } + + for (TraceAddressSnapRange box : toAdd) { + doPutState(box.getRange(), box.getLifespan(), state); + } + } + + protected void doSetState(long snap, Address start, Address end, TraceMemoryState state) { + Objects.requireNonNull(state); + AddressRangeImpl range = new AddressRangeImpl(start, end); + AddressSet remains = new AddressSet(range); + AddressSet toExpand = state.truncates() ? null : new AddressSet(); + List> truncated = new ArrayList<>(); + + // Where the desired state is already present, do nothing + var exist = List.copyOf(stateMapSpace + .reduce(TraceAddressSnapRangeQuery.intersecting(range, Lifespan.at(snap))) + .entries()); + for (Entry ex : exist) { + TraceAddressSnapRange key = ex.getKey(); + if (key.getLifespan().lmin() == snap) { + if (ex.getValue() == state) { + remains.delete(key.getRange()); + } + else { + AddressSet diff = new AddressSet(key.getRange()); + diff.delete(range); + for (AddressRange d : diff) { + truncated.add(Map.entry(new ImmutableTraceAddressSnapRange( + d, key.getLifespan()), ex.getValue())); + } + + stateMapSpace.remove(ex); + if (toExpand != null /* New value does not truncate */ && + ex.getValue().truncates() /* Old value truncates */) { + toExpand.add(key.getRange()); + } + } + } + } + + // Implies nothing needs to change, and toExpand ought be be empty too + if (remains.isEmpty() && (toExpand == null || toExpand.isEmpty())) { + return; + } + + /** + * Truncate or delete the existing entries to make room for the new entries. Every entry is + * truncated at KNOWN. No entry is truncated by ERROR, so queries must sort that out. + */ + if (state.truncates()) { + if (!remains.isEmpty()) { + // Remove existing entries to make room. We'll add them back (truncated) later + for (Entry entry : exist) { + if (remains.intersects(entry.getKey().getRange())) { + stateMapSpace.remove(entry); + truncated.add(Map.entry(entry.getKey(), entry.getValue())); + } + } + } + } + else { + /** + * It's possible truncating values were removed, permitting entries from the previous + * snap to expand forward. Those entries may need to be split, depending on what + * truncating entries are in the future. + */ + if (snap != Long.MIN_VALUE && snap != 0) { // Is there a previous snap? + Lifespan prev = Lifespan.at(snap - 1); + for (AddressRange exp : toExpand) { + for (Entry entry : stateMapSpace + .reduce(TraceAddressSnapRangeQuery.intersecting(exp, prev)) + .entries()) { + stateMapSpace.remove(entry); + doTruncateAndPut(entry.getKey().getLifespan().lmin(), + new AddressSet(entry.getKey().getRange()), entry.getValue()); + } + } + } + } + + if (!state.impliedByNull() && remains != null) { + doTruncateAndPut(snap, remains, state); + } + + for (Entry entry : truncated) { + TraceAddressSnapRange key = entry.getKey(); + doTruncateAndPut(key.getLifespan().lmin(), new AddressSet(key.getRange()), + entry.getValue()); + } + + addressSetStateCache.clear(); + mostRecentStateEntryCache.clear(); + // TODO: Maybe the end snap should extend as far into the future as was actually written? + trace.setChanged(new TraceChangeRecord<>(TraceEvents.BYTES_STATE_CHANGED, space, + new ImmutableTraceAddressSnapRange(start, end, snap, snap), state)); } protected void checkState(TraceMemoryState state) { @@ -242,9 +373,8 @@ public class DBTraceMemorySpace @Override public TraceMemoryState getState(long snap, Address address) { - TraceMemoryState state = - stateMapSpace.reduce(TraceAddressSnapRangeQuery.at(address, snap)).firstValue(); - return state == null ? TraceMemoryState.UNKNOWN : state; + return TraceMemoryState.orImplied( + stateMapSpace.reduce(TraceAddressSnapRangeQuery.minAt(address, snap)).firstValue()); } @Override @@ -252,10 +382,11 @@ public class DBTraceMemorySpace for (Lifespan span : viewport.getOrderedSpans(snap)) { TraceMemoryState state = getState(span.lmax(), address); switch (state) { - case KNOWN: - case ERROR: + case KNOWN, ERROR -> { return Map.entry(span.lmax(), state); - default: // fall through + } + default -> { // fall through + } } // Only the snap with the schedule specified gets the source snap's states if (span.lmax() - span.lmin() > 0) { @@ -268,14 +399,22 @@ public class DBTraceMemorySpace @Override public Entry getMostRecentStateEntry(long snap, Address address) { - return stateMapSpace.reduce( - TraceAddressSnapRangeQuery.mostRecent(address, snap)).firstEntry(); + return stateMapSpace.reduce(TraceAddressSnapRangeQuery.mostRecent(address, snap)) + // not .firstEntry(), because TOPMOST sorts by Y2 + .entries() + .stream() + .sorted(Comparator.comparing(e -> -e.getKey().getY1())) + .findFirst() + .orElse(null); } @Override public Entry getViewMostRecentStateEntry(long snap, Address address) { - return getViewMostRecentStateEntry(snap, new AddressRangeImpl(address, address), s -> true); + // LATER: Cache here or on the delegate? + return mostRecentStateEntryCache.computeIfAbsent(new MostRecentStateCacheKey(snap, address), + k -> getViewMostRecentStateEntry(snap, new AddressRangeImpl(address, address), + StatePredicate.IS_KNOWN_OR_ERROR)); } @Override @@ -284,9 +423,14 @@ public class DBTraceMemorySpace assertInSpace(range); for (Lifespan span : viewport.getOrderedSpans(snap)) { var entry = stateMapSpace.reduce(TraceAddressSnapRangeQuery.mostRecent(range, span)) - .firstEntry(); - if (entry != null && predicate.test(entry.getValue())) { - return entry; + .entries() // not ordered, since we're doing our own sort + .stream() + .filter(e -> predicate.test(e.getValue())) + // TOPMOST sorts by Y2 + .sorted(Comparator.comparing(e -> -e.getKey().getY1())) + .findFirst(); + if (entry.isPresent()) { + return entry.get(); } } return null; @@ -295,7 +439,7 @@ public class DBTraceMemorySpace @Override public AddressSetView getAddressesWithState(long snap, Predicate predicate) { return new DBTraceAddressSnapRangePropertyMapAddressSetView<>(space, lock, - stateMapSpace.reduce(TraceAddressSnapRangeQuery.atSnap(snap, space)), + stateMapSpace.reduce(TraceAddressSnapRangeQuery.minAtSnap(snap, space)), predicate); } @@ -303,7 +447,7 @@ public class DBTraceMemorySpace public AddressSetView getAddressesWithState(Lifespan lifespan, Predicate predicate) { return new DBTraceAddressSnapRangePropertyMapAddressSetView<>(space, lock, - stateMapSpace.reduce(TraceAddressSnapRangeQuery.intersecting(lifespan, space)), + stateMapSpace.reduce(TraceAddressSnapRangeQuery.minWithin(lifespan, space)), predicate); } @@ -311,31 +455,71 @@ public class DBTraceMemorySpace public AddressSetView getAddressesWithState(Lifespan span, AddressSetView set, Predicate predicate) { try (LockHold hold = LockHold.lock(lock.readLock())) { + if (!(predicate instanceof StatePredicate stock)) { + return doGetAddressesWithState(span, set, predicate); + } AddressSet remains = new AddressSet(set); AddressSet result = new AddressSet(); while (!remains.isEmpty()) { - AddressRange range = remains.getFirstRange(); - remains.delete(range); - for (Entry entry : doGetStates(span, - range)) { - AddressRange foundRange = entry.getKey().getRange(); - remains.delete(foundRange); - if (predicate.test(entry.getValue())) { - result.add(foundRange); + Address min = remains.getMinAddress(); + long blockMinOffset = min.getOffset() & STATE_BLOCK_MASK; + Address blockMin = min.getNewAddress(blockMinOffset); + Address blockMax = blockMin.add(STATE_BLOCK_SIZE - 1); + remains.delete(min, blockMax); + + result.add(addressSetStateCache.computeIfAbsent( + new AddressSetStateCacheKey(span, blockMinOffset, stock), + k -> doGetAddressesWithState(span, new AddressSet(blockMin, blockMax), stock))); + } + return result.intersect(set); + } + } + + protected AddressSetView doGetAddressesWithState(Lifespan span, AddressSetView set, + Predicate predicate) { + AddressSet remains = new AddressSet(set); + AddressSet result = new AddressSet(); + while (!remains.isEmpty()) { + AddressRange range = remains.getFirstRange(); + AddressSet subRem = new AddressSet(range); + remains.delete(range); + for (Entry entry : doGetStates(span, range)) { + AddressRange foundRange = entry.getKey().getRange(); + if (predicate.test(entry.getValue())) { + result.add(foundRange); + remains.delete(foundRange); // Could intersect a later range + subRem.delete(foundRange); + if (subRem.isEmpty()) { + break; } } } - return result; + if (remains.isEmpty()) { + return result; + } } + return result; } protected Collection> doGetStates(Lifespan span, AddressRange range) { // LATER: A better way to handle memory-mapped registers? - if (getAddressSpace().isRegisterSpace() && !range.getAddressSpace().isRegisterSpace()) { + AddressSpace thisSpace = getAddressSpace(); + if (thisSpace.isRegisterSpace() && !range.getAddressSpace().isRegisterSpace()) { return trace.getMemoryManager().doGetStates(span, range); } - return stateMapSpace.reduce(TraceAddressSnapRangeQuery.intersecting(range, span)).entries(); + if (thisSpace != range.getAddressSpace()) { + range = new AddressRangeImpl( + thisSpace.getOverlayAddress(range.getMinAddress()), + thisSpace.getOverlayAddress(range.getMaxAddress())); + } + var subMap = + stateMapSpace.reduce(TraceAddressSnapRangeQuery.intersectingMinWithin(range, span)); + return subMap.entries(); + /*@SuppressWarnings({ "unchecked", "rawtypes" }) + List> asList = + (List) Arrays.asList(subMap.entries().toArray()); + return asList;*/ } @Override @@ -348,8 +532,10 @@ public class DBTraceMemorySpace @Override public Iterable> getMostRecentStates( TraceAddressSnapRange within) { - return new DBTraceAddressSnapRangePropertyMapOcclusionIntoPastIterable<>(this.stateMapSpace, - within); + return stateMapSpace + .reduce( + TraceAddressSnapRangeQuery.mostRecent(within.getRange(), within.getLifespan())) + .orderedEntries(); } protected DBTraceMemoryBlockEntry findMostRecentBlockEntry(OffsetSnap loc, boolean inclusive) { @@ -470,7 +656,7 @@ public class DBTraceMemorySpace break; } AddressSetView withState = - getAddressesWithState(next.getSnap(), remaining, state -> true); + getAddressesWithState(next.getSnap(), remaining, StatePredicate.IS_KNOWN_OR_ERROR); remaining = remaining.subtract(withState); long endSnap = next.getSnap() - 1; for (AddressRange rng : withState) { @@ -626,8 +812,7 @@ public class DBTraceMemorySpace spans: for (Lifespan span : viewport.getOrderedSpans(snap)) { Iterator arit = - getAddressesWithState(span, remains, s -> s == TraceMemoryState.KNOWN) - .iterator(start, true); + getAddressesWithState(span, remains, StatePredicate.IS_KNOWN).iterator(start, true); while (arit.hasNext()) { AddressRange rng = arit.next(); if (rng.getMinAddress().compareTo(toRead.getMaxAddress()) > 0) { @@ -710,12 +895,11 @@ public class DBTraceMemorySpace return null; } - // TODO: Could do better, but have to worry about viewport, too - // This will reduce the search to ranges that have been written at any snap - // We could do for this and previous snaps, but that's where the viewport comes in. - // TODO: Potentially costly to pre-compute the set concretely + // LATER: Worry about the viewport, too? + // This will reduce the search to ranges that have any once-known value at the snap. + // NOTE: Potentially costly to pre-compute the set concretely AddressSet known = new AddressSet( - stateMapSpace.getAddressSetView(Lifespan.ALL, s -> s == TraceMemoryState.KNOWN)) + stateMapSpace.getAddressSetView(Lifespan.at(snap), StatePredicate.IS_KNOWN)) .intersect(new AddressSet(range)); monitor.initialize(known.getNumAddresses()); for (AddressRange knownRange : known.getAddressRanges(forward)) { @@ -829,9 +1013,8 @@ public class DBTraceMemorySpace ByteBuffer buf1 = ByteBuffer.allocate(BLOCK_SIZE); ByteBuffer buf2 = ByteBuffer.allocate(BLOCK_SIZE); try (LockHold hold = LockHold.lock(lock.readLock())) { - for (TraceAddressSnapRange tasr : stateMapSpace.reduce( - TraceAddressSnapRangeQuery.intersecting(range, fwdOne) - .starting(Rectangle2DDirection.BOTTOMMOST)) + for (TraceAddressSnapRange tasr : stateMapSpace + .reduce(TraceAddressSnapRangeQuery.leastRecent(range, fwdOne)) .orderedKeys()) { AddressRange toExamine = range.intersect(tasr.getRange()); if (doCheckBytesChanged(tasr.getY1(), toExamine, buf1, buf2)) { @@ -917,6 +1100,8 @@ public class DBTraceMemorySpace bufferStore.invalidateCache(); blockStore.invalidateCache(); blockCacheMostRecent.clear(); + addressSetStateCache.clear(); + mostRecentStateEntryCache.clear(); } } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemoryStateEntry.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemoryStateEntry.java index 7d43f6f035..cbe5913729 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemoryStateEntry.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemoryStateEntry.java @@ -18,6 +18,7 @@ package ghidra.trace.database.memory; import javax.help.UnsupportedOperationException; import db.DBRecord; +import ghidra.lifecycle.Internal; import ghidra.program.model.address.AddressSpace; import ghidra.trace.database.DBTraceUtils; import ghidra.trace.database.map.DBTraceAddressSnapRangePropertyMapTree; @@ -30,7 +31,8 @@ import ghidra.util.database.annot.*; /** * INTERNAL: An entry to record memory observation states in the database */ -@DBAnnotatedObjectInfo(version = 0) +@DBAnnotatedObjectInfo(version = 1) +@Internal class DBTraceMemoryStateEntry extends AbstractDBTraceAddressSnapRangePropertyMapData { diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/InternalTraceMemoryOperations.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/InternalTraceMemoryOperations.java index 3342019583..26e1d15783 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/InternalTraceMemoryOperations.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/InternalTraceMemoryOperations.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.Map.Entry; import java.util.concurrent.locks.ReadWriteLock; +import ghidra.lifecycle.Internal; import ghidra.program.model.address.AddressRange; import ghidra.program.model.address.AddressSpace; import ghidra.program.model.lang.Register; @@ -31,17 +32,17 @@ import ghidra.trace.model.memory.TraceMemoryState; import ghidra.trace.util.TraceRegisterUtils; import ghidra.util.LockHold; +@Internal public interface InternalTraceMemoryOperations extends TraceMemoryOperations { - static TraceMemoryState requireOne( + static TraceMemoryState requireOne(AddressRange range, Collection> states, Register register) { - if (states.isEmpty()) { - return TraceMemoryState.UNKNOWN; - } - if (states.size() != 1) { + TraceMemoryState state = + TraceMemoryOperations.oneState(range, states); + if (state == null) { throw new IllegalStateException("More than one state is present in " + register); } - return states.iterator().next().getValue(); + return state; } /** @@ -63,7 +64,7 @@ public interface InternalTraceMemoryOperations extends TraceMemoryOperations { @Override default TraceMemoryState getState(TracePlatform platform, long snap, Register register) { AddressRange range = platform.getConventionalRegisterRange(getSpace(), register); - return requireOne(getStates(snap, range), register); + return requireOne(range, getStates(snap, range), register); } @Override diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/program/AbstractDBTraceProgramViewListing.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/program/AbstractDBTraceProgramViewListing.java index 87c457c233..ac0af52788 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/program/AbstractDBTraceProgramViewListing.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/program/AbstractDBTraceProgramViewListing.java @@ -41,8 +41,8 @@ import ghidra.trace.database.memory.DBTraceMemorySpace; import ghidra.trace.database.program.DBTraceProgramViewMemory.RegionEntry; import ghidra.trace.model.*; import ghidra.trace.model.listing.*; +import ghidra.trace.model.memory.TraceMemoryOperations.StatePredicate; import ghidra.trace.model.memory.TraceMemoryRegion; -import ghidra.trace.model.memory.TraceMemoryState; import ghidra.trace.model.program.TraceProgramView; import ghidra.trace.model.program.TraceProgramViewListing; import ghidra.trace.model.property.TracePropertyMapOperations; @@ -721,7 +721,7 @@ public abstract class AbstractDBTraceProgramViewListing implements TraceProgramV throw new CodeUnitInsertionException("Code unit would extend beyond address space"); } var mostRecent = program.memory.memoryManager.getViewMostRecentStateEntry(program.snap, - range, s -> s == TraceMemoryState.KNOWN); + range, StatePredicate.IS_KNOWN); long snap = mostRecent == null ? program.snap : mostRecent.getKey().getY2(); return codeOperations.instructions() .create(Lifespan.nowOn(snap), addr, program.platform, prototype, context, diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/program/DBTraceProgramView.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/program/DBTraceProgramView.java index 20746c66cf..721a5808e8 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/program/DBTraceProgramView.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/program/DBTraceProgramView.java @@ -58,6 +58,7 @@ import ghidra.trace.model.data.TraceBasedDataTypeManager; import ghidra.trace.model.listing.*; import ghidra.trace.model.memory.TraceMemoryRegion; import ghidra.trace.model.memory.TraceMemoryState; +import ghidra.trace.model.memory.TraceMemoryOperations.StatePredicate; import ghidra.trace.model.program.TraceProgramView; import ghidra.trace.model.symbol.*; import ghidra.trace.util.TraceEvents; @@ -598,7 +599,8 @@ public class DBTraceProgramView implements TraceProgramView { } protected static class OverlappingAddressRangeKeyIteratorMerger extends - PairingIteratorMerger, Entry, Entry> { + PairingIteratorMerger, Entry, + Entry> { protected static Iterable, Entry>> iter( Iterable> left, Iterable> right) { @@ -1452,7 +1454,7 @@ public class DBTraceProgramView implements TraceProgramView { return RangeQueryOcclusion.super.occluded(cu, range, span); } AddressSetView known = - memSpace.getAddressesWithState(span, s -> s == TraceMemoryState.KNOWN); + memSpace.getAddressesWithState(span, StatePredicate.IS_KNOWN); if (!known.intersects(range.getMinAddress(), range.getMaxAddress())) { return RangeQueryOcclusion.super.occluded(cu, range, span); } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/symbol/DBTraceReferenceSpace.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/symbol/DBTraceReferenceSpace.java index 9e25bbd199..aa572c08a0 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/symbol/DBTraceReferenceSpace.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/symbol/DBTraceReferenceSpace.java @@ -17,6 +17,7 @@ package ghidra.trace.database.symbol; import java.io.IOException; import java.util.Collection; +import java.util.List; import java.util.concurrent.locks.ReadWriteLock; import java.util.stream.Stream; @@ -50,6 +51,8 @@ import ghidra.util.database.spatial.rect.Rectangle2DDirection; import ghidra.util.exception.VersionException; public class DBTraceReferenceSpace implements DBTraceSpaceBased, TraceReferenceSpace { + protected final static int CHUNK_SIZE = DBTrace.CHUNK_SIZE; + protected enum TypeEnum { MEMORY { /** @@ -142,9 +145,15 @@ public class DBTraceReferenceSpace implements DBTraceSpaceBased, TraceReferenceS return DBTraceUtils.tableName(TABLE_NAME, space); } - @DBAnnotatedField(column = TO_ADDR_MIN_COLUMN_NAME, indexed = true, codec = AddressDBFieldCodec.class) + @DBAnnotatedField( + column = TO_ADDR_MIN_COLUMN_NAME, + indexed = true, + codec = AddressDBFieldCodec.class) protected Address toAddrMin = Address.NO_ADDRESS; - @DBAnnotatedField(column = TO_ADDR_MAX_COLUMN_NAME, indexed = true, codec = AddressDBFieldCodec.class) + @DBAnnotatedField( + column = TO_ADDR_MAX_COLUMN_NAME, + indexed = true, + codec = AddressDBFieldCodec.class) protected Address toAddrMax = Address.NO_ADDRESS; @DBAnnotatedField(column = SYMBOL_ID_COLUMN_NAME, indexed = true) protected long symbolId; // TODO: Is this at the from or to address? I think TO... @@ -343,10 +352,12 @@ public class DBTraceReferenceSpace implements DBTraceSpaceBased, TraceReferenceS protected final AddressRangeImpl fullSpace; - protected final DBTraceAddressSnapRangePropertyMapSpace referenceMapSpace; + protected final DBTraceAddressSnapRangePropertyMapSpace referenceMapSpace; protected final DBCachedObjectIndex refsBySymbolId; - protected final DBTraceAddressSnapRangePropertyMapSpace xrefMapSpace; + protected final DBTraceAddressSnapRangePropertyMapSpace xrefMapSpace; protected final DBCachedObjectIndex xrefsByRefKey; public DBTraceReferenceSpace(DBTraceReferenceManager manager, DBHandle dbh, AddressSpace space, @@ -444,10 +455,10 @@ public class DBTraceReferenceSpace implements DBTraceSpaceBased, TraceReferenceS int operandIndex) { // Do I consider "compatibility?" as in ReferenceDBManager? // NOTE: Always call with the write lock - for (DBTraceReferenceEntry ent : referenceMapSpace + for (DBTraceReferenceEntry ent : List.copyOf(referenceMapSpace .reduce(TraceAddressSnapRangeQuery .intersecting(new AddressRangeImpl(fromAddress, fromAddress), span)) - .values()) { + .values())) { if (!ent.toRange.equals(toRange) || ent.opIndex != operandIndex) { continue; } @@ -642,10 +653,18 @@ public class DBTraceReferenceSpace implements DBTraceSpaceBased, TraceReferenceS public void clearReferencesFrom(Lifespan span, AddressRange range) { try (LockHold hold = manager.getTrace().lockWrite()) { long startSnap = span.lmin(); - for (DBTraceReferenceEntry ref : referenceMapSpace - .reduce(TraceAddressSnapRangeQuery.intersecting(range, span)) - .values()) { - truncateOrDeleteEntry(ref, startSnap); + + var submap = + referenceMapSpace.reduce(TraceAddressSnapRangeQuery.intersecting(range, span)); + while (true) { + List chunk = + submap.values().stream().limit(CHUNK_SIZE).toList(); + for (DBTraceReferenceEntry ref : chunk) { + truncateOrDeleteEntry(ref, startSnap); + } + if (chunk.size() < CHUNK_SIZE) { + break; + } } // TODO: Coalesce events? } @@ -698,11 +717,17 @@ public class DBTraceReferenceSpace implements DBTraceSpaceBased, TraceReferenceS public void clearReferencesTo(Lifespan span, AddressRange range) { try (LockHold hold = manager.getTrace().lockWrite()) { long startSnap = span.lmin(); - for (DBTraceXRefEntry xref : xrefMapSpace - .reduce(TraceAddressSnapRangeQuery.intersecting(range, span)) - .values()) { - DBTraceReferenceEntry ref = getRefEntryForXRefEntry(xref); - truncateOrDeleteEntry(ref, startSnap); + + var submap = xrefMapSpace.reduce(TraceAddressSnapRangeQuery.intersecting(range, span)); + while (true) { + List chunk = submap.values().stream().limit(CHUNK_SIZE).toList(); + for (DBTraceXRefEntry xref : chunk) { + DBTraceReferenceEntry ref = getRefEntryForXRefEntry(xref); + truncateOrDeleteEntry(ref, startSnap); + } + if (chunk.size() < CHUNK_SIZE) { + break; + } } // TODO: Coalesce events? } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/Lifespan.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/Lifespan.java index 76b256cac1..4ed5cbb90f 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/Lifespan.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/Lifespan.java @@ -57,13 +57,14 @@ public sealed interface Lifespan extends Span, Iterable { * Get the lifespan from 0 to the given snap. * *

- * The lower bound is 0 to exclude scratch space. + * If the snapshot is in scratch space, then the span will have a lower endpoint of + * {@link Long#MIN_VALUE}. Otherwise, the lower endpoint is 0 to exclude scratch space. * * @param snap the snapshot key * @return the lifespan */ static Lifespan since(long snap) { - return new Impl(0, snap); + return new Impl(isScratch(snap) ? Long.MIN_VALUE : 0, snap); } /** @@ -87,10 +88,7 @@ public sealed interface Lifespan extends Span, Iterable { * @return the lifespan */ static Lifespan nowOnMaybeScratch(long snap) { - if (isScratch(snap)) { - return new Impl(snap, -1); - } - return new Impl(snap, Long.MAX_VALUE); + return new Impl(snap, isScratch(snap) ? -1 : Long.MAX_VALUE); } /** diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/memory/TraceMemoryOperations.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/memory/TraceMemoryOperations.java index acddf1fa6f..4cf60eeea2 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/memory/TraceMemoryOperations.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/memory/TraceMemoryOperations.java @@ -18,6 +18,7 @@ package ghidra.trace.model.memory; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Collection; +import java.util.Iterator; import java.util.Map.Entry; import java.util.function.Predicate; @@ -32,7 +33,6 @@ import ghidra.util.task.TaskMonitor; /** * Operations for mutating memory regions, values, and state within a trace - * *

* This models memory over the course of an arbitrary number of snaps. The duration between snaps is * unspecified. However, the mapping of snaps to real time ought to be strictly monotonic. @@ -45,7 +45,11 @@ import ghidra.util.task.TaskMonitor; * states can be manipulated directly; however, this is recommended only to record read failures, * using the state {@link TraceMemoryState#ERROR}. A state of {@code null} is equivalent to * {@link TraceMemoryState#UNKNOWN} and indicates no observation has been made. - * + *

+ * To support queries for the "most recent" bytes and states, entries are extended as far into the + * future until it would collide with another entry, splitting the entry so it can be extended + * piecewise. Note that the state itself is only effective at the starting snap of the + * lifespan. Otherwise, the effective state is {@link TraceMemoryState#UNKNOWN}. *

* Negative snaps may have different semantics than positive, since negative snaps are used as * "scratch space". These snaps are not presumed to have any temporal relation to their neighbors, @@ -60,14 +64,56 @@ import ghidra.util.task.TaskMonitor; * accidentally rely on implied temporal relationships in scratch space. */ public interface TraceMemoryOperations { + /** * Check if the return value of {@link #getStates(long, AddressRange)} or similar represents a - * single entry of the given state. - * + * single state across the given range. *

- * This method returns false if there is not exactly one entry of the given state whose range - * covers the given range. As a special case, an empty collection will cause this method to - * return true iff state is {@link TraceMemoryState#UNKNOWN}. + * This method returns null if any of the entries indicate a state that differs from the others, + * or if the given entries do not completely cover the given range. As a special case, an empty + * collection will result in {@link TraceMemoryState#UNKNOWN}. + *

+ * Each of the given entries must intersect the given range, or else the behavior is + * undefined. If the same range is given to {@link #getStates(long, AddressRange)}, this + * requirement is satisfied. + * + * @param range the range to check, usually that passed to + * {@link #getStates(long, AddressRange)}. + * @param stateEntries the collection returned by {@link #getStates(long, AddressRange)}. + * @param states + * @return the uniform state, or null + */ + static TraceMemoryState oneState(AddressRange range, + Collection> states) { + Iterator> it = states.iterator(); + if (!it.hasNext()) { + return TraceMemoryState.IMPLIED_BY_NULL; + } + Entry entry = it.next(); + TraceMemoryState state = entry.getValue(); + AddressSet remains = new AddressSet(range); + remains.delete(entry.getKey().getRange()); + while (it.hasNext()) { + if (entry.getValue() != state) { + return null; + } + remains.delete(entry.getKey().getRange()); + } + return remains.isEmpty() ? state : null; + } + + /** + * Check if the return value of {@link #getStates(long, AddressRange)} or similar represents a + * given state across the given range. + *

+ * This method returns false if any of the entries indicate a state other than the given one, or + * if the given entries do not completely cover the given range. As a special case, an empty + * collection will cause this method to return true iff state is + * {@link TraceMemoryState#UNKNOWN}. + *

+ * Each of the given entries must intersect the given range, or else the behavior is + * undefined. If the same range is given to {@link #getStates(long, AddressRange)}, this + * requirement is satisfied. * * @param range the range to check, usually that passed to * {@link #getStates(long, AddressRange)}. @@ -78,18 +124,7 @@ public interface TraceMemoryOperations { static boolean isStateEntirely(AddressRange range, Collection> stateEntries, TraceMemoryState state) { - if (stateEntries.isEmpty()) { - return state == TraceMemoryState.UNKNOWN; - } - if (stateEntries.size() != 1) { - return false; - } - Entry ent = stateEntries.iterator().next(); - if (ent.getValue() != state) { - return false; - } - AddressRange entRange = ent.getKey().getRange(); - return entRange.contains(range.getMinAddress()) && entRange.contains(range.getMaxAddress()); + return oneState(range, stateEntries) == state; } /** @@ -101,7 +136,6 @@ public interface TraceMemoryOperations { /** * Set the state of memory over a given time and address range - * *

* Setting state to {@link TraceMemoryState#KNOWN} via this method is not recommended. Setting * bytes will automatically update the state accordingly. @@ -131,7 +165,6 @@ public interface TraceMemoryOperations { /** * Get the state of memory at a given snap and address - * *

* If the location's state has not been set, the result is {@code null}, which implies * {@link TraceMemoryState#UNKNOWN}. @@ -153,14 +186,15 @@ public interface TraceMemoryOperations { /** * Get the entry recording the most recent state at the given snap and address - * *

- * The entry includes the entire entry at that snap. Parts occluded by more recent snaps are not - * subtracted from the entry's address range. + * The entry may include more addresses and snaps than requested. The address range is largely + * circumstantial, but may be useful if the client is performing multiple queries in a locality. + * The lifespan is more important as it indicates the actual effective snapshot (the lower + * bound) as well as when the next change is (one after the upper bound.) * * @param snap the time * @param address the location - * @return the entry including the entire recorded range + * @return the most-recent entry */ Entry getMostRecentStateEntry(long snap, Address address); @@ -176,6 +210,36 @@ public interface TraceMemoryOperations { Entry getViewMostRecentStateEntry(long snap, Address address); + /** + * Built-in predicates for filtering/testing state entries + *

+ * Many methods accept an unrestricted {@link Predicate} on {@link TraceMemoryState}. However, + * use of these built-ins is strongly recommended for two reasons: 1) They handle conventional + * cases, e.g., null representing {@link TraceMemoryState#UNKNOWN}. 2) Caching depends on the + * implementation being able to recognize the identity of the predicate. This generally cannot + * be done with lambda methods. + */ + enum StatePredicate implements Predicate { + IS_KNOWN { + @Override + public boolean test(TraceMemoryState state) { + return state == TraceMemoryState.KNOWN; + } + }, + IS_ERROR { + @Override + public boolean test(TraceMemoryState state) { + return state == TraceMemoryState.ERROR; + } + }, + IS_KNOWN_OR_ERROR { + @Override + public boolean test(TraceMemoryState state) { + return state != null && state != TraceMemoryState.UNKNOWN; + } + } + } + /** * Get the entry recording the most recent state since the given snap within the given range * that satisfies a given predicate, following schedule forks @@ -183,13 +247,13 @@ public interface TraceMemoryOperations { * @param snap the latest time to consider * @param range the range of addresses * @param predicate a predicate on the state - * @return the most-recent entry + * @return the most-recent entry or null */ Entry getViewMostRecentStateEntry(long snap, AddressRange range, Predicate predicate); /** - * Get at least the subset of addresses having state satisfying the given predicate + * Get at least the intersection of addresses having state satisfying the given predicate * * @param snap the time * @param set the set to examine @@ -203,19 +267,16 @@ public interface TraceMemoryOperations { } /** - * Get at least the subset of addresses having state satisfying the given predicate - * + * Get at least the intersection of addresses having state satisfying the given predicate *

- * The implementation may provide a larger view than requested, but within the requested set, + * The implementation may provide a larger set than requested, but within the requested set, * only ranges satisfying the predicate may be present. Use - * {@link AddressSetView#intersect(AddressSetView)} with {@code set} if a strict subset is + * {@link AddressSetView#intersect(AddressSetView)} with {@code set} if a strict intersection is * required. - * *

* Because {@link TraceMemoryState#UNKNOWN} is not explicitly stored in the map, to compute the * set of {@link TraceMemoryState#UNKNOWN} addresses, use the predicate - * {@code state -> state != null && state != TraceMemoryState.UNKNOWN} and subtract the result - * from {@code set}. + * {@link StatePredicate#IS_KNOWN_OR_ERROR} and subtract the result from {@code set}. * * @param span the range of time * @param set the set to examine @@ -227,7 +288,6 @@ public interface TraceMemoryOperations { /** * Get the addresses having state satisfying the given predicate - * *

* The implementation may provide a view that updates with changes. Behavior is not well defined * for predicates testing for {@link TraceMemoryState#UNKNOWN}. @@ -241,7 +301,6 @@ public interface TraceMemoryOperations { /** * Get the addresses having state satisfying the given predicate at any time in the specified * lifespan - * *

* The implementation may provide a view that updates with changes. Behavior is not well defined * for predicates testing for {@link TraceMemoryState#UNKNOWN} . @@ -250,12 +309,10 @@ public interface TraceMemoryOperations { * @param predicate a predicate on state to search for * @return the address set */ - AddressSetView getAddressesWithState(Lifespan lifespan, - Predicate predicate); + AddressSetView getAddressesWithState(Lifespan lifespan, Predicate predicate); /** - * Break a range of addresses into smaller ranges each mapped to its state at the given snap - * + * Get all the entries covering the given range effective at the given snap *

* Note that {@link TraceMemoryState#UNKNOWN} entries will not appear in the result. Gaps in the * returned entries are implied to be {@link TraceMemoryState#UNKNOWN}. @@ -276,31 +333,24 @@ public interface TraceMemoryOperations { */ default boolean isKnown(long snap, AddressRange range) { Collection> states = getStates(snap, range); - if (states.isEmpty()) { - return false; - } - if (states.size() != 1) { - return false; - } - AddressRange entryRange = states.iterator().next().getKey().getRange(); - if (!entryRange.contains(range.getMinAddress()) || - !entryRange.contains(range.getMaxAddress())) { - return false; - } - return true; + return isStateEntirely(range, states, TraceMemoryState.KNOWN); } /** - * Break a range of addresses into smaller ranges each mapped to its most recent state at the - * given time - * + * Get all the entries covering the given range, effective at or extending as "most recent" to + * the given snap. *

- * Typically {@code within} is the box whose width is the address range to break down and whose - * height is from "negative infinity" to the "current" snap. - * + * Typically {@code within} is the box whose width is the address range and whose height is + * {@link Lifespan#since(long)} a desired snap. Queries that respect forking should only extend + * as far back as the most recent forked snapshot. *

* In this context, "most recent" means the latest state other than - * {@link TraceMemoryState#UNKNOWN}. + * {@link TraceMemoryState#UNKNOWN}. The entries returned can overlap. The rule is that + * {@link TraceMemoryState#KNOWN} entries may not overlap one another. They are split + * and truncated so that they extend as far as possible into the future without overlapping. A + * {@link TraceMemoryState#ERROR} entry can overlap a less-recent {@link TraceMemoryState#KNOWN} + * entry, but not a more-recent one. This is to ensure the most-recent-known values can still be + * obtained when the most recent attempt to read bytes resulted in an error. * * @param within a box intersecting entries to consider * @return an iterable over the snap ranges and states @@ -319,7 +369,6 @@ public interface TraceMemoryOperations { /** * Write bytes at the given snap and address - * *

* This will attempt to read {@link ByteBuffer#remaining()} bytes starting at * {@link ByteBuffer#position()} from the source buffer {@code buf} and write them into memory @@ -336,7 +385,6 @@ public interface TraceMemoryOperations { /** * Read the most recent bytes from the given snap and address - * *

* This will attempt to read {@link ByteBuffer#remaining()} of the most recent bytes from memory * at the specified time and location and write them into the destination buffer {@code buf} @@ -353,7 +401,6 @@ public interface TraceMemoryOperations { /** * Read the most recent bytes from the given snap and address, following schedule forks - * *

* This behaves similarly to {@link #getBytes(long, Address, ByteBuffer)}, except it checks for * the {@link TraceMemoryState#KNOWN} state among each involved snap range and reads the @@ -384,14 +431,12 @@ public interface TraceMemoryOperations { /** * Remove bytes from the given time and location - * *

* This deletes all observed bytes from the given address through length at the given snap. If * there were no observations in the range at exactly the given snap, this has no effect. If * there were, then those observations are removed. The next time those bytes are read, they * will have a value from a previous snap, or no value at all. The affected region's state is - * also deleted, i.e., set to {@code null}, implying {@link TraceMemoryState#UNKNOWN}. - * + * also deleted, i.e., set to {@link TraceMemoryState#UNKNOWN}. *

* Note, use of this method is discouraged. The more observations within the same range that * follow the deleted observation, the more expensive this operation typically is, since all of @@ -405,7 +450,6 @@ public interface TraceMemoryOperations { /** * Get a view of a particular snap as a memory buffer - * *

* The bytes read by this buffer are the most recent bytes written before the given snap * @@ -430,7 +474,6 @@ public interface TraceMemoryOperations { /** * Find the internal storage block that most-recently defines the value at the given snap and * address, and return the block's snap. - * *

* This method reveals portions of the internal storage so that clients can optimize difference * computations by eliminating corresponding ranges defined by the same block. If the underlying @@ -444,7 +487,6 @@ public interface TraceMemoryOperations { /** * Get the block size used by internal storage. - * *

* This method reveals portions of the internal storage so that clients can optimize searches. * If the underlying implementation cannot answer this question, this returns 0. @@ -455,7 +497,6 @@ public interface TraceMemoryOperations { /** * Optimize storage space - * *

* This gives the implementation an opportunity to clean up garbage, apply compression, etc., in * order to best use the storage space. Because memory observations can be sparse, a trace's @@ -466,7 +507,6 @@ public interface TraceMemoryOperations { /** * Set the state of a given register at a given time - * *

* Setting state to {@link TraceMemoryState#KNOWN} via this method is not recommended. Setting * bytes will automatically update the state accordingly. @@ -480,7 +520,6 @@ public interface TraceMemoryOperations { /** * Set the state of a given register at a given time - * *

* Setting state to {@link TraceMemoryState#KNOWN} via this method is not recommended. Setting * bytes will automatically update the state accordingly. @@ -519,8 +558,7 @@ public interface TraceMemoryOperations { } /** - * Break the register's range into smaller ranges each mapped to its state at the given snap - * + * Get all the entries covering the given register at the given snap *

* If the register is memory mapped, this will delegate to the appropriate space. * @@ -533,8 +571,7 @@ public interface TraceMemoryOperations { long snap, Register register); /** - * Break the register's range into smaller ranges each mapped to its state at the given snap - * + * Get all the entries covering the given register at the given snap *

* If the register is memory mapped, this will delegate to the appropriate space. * @@ -558,11 +595,9 @@ public interface TraceMemoryOperations { /** * Set the value of a register at the given snap - * *

* If the register is memory mapped, this will delegate to the appropriate space. In those * cases, the assignment affects all threads. - * *

* IMPORTANT: The trace database cannot track the state ({@link TraceMemoryState#KNOWN}, * etc.) with per-bit accuracy. It only has byte precision. If the given value specifies, e.g., @@ -589,11 +624,9 @@ public interface TraceMemoryOperations { /** * Write bytes at the given snap and register address - * *

* If the register is memory mapped, this will delegate to the appropriate space. In those * cases, the assignment affects all threads. - * *

* Note that bit-masked registers are not properly heeded. If the caller wishes to preserve * non-masked bits, it must first retrieve the current value and combine it with the desired @@ -611,7 +644,6 @@ public interface TraceMemoryOperations { /** * Get the most-recent value of a given register at the given time - * *

* If the register is memory mapped, this will delegate to the appropriate space. * @@ -624,7 +656,6 @@ public interface TraceMemoryOperations { /** * Get the most-recent value of a given register at the given time - * *

* If the register is memory mapped, this will delegate to the appropriate space. * @@ -637,8 +668,7 @@ public interface TraceMemoryOperations { } /** - * Get the most-recent value of a given register at the given time - * + * Get the most-recent value of a given register at the given time, following schedule forks *

* If the register is memory mapped, this will delegate to the appropriate space. * @@ -651,7 +681,6 @@ public interface TraceMemoryOperations { /** * Get the most-recent value of a given register at the given time, following schedule forks - * *

* If the register is memory mapped, this will delegate to the appropriate space. * @@ -665,7 +694,6 @@ public interface TraceMemoryOperations { /** * Get the most-recent bytes of a given register at the given time - * *

* If the register is memory mapped, this will delegate to the appropriate space. * @@ -679,7 +707,6 @@ public interface TraceMemoryOperations { /** * Get the most-recent bytes of a given register at the given time - * *

* If the register is memory mapped, this will delegate to the appropriate space. * @@ -702,10 +729,8 @@ public interface TraceMemoryOperations { /** * Remove a value from the given time and register - * *

* If the register is memory mapped, this will delegate to the appropriate space. - * *

* IMPORANT: The trace database cannot track the state ({@link TraceMemoryState#KNOWN}, * etc.) with per-bit accuracy. It only has byte precision. If the given register specifies, diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/memory/TraceMemoryState.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/memory/TraceMemoryState.java index 694fe598e7..625048add5 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/memory/TraceMemoryState.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/memory/TraceMemoryState.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,17 +15,42 @@ */ package ghidra.trace.model.memory; +import java.util.stream.Stream; + public enum TraceMemoryState { /** * The value was not observed at the snapshot */ - UNKNOWN, + UNKNOWN(true, false), /** * The value was observed at the snapshot */ - KNOWN, + KNOWN(false, true), /** * The value could not be observed at the snapshot */ - ERROR, + ERROR(false, false); + + public static final TraceMemoryState IMPLIED_BY_NULL = + Stream.of(values()).filter(TraceMemoryState::impliedByNull).findFirst().orElseThrow(); + + public static TraceMemoryState orImplied(TraceMemoryState s) { + return s == null ? IMPLIED_BY_NULL : s; + } + + private final boolean impliedByNull; + private final boolean truncates; + + private TraceMemoryState(boolean impliedByNull, boolean truncates) { + this.impliedByNull = impliedByNull; + this.truncates = truncates; + } + + public boolean impliedByNull() { + return impliedByNull; + } + + public boolean truncates() { + return truncates; + } } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/test/java/docking/widgets/model/DemoFieldsTest.java b/Ghidra/Debug/Framework-TraceModeling/src/test/java/docking/widgets/model/DemoFieldsTest.java index 2b49385205..faae03bb55 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/test/java/docking/widgets/model/DemoFieldsTest.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/test/java/docking/widgets/model/DemoFieldsTest.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,13 +22,13 @@ import java.awt.BorderLayout; import javax.swing.JButton; import javax.swing.JFrame; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import ghidra.test.AbstractGhidraHeadedIntegrationTest; import ghidra.util.Msg; import ghidra.util.SystemUtilities; +@Ignore public class DemoFieldsTest extends AbstractGhidraHeadedIntegrationTest { @Before public void checkNotBatch() { diff --git a/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/pcode/exec/trace/BytesTracePcodeEmulatorTest.java b/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/pcode/exec/trace/BytesTracePcodeEmulatorTest.java index 55adbeaf23..e5086beeba 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/pcode/exec/trace/BytesTracePcodeEmulatorTest.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/pcode/exec/trace/BytesTracePcodeEmulatorTest.java @@ -33,8 +33,7 @@ import ghidra.pcode.emu.PcodeThread; import ghidra.pcode.exec.*; import ghidra.pcode.exec.PcodeExecutorStatePiece.Reason; import ghidra.pcode.exec.trace.TraceEmulationIntegration.Writer; -import ghidra.program.model.address.Address; -import ghidra.program.model.address.AddressRangeImpl; +import ghidra.program.model.address.*; import ghidra.program.model.lang.*; import ghidra.trace.database.ToyDBTraceBuilder; import ghidra.trace.database.context.DBTraceRegisterContextManager; @@ -1089,4 +1088,27 @@ public class BytesTracePcodeEmulatorTest extends AbstractTracePcodeEmulatorTest TraceSleighUtils.evaluate("r2", tb.trace, 1, thread, 0)); } } + + @Test + public void testReadUninit() throws Throwable { + try (ToyDBTraceBuilder tb = new ToyDBTraceBuilder("Test", "Toy:BE:64:default")) { + initTrace(tb, """ + r0 = 0x00600000; + *:8 r0 = 0x1122334455667788; + """, + List.of()); + + Writer writer = createWriter(tb.host, 0); + PcodeEmulator emu = createEmulator(tb.host, writer); + + AddressSpace space = emu.getLanguage().getDefaultDataSpace(); + + // Cause a gap in "known-but-uninitialized" + assertArrayEquals(bytes(0x33, 0x44, 0x55, 0x66), + emu.getSharedState().getVar(space, 0x00600002, 4, false, Reason.EXECUTE_READ)); + // Now validate the read across that gap + assertArrayEquals(bytes(0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88), + emu.getSharedState().getVar(space, 0x00600000, 8, false, Reason.EXECUTE_READ)); + } + } } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/trace/database/ToyDBTraceBuilder.java b/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/trace/database/ToyDBTraceBuilder.java index 8b32ff73e5..0f461cef41 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/trace/database/ToyDBTraceBuilder.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/trace/database/ToyDBTraceBuilder.java @@ -467,15 +467,15 @@ public class ToyDBTraceBuilder implements AutoCloseable { } /** - * Create an address-span box in the trace's default space with a singleton snap + * Create an address-span box in the trace's default space with a span * - * @param snap the snap + * @param span the span * @param start the start address offset * @param end the end address offset * @return the box */ - public TraceAddressSnapRange srange(long snap, long start, long end) { - return new ImmutableTraceAddressSnapRange(addr(start), addr(end), snap, snap); + public TraceAddressSnapRange srange(Lifespan span, long start, long end) { + return new ImmutableTraceAddressSnapRange(addr(start), addr(end), span); } /** diff --git a/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/trace/database/memory/AbstractDBTraceMemoryManagerMemoryTest.java b/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/trace/database/memory/AbstractDBTraceMemoryManagerMemoryTest.java index e6f4f9d5dc..549fbe316e 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/trace/database/memory/AbstractDBTraceMemoryManagerMemoryTest.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/trace/database/memory/AbstractDBTraceMemoryManagerMemoryTest.java @@ -23,7 +23,9 @@ import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.Map.Entry; +import org.junit.Ignore; import org.junit.Test; import db.DBHandle; @@ -36,6 +38,7 @@ import ghidra.trace.database.DBTrace; import ghidra.trace.model.Lifespan; import ghidra.trace.model.TraceAddressSnapRange; import ghidra.trace.model.memory.*; +import ghidra.trace.model.memory.TraceMemoryOperations.StatePredicate; import ghidra.trace.model.thread.TraceThread; import ghidra.trace.model.thread.TraceThreadManager; import ghidra.util.SystemUtilities; @@ -336,17 +339,18 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest collectAsMap(memory.getMostRecentStates(2, b.range(0x2800, 0x9000)))); expected = new HashMap<>(); - expected.put(b.srange(3, 0x4000, 0x4fff), TraceMemoryState.KNOWN); - expected.put(b.srange(3, 0x5000, 0x6000), TraceMemoryState.ERROR); - expected.put(b.srange(3, 0x6001, 0x7000), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.at(3), 0x4000, 0x4800), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4801, 0x4fff), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x5000, 0x6000), TraceMemoryState.ERROR); + expected.put(b.srange(Lifespan.nowOn(3), 0x6001, 0x7000), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getMostRecentStates(3, b.range(0x2800, 0x9000)))); expected = new HashMap<>(); - expected.put(b.srange(4, 0x3000, 0x4800), TraceMemoryState.KNOWN); - expected.put(b.srange(3, 0x4801, 0x4fff), TraceMemoryState.KNOWN); - expected.put(b.srange(3, 0x5000, 0x6000), TraceMemoryState.ERROR); - expected.put(b.srange(3, 0x6001, 0x7000), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(4), 0x3000, 0x4800), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4801, 0x4fff), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x5000, 0x6000), TraceMemoryState.ERROR); + expected.put(b.srange(Lifespan.nowOn(3), 0x6001, 0x7000), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getMostRecentStates(4, b.range(0x2800, 0x9000)))); assertEquals(expected, @@ -373,14 +377,14 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest expected = new AddressSet(); expected.add(b.range(0x3000, 0x3800)); expected.add(b.range(0x3c00, 0x4800)); - result = memory.getAddressesWithState(4, set, state -> state == TraceMemoryState.KNOWN); + result = memory.getAddressesWithState(4, set, StatePredicate.IS_KNOWN); assertEquals(expected, set.intersect(result)); expected = new AddressSet(); expected.add(b.range(0x4000, 0x4800)); expected.add(b.range(0x4c00, 0x4fff)); expected.add(b.range(0x6001, 0x6100)); - result = memory.getAddressesWithState(3, set, state -> state == TraceMemoryState.KNOWN); + result = memory.getAddressesWithState(3, set, StatePredicate.IS_KNOWN); assertEquals(expected, set.intersect(result)); // Test gaps @@ -388,7 +392,7 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest expected.add(b.range(0x2800, 0x3800)); expected.add(b.range(0x3c00, 0x3fff)); expected.add(b.range(0x8000, 0x9000)); - result = memory.getAddressesWithState(3, set, state -> true); + result = memory.getAddressesWithState(3, set, StatePredicate.IS_KNOWN_OR_ERROR); assertEquals(expected, set.subtract(result)); } @@ -402,9 +406,9 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest Map expected; expected = new HashMap<>(); - expected.put(b.srange(3, 0x4000, 0x4fff), TraceMemoryState.KNOWN); - expected.put(b.srange(3, 0x5000, 0x6000), TraceMemoryState.ERROR); - expected.put(b.srange(3, 0x6001, 0x7000), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4000, 0x4fff), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x5000, 0x6000), TraceMemoryState.ERROR); + expected.put(b.srange(Lifespan.nowOn(3), 0x6001, 0x7000), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getStates(3, b.range(0x3000, 0x8000)))); } @@ -438,7 +442,7 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest // verify the corresponding change in state; Map expected; expected = new HashMap<>(); - expected.put(b.srange(3, 0x4000, 0x4003), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4000, 0x4003), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getStates(3, b.range(0x3000, 0x5000)))); ByteBuffer read = b.buf(-1, -2, -3, -4); // Verify zeros actually written @@ -459,7 +463,7 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest // verify the corresponding change in state; Map expected; expected = new HashMap<>(); - expected.put(b.srange(3, 0x4000, 0x4003), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4000, 0x4003), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getStates(3, b.range(0x3000, 0x5000)))); ByteBuffer read = ByteBuffer.allocate(4); @@ -534,10 +538,10 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest assertEquals(expected, collectAsMap(memory.getStates(6, b.range(0x3000, 0x5000)))); expected = new HashMap<>(); - expected.put(b.srange(3, 0x4000, 0x4003), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.span(3, 4), 0x4000, 0x4003), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getStates(3, b.range(0x3000, 0x5000)))); expected = new HashMap<>(); - expected.put(b.srange(5, 0x4000, 0x4003), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(5), 0x4000, 0x4003), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getStates(5, b.range(0x3000, 0x5000)))); ByteBuffer read = b.buf(0, 0, 0, 0); @@ -735,6 +739,16 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest b.buf(-1, -1, -1), true, TaskMonitor.DUMMY)); } + protected void dumpStates() { + System.err.println("STATES"); + for (DBTraceMemorySpace space : memory.getActiveSpaces()) { + for (Entry entry : space.stateMapSpace + .entries()) { + System.err.println(" " + entry); + } + } + } + @Test public void testRemoveBytes() { try (Transaction tx = b.startTransaction()) { @@ -783,9 +797,9 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest // Check overall effect on state Map expected; expected = new HashMap<>(); - expected.put(b.srange(2, 0x47fe, 0x47fe), TraceMemoryState.KNOWN); - expected.put(b.srange(4, 0x4800, 0x4803), TraceMemoryState.KNOWN); - expected.put(b.srange(3, 0x4804, 0x4805), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(2), 0x47fe, 0x47fe), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(4), 0x4800, 0x4803), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4804, 0x4805), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getMostRecentStates(6, b.range(0x4700, 0x4900)))); } @@ -814,7 +828,7 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest // verify the corresponding change in state; Map expected; expected = new HashMap<>(); - expected.put(b.srange(3, 0x4000, 0x4003), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4000, 0x4003), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getStates(3, b.range(0x3000, 0x5000)))); ByteBuffer read = ByteBuffer.allocate(4); @@ -838,7 +852,7 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest // verify the corresponding change in state; Map expected; expected = new HashMap<>(); - expected.put(b.srange(3, 0x4000, 0x4003), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4000, 0x4003), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getStates(3, b.range(0x3000, 0x5000)))); tx.abort(); @@ -866,7 +880,7 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest // verify the corresponding change in state; Map expected; expected = new HashMap<>(); - expected.put(b.srange(3, 0x4000, 0x4003), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4000, 0x4003), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getStates(3, b.range(0x3000, 0x5000)))); } b.trace.undo(); @@ -894,7 +908,7 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest // verify the corresponding change in state; Map expected; expected = new HashMap<>(); - expected.put(b.srange(3, 0x4000, 0x4003), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4000, 0x4003), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getStates(3, b.range(0x3000, 0x5000)))); } b.trace.undo(); @@ -917,7 +931,7 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest assertEquals(1, getBufferRecordCount()); expected = new HashMap<>(); - expected.put(b.srange(3, 0x4000, 0x4003), TraceMemoryState.KNOWN); + expected.put(b.srange(Lifespan.nowOn(3), 0x4000, 0x4003), TraceMemoryState.KNOWN); assertEquals(expected, collectAsMap(memory.getStates(3, b.range(0x3000, 0x5000)))); read.position(0); @@ -1003,6 +1017,7 @@ public abstract class AbstractDBTraceMemoryManagerMemoryTest * @throws Exception because */ @Test + @Ignore("Developer's desk") public void testReplicateClassCastExceptionScenario() throws Exception { final int TICKS = 100_000; diff --git a/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/AbstractConstraintsTree.java b/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/AbstractConstraintsTree.java index 2d25325e1d..3edd80174d 100644 --- a/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/AbstractConstraintsTree.java +++ b/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/AbstractConstraintsTree.java @@ -22,7 +22,6 @@ import java.util.function.Consumer; import java.util.stream.Stream; import db.DBRecord; -import generic.util.FlattenedIterator; import generic.util.PeekableIterator; import ghidra.util.LockHold; import ghidra.util.database.*; @@ -44,9 +43,9 @@ public abstract class AbstractConstraintsTree< protected final DBCachedObjectStore dataStore; protected final DBCachedObjectStore nodeStore; - protected final Map> cachedDataChildren = Collections.synchronizedMap( + protected final Map> cachedDataChildren = Collections.synchronizedMap( new FixedSizeHashMap<>(MAX_CACHE_ENTRIES)); - protected final Map> cachedNodeChildren = Collections.synchronizedMap( + protected final Map> cachedNodeChildren = Collections.synchronizedMap( new FixedSizeHashMap<>(MAX_CACHE_ENTRIES)); protected NR root; @@ -95,8 +94,8 @@ public abstract class AbstractConstraintsTree< * @return a collection of the children */ protected Collection getNodeChildrenOf(NR parent) { - return cachedNodeChildren.computeIfAbsent(parent.getKey(), - k -> new ArrayList<>(getNodeChildrenOf(k))); + return Collections.unmodifiableList(cachedNodeChildren.computeIfAbsent(parent.getKey(), + k -> new ArrayList<>(getNodeChildrenOf(k)))); } /** @@ -121,9 +120,9 @@ public abstract class AbstractConstraintsTree< * @param parent the parent node * @return a collection of the children */ - protected Collection getDataChildrenOf(NR parent) { - return cachedDataChildren.computeIfAbsent(parent.getKey(), - k -> new ArrayList<>(getDataChildrenOf(k))); + protected List getDataChildrenOf(NR parent) { + return Collections.unmodifiableList(cachedDataChildren.computeIfAbsent(parent.getKey(), + k -> new ArrayList<>(getDataChildrenOf(k)))); } /** @@ -209,51 +208,34 @@ public abstract class AbstractConstraintsTree< } protected abstract VisitResult visitData(NR parent, DR d, boolean included); + + protected Iterator iterateNodes(Stream nodes) { + return nodes.iterator(); + } + + protected Iterator iterateData(Stream data) { + return data.iterator(); + } } protected VisitResult visit(Q query, TreeRecordVisitor visitor, boolean ordered) { return visit(null, root, query, visitor, ordered); } - protected VisitResult visit(NR parent, NR node, Q query, TreeRecordVisitor visitor, - boolean ordered) { - QueryInclusion inclusion = - query == null ? QueryInclusion.ALL : query.testNode(node.getShape()); - VisitResult r = visitor.beginNode(parent, node, inclusion); - if (r != VisitResult.DESCEND) { - return r; - } - if (node.getType().isLeaf()) { - List data = new ArrayList<>(getDataChildrenOf(node)); - if (query != null && ordered) { - data.sort(Comparator.comparing(DR::getBounds, query.getBoundsComparator())); + protected VisitResult visitLeaf(NR parent, NR node, Q query, TreeRecordVisitor visitor, + boolean ordered, QueryInclusion inclusion) { + Stream data = getDataChildrenOf(node).stream(); + if (query != null) { + data = data.filter(d -> query.testData(d.getShape())); + if (ordered) { + data = + data.sorted(Comparator.comparing(DR::getBounds, query.getBoundsComparator())); } - for (DR d : data) { - if (query != null && ordered && query.terminateEarlyData(d.getShape())) { - break; - } - boolean included = query == null || query.testData(d.getShape()); - r = visitor.visitData(node, d, included); - if (r == VisitResult.ASCEND) { - return visitor.endNode(parent, node, inclusion); - } - if (r == VisitResult.TERMINATE) { - visitor.endNode(parent, node, inclusion); - return r; - } - } - return visitor.endNode(parent, node, inclusion); } - assert node.getType().isDirectory(); - List nodes = new ArrayList<>(getNodeChildrenOf(node)); - if (query != null && ordered) { - nodes.sort(Comparator.comparing(NR::getBounds, query.getBoundsComparator())); - } - for (NR n : nodes) { - if (query != null && ordered && query.terminateEarlyNode(n.getShape())) { - break; - } - r = visit(node, n, query, visitor, ordered); + Iterator dit = visitor.iterateData(data); + while (dit.hasNext()) { + DR d = dit.next(); + VisitResult r = visitor.visitData(node, d, true); if (r == VisitResult.ASCEND) { return visitor.endNode(parent, node, inclusion); } @@ -265,29 +247,66 @@ public abstract class AbstractConstraintsTree< return visitor.endNode(parent, node, inclusion); } + protected VisitResult visitDirectory(NR parent, NR node, Q query, TreeRecordVisitor visitor, + boolean ordered, QueryInclusion inclusion) { + Stream nodes = getNodeChildrenOf(node).stream(); + if (query != null) { + nodes = nodes.filter(n -> query.testNode(n.getShape()) != QueryInclusion.NONE); + if (ordered) { + nodes = + nodes.sorted(Comparator.comparing(NR::getBounds, query.getBoundsComparator())); + } + } + Iterator nit = visitor.iterateNodes(nodes); + while (nit.hasNext()) { + NR n = nit.next(); + VisitResult r = visit(node, n, query, visitor, ordered); + if (r == VisitResult.ASCEND) { + return visitor.endNode(parent, node, inclusion); + } + if (r == VisitResult.TERMINATE) { + visitor.endNode(parent, node, inclusion); + return r; + } + } + return visitor.endNode(parent, node, inclusion); + } + + protected VisitResult visit(NR parent, NR node, Q query, TreeRecordVisitor visitor, + boolean ordered) { + QueryInclusion inclusion = + query == null ? QueryInclusion.ALL : query.testNode(node.getShape()); + VisitResult r = visitor.beginNode(parent, node, inclusion); + if (r != VisitResult.DESCEND) { + return r; + } + if (node.getType().isLeaf()) { + return visitLeaf(parent, node, query, visitor, ordered, inclusion); + } + assert node.getType().isDirectory(); + return visitDirectory(parent, node, query, visitor, ordered, inclusion); + } + protected Iterator iterator(Q query) { return iterator(root, query); } - protected Iterator iterator(NR node, Q query) { + protected Stream stream(NR node, Q query) { if (node.getType().isLeaf()) { - List data = new ArrayList<>(node.getChildCount()); - for (DR d : getDataChildrenOf(node)) { - if (query != null && !query.testData(d.getShape())) { - continue; - } - data.add(d); + Collection data = getDataChildrenOf(node); + if (query == null) { + return data.stream(); } - return data.iterator(); + return data.stream().filter(d -> query.testData(d.getShape())); } - List nodes = new ArrayList<>(node.getChildCount()); - for (NR n : getNodeChildrenOf(node)) { - if (query != null && query.testNode(n.getShape()) == QueryInclusion.NONE) { - continue; - } - nodes.add(n); - } - return FlattenedIterator.start(nodes.iterator(), n -> iterator(n, query)); + Collection nodes = getNodeChildrenOf(node); + Stream passing = query == null ? nodes.stream() + : nodes.stream().filter(n -> query.testNode(n.getShape()) != QueryInclusion.NONE); + return passing.flatMap(n -> stream(n, query)); + } + + protected Iterator iterator(NR node, Q query) { + return stream(node, query).iterator(); } protected Iterator orderedIterator(Q query) { @@ -574,7 +593,7 @@ public abstract class AbstractConstraintsTree< } protected void doRemoveFromCachedChildren(long parentKey, R child, - Map> cache) { + Map> cache) { Collection children = cache.get(parentKey); if (children == null) { return; @@ -585,7 +604,7 @@ public abstract class AbstractConstraintsTree< } protected void doAddToCachedChildren(long parentKey, R child, - Map> cache) { + Map> cache) { Collection children = cache.get(parentKey); if (children == null) { return; @@ -596,7 +615,7 @@ public abstract class AbstractConstraintsTree< } protected > void doSetParentKey(R child, long key, - Map> cache) { + Map> cache) { doRemoveFromCachedChildren(child.getParentKey(), child, cache); child.setParentKey(key); doAddToCachedChildren(key, child, cache); @@ -721,6 +740,16 @@ public abstract class AbstractConstraintsTree< } return VisitResult.NEXT; } + + @Override + protected Iterator iterateNodes(Stream nodes) { + return nodes.toList().iterator(); + } + + @Override + protected Iterator iterateData(Stream data) { + return data.toList().iterator(); + } }, false); } @@ -876,7 +905,7 @@ public abstract class AbstractConstraintsTree< public void checkIntegrity() { synchronized (cachedDataChildren) { // Before we visit, integrity check that cache. Visiting will affect cache. - for (Entry> ent : cachedDataChildren.entrySet()) { + for (Entry> ent : cachedDataChildren.entrySet()) { Set databasedChildren = new TreeSet<>(Comparator.comparing(DR::getKey)); // NOTE: Bypass the cache by using the variant with a key parameter databasedChildren.addAll(getDataChildrenOf(ent.getKey())); @@ -889,7 +918,7 @@ public abstract class AbstractConstraintsTree< } } synchronized (cachedNodeChildren) { - for (Entry> ent : cachedNodeChildren.entrySet()) { + for (Entry> ent : cachedNodeChildren.entrySet()) { Set databasedChildren = new TreeSet<>(Comparator.comparing(NR::getKey)); // NOTE: Bypass the cache by using the variant with a key parameter databasedChildren.addAll(getNodeChildrenOf(ent.getKey())); diff --git a/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/AbstractConstraintsTreeSpatialMap.java b/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/AbstractConstraintsTreeSpatialMap.java index 7f0145224f..ec8ae83435 100644 --- a/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/AbstractConstraintsTreeSpatialMap.java +++ b/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/AbstractConstraintsTreeSpatialMap.java @@ -149,15 +149,21 @@ public abstract class AbstractConstraintsTreeSpatialMap< public Object[] toArray() { try (LockHold hold = LockHold.lock(tree.dataStore.readLock())) { - // Note, computing size requires a traversal. Bad idea, I think. - List> result = new ArrayList<>(); - tree.visitAllData(query, new ToListConsumer<>(result) { - @Override - protected Entry transformed(DR t) { + /** + * One one hand, traversing twice (one to compute the size, and again to + * retrieve the elements) is not great. On the other, resizing the array list + * many times is also not great. Which is worse? IDK. LATER: Take some + * measurements with large traces. + */ + int size = AbstractConstraintsTreeSpatialMap.this.size(); + Object[] a = new Object[size]; + ToArrayConsumer consumer = new ToArrayConsumer<>(a) { + protected Object transformed(DR t) { return t.asEntry(); - } - }, false); - return result.toArray(); + }; + }; + tree.visitAllData(query, consumer, false); + return a; } } diff --git a/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/DBTreeDataRecord.java b/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/DBTreeDataRecord.java index f54f84ebc6..67dbb89608 100644 --- a/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/DBTreeDataRecord.java +++ b/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/DBTreeDataRecord.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -123,7 +123,7 @@ public abstract class DBTreeDataRecord, NS extends B protected abstract T getRecordValue(); protected RecordEntry asEntry() { - return entry; + return Objects.requireNonNull(entry); } @Override diff --git a/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/rect/AbstractRectangle2DQuery.java b/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/rect/AbstractRectangle2DQuery.java index 4d0eb39738..28e80d1534 100644 --- a/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/rect/AbstractRectangle2DQuery.java +++ b/Ghidra/Debug/ProposedUtils/src/main/java/ghidra/util/database/spatial/rect/AbstractRectangle2DQuery.java @@ -59,6 +59,15 @@ public abstract class AbstractRectangle2DQuery< return factory.create(r1, r2, direction); } + protected static , + Q extends AbstractRectangle2DQuery> Q intersectingEnclosed( + NS rect, Rectangle2DDirection direction, QueryFactory factory) { + Rectangle2D full = rect.getSpace().getFull(); + NS r1 = rect.immutable(full.getX1(), rect.getX2(), rect.getY1(), full.getY2()); + NS r2 = rect.immutable(rect.getX1(), full.getX2(), full.getY1(), rect.getY2()); + return factory.create(r1, r2, direction); + } + protected static , Q extends AbstractRectangle2DQuery> Q equalTo( NS rect, Rectangle2DDirection direction, QueryFactory factory) { diff --git a/Ghidra/Debug/ProposedUtils/src/main/java/utilities/util/DbgMsgTracer.java b/Ghidra/Debug/ProposedUtils/src/main/java/utilities/util/DbgMsgTracer.java index 6a21167865..76ec26c7b8 100644 --- a/Ghidra/Debug/ProposedUtils/src/main/java/utilities/util/DbgMsgTracer.java +++ b/Ghidra/Debug/ProposedUtils/src/main/java/utilities/util/DbgMsgTracer.java @@ -44,7 +44,7 @@ public class DbgMsgTracer { } private void doMsg(Object obj, String message) { - Msg.info(obj, "%s %s".formatted(prefixStack(), message)); + Msg.info(obj, "%s %s %s".formatted(Thread.currentThread(), prefixStack(), message)); } public record CallRec(DbgMsgTracer tracer, Object obj, String name, long start) @@ -53,7 +53,15 @@ public class DbgMsgTracer { public void close() { long stop = System.currentTimeMillis(); long elapsedMs = stop - start; - tracer.doMsg(obj, "%d: (EXITED) after %f s".formatted(stop, elapsedMs / 1000.0)); + String extra; + if (elapsedMs > 100) { + extra = " (LONG)"; + } + else { + extra = ""; + } + tracer.doMsg(obj, + "%d: (EXITED) after %f s%s".formatted(stop, elapsedMs / 1000.0, extra)); CallRec popped = tracer.stack.pop(); assert popped == this; } diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressRangeChunker.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressRangeChunker.java index 12c58d37fc..0a1122486b 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressRangeChunker.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressRangeChunker.java @@ -19,6 +19,8 @@ import java.util.*; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import ghidra.util.MathUtilities; + /** * A class to break a range of addresses into 'chunks' of a give size. This is useful to break-up * processing of large swaths of addresses, such as when performing work in a background thread. @@ -29,13 +31,14 @@ public class AddressRangeChunker implements Iterable { private Address end; private Address nextStartAddress; - private int chunkSize; + private long chunkSizeUnsigned; - public AddressRangeChunker(AddressRange range, int chunkSize) throws IllegalArgumentException { - this(range.getMinAddress(), range.getMaxAddress(), chunkSize); + public AddressRangeChunker(AddressRange range, long chunkSizeUnsigned) + throws IllegalArgumentException { + this(range.getMinAddress(), range.getMaxAddress(), chunkSizeUnsigned); } - public AddressRangeChunker(Address start, Address end, int chunkSize) + public AddressRangeChunker(Address start, Address end, long chunkSizeUnsigned) throws IllegalArgumentException { if (start == null) { @@ -58,13 +61,13 @@ public class AddressRangeChunker implements Iterable { throw new IllegalArgumentException("Address must be in the same address space"); } - if (chunkSize < 1) { + if (chunkSizeUnsigned == 0) { throw new IllegalArgumentException("Chunk size must be greater than 0"); } this.end = end; this.nextStartAddress = start; - this.chunkSize = chunkSize; + this.chunkSizeUnsigned = chunkSizeUnsigned; } @Override @@ -82,15 +85,12 @@ public class AddressRangeChunker implements Iterable { return null; } - long available = end.subtract(nextStartAddress) + 1; // +1 to be inclusive + long availableLess1 = end.subtract(nextStartAddress); - int size = chunkSize; - if (available >= 0 && available < chunkSize) { - size = (int) available; - } + long sizeLess1 = MathUtilities.unsignedMin(chunkSizeUnsigned - 1, availableLess1); Address currentStart = nextStartAddress; - Address currentEnd = nextStartAddress.add(size - 1); // -1 since inclusive + Address currentEnd = nextStartAddress.addWrap(sizeLess1); if (currentEnd.compareTo(end) == 0) { nextStartAddress = null; // no more } @@ -111,8 +111,14 @@ public class AddressRangeChunker implements Iterable { @Override public Spliterator spliterator() { - long countAddrs = end.subtract(nextStartAddress) + 1; - long size = Long.divideUnsigned(countAddrs + chunkSize - 1, chunkSize); + long countAddrsLess1 = end.subtract(nextStartAddress); + // Can't do the (count+size-1)/size thing since count+size may overflow + long size = Long.divideUnsigned(countAddrsLess1, chunkSizeUnsigned) + 1; + if (size <= 0) { + // Known but too big to encode in (signed) long. 0 is actually 2**64. + return Spliterators.spliteratorUnknownSize(iterator(), Spliterator.DISTINCT | + Spliterator.NONNULL | Spliterator.ORDERED | Spliterator.SORTED); + } return Spliterators.spliterator(iterator(), size, Spliterator.DISTINCT | Spliterator.NONNULL | Spliterator.ORDERED | Spliterator.SORTED | Spliterator.SIZED); diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressSetView.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressSetView.java index c73164547b..189e79f4ef 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressSetView.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressSetView.java @@ -234,7 +234,7 @@ public interface AddressSetView extends Iterable { public boolean intersects(AddressSetView addrSet); /** - * Determine if the start and end range intersects with the specified address set. + * Determine if the start and end range intersects with this address set. *

* The specified start and end addresses must form a valid range within a single * {@link AddressSpace}. @@ -245,6 +245,16 @@ public interface AddressSetView extends Iterable { */ public boolean intersects(Address start, Address end); + /** + * Determine if the range intersects with this address set. + * + * @param range the range + * @return true if the given range intersects this address set. + */ + default public boolean intersects(AddressRange range) { + return intersects(range.getMinAddress(), range.getMaxAddress()); + } + /** * Computes the intersection of this address set with the given address set. *

@@ -269,6 +279,19 @@ public interface AddressSetView extends Iterable { */ public AddressSet intersectRange(Address start, Address end); + /** + * Computes the intersection of this address set with the given address range. + *

+ * This method does not modify this address set. + * + * @param range the range + * @return AddressSet a new address set that contains all addresses that are contained in both + * this set and the given range. + */ + default public AddressSet intersectRange(AddressRange range) { + return intersectRange(range.getMinAddress(), range.getMaxAddress()); + } + /** * Computes the union of this address set with the given address set. *

diff --git a/Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/address/AddressRangeImplTest.java b/Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/address/AddressRangeImplTest.java index 55c799225d..485a212a13 100644 --- a/Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/address/AddressRangeImplTest.java +++ b/Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/address/AddressRangeImplTest.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package ghidra.program.model.address; import static org.junit.Assert.*; import java.util.Iterator; +import java.util.List; import org.junit.*; @@ -28,6 +29,7 @@ public class AddressRangeImplTest extends AbstractGenericTest { /** * Constructor for AddressRangeImplTest. + * * @param name */ public AddressRangeImplTest() { @@ -37,7 +39,7 @@ public class AddressRangeImplTest extends AbstractGenericTest { @Before public void setUp() throws Exception { - space = new GenericAddressSpace("xx", 32, AddressSpace.TYPE_RAM, 0); + space = new GenericAddressSpace("xx", 64, AddressSpace.TYPE_RAM, 0); } @Test @@ -159,6 +161,37 @@ public class AddressRangeImplTest extends AbstractGenericTest { assertFalse(it.hasNext()); } + @Test + public void testAddressRangeChunker_GiganticIn2Chunks() { + AddressRangeChunker chunker = new AddressRangeChunker(rng(0, -1), Long.MIN_VALUE); + assertEqualRanges(List.of(rng(0, Long.MAX_VALUE), rng(Long.MIN_VALUE, -1)), chunker); + } + + @Test + public void testAddressRangeChunker_GiganticWithSmallChunks_DontExhaust() { + AddressRangeChunker chunker = new AddressRangeChunker(rng(0, -1), 4096); + // 1 is "64th" bit, >>> 12 into 52nd, so 0x 1 with 13 trailing 0s. + assertEquals(0x0010_0000_0000_0000L, chunker.spliterator().estimateSize()); + List first4 = chunker.stream().limit(4).toList(); + assertEquals( + List.of(rng(0, 0x0fff), rng(0x1000, 0x1fff), rng(0x2000, 0x2fff), rng(0x3000, 0x3fff)), + first4); + } + + @Test + public void testAddressRangeChunker_GiganticWithMaxChunkSize() { + AddressRangeChunker chunker = new AddressRangeChunker(rng(0, -1), -1); + assertEqualRanges(List.of(rng(0, -2), rng(-1, -1)), chunker); + } + + @Test + public void testAddressRangeChunker_GiganticWithUnsignedChunkSize() { + AddressRangeChunker chunker = new AddressRangeChunker(rng(0, -1), Long.MIN_VALUE + 1); + assertEqualRanges( + List.of(rng(0, Long.MIN_VALUE), rng(Long.MIN_VALUE + 1, -1)), + chunker); + } + @Test public void testAddressRangeChunker_NullAddresses() { try { @@ -180,13 +213,7 @@ public class AddressRangeImplTest extends AbstractGenericTest { @Test public void testAddressRangeChunker_BadChunkSize() { - try { - new AddressRangeChunker(addr(0), addr(1), -1); - Assert.fail("Did not get exception when passing bad chunk size to chunker."); - } - catch (IllegalArgumentException e) { - // good! - } + // NOTE: "Negative" chunk sizes are not possible, because chunkSize is treated unsigned try { new AddressRangeChunker(addr(0), addr(1), 0); @@ -220,8 +247,8 @@ public class AddressRangeImplTest extends AbstractGenericTest { try { new AddressRangeChunker(a1, a2, 10); - Assert.fail("Did not get exception when passing addresses from different address " - + "spaces to chunker."); + Assert.fail("Did not get exception when passing addresses from different address " + + "spaces to chunker."); } catch (IllegalArgumentException e) { // good! @@ -260,8 +287,8 @@ public class AddressRangeImplTest extends AbstractGenericTest { assertTrue( "Address Iterator does not properly enumerate address range: " + - String.format("%s (%d long) -- found %d", r1.toString(), r1.getLength(), addrCount), - addrCount == (size + 1)); + String.format("%s (%d long) -- found %d", r1.toString(), r1.getLength(), addrCount), + addrCount == (size + 1)); } @Test @@ -281,8 +308,16 @@ public class AddressRangeImplTest extends AbstractGenericTest { assertTrue("Address Iterator extent does not match end of range", lastAddr.equals(limit)); } - private Address addr(int a) { + private Address addr(long a) { return new GenericAddress(space, a); } + private AddressRange rng(long s, long e) { + return new AddressRangeImpl(addr(s), addr(e)); + } + + private void assertEqualRanges(List expected, AddressRangeChunker actual) { + List justEnough = actual.stream().limit(expected.size() + 1).toList(); + assertEquals(expected, justEnough); + } } diff --git a/Ghidra/Test/DebuggerIntegrationTest/src/test.slow/java/agent/gdb/rmi/GdbCommandsTest.java b/Ghidra/Test/DebuggerIntegrationTest/src/test.slow/java/agent/gdb/rmi/GdbCommandsTest.java index 16bf179c5f..548440a340 100644 --- a/Ghidra/Test/DebuggerIntegrationTest/src/test.slow/java/agent/gdb/rmi/GdbCommandsTest.java +++ b/Ghidra/Test/DebuggerIntegrationTest/src/test.slow/java/agent/gdb/rmi/GdbCommandsTest.java @@ -17,7 +17,7 @@ package agent.gdb.rmi; import static org.hamcrest.Matchers.*; import static org.junit.Assert.*; -import static org.junit.Assume.*; +import static org.junit.Assume.assumeFalse; import java.nio.ByteBuffer; import java.util.*; @@ -247,13 +247,11 @@ public class GdbCommandsTest extends AbstractGdbTraceRmiTest { quit """.formatted(PREAMBLE, target)); String importSection = extractOutSection(out, "---Import---"); - assertTrue(importSection.contains( - """ + assertTrue(importSection.contains(""" Selected Ghidra language: x86:LE:32:default Selected Ghidra compiler: %s""".formatted(PLAT.cSpec()))); String fileSection = extractOutSection(out, "---File---"); - assertTrue(fileSection.contains( - """ + assertTrue(fileSection.contains(""" Selected Ghidra language: %s Selected Ghidra compiler: %s""".formatted(PLAT.lang(), PLAT.cSpec()))); assertEquals(""" @@ -425,7 +423,7 @@ public class GdbCommandsTest extends AbstractGdbTraceRmiTest { Entry entry = tb.trace.getMemoryManager().getMostRecentStateEntry(snap, addr); assertEquals(Map.entry(new ImmutableTraceAddressSnapRange( - quantize(rng(addr, 10), 4096), Lifespan.at(0)), TraceMemoryState.ERROR), entry); + quantize(rng(addr, 10), 4096), Lifespan.nowOn(0)), TraceMemoryState.ERROR), entry); } } diff --git a/Ghidra/Test/DebuggerIntegrationTest/src/test.slow/java/agent/lldb/rmi/LldbCommandsTest.java b/Ghidra/Test/DebuggerIntegrationTest/src/test.slow/java/agent/lldb/rmi/LldbCommandsTest.java index 09195ce840..b2fd8285ea 100644 --- a/Ghidra/Test/DebuggerIntegrationTest/src/test.slow/java/agent/lldb/rmi/LldbCommandsTest.java +++ b/Ghidra/Test/DebuggerIntegrationTest/src/test.slow/java/agent/lldb/rmi/LldbCommandsTest.java @@ -17,7 +17,7 @@ package agent.lldb.rmi; import static org.hamcrest.Matchers.*; import static org.junit.Assert.*; -import static org.junit.Assume.*; +import static org.junit.Assume.assumeFalse; import java.nio.ByteBuffer; import java.util.*; @@ -384,7 +384,7 @@ public class LldbCommandsTest extends AbstractLldbTraceRmiTest { Entry entry = tb.trace.getMemoryManager().getMostRecentStateEntry(snap, addr); assertEquals(Map.entry(new ImmutableTraceAddressSnapRange( - quantize(rng(addr, 10), 4096), Lifespan.at(0)), TraceMemoryState.ERROR), entry); + quantize(rng(addr, 10), 4096), Lifespan.nowOn(0)), TraceMemoryState.ERROR), entry); } } @@ -900,18 +900,17 @@ public class LldbCommandsTest extends AbstractLldbTraceRmiTest { """.formatted(PREAMBLE, addr, getSpecimenPrint())); try (ManagedDomainObject mdo = openDomainObject(projectName("expPrint"))) { tb = new ToyDBTraceBuilder((Trace) mdo.get()); - assertEquals( - """ - Parent Key Span Value Type - Test.Objects[1] vaddr [0,+inf) ram:0000dead ADDRESS - Test.Objects[1] vbool [0,+inf) True BOOL - Test.Objects[1] vbyte [0,+inf) 1 BYTE - Test.Objects[1] vchar [0,+inf) 'A' CHAR - Test.Objects[1] vint [0,+inf) 3 INT - Test.Objects[1] vlong [0,+inf) 4 LONG - Test.Objects[1] vobj [0,+inf) Test.Objects[1] OBJECT - Test.Objects[1] vshort [0,+inf) 2 SHORT\ - """, + assertEquals(""" + Parent Key Span Value Type + Test.Objects[1] vaddr [0,+inf) ram:0000dead ADDRESS + Test.Objects[1] vbool [0,+inf) True BOOL + Test.Objects[1] vbyte [0,+inf) 1 BYTE + Test.Objects[1] vchar [0,+inf) 'A' CHAR + Test.Objects[1] vint [0,+inf) 3 INT + Test.Objects[1] vlong [0,+inf) 4 LONG + Test.Objects[1] vobj [0,+inf) Test.Objects[1] OBJECT + Test.Objects[1] vshort [0,+inf) 2 SHORT\ + """, extractOutSection(out, "---GetValues---")); } } @@ -939,11 +938,10 @@ public class LldbCommandsTest extends AbstractLldbTraceRmiTest { """.formatted(PREAMBLE, addr, getSpecimenPrint())); try (ManagedDomainObject mdo = openDomainObject(projectName("expPrint"))) { tb = new ToyDBTraceBuilder((Trace) mdo.get()); - assertTrue(extractOutSectionWithPrompt(out, "---GetValues---").contains( - """ - Parent Key Span Value Type - Test.Objects[1] vaddr [0,+inf) ram:0000dead ADDRESS\ - """)); + assertTrue(extractOutSectionWithPrompt(out, "---GetValues---").contains(""" + Parent Key Span Value Type + Test.Objects[1] vaddr [0,+inf) ram:0000dead ADDRESS\ + """)); } }