diff --git a/Ghidra/Extensions/MachineLearning/ghidra_scripts/FindFunctionsRFExampleScript.java b/Ghidra/Extensions/MachineLearning/ghidra_scripts/FindFunctionsRFExampleScript.java index 74fbb5c98f..60803cffd3 100644 --- a/Ghidra/Extensions/MachineLearning/ghidra_scripts/FindFunctionsRFExampleScript.java +++ b/Ghidra/Extensions/MachineLearning/ghidra_scripts/FindFunctionsRFExampleScript.java @@ -113,7 +113,8 @@ public class FindFunctionsRFExampleScript extends GhidraScript { //show the true function starts most similar to one of the false positives if (!falsePositives.isEmpty()) { - SimilarStartsFinder finder = new SimilarStartsFinder(currentProgram, best); + SimilarStartsFinder finder = + new SimilarStartsFinder(currentProgram, currentProgram, best); List neighbors = finder.getSimilarFunctionStarts(falsePositives.get(0).getKey(), 10); printf("\nClosest function starts to false positive at %s :\n", diff --git a/Ghidra/Extensions/MachineLearning/src/main/help/help/topics/RandomForestFunctionFinderPlugin/RandomForestFunctionFinderPlugin.htm b/Ghidra/Extensions/MachineLearning/src/main/help/help/topics/RandomForestFunctionFinderPlugin/RandomForestFunctionFinderPlugin.htm index c13e0aab22..ae449f5f1c 100644 --- a/Ghidra/Extensions/MachineLearning/src/main/help/help/topics/RandomForestFunctionFinderPlugin/RandomForestFunctionFinderPlugin.htm +++ b/Ghidra/Extensions/MachineLearning/src/main/help/help/topics/RandomForestFunctionFinderPlugin/RandomForestFunctionFinderPlugin.htm @@ -181,8 +181,8 @@ is measured using random forest proximity. Given a potential start p and a known start s, the similarity of p and s is the proportion of trees which end up in the same leaf node when processing p and s.

-

For convenience, the potential start is also displayed as a row in the table. In - the Address column, its address is surrounded by asterisks.

+

For convenience, the potential start is displayed in a table with a single row directly + above the similar starts table.

Debug Model Table

This table has the same format as the Potential Functions Table diff --git a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/FunctionStartRFParamsDialog.java b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/FunctionStartRFParamsDialog.java index 62974ed784..a5cf021edf 100644 --- a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/FunctionStartRFParamsDialog.java +++ b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/FunctionStartRFParamsDialog.java @@ -16,7 +16,6 @@ package ghidra.machinelearning.functionfinding; import java.awt.BorderLayout; -import java.io.IOException; import java.util.*; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -35,13 +34,12 @@ import docking.widgets.table.threaded.GThreadedTablePanel; import docking.widgets.textfield.IntegerTextField; import ghidra.app.services.ProgramManager; import ghidra.framework.main.DataTreeDialog; +import ghidra.framework.model.DomainFile; import ghidra.framework.preferences.Preferences; import ghidra.program.model.address.AddressSet; import ghidra.program.model.listing.*; import ghidra.util.HelpLocation; import ghidra.util.Msg; -import ghidra.util.exception.CancelledException; -import ghidra.util.exception.VersionException; import ghidra.util.layout.PairLayout; import ghidra.util.table.SelectionNavigationAction; import ghidra.util.table.actions.MakeProgramSelectionAction; @@ -145,7 +143,6 @@ public class FunctionStartRFParamsDialog extends DialogComponentProvider { private RandomForestTableModel tableModel; private Program trainingSource; private FunctionStartRFParams params; - private Set openPrograms; private Vector moduli = new Vector<>(Arrays.asList(new Long[] { 4l, 8l, 16l, 32l })); private GComboBox modBox; private JButton trainButton; @@ -164,7 +161,6 @@ public class FunctionStartRFParamsDialog extends DialogComponentProvider { true, true); this.plugin = plugin; rowObjects = new ArrayList<>(); - openPrograms = new HashSet<>(); trainingSource = plugin.getCurrentProgram(); JPanel panel = createPanel(); addWorkPanel(panel); @@ -487,24 +483,37 @@ public class FunctionStartRFParamsDialog extends DialogComponentProvider { } private void searchOtherProgram(RandomForestRowObject modelRow) { - Program p = selectProgram(); - if (p == null) { + DataTreeDialog dtd = new DataTreeDialog(null, "Select Program", DataTreeDialog.OPEN, f -> { + Class c = f.getDomainObjectClass(); + return Program.class.isAssignableFrom(c); + }); + dtd.show(); + DomainFile dFile = dtd.getDomainFile(); + if (dFile == null) { return; } ProgramManager pm = plugin.getTool().getService(ProgramManager.class); - pm.openProgram(p, ProgramManager.OPEN_VISIBLE); + Program p = pm.openProgram(dFile, DomainFile.DEFAULT_VERSION, ProgramManager.OPEN_VISIBLE); + if (p == null) { + return; + } + if (!isProgramCompatible(p)) { + Msg.showWarn(this, null, "Incompatible Program", p.getName() + + " is not compatible with training source program " + trainingSource.getName()); + return; + } searchProgram(p, modelRow); } private void showTestErrors(RandomForestRowObject modelRow) { FunctionStartTableProvider provider = new FunctionStartTableProvider(plugin, trainingSource, modelRow.getTestErrors(), modelRow, true); - addGeneralActions(provider); + addGeneralActions(provider, trainingSource); } - private void searchProgram(Program prog, RandomForestRowObject modelRow) { + private void searchProgram(Program targetProgram, RandomForestRowObject modelRow) { GetAddressesToClassifyTask getTask = - new GetAddressesToClassifyTask(prog, plugin.getMinUndefinedRangeSize()); + new GetAddressesToClassifyTask(targetProgram, plugin.getMinUndefinedRangeSize()); //don't want to use the dialog's progress bar TaskLauncher.launchModal("Gathering Addresses To Classify", getTask); if (getTask.isCancelled()) { @@ -518,25 +527,25 @@ public class FunctionStartRFParamsDialog extends DialogComponentProvider { execNonFunc = getTask.getAddressesToClassify(); } FunctionStartTableProvider provider = - new FunctionStartTableProvider(plugin, prog, execNonFunc, modelRow, false); - addGeneralActions(provider); + new FunctionStartTableProvider(plugin, targetProgram, execNonFunc, modelRow, false); + addGeneralActions(provider, targetProgram); DisassembleFunctionStartsAction disassembleAction = null; if (params.isRestrictedByContext()) { - disassembleAction = new DisassembleAndApplyContextAction(plugin, prog, + disassembleAction = new DisassembleAndApplyContextAction(plugin, targetProgram, provider.getTable(), provider.getTableModel()); } else { - disassembleAction = new DisassembleFunctionStartsAction(plugin, prog, + disassembleAction = new DisassembleFunctionStartsAction(plugin, targetProgram, provider.getTable(), provider.getTableModel()); } plugin.getTool().addLocalAction(provider, disassembleAction); - CreateFunctionsAction createActions = - new CreateFunctionsAction(plugin, prog, provider.getTable(), provider.getTableModel()); + CreateFunctionsAction createActions = new CreateFunctionsAction(plugin, targetProgram, + provider.getTable(), provider.getTableModel()); plugin.getTool().addLocalAction(provider, createActions); } - private void addGeneralActions(FunctionStartTableProvider provider) { + private void addGeneralActions(FunctionStartTableProvider provider, Program targetProgram) { plugin.addProvider(provider); DockingAction programSelectAction = new MakeProgramSelectionAction(plugin, provider.getTable()); @@ -545,35 +554,11 @@ public class FunctionStartRFParamsDialog extends DialogComponentProvider { DockingAction selectNavigationAction = new SelectionNavigationAction(plugin, provider.getTable()); plugin.getTool().addLocalAction(provider, selectNavigationAction); - ShowSimilarStartsAction similarStarts = new ShowSimilarStartsAction(plugin, - plugin.getCurrentProgram(), provider.getTable(), provider.getTableModel()); + ShowSimilarStartsAction similarStarts = new ShowSimilarStartsAction(plugin, trainingSource, + targetProgram, provider.getTable(), provider.getTableModel()); plugin.getTool().addLocalAction(provider, similarStarts); } - private Program selectProgram() { - DataTreeDialog dtd = new DataTreeDialog(null, "Select Program", DataTreeDialog.OPEN, f -> { - Class c = f.getDomainObjectClass(); - return Program.class.isAssignableFrom(c); - }); - dtd.show(); - if (dtd.wasCancelled()) { - return null; - } - Program otherProgram = null; - try { - otherProgram = (Program) dtd.getDomainFile() - .getDomainObject(plugin, true, true, getTaskMonitorComponent()); - openPrograms.add(otherProgram); - } - catch (VersionException | CancelledException | IOException e) { - return null; - } - if (isProgramCompatible(otherProgram)) { - return otherProgram; - } - return null; - } - //checks whether otherProgram contains any specified context registers //at some point might be worth adding more restrictions private boolean isProgramCompatible(Program otherProgram) { diff --git a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/RandomForestFunctionFinderPlugin.java b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/RandomForestFunctionFinderPlugin.java index b73431cb2f..d38f90c08b 100644 --- a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/RandomForestFunctionFinderPlugin.java +++ b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/RandomForestFunctionFinderPlugin.java @@ -30,6 +30,7 @@ import ghidra.app.events.ProgramLocationPluginEvent; import ghidra.app.plugin.PluginCategoryNames; import ghidra.app.plugin.ProgramPlugin; import ghidra.app.services.GoToService; +import ghidra.app.services.ProgramManager; import ghidra.framework.options.OptionsChangeListener; import ghidra.framework.options.ToolOptions; import ghidra.framework.plugintool.PluginInfo; @@ -43,12 +44,12 @@ import ghidra.util.bean.opteditor.OptionsVetoException; //@formatter:off @PluginInfo( - status = PluginStatus.UNSTABLE, + status = PluginStatus.RELEASED, packageName = MiscellaneousPluginPackage.NAME, category = PluginCategoryNames.ANALYSIS, shortDescription = "Function Finder", description = "Trains a random forest model to find function starts.", - servicesRequired = { GoToService.class}, + servicesRequired = { GoToService.class, ProgramManager.class}, eventsProduced = { ProgramLocationPluginEvent.class }, eventsConsumed = { ProgramClosedPluginEvent.class} ) @@ -212,10 +213,6 @@ public class RandomForestFunctionFinderPlugin extends ProgramPlugin if (paramsDialog == null) { paramsDialog = new FunctionStartRFParamsDialog(this); } - if (!paramsDialog.getTrainingSource().equals(this.getCurrentProgram())) { - paramsDialog.dismissCallback(); - paramsDialog = new FunctionStartRFParamsDialog(this); - } tool.showDialog(paramsDialog, c.getComponentProvider()); } diff --git a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/ShowSimilarStartsAction.java b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/ShowSimilarStartsAction.java index fbee5dfcf0..e0a8900496 100644 --- a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/ShowSimilarStartsAction.java +++ b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/ShowSimilarStartsAction.java @@ -27,13 +27,14 @@ import ghidra.util.table.GhidraTable; /** * A {@link DockingAction} for showing the most similar function starts in the training - * set to a possible function start + * set to a possible function start */ public class ShowSimilarStartsAction extends DockingAction { private static final String MENU_TEXT = "Show Similar Function Starts"; private static final String ACTION_NAME = "ShowSimilarStartsAction"; private static final int NUM_NEIGHBORS = 10; - private Program program; + private Program trainingSource; + private Program targetProgram; private FunctionStartTableModel model; private GhidraTable table; private RandomForestRowObject modelAndParams; @@ -43,20 +44,22 @@ public class ShowSimilarStartsAction extends DockingAction { /** * Constructs an action display similar function starts * @param plugin plugin - * @param program source program + * @param trainingSource source of training data + * @param targetProgram program being searched * @param table table * @param model table with action */ - public ShowSimilarStartsAction(RandomForestFunctionFinderPlugin plugin, Program program, - GhidraTable table, FunctionStartTableModel model) { + public ShowSimilarStartsAction(RandomForestFunctionFinderPlugin plugin, Program trainingSource, + Program targetProgram, GhidraTable table, FunctionStartTableModel model) { super(ACTION_NAME, plugin.getName()); - this.program = program; + this.trainingSource = trainingSource; + this.targetProgram = targetProgram; this.model = model; this.table = table; this.plugin = plugin; this.modelAndParams = model.getRandomForestRowObject(); init(); - finder = new SimilarStartsFinder(program, modelAndParams); + finder = new SimilarStartsFinder(trainingSource, targetProgram, modelAndParams); } @Override @@ -74,8 +77,8 @@ public class ShowSimilarStartsAction extends DockingAction { Address potential = model.getAddress(table.getSelectedRow()); List closeNeighbors = finder.getSimilarFunctionStarts(potential, NUM_NEIGHBORS); - SimilarStartsTableProvider provider = new SimilarStartsTableProvider(plugin, program, - potential, closeNeighbors, modelAndParams); + SimilarStartsTableProvider provider = new SimilarStartsTableProvider(plugin, trainingSource, + targetProgram, potential, closeNeighbors, modelAndParams); plugin.addProvider(provider); } diff --git a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/SimilarStartsFinder.java b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/SimilarStartsFinder.java index 6b097eba65..18b76420b7 100644 --- a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/SimilarStartsFinder.java +++ b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/SimilarStartsFinder.java @@ -35,11 +35,13 @@ import ghidra.program.model.listing.Program; * function starts, this class is used to find the function starts in the training set * most similar to {@code S}. Here "similar" is defined in terms of proximity in a * random forest (i.e., proportion of trees which agree on two feature vectors). + * Note that {@code S} may or may not be in the training source program. */ public class SimilarStartsFinder { private RandomForestRowObject modelAndParams; - private Program program; + private Program trainingSource; + private Program targetProgram; private int preBytes; private int initialBytes; private boolean includeBitFeatures; @@ -48,11 +50,14 @@ public class SimilarStartsFinder { /** * Creates a {@link SimilarStartsFinder} for the given program and model - * @param program program + * @param trainingSource source of training data + * @param targetProgram program being searched * @param modelAndParams model and params */ - public SimilarStartsFinder(Program program, RandomForestRowObject modelAndParams) { - this.program = program; + public SimilarStartsFinder(Program trainingSource, Program targetProgram, + RandomForestRowObject modelAndParams) { + this.trainingSource = trainingSource; + this.targetProgram = targetProgram; this.modelAndParams = modelAndParams; preBytes = modelAndParams.getNumPreBytes(); initialBytes = modelAndParams.getNumInitialBytes(); @@ -69,7 +74,7 @@ public class SimilarStartsFinder { * @return similar starts (in descending order) */ public List getSimilarFunctionStarts(Address potential, int numStarts) { - List> leafNodes = getLeafNodes(potential); + List> leafNodes = getLeafNodes(potential, targetProgram); List neighbors = new ArrayList<>(startsToLeafList.size()); for (Entry>> entry : startsToLeafList.entrySet()) { Address start = entry.getKey(); @@ -99,7 +104,7 @@ public class SimilarStartsFinder { AddressIterator addrIter = knownStarts.getAddresses(true); while (addrIter.hasNext()) { Address start = addrIter.next(); - List> nodeList = getLeafNodes(start); + List> nodeList = getLeafNodes(start, trainingSource); startsToLeafList.put(start, nodeList); } } @@ -107,10 +112,11 @@ public class SimilarStartsFinder { /** * Creates a feature vector for {@code addr}, runs it down each tree in the forest, * and records the leaf node reached. - * @param addr potential function start + * @param addr (potential) function start + * @param program program containing {@code addr} * @return list of leaf nodes */ - List> getLeafNodes(Address addr) { + List> getLeafNodes(Address addr, Program program) { List> leafNodes = new ArrayList<>(randomForest.getNumModels()); List potentialFeatureVector = ModelTrainingUtils.getFeatureVector(program, addr, preBytes, initialBytes, includeBitFeatures); diff --git a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/SimilarStartsTableModel.java b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/SimilarStartsTableModel.java index f0d33a331b..1469b10716 100644 --- a/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/SimilarStartsTableModel.java +++ b/Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/SimilarStartsTableModel.java @@ -41,7 +41,6 @@ import ghidra.util.task.TaskMonitor; */ public class SimilarStartsTableModel extends AddressBasedTableModel { - private Address potentialStart; private List rows; private RandomForestRowObject randomForestRow; @@ -50,14 +49,12 @@ public class SimilarStartsTableModel extends AddressBasedTableModel rows, RandomForestRowObject randomForestRow) { - super("test", plugin, program, null, false); - this.potentialStart = potentialStart; + super("Similar Starts", plugin, program, null, false); this.rows = rows; this.randomForestRow = randomForestRow; } @@ -70,10 +67,6 @@ public class SimilarStartsTableModel extends AddressBasedTableModel accumulator, TaskMonitor monitor) throws CancelledException { - //add a special row corresponding to the potential function start - //want it in the table to facilitate (visual) byte string comparisons - accumulator.add(new SimilarStartRowObject(potentialStart, - randomForestRow.getRandomForest().getNumModels())); accumulator.addAll(rows); } @@ -99,11 +92,6 @@ public class SimilarStartsTableModel extends AddressBasedTableModel rows; - private JComponent component; + private JSplitPane component; private RandomForestRowObject randomForestRow; /** * Create a table provider * @param plugin owning plugin - * @param program program being search + * @param trainingSource source of training data + * @param targetProgram program being searched * @param potentialStart address of potential start * @param rows closest potential starts * @param randomForestRow model and params */ - public SimilarStartsTableProvider(RandomForestFunctionFinderPlugin plugin, Program program, - Address potentialStart, List rows, - RandomForestRowObject randomForestRow) { - super(program.getName() + ": Similar Function Starts", plugin.getName(), program, plugin); - this.program = program; + public SimilarStartsTableProvider(RandomForestFunctionFinderPlugin plugin, + Program trainingSource, Program targetProgram, Address potentialStart, + List rows, RandomForestRowObject randomForestRow) { + super("Potential Start in " + targetProgram.getName(), plugin.getName(), targetProgram, + plugin); + this.trainingSource = trainingSource; + this.targetProgram = targetProgram; this.potentialStart = potentialStart; this.rows = rows; this.randomForestRow = randomForestRow; - this.setSubTitle("Function Starts Similar to " + potentialStart.toString()); + this.setSubTitle( + potentialStart.toString() + " compared to closest known starts in training set (from " + + trainingSource.getName() + ")"); build(); setHelpLocation(new HelpLocation(plugin.getName(), "SimilarStartsTable")); } @@ -65,23 +71,51 @@ public class SimilarStartsTableProvider extends ProgramAssociatedComponentProvid return component; } + /** + * Builds the main component for this provider. + *

+ * The component is a {@code JSplitPanel} with two {@link GhidraTable}s. The upper + * table consists of a single row containing the potential function start. The rows + * of the lower table contain the function starts in the training source program closest to + * the potential function start. Both tables are navigable; note that the potential + * function start may or may not be in training source program. + */ private void build() { - component = new JPanel(new BorderLayout()); - SimilarStartsTableModel model = - new SimilarStartsTableModel(tool, program, potentialStart, rows, randomForestRow); + SimilarStartsTableModel similarStartsModel = + new SimilarStartsTableModel(tool, trainingSource, rows, randomForestRow); GhidraThreadedTablePanel similarStartsPanel = - new GhidraThreadedTablePanel<>(model, 1000); + new GhidraThreadedTablePanel<>(similarStartsModel, 1000); GhidraTable similarStartsTable = similarStartsPanel.getTable(); - similarStartsTable.setName( - program.getName() + ": Known Starts Similar to " + potentialStart.toString()); + similarStartsPanel.setName( + targetProgram.getName() + ": Known Starts Similar to " + potentialStart.toString()); GoToService goToService = tool.getService(GoToService.class); if (goToService != null) { similarStartsTable.installNavigation(goToService, goToService.getDefaultNavigatable()); } similarStartsTable.setNavigateOnSelectionEnabled(true); similarStartsTable.setAutoResizeMode(JTable.AUTO_RESIZE_SUBSEQUENT_COLUMNS); - similarStartsTable.setPreferredScrollableViewportSize(new Dimension(900, 300)); - component.add(similarStartsPanel, BorderLayout.CENTER); + similarStartsTable.setPreferredScrollableViewportSize(new Dimension(700, 200)); + similarStartsTable.setToolTipText("Known Starts in " + trainingSource.getName()); + + List singleton = new ArrayList<>(); + singleton.add(new SimilarStartRowObject(potentialStart, + randomForestRow.getRandomForest().getNumModels())); + SimilarStartsTableModel potentialStartSingletonModel = + new SimilarStartsTableModel(tool, targetProgram, singleton, randomForestRow); + GhidraThreadedTablePanel potentialStartPanel = + new GhidraThreadedTablePanel<>(potentialStartSingletonModel, 1000); + GhidraTable potentialStartTable = potentialStartPanel.getTable(); + potentialStartTable + .setToolTipText("Potential Function Start in " + targetProgram.getName()); + if (goToService != null) { + potentialStartTable.installNavigation(goToService, goToService.getDefaultNavigatable()); + } + potentialStartTable.setNavigateOnSelectionEnabled(true); + potentialStartTable.setPreferredScrollableViewportSize(new Dimension(700, 30)); + potentialStartTable.setAutoResizeMode(JTable.AUTO_RESIZE_SUBSEQUENT_COLUMNS); + component = + new JSplitPane(JSplitPane.VERTICAL_SPLIT, potentialStartPanel, similarStartsPanel); + component.setResizeWeight(.1); } } diff --git a/Ghidra/Extensions/MachineLearning/src/test.slow/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTaskTest.java b/Ghidra/Extensions/MachineLearning/src/test.slow/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTaskTest.java index e3a80fca9e..97d0b94a10 100644 --- a/Ghidra/Extensions/MachineLearning/src/test.slow/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTaskTest.java +++ b/Ghidra/Extensions/MachineLearning/src/test.slow/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTaskTest.java @@ -117,7 +117,7 @@ public class RandomForestTrainingTaskTest extends AbstractProgramBasedTest { RandomForestFunctionFinderPlugin.NON_START, 1, 1, true, TaskMonitor.DUMMY)); LabelFactory lf = new LabelFactory(); ListDataSource