From 1574262722461ba86a4ae5e7d11512d367b6dcc5 Mon Sep 17 00:00:00 2001 From: Ryan Kurtz Date: Mon, 13 Feb 2023 16:58:30 -0500 Subject: [PATCH] GP-2877: Refactoring Loader and AutoImporter to better accommodate loading more than one thing --- .../FixupELFExternalSymbolsScript.java | 69 ++- ...ImportAllProgramsFromADirectoryScript.java | 40 +- .../java/ghidra/app/script/GhidraScript.java | 37 +- .../app/services/ProgramCoordinator.java | 15 +- .../src/main/java/ghidra/app/util/Option.java | 8 + .../ghidra/app/util/OptionsEditorPanel.java | 30 +- .../app/util/headless/HeadlessAnalyzer.java | 256 ++++---- .../app/util/importer/AutoImporter.java | 562 ++++++++++++++---- .../importer/MultipleProgramsStrategy.java | 58 -- .../opinion/AbstractLibrarySupportLoader.java | 467 ++++++++------- .../opinion/AbstractOrdinalSupportLoader.java | 46 +- .../util/opinion/AbstractProgramLoader.java | 274 ++++----- .../opinion/AbstractProgramWrapperLoader.java | 29 +- .../ghidra/app/util/opinion/BinaryLoader.java | 38 +- .../ghidra/app/util/opinion/CoffLoader.java | 6 - .../ghidra/app/util/opinion/ElfLoader.java | 13 +- .../ghidra/app/util/opinion/GdtLoader.java | 102 +++- .../ghidra/app/util/opinion/GzfLoader.java | 81 +-- .../app/util/opinion/IntelHexLoader.java | 36 +- .../app/util/opinion/LoadException.java | 40 ++ .../ghidra/app/util/opinion/LoadResults.java | 186 ++++++ .../java/ghidra/app/util/opinion/Loaded.java | 221 +++++++ .../java/ghidra/app/util/opinion/Loader.java | 88 ++- .../app/util/opinion/MotorolaHexLoader.java | 36 +- .../ghidra/app/util/opinion/OmfLoader.java | 6 - .../ghidra/app/util/opinion/XmlLoader.java | 32 +- .../ghidra/base/project/GhidraProject.java | 50 +- .../ghidra/framework/main/DataTreeDialog.java | 5 +- .../plugin/importer/AddToProgramDialog.java | 4 +- .../plugin/importer/ImporterDialog.java | 99 +-- .../plugin/importer/ImporterUtilities.java | 64 +- .../importer/tasks/ImportBatchTask.java | 48 +- .../program/examiner/ProgramExaminer.java | 28 +- .../util/ELFExternalSymbolResolver.java | 110 ++-- .../ghidra/app/util/opinion/ApkLoader.java | 28 +- .../ios/prelink/MachoPrelinkFileSystem.java | 2 +- .../ghidra_scripts/ImportMSLibs.java | 42 +- .../MSLibBatchImportWorker.java | 42 +- .../RecursiveRecursiveMSLibImport.java | 28 +- .../ReferencesPluginScreenShots.java | 23 +- 40 files changed, 2049 insertions(+), 1300 deletions(-) delete mode 100644 Ghidra/Features/Base/src/main/java/ghidra/app/util/importer/MultipleProgramsStrategy.java create mode 100644 Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/LoadException.java create mode 100644 Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/LoadResults.java create mode 100644 Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loaded.java diff --git a/Ghidra/Features/Base/ghidra_scripts/FixupELFExternalSymbolsScript.java b/Ghidra/Features/Base/ghidra_scripts/FixupELFExternalSymbolsScript.java index dd5f4919a7..a46c24a9c1 100644 --- a/Ghidra/Features/Base/ghidra_scripts/FixupELFExternalSymbolsScript.java +++ b/Ghidra/Features/Base/ghidra_scripts/FixupELFExternalSymbolsScript.java @@ -22,11 +22,20 @@ // list. // //@category Symbol +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + import ghidra.app.script.GhidraScript; import ghidra.app.util.importer.MessageLog; import ghidra.app.util.opinion.ElfLoader; +import ghidra.app.util.opinion.Loaded; +import ghidra.framework.model.*; +import ghidra.program.model.listing.Library; +import ghidra.program.model.listing.Program; import ghidra.program.util.ELFExternalSymbolResolver; import ghidra.util.Msg; +import ghidra.util.exception.VersionException; public class FixupELFExternalSymbolsScript extends GhidraScript { @@ -38,10 +47,62 @@ public class FixupELFExternalSymbolsScript extends GhidraScript { ")"); return; } - MessageLog msgLog = new MessageLog(); - ELFExternalSymbolResolver.fixUnresolvedExternalSymbols(currentProgram, false, msgLog, - monitor); - Msg.info(this, msgLog.toString()); + MessageLog messageLog = new MessageLog(); + Object consumer = new Object(); + ProjectData projectData = currentProgram.getDomainFile().getParent().getProjectData(); + List> loadedPrograms = new ArrayList<>(); + + // Add current program to list + loadedPrograms.add(new Loaded<>(currentProgram, currentProgram.getName(), + currentProgram.getDomainFile().getPathname())); + + // Add external libraries to list + for (Library extLibrary : ELFExternalSymbolResolver.getLibrarySearchList(currentProgram)) { + monitor.checkCanceled(); + String libName = extLibrary.getName(); + String libPath = extLibrary.getAssociatedProgramPath(); + if (libPath == null) { + continue; + } + + DomainFile libDomainFile = projectData.getFile(libPath); + if (libDomainFile == null) { + messageLog.appendMsg("Referenced external program not found: " + libPath); + continue; + } + + DomainObject libDomainObject = null; + try { + libDomainObject = + libDomainFile.getDomainObject(consumer, false, false, monitor); + if (libDomainObject instanceof Program program) { + loadedPrograms.add(new Loaded<>(program, libName, libPath)); + } + else { + messageLog + .appendMsg("Referenced external program is not a program: " + libPath); + } + } + catch (IOException e) { + // failed to open library + messageLog.appendMsg("Failed to open library dependency project file: " + + libDomainFile.getPathname()); + } + catch (VersionException e) { + messageLog.appendMsg( + "Referenced external program requires updgrade, unable to consider symbols: " + + libPath); + } + } + + // Resolve symbols + ELFExternalSymbolResolver.fixUnresolvedExternalSymbols(loadedPrograms, messageLog, monitor); + + // Cleanup + for (int i = 1; i < loadedPrograms.size(); i++) { + loadedPrograms.get(i).release(consumer); + } + Msg.info(this, messageLog.toString()); } } diff --git a/Ghidra/Features/Base/ghidra_scripts/ImportAllProgramsFromADirectoryScript.java b/Ghidra/Features/Base/ghidra_scripts/ImportAllProgramsFromADirectoryScript.java index 3608c875dd..0d6e8d89a4 100644 --- a/Ghidra/Features/Base/ghidra_scripts/ImportAllProgramsFromADirectoryScript.java +++ b/Ghidra/Features/Base/ghidra_scripts/ImportAllProgramsFromADirectoryScript.java @@ -1,6 +1,5 @@ /* ### * IP: GHIDRA - * REVIEWED: YES * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,15 +16,17 @@ //Imports all programs from a selected directory. //@category Import +import java.io.File; +import java.io.IOException; + import ghidra.app.script.GhidraScript; import ghidra.app.util.importer.AutoImporter; import ghidra.app.util.importer.MessageLog; +import ghidra.app.util.opinion.LoadResults; import ghidra.framework.model.DomainFolder; import ghidra.program.model.lang.LanguageCompilerSpecPair; import ghidra.program.model.listing.Program; -import java.io.File; - public class ImportAllProgramsFromADirectoryScript extends GhidraScript { @Override @@ -52,36 +53,23 @@ public class ImportAllProgramsFromADirectoryScript extends GhidraScript { continue; } - Program program = null; - + LoadResults loadResults = null; try { - program = importFile(file); + loadResults = AutoImporter.importByLookingForLcs(file, state.getProject(), + folder.getPathname(), language.getLanguage(), language.getCompilerSpec(), this, + log, monitor); + loadResults.getPrimary().save(state.getProject(), log, monitor); } - catch (Exception e) { - e.printStackTrace(); - } - - if (program == null) { - try { - program = - AutoImporter.importByLookingForLcs(file, folder, language.getLanguage(), - language.getCompilerSpec(), this, log, monitor); - } - catch (Exception e) { - e.printStackTrace(); - } - } - - if (program == null) { + catch (IOException e) { println("Unable to import program from file " + file.getName()); } - else { - //openProgram( program ); - program.release(this); + finally { + if (loadResults != null) { + loadResults.release(this); + } } println(log.toString()); - log.clear(); } } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/script/GhidraScript.java b/Ghidra/Features/Base/src/main/java/ghidra/app/script/GhidraScript.java index 4ac81d9bd5..f27a5c538d 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/script/GhidraScript.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/script/GhidraScript.java @@ -39,6 +39,7 @@ import ghidra.app.util.demangler.DemanglerUtil; import ghidra.app.util.dialog.AskAddrDialog; import ghidra.app.util.importer.AutoImporter; import ghidra.app.util.importer.MessageLog; +import ghidra.app.util.opinion.*; import ghidra.app.util.query.TableService; import ghidra.app.util.viewer.field.BrowserCodeUnitFormat; import ghidra.app.util.viewer.field.CommentUtils; @@ -3382,18 +3383,38 @@ public abstract class GhidraScript extends FlatProgramAPI { /** * Attempts to import the specified file. It attempts to detect the format and * automatically import the file. If the format is unable to be determined, then - * null is returned. + * null is returned. For more control over the import process, {@link AutoImporter} may be + * directly called. + *

+ * NOTE: The returned {@link Program} is not automatically saved into the current project. + *

+ * NOTE: It is the responsibility of the script that calls this method to release the returned + * {@link Program} with {@link DomainObject#release(Object consumer)} when it is no longer + * needed, where consumer is this. * * @param file the file to import * @return the newly imported program, or null * @throws Exception if any exceptions occur while importing */ public Program importFile(File file) throws Exception { - return AutoImporter.importByUsingBestGuess(file, null, this, new MessageLog(), monitor); + try { + LoadResults loadResults = AutoImporter.importByUsingBestGuess(file, + state.getProject(), null, this, new MessageLog(), monitor); + loadResults.releaseNonPrimary(this); + return loadResults.getPrimaryDomainObject(); + } + catch (LoadException e) { + return null; + } } /** - * Imports the specified file as raw binary. + * Imports the specified file as raw binary. For more control over the import process, + * {@link AutoImporter} may be directly called. + *

+ * NOTE: It is the responsibility of the script that calls this method to release the returned + * {@link Program} with {@link DomainObject#release(Object consumer)} when it is no longer + * needed, where consumer is this. * * @param file the file to import * @param language the language of the new program @@ -3403,8 +3424,14 @@ public abstract class GhidraScript extends FlatProgramAPI { */ public Program importFileAsBinary(File file, Language language, CompilerSpec compilerSpec) throws Exception { - return AutoImporter.importAsBinary(file, null, language, compilerSpec, this, - new MessageLog(), monitor); + try { + Loaded loaded = AutoImporter.importAsBinary(file, state.getProject(), null, + language, compilerSpec, this, new MessageLog(), monitor); + return loaded.getDomainObject(); + } + catch (LoadException e) { + return null; + } } /** diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/services/ProgramCoordinator.java b/Ghidra/Features/Base/src/main/java/ghidra/app/services/ProgramCoordinator.java index 9d095cba7c..7a5bc4da29 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/services/ProgramCoordinator.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/services/ProgramCoordinator.java @@ -21,8 +21,7 @@ import java.util.StringTokenizer; import ghidra.app.util.importer.AutoImporter; import ghidra.app.util.importer.MessageLog; -import ghidra.app.util.opinion.Loader; -import ghidra.app.util.opinion.PeLoader; +import ghidra.app.util.opinion.*; import ghidra.framework.main.AppInfo; import ghidra.framework.model.*; import ghidra.program.model.address.Address; @@ -128,21 +127,25 @@ public abstract class ProgramCoordinator { MessageLog messageLog = new MessageLog(); DomainFolder folder = getFolder(file.getParent()); Class loaderClass = PeLoader.class; + LoadResults loadResults = null; try { Language language = languageService.getDefaultLanguage( Processor.findOrPossiblyCreateProcessor("x86")); CompilerSpec compilerSpec = language.getCompilerSpecByID(new CompilerSpecID("windows")); - importProgram = AutoImporter.importByUsingSpecificLoaderClassAndLcs(file, - folder, loaderClass, null, language, compilerSpec, consumer, messageLog, - monitor); + loadResults = AutoImporter.importByUsingSpecificLoaderClassAndLcs(file, + AppInfo.getActiveProject(), folder.getPathname(), loaderClass, null, + language, compilerSpec, consumer, messageLog, monitor); + importProgram = loadResults.getPrimaryDomainObject(); programManager.openProgram(importProgram); - importProgram.release(this); } catch (Exception e) { e.printStackTrace();//TODO } finally { + if (loadResults != null) { + loadResults.release(this); + } importSemaphore.notify(); } } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/Option.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/Option.java index e5262f5181..e8e8920e0a 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/Option.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/Option.java @@ -190,6 +190,14 @@ public class Option { return false; } } + else if (Integer.class.isAssignableFrom(getValueClass())) { + try { + setValue(Integer.decode(str)); + } + catch (NumberFormatException e) { + return false; + } + } else if (Address.class.isAssignableFrom(getValueClass())) { try { Address origAddr = (Address) getValue(); diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/OptionsEditorPanel.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/OptionsEditorPanel.java index 32b8f77f43..cabbef76bd 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/OptionsEditorPanel.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/OptionsEditorPanel.java @@ -33,10 +33,12 @@ import docking.widgets.checkbox.GCheckBox; import docking.widgets.combobox.GComboBox; import docking.widgets.label.GLabel; import docking.widgets.textfield.IntegerTextField; -import ghidra.app.util.opinion.AbstractLibrarySupportLoader; -import ghidra.app.util.opinion.LibraryPathsDialog; +import ghidra.app.util.opinion.*; +import ghidra.framework.main.AppInfo; import ghidra.framework.main.DataTreeDialog; import ghidra.framework.model.DomainFolder; +import ghidra.framework.model.Project; +import ghidra.framework.options.SaveState; import ghidra.program.model.address.*; import ghidra.util.exception.AssertException; import ghidra.util.layout.*; @@ -251,21 +253,35 @@ public class OptionsEditorPanel extends JPanel { } private Component buildProjectFolderEditor(Option option) { - JPanel panel = new JPanel(new BorderLayout()); - JTextField textField = new JTextField(); + Project project = AppInfo.getActiveProject(); + final SaveState state; + SaveState existingState = project.getSaveableData(Loader.OPTIONS_PROJECT_SAVE_STATE_KEY); + if (existingState != null) { + state = existingState; + } + else { + state = new SaveState(); + project.setSaveableData(Loader.OPTIONS_PROJECT_SAVE_STATE_KEY, state); + } + String lastFolderPath = state.getString(option.getName(), ""); + option.setValue(lastFolderPath); + JTextField textField = new JTextField(lastFolderPath); textField.setEditable(false); JButton button = new BrowseButton(); button.addActionListener(e -> { DataTreeDialog dataTreeDialog = new DataTreeDialog(this, "Choose a project folder", DataTreeDialog.CHOOSE_FOLDER); - dataTreeDialog.setSelectedFolder(null); + dataTreeDialog.setSelectedFolder(project.getProjectData().getFolder(lastFolderPath)); dataTreeDialog.showComponent(); DomainFolder folder = dataTreeDialog.getDomainFolder(); if (folder != null) { - textField.setText(folder.getPathname()); - option.setValue(folder.getPathname()); + String newFolderPath = folder.getPathname(); + textField.setText(newFolderPath); + option.setValue(newFolderPath); + state.putString(option.getName(), newFolderPath); } }); + JPanel panel = new JPanel(new BorderLayout()); panel.add(textField, BorderLayout.CENTER); panel.add(button, BorderLayout.EAST); return panel; diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessAnalyzer.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessAnalyzer.java index e042e7b4b9..504f3c9823 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessAnalyzer.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessAnalyzer.java @@ -31,7 +31,7 @@ import ghidra.app.script.*; import ghidra.app.util.headless.HeadlessScript.HeadlessContinuationOption; import ghidra.app.util.importer.AutoImporter; import ghidra.app.util.importer.MessageLog; -import ghidra.app.util.opinion.BinaryLoader; +import ghidra.app.util.opinion.*; import ghidra.framework.*; import ghidra.framework.client.ClientUtil; import ghidra.framework.client.RepositoryAdapter; @@ -1382,7 +1382,15 @@ public class HeadlessAnalyzer { return p; } - private boolean checkOverwrite(DomainFile df) throws IOException { + private boolean checkOverwrite(Loaded loaded) throws IOException { + DomainFolder folder = project.getProjectData().getFolder(loaded.getProjectFolderPath()); + if (folder == null) { + return true; + } + DomainFile df = folder.getFile(loaded.getName()); + if (df == null) { + return true; + } if (options.overwrite) { try { if (df.isHijacked()) { @@ -1500,192 +1508,148 @@ public class HeadlessAnalyzer { Msg.info(this, "IMPORTING: " + file.getAbsolutePath()); - Program program = null; - + LoadResults loadResults = null; + Loaded primary = null; try { - String dfName = null; - DomainFile df = null; - DomainFolder domainFolder = null; - try { - // Gets parent folder for import (creates path if doesn't exist) - domainFolder = getDomainFolder(folderPath, false); - dfName = file.getName(); + // Perform the load. Note that loading 1 file may result in more than 1 thing getting + // loaded. + loadResults = loadPrograms(file, folderPath); + Msg.info(this, "IMPORTING: Loaded " + (loadResults.size() - 1) + " additional files"); - if (dfName.toLowerCase().endsWith(".gzf") || - dfName.toLowerCase().endsWith(".xml")) { - // Use filename without .gzf - int index = dfName.lastIndexOf('.'); - dfName = dfName.substring(0, index); - } + primary = loadResults.getPrimary(); + Program primaryProgram = primary.getDomainObject(); - if (!options.readOnly) { - if (domainFolder != null) { - df = domainFolder.getFile(dfName); - } - if (df != null && !checkOverwrite(df)) { + // Make sure we are allowed to save ALL programs to the project. If not, save none and + // fail. + if (!options.readOnly) { + for (Loaded loaded : loadResults) { + if (!checkOverwrite(loaded)) { return false; } - df = null; - } - - program = loadProgram(file); - if (program == null) { - return false; - } - - // Check if there are defined memory blocks; abort if not (there is nothing - // to work with!) - if (program.getMemory().getAllInitializedAddressSet().isEmpty()) { - Msg.error(this, "REPORT: Error: No memory blocks were defined for file '" + - file.getAbsolutePath() + "'."); - return false; } } - catch (Exception exc) { - Msg.error(this, "REPORT: " + exc.getMessage(), exc); - exc.printStackTrace(); + + // Check if there are defined memory blocks in the primary program. + // Abort if not (there is nothing to work with!). + if (primaryProgram.getMemory().getAllInitializedAddressSet().isEmpty()) { + Msg.error(this, "REPORT: Error: No memory blocks were defined for file " + + file.getAbsolutePath()); return false; } - Msg.info(this, - "REPORT: Import succeeded with language \"" + - program.getLanguageID().getIdAsString() + "\" and cspec \"" + - program.getCompilerSpec().getCompilerSpecID().getIdAsString() + - "\" for file: " + file.getAbsolutePath()); + // Analyze the primary program, and determine if we should save. + // TODO: Analyze non-primary programs (GP-2965). + boolean doSave = + analyzeProgram(file.getAbsolutePath(), primaryProgram) && !options.readOnly; - boolean doSave; - try { + // The act of marking the program as temporary by a script will signal + // us to discard any changes + if (!doSave) { + loadResults.forEach(e -> e.getDomainObject().setTemporary(true)); + } - doSave = analyzeProgram(file.getAbsolutePath(), program) && !options.readOnly; - - if (!doSave) { - program.setTemporary(true); + // Apply saveDomainFolder to the primary program, if applicable. + // We don't support changing the save folder on any non-primary loaded programs. + // Note that saveDomainFolder is set by pre/post-scripts, so it can only be used + // after analysis happens. + if (saveDomainFolder != null) { + primary.setProjectFolderPath(saveDomainFolder.getPathname()); + if (!checkOverwrite(primary)) { + return false; } - - // The act of marking the program as temporary by a script will signal - // us to discard any program changes. - if (program.isTemporary()) { + } + + // Save + for (Loaded loaded : loadResults) { + if (!loaded.getDomainObject().isTemporary()) { + try { + DomainFile domainFile = + loaded.save(project, new MessageLog(), TaskMonitor.DUMMY); + Msg.info(this, String.format("REPORT: Save succeeded for: %s (%s)", loaded, + domainFile)); + } + catch (IOException e) { + Msg.info(this, "REPORT: Save failed for: " + loaded); + } + } + else { if (options.readOnly) { Msg.info(this, "REPORT: Discarded file import due to readOnly option: " + - file.getAbsolutePath()); + loaded); } else { - Msg.info(this, "REPORT: Discarded file import as a result of script " + - "activity or analysis timeout: " + file.getAbsolutePath()); + Msg.info(this, + "REPORT: Discarded file import as a result of script " + + "activity or analysis timeout: " + loaded); } - return true; } + } - try { - if (saveDomainFolder != null) { - - df = saveDomainFolder.getFile(dfName); - - // Return if file already exists and overwrite == false - if (df != null && !checkOverwrite(df)) { - return false; + // Commit changes + if (options.commit) { + for (Loaded loaded : loadResults) { + if (!loaded.getDomainObject().isTemporary()) { + if (loaded == primary) { + AutoAnalysisManager.getAnalysisManager(primaryProgram).dispose(); } - - domainFolder = saveDomainFolder; + loaded.release(this); + commitProgram(loaded.getSavedDomainFile()); } - else if (domainFolder == null) { - domainFolder = getDomainFolder(folderPath, true); - } - df = domainFolder.createFile(dfName, program, TaskMonitor.DUMMY); - Msg.info(this, "REPORT: Save succeeded for file: " + df.getPathname()); - - if (options.commit) { - - AutoAnalysisManager.getAnalysisManager(program).dispose(); - program.release(this); - program = null; - - commitProgram(df); - } - } - catch (IOException e) { - e.printStackTrace(); - throw new IOException("Cannot create file: " + domainFolder.getPathname() + - DomainFolder.SEPARATOR + dfName, e); - } - } - catch (Exception exc) { - String logErrorMsg = - file.getAbsolutePath() + " Error during analysis: " + exc.getMessage(); - Msg.info(this, logErrorMsg); - return false; - } - finally { - if (program != null) { - AutoAnalysisManager.getAnalysisManager(program).dispose(); } } + Msg.info(this, "REPORT: Import succeeded"); return true; } - finally { - // Program must be released here, since the AutoAnalysisManager uses program to - // call dispose() in the finally() block above. - if (program != null) { - program.release(this); - program = null; - } - } - } - - private Program loadProgram(File file) throws VersionException, InvalidNameException, - DuplicateNameException, CancelledException, IOException { - - MessageLog messageLog = new MessageLog(); - Program program = null; - - // NOTE: we must pass a null DomainFolder to the AutoImporter so as not to - // allow the DomainFile to be saved at this point. DomainFile should be - // saved after all applicable analysis/scripts are run. - - if (options.loaderClass == null) { - // User did not specify a loader - if (options.language == null) { - program = AutoImporter.importByUsingBestGuess(file, null, this, messageLog, - TaskMonitor.DUMMY); - } - else { - program = AutoImporter.importByLookingForLcs(file, null, options.language, - options.compilerSpec, this, messageLog, TaskMonitor.DUMMY); - } - } - else { - // User specified a loader - if (options.language == null) { - program = AutoImporter.importByUsingSpecificLoaderClass(file, null, - options.loaderClass, options.loaderArgs, this, messageLog, TaskMonitor.DUMMY); - } - else { - program = AutoImporter.importByUsingSpecificLoaderClassAndLcs(file, null, - options.loaderClass, options.loaderArgs, options.language, options.compilerSpec, - this, messageLog, TaskMonitor.DUMMY); - } - } - - if (program == null) { + catch (LoadException e) { Msg.error(this, "The AutoImporter could not successfully load " + file.getAbsolutePath() + " with the provided import parameters. Please ensure that any specified" + " processor/cspec arguments are compatible with the loader that is used during" + " import and try again."); - if (options.loaderClass != null && options.loaderClass != BinaryLoader.class) { Msg.error(this, "NOTE: Import failure may be due to missing opinion for \"" + options.loaderClass.getSimpleName() + "\". If so, please contact Ghidra team for assistance."); } + return false; + } + catch (Exception e) { + Msg.error(this, "REPORT: " + e.getMessage(), e); + return false; + } + finally { + if (loadResults != null) { + loadResults.release(this); + } + } + } - return null; + private LoadResults loadPrograms(File file, String folderPath) throws VersionException, + InvalidNameException, DuplicateNameException, CancelledException, IOException, + LoadException { + MessageLog messageLog = new MessageLog(); + + if (options.loaderClass == null) { + // User did not specify a loader + if (options.language == null) { + return AutoImporter.importByUsingBestGuess(file, project, folderPath, this, + messageLog, TaskMonitor.DUMMY); + } + return AutoImporter.importByLookingForLcs(file, project, folderPath, options.language, + options.compilerSpec, this, messageLog, TaskMonitor.DUMMY); } - return program; + // User specified a loader + if (options.language == null) { + return AutoImporter.importByUsingSpecificLoaderClass(file, project, folderPath, + options.loaderClass, options.loaderArgs, this, messageLog, TaskMonitor.DUMMY); + } + return AutoImporter.importByUsingSpecificLoaderClassAndLcs(file, project, folderPath, + options.loaderClass, options.loaderArgs, options.language, options.compilerSpec, this, + messageLog, TaskMonitor.DUMMY); } private void processWithImport(File file, String folderPath, boolean isFirstTime) diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/importer/AutoImporter.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/importer/AutoImporter.java index 4eaa20dac5..ae2984a4c4 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/importer/AutoImporter.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/importer/AutoImporter.java @@ -17,6 +17,7 @@ package ghidra.app.util.importer; import java.io.File; import java.io.IOException; +import java.nio.file.AccessMode; import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; @@ -24,10 +25,10 @@ import java.util.function.Predicate; import generic.stl.Pair; import ghidra.app.util.Option; import ghidra.app.util.bin.ByteProvider; -import ghidra.app.util.bin.RandomAccessByteProvider; +import ghidra.app.util.bin.FileByteProvider; import ghidra.app.util.opinion.*; -import ghidra.framework.model.DomainFolder; -import ghidra.framework.model.DomainObject; +import ghidra.formats.gfilesystem.FileSystemService; +import ghidra.framework.model.*; import ghidra.program.model.address.AddressFactory; import ghidra.program.model.lang.*; import ghidra.program.model.listing.Program; @@ -38,146 +39,443 @@ import ghidra.util.exception.*; import ghidra.util.task.TaskMonitor; /** - * Utility methods to do imports automatically (without requiring user interaction). + * Utility methods to do {@link Program} imports automatically (without requiring user interaction) */ public final class AutoImporter { private AutoImporter() { // service class; cannot instantiate } - public static Program importByUsingBestGuess(File file, DomainFolder programFolder, - Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, - CancelledException, DuplicateNameException, InvalidNameException, VersionException { - List programs = importFresh(file, programFolder, consumer, messageLog, monitor, - LoaderService.ACCEPT_ALL, LoadSpecChooser.CHOOSE_THE_FIRST_PREFERRED, null, - OptionChooser.DEFAULT_OPTIONS, MultipleProgramsStrategy.ONE_PROGRAM_OR_NULL); - if (programs != null && programs.size() == 1) { - return programs.get(0); - } - return null; + /** + * Automatically imports the given {@link File} with the best matching {@link Loader} for the + * {@link File}'s format. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param file The {@link File} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static LoadResults importByUsingBestGuess(File file, Project project, + String projectFolderPath, Object consumer, MessageLog messageLog, TaskMonitor monitor) + throws IOException, CancelledException, DuplicateNameException, InvalidNameException, + VersionException, LoadException { + return importFresh(file, project, projectFolderPath, consumer, messageLog, + monitor, LoaderService.ACCEPT_ALL, LoadSpecChooser.CHOOSE_THE_FIRST_PREFERRED, null, + OptionChooser.DEFAULT_OPTIONS); } - public static Program importByUsingBestGuess(ByteProvider provider, DomainFolder programFolder, - Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, - CancelledException, DuplicateNameException, InvalidNameException, VersionException { - List programs = importFresh(provider, programFolder, consumer, messageLog, monitor, - LoaderService.ACCEPT_ALL, LoadSpecChooser.CHOOSE_THE_FIRST_PREFERRED, null, - OptionChooser.DEFAULT_OPTIONS, MultipleProgramsStrategy.ONE_PROGRAM_OR_NULL); - if (programs != null && programs.size() == 1) { - return programs.get(0); - } - return null; - } - - public static Program importByUsingSpecificLoaderClass(File file, DomainFolder programFolder, - Class loaderClass, List> loaderArgs, - Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, - CancelledException, DuplicateNameException, InvalidNameException, VersionException { - SingleLoaderFilter loaderFilter = new SingleLoaderFilter(loaderClass, loaderArgs); - List programs = importFresh(file, programFolder, consumer, messageLog, monitor, - loaderFilter, LoadSpecChooser.CHOOSE_THE_FIRST_PREFERRED, null, - new LoaderArgsOptionChooser(loaderFilter), - MultipleProgramsStrategy.ONE_PROGRAM_OR_NULL); - if (programs != null && programs.size() == 1) { - return programs.get(0); - } - return null; - } - - public static Program importByLookingForLcs(File file, DomainFolder programFolder, - Language language, CompilerSpec compilerSpec, Object consumer, MessageLog messageLog, + /** + * Automatically imports the give {@link ByteProvider bytes} with the best matching + * {@link Loader} for the {@link ByteProvider}'s format. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param provider The bytes to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static LoadResults importByUsingBestGuess(ByteProvider provider, + Project project, String projectFolderPath, Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, CancelledException, DuplicateNameException, - InvalidNameException, VersionException { - List programs = importFresh(file, programFolder, consumer, messageLog, monitor, - LoaderService.ACCEPT_ALL, new LcsHintLoadSpecChooser(language, compilerSpec), null, - OptionChooser.DEFAULT_OPTIONS, MultipleProgramsStrategy.ONE_PROGRAM_OR_NULL); - if (programs != null && programs.size() == 1) { - return programs.get(0); - } - return null; + InvalidNameException, VersionException, LoadException { + return importFresh(provider, project, projectFolderPath, consumer, messageLog, monitor, + LoaderService.ACCEPT_ALL, LoadSpecChooser.CHOOSE_THE_FIRST_PREFERRED, null, + OptionChooser.DEFAULT_OPTIONS); } - public static Program importByUsingSpecificLoaderClassAndLcs(File file, - DomainFolder programFolder, Class loaderClass, + /** + * Automatically imports the given {@link File} with the given type of {@link Loader}. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param file The {@link File} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param loaderClass The {@link Loader} class to use + * @param loaderArgs A {@link List} of optional {@link Loader}-specific arguments + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static LoadResults importByUsingSpecificLoaderClass(File file, + Project project, String projectFolderPath, Class loaderClass, + List> loaderArgs, Object consumer, MessageLog messageLog, + TaskMonitor monitor) throws IOException, CancelledException, DuplicateNameException, + InvalidNameException, VersionException, LoadException { + SingleLoaderFilter loaderFilter = new SingleLoaderFilter(loaderClass, loaderArgs); + return importFresh(file, project, projectFolderPath, consumer, messageLog, + monitor, loaderFilter, LoadSpecChooser.CHOOSE_THE_FIRST_PREFERRED, null, + new LoaderArgsOptionChooser(loaderFilter)); + } + + /** + * Automatically imports the given {@link File} with the best matching {@link Loader} that + * supports the given language and compiler specification. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param file The {@link File} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param language The desired {@link Language} + * @param compilerSpec The desired {@link CompilerSpec compiler specification} + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static LoadResults importByLookingForLcs(File file, Project project, + String projectFolderPath, Language language, CompilerSpec compilerSpec, Object consumer, + MessageLog messageLog, TaskMonitor monitor) throws IOException, CancelledException, + DuplicateNameException, InvalidNameException, VersionException, LoadException { + return importFresh(file, project, projectFolderPath, consumer, messageLog, + monitor, LoaderService.ACCEPT_ALL, new LcsHintLoadSpecChooser(language, compilerSpec), + null, OptionChooser.DEFAULT_OPTIONS); + } + + /** + * Automatically imports the given {@link File} with the given type of {@link Loader}, language, + * and compiler specification. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param file The {@link File} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param loaderClass The {@link Loader} class to use + * @param loaderArgs A {@link List} of optional {@link Loader}-specific arguments + * @param language The desired {@link Language} + * @param compilerSpec The desired {@link CompilerSpec compiler specification} + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + */ + public static LoadResults importByUsingSpecificLoaderClassAndLcs(File file, + Project project, String projectFolderPath, Class loaderClass, List> loaderArgs, Language language, CompilerSpec compilerSpec, Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, CancelledException, DuplicateNameException, InvalidNameException, VersionException { SingleLoaderFilter loaderFilter = new SingleLoaderFilter(loaderClass, loaderArgs); - List programs = importFresh(file, programFolder, consumer, messageLog, monitor, - loaderFilter, new LcsHintLoadSpecChooser(language, compilerSpec), null, - new LoaderArgsOptionChooser(loaderFilter), - MultipleProgramsStrategy.ONE_PROGRAM_OR_NULL); - if (programs != null && programs.size() == 1) { - return programs.get(0); - } - return null; + return importFresh(file, project, projectFolderPath, consumer, messageLog, + monitor, loaderFilter, new LcsHintLoadSpecChooser(language, compilerSpec), null, + new LoaderArgsOptionChooser(loaderFilter)); } private static final Predicate BINARY_LOADER = new SingleLoaderFilter(BinaryLoader.class); - public static Program importAsBinary(File file, DomainFolder programFolder, Language language, - CompilerSpec compilerSpec, Object consumer, MessageLog messageLog, TaskMonitor monitor) - throws IOException, CancelledException, DuplicateNameException, InvalidNameException, - VersionException { - List programs = importFresh(file, programFolder, consumer, messageLog, monitor, - BINARY_LOADER, new LcsHintLoadSpecChooser(language, compilerSpec), null, - OptionChooser.DEFAULT_OPTIONS, MultipleProgramsStrategy.ONE_PROGRAM_OR_NULL); - if (programs != null && programs.size() == 1) { - return programs.get(0); - } - return null; + /** + * Automatically imports the given {@link File} with the {@link BinaryLoader}, using the given + * language and compiler specification. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program} is + * not saved to a project. That is the responsibility of the caller (see + * {@link Loaded#save(Project, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program} with {@link Loaded#release(Object)} when it is no longer needed. + * + * @param file The {@link File} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for the {@link Loaded} result. The {@link Loaded} result + * should be queried for its true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param language The desired {@link Language} + * @param compilerSpec The desired {@link CompilerSpec compiler specification} + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link Loaded} {@link Program} (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static Loaded importAsBinary(File file, Project project, + String projectFolderPath, Language language, CompilerSpec compilerSpec, Object consumer, + MessageLog messageLog, TaskMonitor monitor) throws IOException, CancelledException, + DuplicateNameException, InvalidNameException, VersionException, LoadException { + LoadResults loadResults = importFresh(file, project, projectFolderPath, consumer, + messageLog, monitor, BINARY_LOADER, new LcsHintLoadSpecChooser(language, compilerSpec), + null, OptionChooser.DEFAULT_OPTIONS); + loadResults.releaseNonPrimary(consumer); + return loadResults.getPrimary(); } - public static Program importAsBinary(ByteProvider bytes, DomainFolder programFolder, - Language language, CompilerSpec compilerSpec, Object consumer, MessageLog messageLog, - TaskMonitor monitor) throws IOException, CancelledException, DuplicateNameException, - InvalidNameException, VersionException { - List programs = importFresh(bytes, programFolder, consumer, messageLog, monitor, - BINARY_LOADER, new LcsHintLoadSpecChooser(language, compilerSpec), null, - OptionChooser.DEFAULT_OPTIONS, MultipleProgramsStrategy.ONE_PROGRAM_OR_NULL); - if (programs != null && programs.size() == 1) { - return programs.get(0); - } - return null; + /** + * Automatically imports the given {@link ByteProvider} bytes with the {@link BinaryLoader}, + * using the given language and compiler specification. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program} is + * not saved to a project. That is the responsibility of the caller (see + * {@link Loaded#save(Project, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program} with {@link Loaded#release(Object)} when it is no longer needed. + * + * @param bytes The bytes to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it the {@link Loaded} result. The {@link Loaded} result + * should be queried for its true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param language The desired {@link Language} + * @param compilerSpec The desired {@link CompilerSpec compiler specification} + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link Loaded} {@link Program} (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static Loaded importAsBinary(ByteProvider bytes, Project project, + String projectFolderPath, Language language, CompilerSpec compilerSpec, + Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, + CancelledException, DuplicateNameException, InvalidNameException, VersionException, + LoadException { + LoadResults loadResults = importFresh(bytes, project, projectFolderPath, consumer, + messageLog, monitor, BINARY_LOADER, new LcsHintLoadSpecChooser(language, compilerSpec), + null, OptionChooser.DEFAULT_OPTIONS); + loadResults.releaseNonPrimary(consumer); + return loadResults.getPrimary(); } - public static List importFresh(File file, DomainFolder programFolder, Object consumer, - MessageLog messageLog, TaskMonitor monitor, Predicate loaderFilter, - LoadSpecChooser loadSpecChooser, String programNameOverride, - OptionChooser optionChooser, MultipleProgramsStrategy multipleProgramsStrategy) - throws IOException, CancelledException, DuplicateNameException, InvalidNameException, - VersionException { - if (file == null) { - return null; - } - - try (ByteProvider provider = new RandomAccessByteProvider(file)) { - return importFresh(provider, programFolder, consumer, messageLog, monitor, loaderFilter, - loadSpecChooser, programNameOverride, optionChooser, multipleProgramsStrategy); - } - } - - public static List importFresh(ByteProvider provider, DomainFolder programFolder, - Object consumer, MessageLog messageLog, TaskMonitor monitor, + /** + * Automatically imports the given {@link File} with advanced options. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param file The {@link File} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param loaderFilter A {@link Predicate} used to choose what {@link Loader}(s) get used + * @param loadSpecChooser A {@link LoadSpecChooser} used to choose what {@link LoadSpec}(s) get + * used + * @param importNameOverride The name to use for the imported thing. Null to use the + * {@link Loader}'s preferred name. + * @param optionChooser A {@link OptionChooser} used to choose what {@link Loader} options get + * used + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static LoadResults importFresh(File file, Project project, + String projectFolderPath, Object consumer, MessageLog messageLog, TaskMonitor monitor, Predicate loaderFilter, LoadSpecChooser loadSpecChooser, - String programNameOverride, OptionChooser optionChooser, - MultipleProgramsStrategy multipleProgramsStrategy) throws IOException, - CancelledException, DuplicateNameException, InvalidNameException, VersionException { + String importNameOverride, OptionChooser optionChooser) throws IOException, + CancelledException, DuplicateNameException, InvalidNameException, VersionException, + LoadException { + if (file == null) { + throw new LoadException("Cannot load null file"); + } + + try (ByteProvider provider = new FileByteProvider(file, + FileSystemService.getInstance().getLocalFSRL(file), AccessMode.READ)) { + return importFresh(provider, project, projectFolderPath, consumer, messageLog, monitor, + loaderFilter, loadSpecChooser, importNameOverride, optionChooser); + } + } + + /** + * Automatically imports the given {@link ByteProvider bytes} with advanced options. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param provider The bytes to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param loaderFilter A {@link Predicate} used to choose what {@link Loader}(s) get used + * @param loadSpecChooser A {@link LoadSpecChooser} used to choose what {@link LoadSpec}(s) get + * used + * @param importNameOverride The name to use for the imported thing. Null to use the + * {@link Loader}'s preferred name. + * @param optionChooser A {@link OptionChooser} used to choose what {@link Loader} options get + * used + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static LoadResults importFresh(ByteProvider provider, Project project, + String projectFolderPath, Object consumer, MessageLog messageLog, TaskMonitor monitor, + Predicate loaderFilter, LoadSpecChooser loadSpecChooser, + String importNameOverride, OptionChooser optionChooser) throws IOException, + CancelledException, DuplicateNameException, InvalidNameException, VersionException, + LoadException { + if (provider == null) { - return null; + throw new LoadException("Cannot load null provider"); } // Get the load spec LoadSpec loadSpec = getLoadSpec(loaderFilter, loadSpecChooser, provider); if (loadSpec == null) { - return null; + throw new LoadException("No load spec found"); } - // Get the program name - String programName = loadSpec.getLoader().getPreferredFileName(provider); - if (programNameOverride != null) { - programName = programNameOverride; + // Get the preferred import name + String importName = loadSpec.getLoader().getPreferredFileName(provider); + if (importNameOverride != null) { + importName = importNameOverride; } // Collect options @@ -185,21 +483,40 @@ public final class AutoImporter { AddressFactory addrFactory = null;// Address type options not permitted if null if (languageCompilerSpecPair != null) { // It is assumed that if languageCompilerSpecPair exists, then language will be found - addrFactory = DefaultLanguageService.getLanguageService().getLanguage( - languageCompilerSpecPair.languageID).getAddressFactory(); + addrFactory = DefaultLanguageService.getLanguageService() + .getLanguage( + languageCompilerSpecPair.languageID) + .getAddressFactory(); } List

- * Other programs in the given list are matched first, then the ghidraLibSearchFolders are - * searched for matches. + * Other {@link Program}s in the given list are matched first, then the given + * {@link DomainFolder search folder} is searched for matches. * - * @param programs the list of programs to resolve against each other. Programs not saved - * to the project will be considered as a valid external library. - * @param searchFolder the {@link DomainFolder} which imported libraries will be searched. - * This folder will be searched if a library is not found within the list of - * programs supplied. If null, only the list of programs will be considered. - * @param saveIfModified flag to have this method save any programs it modifies + * @param loadedPrograms the list of {@link Loaded} {@link Program}s + * @param searchFolders an ordered list of {@link DomainFolder}s which imported libraries will + * be searched. These folders will be searched if a library is not found within the list of + * programs supplied. * @param messageLog log for messages. * @param monitor the task monitor * @throws IOException if there was an IO-related problem resolving. * @throws CancelledException if the user cancelled the load. */ - private void fixupExternalLibraries(List programs, DomainFolder searchFolder, - boolean saveIfModified, MessageLog messageLog, TaskMonitor monitor) + private void fixupExternalLibraries(List> loadedPrograms, + List searchFolders, MessageLog messageLog, TaskMonitor monitor) throws CancelledException, IOException { - Map progsByName = programs.stream() - .filter(Objects::nonNull) + Map> loadedByName = loadedPrograms.stream() .collect( - Collectors.toMap((p) -> p.getDomainFile().getName(), (p) -> p)); + Collectors.toMap(loaded -> loaded.getName(), loaded -> loaded)); - monitor.initialize(progsByName.size()); - for (Program program : progsByName.values()) { + monitor.initialize(loadedByName.size()); + for (Loaded loadedProgram : loadedByName.values()) { monitor.incrementProgress(1); - if (monitor.isCancelled()) { - return; - } + monitor.checkCanceled(); + Program program = loadedProgram.getDomainObject(); ExternalManager extManager = program.getExternalManager(); String[] extLibNames = extManager.getExternalLibraryNames(); if (extLibNames.length == 0 || @@ -795,38 +819,35 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader monitor.setMessage("Resolving..." + program.getName()); int id = program.startTransaction("Resolving external references"); try { - resolveExternalLibraries(program, progsByName, searchFolder, monitor, messageLog); + resolveExternalLibraries(program, loadedByName, searchFolders, monitor, messageLog); } finally { program.endTransaction(id, true); - if (saveIfModified && program.canSave() && program.isChanged()) { - program.save("Resolve external references", monitor); - } } } } /** - * Fix up program's external library entries so that they point to a path in the project. + * Fix up program's external library entries so that they point to a path in the project. *

* Other programs in the map are matched first, then the ghidraLibSearchFolders * are searched for matches. * - * @param program the program whose Library entries are to be resolved. An open transaction - * on program is required. - * @param progsByName map of recently imported programs to be considered + * @param program the program whose Library entries are to be resolved. An open + * transaction on program is required. + * @param loadedByName map of recently loaded things to be considered * first when resolving external Libraries. Programs not saved to the project * will be ignored. - * @param searchFolder the {@link DomainFolder} which imported libraries will be searched. - * This folder will be searched if a library is not found within the list of - * programs supplied. If null, only the list of programs will be considered. + * @param searchFolders an order list of {@link DomainFolder}s which imported libraries will be + * searched. These folders will be searched if a library is not found within the list of + * programs supplied. * @param messageLog log for messages. * @param monitor the task monitor * @throws CancelledException if the user cancelled the load. */ - private void resolveExternalLibraries(Program program, Map progsByName, - DomainFolder searchFolder, TaskMonitor monitor, MessageLog messageLog) - throws CancelledException { + private void resolveExternalLibraries(Program program, + Map> loadedByName, List searchFolders, + TaskMonitor monitor, MessageLog messageLog) throws CancelledException { ExternalManager extManager = program.getExternalManager(); String[] extLibNames = extManager.getExternalLibraryNames(); messageLog.appendMsg("Linking external programs to " + program.getName() + "..."); @@ -837,22 +858,27 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader monitor.checkCanceled(); try { String externalFileName = FilenameUtils.getName(externalLibName); - DomainObject matchingExtProgram = findLibrary(progsByName, externalFileName); - if (matchingExtProgram != null && matchingExtProgram.getDomainFile().exists()) { - extManager.setExternalPath(externalLibName, - matchingExtProgram.getDomainFile().getPathname(), false); - messageLog.appendMsg(" [" + externalLibName + "] -> [" + - matchingExtProgram.getDomainFile().getPathname() + "]"); + Loaded matchingExtProgram = findLibrary(loadedByName, externalFileName); + if (matchingExtProgram != null) { + String path = + matchingExtProgram.getProjectFolderPath() + matchingExtProgram.getName(); + extManager.setExternalPath(externalLibName, path, false); + messageLog.appendMsg(" [" + externalLibName + "] -> [" + path + "]"); } else { - DomainFile alreadyImportedLib = findLibrary(externalLibName, searchFolder); - if (alreadyImportedLib != null) { - extManager.setExternalPath(externalLibName, - alreadyImportedLib.getPathname(), false); - messageLog.appendMsg(" [" + externalLibName + "] -> [" + - alreadyImportedLib.getPathname() + "] (previously imported)"); + boolean found = false; + for (DomainFolder searchFolder : searchFolders) { + DomainFile alreadyImportedLib = findLibrary(externalLibName, searchFolder); + if (alreadyImportedLib != null) { + extManager.setExternalPath(externalLibName, + alreadyImportedLib.getPathname(), false); + messageLog.appendMsg(" [" + externalLibName + "] -> [" + + alreadyImportedLib.getPathname() + "] (previously imported)"); + found = true; + break; + } } - else { + if (!found) { messageLog.appendMsg(" [" + externalLibName + "] -> not found"); } } @@ -908,53 +934,26 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader /** * Find the library within the given {@link Map} of {@link Program}s * - * @param programsByName The map to search + * @param loadedByName The map to search * @param libraryName The library name to lookup - * @return The found {@link Program} or null if not found + * @return The found {@link Loaded} {@link Program} or null if not found */ - private Program findLibrary(Map programsByName, String libraryName) { + private Loaded findLibrary(Map> loadedByName, + String libraryName) { Comparator comparator = getLibraryNameComparator(); boolean noExtension = FilenameUtils.getExtension(libraryName).equals(""); - for (String key : programsByName.keySet()) { + for (String key : loadedByName.keySet()) { String candidateName = key; if (isOptionalLibraryFilenameExtensions() && noExtension) { candidateName = FilenameUtils.getBaseName(candidateName); } if (comparator.compare(candidateName, libraryName) == 0) { - return programsByName.get(key); + return loadedByName.get(key); } } return null; } - /** - * Appends the given path elements to form a single path - * - * @param pathElements The path elements to append to one another - * @return A single path consisting of the given path elements appended together - */ - private String appendPath(String... pathElements) { - StringBuilder sb = new StringBuilder(); - for (String pathElement : pathElements) { - if (pathElement == null || pathElement.isEmpty()) { - continue; - } - boolean sbEndsWithSlash = - sb.length() > 0 && "/\\".indexOf(sb.charAt(sb.length() - 1)) != -1; - boolean elementStartsWithSlash = "/\\".indexOf(pathElement.charAt(0)) != -1; - - if (!sbEndsWithSlash && !elementStartsWithSlash && sb.length() > 0) { - sb.append("/"); - } - else if (elementStartsWithSlash && sbEndsWithSlash) { - pathElement = pathElement.substring(1); - } - sb.append(pathElement); - } - - return sb.toString(); - } - /** * Ensures the given {@link LoadSpec} matches one supported by the loader * diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractOrdinalSupportLoader.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractOrdinalSupportLoader.java index c835e89d17..65b87f67a3 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractOrdinalSupportLoader.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractOrdinalSupportLoader.java @@ -21,9 +21,11 @@ import java.util.Iterator; import java.util.List; import ghidra.app.util.Option; +import ghidra.app.util.OptionUtils; import ghidra.app.util.bin.ByteProvider; import ghidra.app.util.importer.MessageLog; import ghidra.framework.model.DomainObject; +import ghidra.framework.model.Project; import ghidra.framework.options.Options; import ghidra.program.model.listing.Function; import ghidra.program.model.listing.Program; @@ -116,33 +118,33 @@ public abstract class AbstractOrdinalSupportLoader extends AbstractLibrarySuppor } @Override - protected void postLoadProgramFixups(List loadedPrograms, List

+ * Note that when the load completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link Loaded#save(Project, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link Loaded#release(Object)} when they are no longer + * needed. * * @param provider The bytes to load. - * @param programName The name of the {@link Program} that's being loaded. - * @param programFolder The {@link DomainFolder} where the loaded thing should be saved. Could - * be null if the thing should not be pre-saved. + * @param loadedName A suggested name for the primary {@link Loaded} {@link Program}. + * This is just a suggestion, and a {@link Loader} implementation reserves the right to change + * it. The {@link Loaded} {@link Program}s should be queried for their true names using + * {@link Loaded#getName()}. + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} + * {@link Program}s should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. * @param loadSpec The {@link LoadSpec} to use during load. * @param options The load options. * @param log The message log. - * @param consumer A consumer object for {@link Program}s generated. - * @param monitor A cancelable task monitor. - * @return A list of {@link LoadedProgram loaded programs} (element 0 corresponds to primary - * loaded {@link Program}). + * @param consumer A consumer object for generated {@link Program}s. + * @param monitor A task monitor. + * @return A {@link List} of one or more {@link Loaded} {@link Program}s (created but not + * saved). + * @throws LoadException if the load failed in an expected way. * @throws IOException if there was an IO-related problem loading. * @throws CancelledException if the user cancelled the load. */ - protected abstract List loadProgram(ByteProvider provider, String programName, - DomainFolder programFolder, LoadSpec loadSpec, List

+ * NOTE: Subclasses should not use this method to release any {@link Program}s they created when + * failure occurs. That should be done by the subclass as soon as it detects failure has + * occurred. + * + * @param success True if the load completed successfully; otherwise, false + */ + protected void postLoadCleanup(boolean success) { + // Default behavior is to do nothing } /** @@ -262,6 +248,35 @@ public abstract class AbstractProgramLoader implements Loader { return false; } + /** + * Concatenates the given path elements to form a single path. Empty and null path elements + * are ignored. + * + * @param pathElements The path elements to append to one another + * @return A single path consisting of the given path elements appended together + */ + protected String concatenatePaths(String... pathElements) { + StringBuilder sb = new StringBuilder(); + for (String pathElement : pathElements) { + if (pathElement == null || pathElement.isEmpty()) { + continue; + } + boolean sbEndsWithSlash = + sb.length() > 0 && "/\\".indexOf(sb.charAt(sb.length() - 1)) != -1; + boolean elementStartsWithSlash = "/\\".indexOf(pathElement.charAt(0)) != -1; + + if (!sbEndsWithSlash && !elementStartsWithSlash && sb.length() > 0) { + sb.append("/"); + } + else if (elementStartsWithSlash && sbEndsWithSlash) { + pathElement = pathElement.substring(1); + } + sb.append(pathElement); + } + + return sb.toString(); + } + /** * Generates a block name. * @@ -307,25 +322,27 @@ public abstract class AbstractProgramLoader implements Loader { Program prog = new ProgramDB(programName, language, compilerSpec, consumer); prog.setEventsEnabled(false); int id = prog.startTransaction("Set program properties"); + boolean success = false; try { setProgramProperties(prog, provider, executableFormatName); - - if (shouldSetImageBase(prog, imageBase)) { - try { + try { + if (shouldSetImageBase(prog, imageBase)) { prog.setImageBase(imageBase, true); } - catch (AddressOverflowException e) { - // can't happen here - } - catch (LockException e) { - // can't happen here - } + success = true; + return prog; + } + catch (LockException | AddressOverflowException e) { + // shouldn't ever happen here + throw new IOException(e); } } finally { - prog.endTransaction(id, true); + prog.endTransaction(id, true); // More efficient to commit when program will be discarded + if (!success) { + prog.release(consumer); + } } - return prog; } /** @@ -453,62 +470,17 @@ public abstract class AbstractProgramLoader implements Loader { } /** - * Releases the given consumer from each of the provided {@link LoadedProgram}s. + * Releases the given consumer from each of the provided {@link Loaded loaded programs} * - * @param loadedPrograms A list of {@link LoadedProgram}s which are no longer being used. - * @param consumer The consumer that was marking the {@link DomainObject}s as being used. + * @param loadedPrograms A list of {@link Loaded loaded programs} which are no longer being used + * @param consumer The consumer that was marking the {@link Program}s as being used */ - protected final void release(List loadedPrograms, Object consumer) { - for (LoadedProgram loadedProgram : loadedPrograms) { - loadedProgram.program().release(consumer); + protected final void release(List> loadedPrograms, Object consumer) { + for (Loaded loadedProgram : loadedPrograms) { + loadedProgram.getDomainObject().release(consumer); } } - private boolean createProgramFile(Program program, DomainFolder programFolder, - String programName, MessageLog messageLog, TaskMonitor monitor) - throws CancelledException, InvalidNameException { - - int uniqueNameIndex = 0; - String uniqueName = programName; - while (!monitor.isCancelled()) { - try { - programFolder.createFile(uniqueName, program, monitor); - break; - } - catch (DuplicateFileException e) { - uniqueName = programName + uniqueNameIndex; - ++uniqueNameIndex; - } - catch (CancelledException | InvalidNameException e) { - throw e; - } - catch (Exception e) { - Throwable t = e.getCause(); - if (t == null) { - t = e; - } - String msg = t.getMessage(); - if (msg == null) { - msg = ""; - } - else { - msg = "\n" + msg; - } - Msg.showError(this, null, "Create Program Failed", - "Failed to create program file: " + uniqueName + msg, e); - messageLog.appendMsg("Unexpected exception creating file: " + uniqueName); - messageLog.appendException(e); - return false; - } - } - - // makes the data tree expand to show new file! - // The following line was disabled as it causes UI updates that are better - // done by the callers to this loader instead of this loader. - //programFolder.setActive(); - return true; - } - private void applyProcessorLabels(List

+ * NOTE: If any fail to save, none will be saved (already saved {@link DomainFile}s will be + * cleaned up/deleted), and all {@link Loaded} {@link DomainObject}s will have been + * {@link #release(Object) released}. + * + * @param project The {@link Project} to save to + * @param consumer the consumer + * @param messageLog The log + * @param monitor A cancelable task monitor + * @throws CancelledException if the operation was cancelled + * @throws IOException If there was a problem saving + * @see Loaded#save(Project, MessageLog, TaskMonitor) + */ + public void save(Project project, Object consumer, MessageLog messageLog, TaskMonitor monitor) + throws CancelledException, IOException { + boolean success = false; + try { + for (Loaded loaded : loadedList) { + loaded.save(project, messageLog, monitor); + } + success = true; + } + finally { + if (!success) { + for (Loaded loaded : this) { + try { + loaded.release(consumer); + loaded.deleteSavedDomainFile(consumer); + } + catch (IOException e1) { + Msg.error(getClass(), "Failed to delete: " + loaded); + } + } + } + } + } + + /** + * Notify all of the {@link Loaded} {@link DomainObject}s that the specified consumer is no + * longer using them. When the last consumer invokes this method, the {@link Loaded} + * {@link DomainObject}s will be closed and will become invalid. + * + * @param consumer the consumer + */ + public void release(Object consumer) { + loadedList.forEach(loaded -> loaded.release(consumer)); + } + + /** + * Notify the filtered {@link Loaded} {@link DomainObject}s that the specified consumer is no + * longer using them. When the last consumer invokes this method, the filtered {@link Loaded} + * {@link DomainObject}s will be closed and will become invalid. + * + * @param consumer the consumer + * @param filter a filter to apply to the {@link Loaded} {@link DomainObject}s prior to the + * release + */ + public void release(Object consumer, Predicate> filter) { + loadedList.stream().filter(filter).forEach(loaded -> loaded.release(consumer)); + } + + /** + * Notify the non-primary {@link Loaded} {@link DomainObject}s that the specified consumer is no + * longer using them. When the last consumer invokes this method, the non-primary {@link Loaded} + * {@link DomainObject}s will be closed and will become invalid. + * + * @param consumer the consumer + */ + public void releaseNonPrimary(Object consumer) { + for (int i = 0; i < loadedList.size(); i++) { + if (i > 0) { + loadedList.get(i).release(consumer); + } + } + } + + @Override + public Iterator> iterator() { + return loadedList.iterator(); + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loaded.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loaded.java new file mode 100644 index 0000000000..853ce88687 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loaded.java @@ -0,0 +1,221 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.app.util.opinion; + +import java.io.FileNotFoundException; +import java.io.IOException; + +import ghidra.app.util.importer.MessageLog; +import ghidra.framework.model.*; +import ghidra.util.InvalidNameException; +import ghidra.util.exception.*; +import ghidra.util.task.TaskMonitor; + +/** + * A loaded {@link DomainObject} produced by a {@link Loader}. In addition to storing the loaded + * {@link DomainObject}, it also stores the {@link Loader}'s desired name and project folder path + * for the loaded {@link DomainObject}, should it get saved to a project. + * + * @param The type of {@link DomainObject} that was loaded + */ +public class Loaded { + + private final T domainObject; + private final String name; + private String projectFolderPath; + + private DomainFile domainFile; + + /** + * Creates a new {@link Loaded} object + * + * @param domainObject The loaded {@link DomainObject} + * @param name The name of the loaded {@link DomainObject}. If a + * {@link #save(Project, MessageLog, TaskMonitor)} occurs, this will attempted to be used for + * the resulting {@link DomainFile}'s name. + * @param projectFolderPath The project folder path this will get saved to during a + * {@link #save(Project, MessageLog, TaskMonitor)} operation. If null or empty, the root + * project folder will be used. + */ + public Loaded(T domainObject, String name, String projectFolderPath) { + this.domainObject = domainObject; + this.name = name; + setProjectFolderPath(projectFolderPath); + } + + /** + * Gets the loaded {@link DomainObject} + * + * @return The loaded {@link DomainObject} + */ + public T getDomainObject() { + return domainObject; + } + + /** + * Gets the name of the loaded {@link DomainObject}. If a + * {@link #save(Project, MessageLog, TaskMonitor)} occurs, this will attempted to be used for + * the resulting {@link DomainFile}'s name. + * + * @return the name of the loaded {@link DomainObject} + */ + public String getName() { + return name; + } + + /** + * Gets the project folder path this will get saved to during a + * {@link #save(Project, MessageLog, TaskMonitor)} operation. + *

+ * NOTE: The returned path will always end with a "/". + * + * @return the project folder path + */ + public String getProjectFolderPath() { + return projectFolderPath; + } + + /** + * Sets the project folder path this will get saved to during a + * {@link #save(Project, MessageLog, TaskMonitor)} operation. + * + * @param projectFolderPath The project folder path this will get saved to during a + * {@link #save(Project, MessageLog, TaskMonitor)} operation. If null or empty, the root + * project folder will be used. + */ + public void setProjectFolderPath(String projectFolderPath) { + if (projectFolderPath == null) { + projectFolderPath = "/"; + } + else if (!projectFolderPath.endsWith("/")) { + projectFolderPath += "/"; + } + this.projectFolderPath = projectFolderPath; + } + + /** + * Notify the loaded {@link DomainObject} that the specified consumer is no longer using it. + * When the last consumer invokes this method, the loaded {@link DomainObject} will be closed + * and will become invalid. + * + * @param consumer the consumer + */ + public void release(Object consumer) { + if (!domainObject.isClosed() && domainObject.isUsedBy(consumer)) { + domainObject.release(consumer); + } + } + + /** + * Saves the loaded {@link DomainObject} to the given {@link Project} at this object's + * project folder path, using this object's name. + *

+ * If a {@link DomainFile} already exists with the same desired name and project folder path, + * the desired name will get a counter value appended to it to avoid a naming conflict. + * Therefore, it should not be assumed that the returned {@link DomainFile} will have the same + * name as a call to {@link #getName()}. + * + * @param project The {@link Project} to save to + * @param messageLog The log + * @param monitor A cancelable task monitor + * @return The {@link DomainFile} where the save happened + * @throws CancelledException if the operation was cancelled + * @throws ClosedException if the loaded {@link DomainObject} was already closed + * @throws IOException If there was an IO-related error, an invalid name was specified, or it + * was already successfully saved and still exists + */ + public DomainFile save(Project project, MessageLog messageLog, TaskMonitor monitor) + throws CancelledException, ClosedException, IOException { + + if (domainObject.isClosed()) { + throw new ClosedException( + "Cannot saved closed DomainObject: " + domainObject.getName()); + } + + try { + if (getSavedDomainFile() != null) { // + throw new IOException("Already saved to " + domainFile); + } + } + catch (FileNotFoundException e) { + // DomainFile was already saved, but no longer exists. + // Allow the save to proceeded. + domainFile = null; + } + + int uniqueNameIndex = 0; + String uniqueName = name; + try { + DomainFolder programFolder = ProjectDataUtils.createDomainFolderPath( + project.getProjectData().getRootFolder(), projectFolderPath); + while (!monitor.isCancelled()) { + try { + domainFile = programFolder.createFile(uniqueName, domainObject, monitor); + return domainFile; + } + catch (DuplicateFileException e) { + uniqueName = name + "." + uniqueNameIndex; + ++uniqueNameIndex; + } + } + } + catch (InvalidNameException e) { + throw new IOException(e); + } + throw new CancelledException(); + } + + /** + * Gets the loaded {@link DomainObject}'s associated {@link DomainFile} that was + * {@link #save(Project, MessageLog, TaskMonitor) saved} + * + * @return The loaded {@link DomainObject}'s associated saved {@link DomainFile}, or null if + * was not saved + * @throws FileNotFoundException If the loaded {@link DomainObject} was saved but the associated + * {@link DomainFile} no longer exists + * @see #save(Project, MessageLog, TaskMonitor) + */ + public DomainFile getSavedDomainFile() throws FileNotFoundException { + if (domainFile != null && !domainFile.exists()) { + throw new FileNotFoundException("Saved DomainFile no longer exists: " + domainFile); + } + return domainFile; + } + + /** + * Deletes the loaded {@link DomainObject}'s associated {@link DomainFile} that was + * {@link #save(Project, MessageLog, TaskMonitor) saved}. This method has no effect if it was + * never saved. + *

+ * NOTE: The loaded {@link DomainObject} must be {@link #release(Object) released} prior to + * calling this method. + * + * @param consumer the consumer + * @throws IOException If there was an issue deleting the saved {@link DomainFile} + * @see #save(Project, MessageLog, TaskMonitor) + */ + void deleteSavedDomainFile(Object consumer) throws IOException { + if (domainFile != null && domainFile.exists()) { + domainFile.delete(); + domainFile = null; + } + } + + @Override + public String toString() { + return getProjectFolderPath() + getName(); + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loader.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loader.java index fb76e3716e..11d34ed9af 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loader.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/Loader.java @@ -23,13 +23,12 @@ import ghidra.app.util.Option; import ghidra.app.util.bin.ByteProvider; import ghidra.app.util.importer.MessageLog; import ghidra.formats.gfilesystem.FSRL; -import ghidra.framework.model.DomainFolder; -import ghidra.framework.model.DomainObject; +import ghidra.framework.model.*; import ghidra.program.model.listing.Program; -import ghidra.util.InvalidNameException; import ghidra.util.classfinder.ClassSearcher; import ghidra.util.classfinder.ExtensionPoint; -import ghidra.util.exception.*; +import ghidra.util.exception.CancelledException; +import ghidra.util.exception.VersionException; import ghidra.util.task.TaskMonitor; /** @@ -41,8 +40,17 @@ import ghidra.util.task.TaskMonitor; */ public interface Loader extends ExtensionPoint, Comparable { + /** + * A string prefixed to each loader headless command line argument to avoid naming conflicts + * with other headless command line argument names + */ public static final String COMMAND_LINE_ARG_PREFIX = "-loader"; + /** + * Key used to lookup and store all loader options in the project's saved state + */ + public static final String OPTIONS_PROJECT_SAVE_STATE_KEY = "LOADER_OPTIONS"; + /** * If this {@link Loader} supports loading the given {@link ByteProvider}, this methods returns * a {@link Collection} of all supported {@link LoadSpec}s that contain discovered load @@ -59,33 +67,50 @@ public interface Loader extends ExtensionPoint, Comparable { public Collection findSupportedLoadSpecs(ByteProvider provider) throws IOException; /** - * Loads bytes in a particular format as a new {@link DomainObject}. - * Multiple {@link DomainObject}s may end up getting created, depending on the nature of the - * format. + * Loads bytes in a particular format as a new {@link Loaded} {@link DomainObject}. Multiple + * {@link DomainObject}s may end up getting created, depending on the nature of the format. + * The {@link Loaded} {@link DomainObject}s are bundled together in a {@link LoadResults} + * object which provides convenience methods to operate on the entire group of {@link Loaded} + * {@link DomainObject}s. + *

+ * Note that when the load completes, the returned {@link Loaded} {@link DomainObject}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link DomainObject}s with {@link LoadResults#release(Object)} when they are no longer + * needed. * * @param provider The bytes to load. - * @param name The name of the thing that's being loaded. - * @param folder The {@link DomainFolder} where the loaded thing should be saved. Could be - * null if the thing should not be pre-saved. + * @param loadedName A suggested name for the primary {@link Loaded} {@link DomainObject}. + * This is just a suggestion, and a {@link Loader} implementation reserves the right to change + * it. The {@link LoadResults} should be queried for their true names using + * {@link Loaded#getName()}. + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link DomainObject}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link LoadResults} + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. * @param loadSpec The {@link LoadSpec} to use during load. * @param options The load options. * @param messageLog The message log. - * @param consumer A consumer object for {@link DomainObject} generated. - * @param monitor A cancelable task monitor. - * @return A list of loaded {@link DomainObject}s (element 0 corresponds to primary loaded - * object). + * @param consumer A consumer object for generated {@link DomainObject}s. + * @param monitor A task monitor. + * @return The {@link LoadResults} which contains one or more {@link Loaded} + * {@link DomainObject}s (created but not saved). + * @throws LoadException if the load failed in an expected way * @throws IOException if there was an IO-related problem loading. * @throws CancelledException if the user cancelled the load. - * @throws DuplicateNameException if the load resulted in a naming conflict with the - * {@link DomainObject}. - * @throws InvalidNameException if an invalid {@link DomainObject} name was used during load. - * @throws VersionException if there was an issue with database versions, probably due to a - * failed language upgrade. + * @throws VersionException if the load process tried to open an existing {@link DomainFile} + * which was created with a newer or unsupported version of Ghidra */ - public List load(ByteProvider provider, String name, DomainFolder folder, - LoadSpec loadSpec, List

* - * @param program ELF {@link Program} to fix. - * @param saveIfModified boolean flag, if true the program will be saved if there was a - * modification. + * @param loadedPrograms ELF {@link Loaded} {@link Program}s to fix.. * @param messageLog {@link MessageLog} to write info message to. * @param monitor {@link TaskMonitor} to watch for cancel and update with progress. * @throws CancelledException if user cancels * @throws IOException if error reading */ - public static void fixUnresolvedExternalSymbols(Program program, boolean saveIfModified, + public static void fixUnresolvedExternalSymbols(List> loadedPrograms, MessageLog messageLog, TaskMonitor monitor) throws CancelledException, IOException { - DomainFolder domainFolder = program.getDomainFile().getParent(); - if (domainFolder == null) { - return; // headless case with nothing pre-saved...currently unsupported - } - ProjectData projectData = domainFolder.getProjectData(); + Map> loadedByName = loadedPrograms.stream() + .collect( + Collectors.toMap(loaded -> loaded.getName(), loaded -> loaded)); - Collection unresolvedExternalFunctionIds = getUnresolvedExternalFunctionIds(program); - if (unresolvedExternalFunctionIds.size() == 0) { - return; - } + monitor.initialize(loadedByName.size()); + for (Loaded loadedProgram : loadedByName.values()) { + Program program = loadedProgram.getDomainObject(); - List libSearchList = getLibrarySearchList(program); - if (libSearchList.isEmpty()) { - return; - } + Collection unresolvedExternalFunctionIds = + getUnresolvedExternalFunctionIds(program); + if (unresolvedExternalFunctionIds.size() == 0) { + return; + } - int transactionID = program.startTransaction("Resolve External Symbols"); - try { + List libSearchList = getLibrarySearchList(program); + if (libSearchList.isEmpty()) { + return; + } - messageLog.appendMsg("----- [" + program.getName() + "] Resolve " + - unresolvedExternalFunctionIds.size() + " external symbols -----"); + int transactionID = program.startTransaction("Resolve External Symbols"); + try { - for (Library extLibrary : libSearchList) { - monitor.checkCanceled(); - String libName = extLibrary.getName(); - String libPath = extLibrary.getAssociatedProgramPath(); - if (libPath == null) { - continue; - } + messageLog.appendMsg("----- [" + program.getName() + "] Resolve " + + unresolvedExternalFunctionIds.size() + " external symbols -----"); - DomainFile libDomainFile = projectData.getFile(libPath); - if (libDomainFile == null) { - messageLog.appendMsg("Referenced external program not found: " + libPath); - continue; - } - - Object consumer = new Object(); - DomainObject libDomainObject = null; - try { - libDomainObject = - libDomainFile.getDomainObject(consumer, false, false, monitor); - if (!(libDomainObject instanceof Program)) { - messageLog.appendMsg( - "Referenced external program is not a program: " + libPath); + for (Library extLibrary : libSearchList) { + monitor.checkCanceled(); + String libName = extLibrary.getName(); + String libPath = extLibrary.getAssociatedProgramPath(); + if (libPath == null) { continue; } + + Loaded loadedLib = loadedByName.get(libName); + if (loadedLib == null) { + messageLog.appendMsg("Referenced external program not found: " + libName); + continue; + } + + DomainObject libDomainObject = loadedLib.getDomainObject(); monitor.setMessage("Resolving symbols published by library " + libName); resolveSymbolsToLibrary(program, unresolvedExternalFunctionIds, extLibrary, (Program) libDomainObject, messageLog, monitor); } - catch (IOException e) { - // failed to open library - messageLog.appendMsg("Failed to open library dependency project file: " + - libDomainFile.getPathname()); - } - catch (VersionException e) { - messageLog.appendMsg( - "Referenced external program requires updgrade, unable to consider symbols: " + - libPath); - } - finally { - if (libDomainObject != null) { - libDomainObject.release(consumer); - } - } + messageLog.appendMsg("Unresolved external symbols which remain: " + + unresolvedExternalFunctionIds.size()); + } + finally { + program.endTransaction(transactionID, true); } - messageLog.appendMsg("Unresolved external symbols which remain: " + - unresolvedExternalFunctionIds.size()); - } - finally { - program.endTransaction(transactionID, true); - } - if (saveIfModified && program.canSave() && program.isChanged()) { - program.save("ExternalSymbolResolver", monitor); } } @@ -223,7 +199,7 @@ public class ELFExternalSymbolResolver { return orderLibraryMap.values(); } - private static List getLibrarySearchList(Program program) { + public static List getLibrarySearchList(Program program) { List result = new ArrayList<>(); ExternalManager externalManager = program.getExternalManager(); for (String libName : getOrderedLibraryNamesNeeded(program)) { diff --git a/Ghidra/Features/FileFormats/src/main/java/ghidra/app/util/opinion/ApkLoader.java b/Ghidra/Features/FileFormats/src/main/java/ghidra/app/util/opinion/ApkLoader.java index f296b9c2e9..69a8fd8452 100644 --- a/Ghidra/Features/FileFormats/src/main/java/ghidra/app/util/opinion/ApkLoader.java +++ b/Ghidra/Features/FileFormats/src/main/java/ghidra/app/util/opinion/ApkLoader.java @@ -37,7 +37,7 @@ import ghidra.file.formats.android.xml.AndroidXmlFileSystem; import ghidra.file.formats.zip.ZipFileSystem; import ghidra.formats.gfilesystem.FileSystemService; import ghidra.formats.gfilesystem.GFile; -import ghidra.framework.model.DomainFolder; +import ghidra.framework.model.Project; import ghidra.program.model.listing.Program; import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskMonitor; @@ -64,12 +64,13 @@ public class ApkLoader extends DexLoader { } @Override - protected List loadProgram(ByteProvider provider, String programName, - DomainFolder programFolder, LoadSpec loadSpec, List