diff --git a/Ghidra/Features/Base/src/main/help/help/topics/HeadlessAnalyzer/HeadlessAnalyzer.htm b/Ghidra/Features/Base/src/main/help/help/topics/HeadlessAnalyzer/HeadlessAnalyzer.htm index a85bf40cfc..6f352876be 100644 --- a/Ghidra/Features/Base/src/main/help/help/topics/HeadlessAnalyzer/HeadlessAnalyzer.htm +++ b/Ghidra/Features/Base/src/main/help/help/topics/HeadlessAnalyzer/HeadlessAnalyzer.htm @@ -50,7 +50,7 @@ The Headless Analyzer can be useful when performing repetitive tasks on a projec
    [-scriptlog <path to script log file>]
    [-log <path to log file>]
    [-overwrite] -
    [-recursive] +
    [-recursive [<depth>]]
    [-readOnly]
    [-deleteProject]
    [-noanalysis] diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/AnalyzeHeadless.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/AnalyzeHeadless.java index 5b44ba640a..3db6eade8d 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/AnalyzeHeadless.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/AnalyzeHeadless.java @@ -212,13 +212,21 @@ public class AnalyzeHeadless implements GhidraLaunchable { options.setPropertiesFileDirectories(args[++argi]); } else if (checkArgument("-import", args, argi)) { - File inputFile = new File(args[++argi]).getAbsoluteFile(); + File inputFile = null; + try { + inputFile = new File(args[++argi]); + inputFile = inputFile.getCanonicalFile(); + } + catch (IOException e) { + throw new InvalidInputException( + "Failed to get canonical form of: " + inputFile); + } if (!inputFile.isDirectory() && !inputFile.isFile()) { throw new InvalidInputException( inputFile + " is not a valid directory or file."); } - HeadlessAnalyzer.checkValidFilename(inputFile); + HeadlessAnalyzer.checkValidFilename(inputFile.toString()); filesToImport.add(inputFile); @@ -240,7 +248,7 @@ public class AnalyzeHeadless implements GhidraLaunchable { otherFile + " is not a valid directory or file."); } - HeadlessAnalyzer.checkValidFilename(otherFile); + HeadlessAnalyzer.checkValidFilename(otherFile.toString()); filesToImport.add(otherFile); } @@ -298,7 +306,21 @@ public class AnalyzeHeadless implements GhidraLaunchable { options.setRunScriptsNoImport(true, processBinary); } else if ("-recursive".equals(args[argi])) { - options.enableRecursiveProcessing(true); + Integer depth = null; + if ((argi + 1) < args.length) { + arg = args[argi + 1]; + if (!arg.startsWith("-")) { + // depth is optional argument after -recursive + try { + depth = Integer.parseInt(arg); + } + catch (NumberFormatException e) { + throw new InvalidInputException("Invalid recursion depth: " + depth); + } + ++argi; + } + } + options.enableRecursiveProcessing(true, depth); } else if ("-readOnly".equalsIgnoreCase(args[argi])) { options.enableReadOnlyProcessing(true); diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessAnalyzer.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessAnalyzer.java index 0d4b0b5274..b8a988da4c 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessAnalyzer.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessAnalyzer.java @@ -20,6 +20,8 @@ import java.net.*; import java.util.*; import java.util.regex.Pattern; +import org.apache.commons.io.FilenameUtils; + import generic.jar.ResourceFile; import generic.stl.Pair; import generic.util.Path; @@ -32,6 +34,7 @@ import ghidra.app.util.headless.HeadlessScript.HeadlessContinuationOption; import ghidra.app.util.importer.AutoImporter; import ghidra.app.util.importer.MessageLog; import ghidra.app.util.opinion.*; +import ghidra.formats.gfilesystem.*; import ghidra.framework.*; import ghidra.framework.client.ClientUtil; import ghidra.framework.client.RepositoryAdapter; @@ -77,6 +80,7 @@ public class HeadlessAnalyzer { private DomainFolder saveDomainFolder; private Map storage; private URLClassLoader classLoaderForDotClassScripts; + private FileSystemService fsService; /** * Gets a headless analyzer, initializing the application if necessary with the specified @@ -206,6 +210,9 @@ public class HeadlessAnalyzer { // Put analyzer in its default state reset(); + + // Initialize FileSytemService + fsService = FileSystemService.getInstance(); } /** @@ -1504,9 +1511,9 @@ public class HeadlessAnalyzer { } } - private boolean processFileWithImport(File file, String folderPath) { + private boolean processFileWithImport(FSRL fsrl, String folderPath) { - Msg.info(this, "IMPORTING: " + file.getAbsolutePath()); + Msg.info(this, "IMPORTING: " + fsrl); LoadResults loadResults = null; Loaded primary = null; @@ -1514,7 +1521,7 @@ public class HeadlessAnalyzer { // Perform the load. Note that loading 1 file may result in more than 1 thing getting // loaded. - loadResults = loadPrograms(file, folderPath); + loadResults = loadPrograms(fsrl, folderPath); Msg.info(this, "IMPORTING: Loaded " + (loadResults.size() - 1) + " additional files"); primary = loadResults.getPrimary(); @@ -1533,15 +1540,14 @@ public class HeadlessAnalyzer { // Check if there are defined memory blocks in the primary program. // Abort if not (there is nothing to work with!). if (primaryProgram.getMemory().getAllInitializedAddressSet().isEmpty()) { - Msg.error(this, "REPORT: Error: No memory blocks were defined for file " + - file.getAbsolutePath()); + Msg.error(this, "REPORT: Error: No memory blocks were defined for file " + fsrl); return false; } // Analyze the primary program, and determine if we should save. // TODO: Analyze non-primary programs (GP-2965). boolean doSave = - analyzeProgram(file.getAbsolutePath(), primaryProgram) && !options.readOnly; + analyzeProgram(fsrl.toString(), primaryProgram) && !options.readOnly; // The act of marking the program as temporary by a script will signal // us to discard any changes @@ -1602,8 +1608,7 @@ public class HeadlessAnalyzer { return true; } catch (LoadException e) { - Msg.error(this, "The AutoImporter could not successfully load " + - file.getAbsolutePath() + + Msg.error(this, "The AutoImporter could not successfully load " + fsrl + " with the provided import parameters. Please ensure that any specified" + " processor/cspec arguments are compatible with the loader that is used during" + " import and try again."); @@ -1626,7 +1631,7 @@ public class HeadlessAnalyzer { } } - private LoadResults loadPrograms(File file, String folderPath) + private LoadResults loadPrograms(FSRL fsrl, String folderPath) throws VersionException, InvalidNameException, DuplicateNameException, CancelledException, IOException, LoadException { MessageLog messageLog = new MessageLog(); @@ -1634,96 +1639,119 @@ public class HeadlessAnalyzer { if (options.loaderClass == null) { // User did not specify a loader if (options.language == null) { - return AutoImporter.importByUsingBestGuess(file, project, folderPath, this, + return AutoImporter.importByUsingBestGuess(fsrl, project, folderPath, this, messageLog, TaskMonitor.DUMMY); } - return AutoImporter.importByLookingForLcs(file, project, folderPath, options.language, + return AutoImporter.importByLookingForLcs(fsrl, project, folderPath, options.language, options.compilerSpec, this, messageLog, TaskMonitor.DUMMY); } // User specified a loader if (options.language == null) { - return AutoImporter.importByUsingSpecificLoaderClass(file, project, folderPath, + return AutoImporter.importByUsingSpecificLoaderClass(fsrl, project, folderPath, options.loaderClass, options.loaderArgs, this, messageLog, TaskMonitor.DUMMY); } - return AutoImporter.importByUsingSpecificLoaderClassAndLcs(file, project, folderPath, + return AutoImporter.importByUsingSpecificLoaderClassAndLcs(fsrl, project, folderPath, options.loaderClass, options.loaderArgs, options.language, options.compilerSpec, this, messageLog, TaskMonitor.DUMMY); } - private void processWithImport(File file, String folderPath, boolean isFirstTime) - throws IOException { - - boolean importSucceeded; - - if (file.isFile()) { - - importSucceeded = processFileWithImport(file, folderPath); - - // Check to see if there are transient programs lying around due - // to programs not being released during Importing - List domainFileContainer = new ArrayList<>(); - TransientDataManager.getTransients(domainFileContainer); - if (domainFileContainer.size() > 0) { - TransientDataManager.releaseFiles(this); + private void processWithImport(FSRL fsrl, String folderPath, int depth) throws IOException { + try (RefdFile refdFile = fsService.getRefdFile(fsrl, TaskMonitor.DUMMY)) { + GFile file = refdFile.file; + if (options.recursive && file.isDirectory()) { + processFS(file.getFilesystem(), file, folderPath, depth); + return; } - - if (!importSucceeded) { - Msg.error(this, "REPORT: Import failed for file: " + file.getAbsolutePath()); + if (options.recursive && depth > 0 && processAsFS(fsrl, folderPath, depth)) { + return; + } + if (!file.isDirectory() && processWithLoader(fsrl, folderPath)) { + return; } - - return; } + catch (CancelledException e) { + Msg.info(this, "REPORT: Importing cancelled"); + } + } - // Looks inside the folder if one of two situations is applicable: - // - If user supplied a directory to import, and it is currently being - // processed (if so, this will be the first time that this method is called) - // - If -recursive is specified - if ((isFirstTime) || (!isFirstTime && options.recursive)) { - // Otherwise, is a directory - Msg.info(this, "REPORT: Importing all files from " + file.getName()); - - File dirFile = file; + private void processFS(GFileSystem fs, GFile startDir, String folderPath, int depth) + throws CancelledException, IOException { + if (!folderPath.endsWith(DomainFolder.SEPARATOR)) { + folderPath += DomainFolder.SEPARATOR; + } + folderPath += startDir.getName(); + for (GFile file : fs.getListing(startDir)) { + String name = file.getName(); + if (name.startsWith(".")) { + Msg.warn(this, "Ignoring file '" + name + "'."); + continue; + } + FSRL fqFSRL; + try { + fqFSRL = fsService.getFullyQualifiedFSRL(file.getFSRL(), TaskMonitor.DUMMY); + } + catch (IOException e) { + Msg.warn(this, "Error getting info for " + file.getFSRL()); + continue; + } + try { + checkValidFilename(fqFSRL.getName()); + processWithImport(fqFSRL, folderPath, depth); + } + catch (InvalidInputException e) { + // Just move on if not valid + } + } + } + + private boolean processAsFS(FSRL fsrl, String folderPath, int depth) throws CancelledException { + try (FileSystemRef fsRef = fsService.probeFileForFilesystem(fsrl, TaskMonitor.DUMMY, + FileSystemProbeConflictResolver.CHOOSEFIRST)) { + if (fsRef == null) { + return false; + } if (!folderPath.endsWith(DomainFolder.SEPARATOR)) { folderPath += DomainFolder.SEPARATOR; } - - String subfolderPath = folderPath + file.getName(); - - String[] names = dirFile.list(); - if (names != null) { - Collections.sort(Arrays.asList(names)); - for (String name : names) { - if (name.charAt(0) == '.') { - Msg.warn(this, "Ignoring file '" + name + "'."); - continue; - } - file = new File(dirFile, name); - - // Even a directory name has to have valid characters -- - // can't create a folder if it's not valid - try { - checkValidFilename(file); - processWithImport(file, subfolderPath, false); - } - catch (InvalidInputException e) { - // Just move on if not valid - } - } - } + folderPath += fsrl.getName(); + processWithImport(fsRef.getFilesystem().getFSRL(), folderPath, depth - 1); + return true; } + catch (IOException e) { + return false; + } + } + + private boolean processWithLoader(FSRL fsrl, String folderPath) { + boolean importSucceeded = processFileWithImport(fsrl, folderPath); + + // Check to see if there are transient programs lying around due + // to programs not being released during Importing + List domainFileContainer = new ArrayList<>(); + TransientDataManager.getTransients(domainFileContainer); + if (domainFileContainer.size() > 0) { + TransientDataManager.releaseFiles(this); + } + + if (!importSucceeded) { + Msg.error(this, "REPORT: Import failed for file: " + fsrl); + } + return importSucceeded; } private void processWithImport(String folderPath, List inputDirFiles) throws IOException { storage.clear(); + if (inputDirFiles != null && !inputDirFiles.isEmpty()) { Msg.info(this, "REPORT: Processing input files: "); Msg.info(this, " project: " + project.getProjectLocator()); - for (File f : inputDirFiles) { - processWithImport(f, folderPath, true); + List fsrls = inputDirFiles.stream().map(f -> fsService.getLocalFSRL(f)).toList(); + for (FSRL fsrl : fsrls) { + processWithImport(fsrl, folderPath, options.recursiveDepth); } } else { @@ -1779,27 +1807,18 @@ public class HeadlessAnalyzer { /** * Checks to make sure the given file contains only valid characters in its name. * - * @param currFile The file to check. - * @throws InvalidInputException if the given file contains invalid characters in it. + * @param path The path of the file to check. + * @throws InvalidInputException if the given file path contains invalid characters in it. */ - static void checkValidFilename(File currFile) throws InvalidInputException { - boolean isDir = currFile.isDirectory(); - String filename = currFile.getName(); + static void checkValidFilename(String path) throws InvalidInputException { + String filename = FilenameUtils.getName(path); for (int i = 0; i < filename.length(); i++) { char c = filename.charAt(i); if (!LocalFileSystem.isValidNameCharacter(c)) { - if (isDir) { - throw new InvalidInputException("The directory '" + filename + - "' contains the invalid characgter: \'" + c + - "\' and can not be created in the project (full path: " + - currFile.getAbsolutePath() + - "). To allow successful import of the directory and its contents, please rename the directory."); - } throw new InvalidInputException( - "The file '" + filename + "' contains the invalid character: \'" + c + - "\' and can not be imported (full path: " + currFile.getAbsolutePath() + - "). Please rename the file."); + "'" + filename + "' contains the invalid character: \'" + c + + "\' and can not be imported (full path: " + path + "). Please rename."); } } } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessOptions.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessOptions.java index 2526ba033a..04d8cc0232 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessOptions.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/headless/HeadlessOptions.java @@ -22,6 +22,7 @@ import generic.jar.ResourceFile; import generic.stl.Pair; import ghidra.app.util.opinion.Loader; import ghidra.app.util.opinion.LoaderService; +import ghidra.formats.gfilesystem.GFileSystem; import ghidra.framework.client.HeadlessClientAuthenticator; import ghidra.program.model.lang.*; import ghidra.program.util.DefaultLanguageService; @@ -62,6 +63,7 @@ public class HeadlessOptions { // -recursive boolean recursive; + int recursiveDepth; // -readOnly boolean readOnly; @@ -128,6 +130,7 @@ public class HeadlessOptions { propertiesFilePaths = new ArrayList<>(); overwrite = false; recursive = false; + recursiveDepth = 0; readOnly = false; deleteProject = false; analyze = true; @@ -319,14 +322,30 @@ public class HeadlessOptions { /** * This method can be used to enable recursive processing of files during - * -import or -process modes. In order for recursive processing of files to - * occur, the user must have specified a directory (and not a specific file) - * for the Headless Analyzer to import or process. + * -import or -process modes. In order for recursive processing of + * files to occur, the user must have specified a project folder to process or a directory or + * supported {@link GFileSystem} container file to import * - * @param enabled if true, enables recursive processing + * @param enabled if true, enables recursive import/processing */ public void enableRecursiveProcessing(boolean enabled) { + enableRecursiveProcessing(enabled, null); + } + + /** + * This method can be used to enable recursive processing of files during + * -import or -process modes. In order for recursive processing of + * files to occur, the user must have specified a project folder to process or a directory or + * supported {@link GFileSystem} container file to import + * + * @param enabled if true, enables recursive import/processing + * @param depth maximum container recursion depth (could be null to use default) + */ + public void enableRecursiveProcessing(boolean enabled, Integer depth) { this.recursive = enabled; + if (depth != null) { + this.recursiveDepth = depth; + } } /** diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/importer/AutoImporter.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/importer/AutoImporter.java index ae2984a4c4..6804fee2a1 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/importer/AutoImporter.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/importer/AutoImporter.java @@ -17,7 +17,6 @@ package ghidra.app.util.importer; import java.io.File; import java.io.IOException; -import java.nio.file.AccessMode; import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; @@ -25,8 +24,8 @@ import java.util.function.Predicate; import generic.stl.Pair; import ghidra.app.util.Option; import ghidra.app.util.bin.ByteProvider; -import ghidra.app.util.bin.FileByteProvider; import ghidra.app.util.opinion.*; +import ghidra.formats.gfilesystem.FSRL; import ghidra.formats.gfilesystem.FileSystemService; import ghidra.framework.model.*; import ghidra.program.model.address.AddressFactory; @@ -83,8 +82,49 @@ public final class AutoImporter { String projectFolderPath, Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, CancelledException, DuplicateNameException, InvalidNameException, VersionException, LoadException { - return importFresh(file, project, projectFolderPath, consumer, messageLog, - monitor, LoaderService.ACCEPT_ALL, LoadSpecChooser.CHOOSE_THE_FIRST_PREFERRED, null, + return importByUsingBestGuess(fileToFsrl(file), project, projectFolderPath, consumer, + messageLog, monitor); + } + + /** + * Automatically imports the given {@link FSRL} with the best matching {@link Loader} for the + * {@link File}'s format. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param fsrl The {@link FSRL} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static LoadResults importByUsingBestGuess(FSRL fsrl, Project project, + String projectFolderPath, Object consumer, MessageLog messageLog, TaskMonitor monitor) + throws IOException, CancelledException, DuplicateNameException, InvalidNameException, + VersionException, LoadException { + return importFresh(fsrl, project, projectFolderPath, consumer, messageLog, monitor, + LoaderService.ACCEPT_ALL, LoadSpecChooser.CHOOSE_THE_FIRST_PREFERRED, null, OptionChooser.DEFAULT_OPTIONS); } @@ -169,9 +209,52 @@ public final class AutoImporter { List> loaderArgs, Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, CancelledException, DuplicateNameException, InvalidNameException, VersionException, LoadException { + return importByUsingSpecificLoaderClass(fileToFsrl(file), project, projectFolderPath, + loaderClass, loaderArgs, consumer, messageLog, monitor); + } + + /** + * Automatically imports the given {@link FSRL} with the given type of {@link Loader}. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param fsrl The {@link FSRL} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param loaderClass The {@link Loader} class to use + * @param loaderArgs A {@link List} of optional {@link Loader}-specific arguments + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static LoadResults importByUsingSpecificLoaderClass(FSRL fsrl, Project project, + String projectFolderPath, Class loaderClass, + List> loaderArgs, Object consumer, MessageLog messageLog, + TaskMonitor monitor) throws IOException, CancelledException, DuplicateNameException, + InvalidNameException, VersionException, LoadException { SingleLoaderFilter loaderFilter = new SingleLoaderFilter(loaderClass, loaderArgs); - return importFresh(file, project, projectFolderPath, consumer, messageLog, - monitor, loaderFilter, LoadSpecChooser.CHOOSE_THE_FIRST_PREFERRED, null, + return importFresh(fsrl, project, projectFolderPath, consumer, messageLog, monitor, + loaderFilter, LoadSpecChooser.CHOOSE_THE_FIRST_PREFERRED, null, new LoaderArgsOptionChooser(loaderFilter)); } @@ -214,9 +297,52 @@ public final class AutoImporter { String projectFolderPath, Language language, CompilerSpec compilerSpec, Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, CancelledException, DuplicateNameException, InvalidNameException, VersionException, LoadException { - return importFresh(file, project, projectFolderPath, consumer, messageLog, - monitor, LoaderService.ACCEPT_ALL, new LcsHintLoadSpecChooser(language, compilerSpec), - null, OptionChooser.DEFAULT_OPTIONS); + return importByLookingForLcs(fileToFsrl(file), project, projectFolderPath, language, + compilerSpec, consumer, messageLog, monitor); + } + + /** + * Automatically imports the given {@link FSRL} with the best matching {@link Loader} that + * supports the given language and compiler specification. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param fsrl The {@link FSRL} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param language The desired {@link Language} + * @param compilerSpec The desired {@link CompilerSpec compiler specification} + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static LoadResults importByLookingForLcs(FSRL fsrl, Project project, + String projectFolderPath, Language language, CompilerSpec compilerSpec, Object consumer, + MessageLog messageLog, TaskMonitor monitor) throws IOException, CancelledException, + DuplicateNameException, InvalidNameException, VersionException, LoadException { + return importFresh(fsrl, project, projectFolderPath, consumer, messageLog, monitor, + LoaderService.ACCEPT_ALL, new LcsHintLoadSpecChooser(language, compilerSpec), null, + OptionChooser.DEFAULT_OPTIONS); } /** @@ -260,9 +386,54 @@ public final class AutoImporter { List> loaderArgs, Language language, CompilerSpec compilerSpec, Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, CancelledException, DuplicateNameException, InvalidNameException, VersionException { + return importByUsingSpecificLoaderClassAndLcs(fileToFsrl(file), project, projectFolderPath, + loaderClass, loaderArgs, language, compilerSpec, consumer, messageLog, monitor); + } + + /** + * Automatically imports the given {@link FSRL} with the given type of {@link Loader}, language, + * and compiler specification. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param fsrl The {@link FSRL} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param loaderClass The {@link Loader} class to use + * @param loaderArgs A {@link List} of optional {@link Loader}-specific arguments + * @param language The desired {@link Language} + * @param compilerSpec The desired {@link CompilerSpec compiler specification} + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + */ + public static LoadResults importByUsingSpecificLoaderClassAndLcs(FSRL fsrl, + Project project, String projectFolderPath, Class loaderClass, + List> loaderArgs, Language language, CompilerSpec compilerSpec, + Object consumer, MessageLog messageLog, TaskMonitor monitor) throws IOException, + CancelledException, DuplicateNameException, InvalidNameException, VersionException { SingleLoaderFilter loaderFilter = new SingleLoaderFilter(loaderClass, loaderArgs); - return importFresh(file, project, projectFolderPath, consumer, messageLog, - monitor, loaderFilter, new LcsHintLoadSpecChooser(language, compilerSpec), null, + return importFresh(fsrl, project, projectFolderPath, consumer, messageLog, monitor, + loaderFilter, new LcsHintLoadSpecChooser(language, compilerSpec), null, new LoaderArgsOptionChooser(loaderFilter)); } @@ -405,12 +576,61 @@ public final class AutoImporter { String importNameOverride, OptionChooser optionChooser) throws IOException, CancelledException, DuplicateNameException, InvalidNameException, VersionException, LoadException { - if (file == null) { - throw new LoadException("Cannot load null file"); + return importFresh(fileToFsrl(file), project, projectFolderPath, consumer, messageLog, + monitor, loaderFilter, loadSpecChooser, importNameOverride, optionChooser); + } + + /** + * Automatically imports the given {@link FSRL} with advanced options. + *

+ * Note that when the import completes, the returned {@link Loaded} {@link Program}s are not + * saved to a project. That is the responsibility of the caller (see + * {@link LoadResults#save(Project, Object, MessageLog, TaskMonitor)}). + *

+ * It is also the responsibility of the caller to release the returned {@link Loaded} + * {@link Program}s with {@link LoadResults#release(Object)} when they are no longer needed. + * + * @param fsrl The {@link FSRL} to import + * @param project The {@link Project}. Loaders can use this to take advantage of existing + * {@link DomainFolder}s and {@link DomainFile}s to do custom behaviors such as loading + * libraries. Could be null if there is no project. + * @param projectFolderPath A suggested project folder path for the {@link Loaded} + * {@link Program}s. This is just a suggestion, and a {@link Loader} implementation + * reserves the right to change it for each {@link Loaded} result. The {@link Loaded} results + * should be queried for their true project folder paths using + * {@link Loaded#getProjectFolderPath()}. + * @param loaderFilter A {@link Predicate} used to choose what {@link Loader}(s) get used + * @param loadSpecChooser A {@link LoadSpecChooser} used to choose what {@link LoadSpec}(s) get + * used + * @param importNameOverride The name to use for the imported thing. Null to use the + * {@link Loader}'s preferred name. + * @param optionChooser A {@link OptionChooser} used to choose what {@link Loader} options get + * used + * @param consumer A consumer + * @param messageLog The log + * @param monitor A task monitor + * @return The {@link LoadResults} which contains one ore more {@link Loaded} {@link Program}s + * (created but not saved) + * @throws IOException if there was an IO-related problem loading + * @throws CancelledException if the operation was cancelled + * @throws DuplicateNameException if the load resulted in a {@link Program} naming conflict + * @throws InvalidNameException if an invalid {@link Program} name was used during load + * @throws VersionException if there was an issue with database versions, probably due to a + * failed language upgrade + * @throws LoadException if nothing was loaded + */ + public static LoadResults importFresh(FSRL fsrl, Project project, + String projectFolderPath, Object consumer, MessageLog messageLog, TaskMonitor monitor, + Predicate loaderFilter, LoadSpecChooser loadSpecChooser, + String importNameOverride, OptionChooser optionChooser) + throws IOException, CancelledException, DuplicateNameException, InvalidNameException, + VersionException, LoadException { + if (fsrl == null) { + throw new LoadException("Cannot load null fsrl"); } - try (ByteProvider provider = new FileByteProvider(file, - FileSystemService.getInstance().getLocalFSRL(file), AccessMode.READ)) { + try (ByteProvider provider = + FileSystemService.getInstance().getByteProvider(fsrl, true, monitor)) { return importFresh(provider, project, projectFolderPath, consumer, messageLog, monitor, loaderFilter, loadSpecChooser, importNameOverride, optionChooser); } @@ -533,4 +753,17 @@ public final class AutoImporter { Msg.info(AutoImporter.class, "No load spec found for import file: " + name); return null; } + + /** + * Converts a {@link File} to a local file system {@link FSRL} + * @param file The {@link File} to convert + * @return A {@link FSRL} that represents the given {@link File} + * @throws LoadException if the given {@link File} is null + */ + private static FSRL fileToFsrl(File file) throws LoadException { + if (file == null) { + throw new LoadException("Cannot load null file"); + } + return FileSystemService.getInstance().getLocalFSRL(file); + } } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractLibrarySupportLoader.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractLibrarySupportLoader.java index a36fb3e2ce..29064b351e 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractLibrarySupportLoader.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/AbstractLibrarySupportLoader.java @@ -1119,7 +1119,8 @@ public abstract class AbstractLibrarySupportLoader extends AbstractProgramLoader for (Path searchPath : searchPaths) { try { - FSRL searchFSRL = fsService.getLocalFSRL(searchPath.toFile()); + FSRL searchFSRL = + fsService.getLocalFSRL(searchPath.toFile().getCanonicalFile()); FileSystemRef fsRef = fsService.probeFileForFilesystem(searchFSRL, monitor, null); if (fsRef != null) { diff --git a/Ghidra/RuntimeScripts/Common/support/analyzeHeadlessREADME.html b/Ghidra/RuntimeScripts/Common/support/analyzeHeadlessREADME.html index 644fe7b45f..737482a746 100644 --- a/Ghidra/RuntimeScripts/Common/support/analyzeHeadlessREADME.html +++ b/Ghidra/RuntimeScripts/Common/support/analyzeHeadlessREADME.html @@ -117,7 +117,7 @@ The Headless Analyzer uses the command-line parameters discussed below. See -scriptlog <path to script log file>] [-log <path to log file>] [-overwrite] - [-recursive] + [-recursive [<depth>]] [-readOnly] [-deleteProject] [-noanalysis] @@ -216,10 +216,11 @@ The Headless Analyzer uses the command-line parameters discussed below. See

Specifies one or more executables (or directories of executables) to import. When importing a - directory, a folder with the same name will be created in the Ghidra project. When using the - -recursive parameter, each executable that is found in a recursive - search through the given directory will be stored in the project in the same relative location - (i.e., any directories found under the import directory will also be created in the project). + directory or supported container format, a folder with the same name will be created in the + Ghidra project. When using the -recursive parameter, each executable + that is found in a recursive search through the given directory or container file will be + stored in the project in the same relative location (i.e., any directories found under the + import directory will also be created in the project).

Operating system-specific wildcard characters can be used when importing files and/or directories. Please see the Wildcards section for more details. @@ -395,11 +396,17 @@ The Headless Analyzer uses the command-line parameters discussed below. See

  • - -recursive
    + -recursive [<depth>]
    If present, enables recursive descent into directories and project sub-folders when a directory/ folder has been specified in -import or -process modes. -
    • +

    + Specifying a positive integer value for the optional <depth> + argument enables recursive descent into supported container files (e.g., zip, tar, .a, etc). + The depth value only applies to nested container files. Intermediate directories found within + each nested container file are not affected by the specified depth value. Specifying a depth + value of 0 has the same effect as not specifying any depth value (i.e., container files will not be recursed into). +

    diff --git a/GhidraDocs/GhidraClass/Intermediate/HeadlessAnalyzer.html b/GhidraDocs/GhidraClass/Intermediate/HeadlessAnalyzer.html index 0c7eee8d3e..de112c8e14 100644 --- a/GhidraDocs/GhidraClass/Intermediate/HeadlessAnalyzer.html +++ b/GhidraDocs/GhidraClass/Intermediate/HeadlessAnalyzer.html @@ -34,7 +34,7 @@ [-scriptPath "<path1>[;<path2>...]"] [-propertiesPath "<path1>[;<path2>...]"] [-log <path to log file>] [-scriptlog <path to script log file>] - [-overwrite] [-recursive] [-readOnly] [-deleteProject] + [-overwrite] [-recursive [<depth>]] [-readOnly] [-deleteProject] [-noanalysis] [-processor <languageID>] [-cspec <compilerSpecID>] [-analysisTimeoutPerFile <timeout in seconds>] @@ -263,11 +263,12 @@
    -
    -recursive
    +
    -recursive [<depth>]


    • Enables recursive descent into directories and project sub-folders when a directory has been specified in -import or -process mode.
    • If not used, only the immediate files contained within the specified directory are imported or processed.
      • +
    • Specify a value for the optional depth argument to recurse into various levels of nested supported container files (e.g., zip, tar, .a, etc.).

    Affects -import or -process modes – used for determining whether to process files in current directory or files in current directory + all its subfolders.