From 7de34e6816397ea827c724ab5ae1bd395ff4e265 Mon Sep 17 00:00:00 2001 From: caheckman <48068198+caheckman@users.noreply.github.com> Date: Fri, 20 Sep 2024 18:12:19 +0000 Subject: [PATCH] GP-4791 Add CompilerSpecID and SourceLanguageID filters to Fid --- .../CreateMultipleLibraries.java | 13 +- .../ghidra_scripts/FidStatistics.java | 15 +- .../ghidra_scripts/RemoveFunctions.java | 18 +- .../Features/FunctionID/src/main/doc/fid.xml | 51 +++- .../help/topics/FunctionID/FunctionID.html | 19 +- .../topics/FunctionID/FunctionIDDebug.html | 2 +- .../topics/FunctionID/FunctionIDPlugin.html | 24 +- .../feature/fid/analyzer/FidAnalyzer.java | 18 +- .../fid/cmd/ApplyFidEntriesCommand.java | 11 +- .../java/ghidra/feature/fid/db/FidDB.java | 8 +- .../java/ghidra/feature/fid/db/FidFile.java | 79 ++--- .../ghidra/feature/fid/db/FidFileManager.java | 25 +- .../java/ghidra/feature/fid/db/FidFilter.java | 269 ++++++++++++++++++ .../ghidra/feature/fid/db/FidProgramID.java | 125 ++++++++ .../feature/fid/db/FidQueryService.java | 19 +- .../ghidra/feature/fid/db/LibrariesTable.java | 33 ++- .../ghidra/feature/fid/db/LibraryRecord.java | 51 +++- .../fid/debug/FidSearchDebugDialog.java | 10 +- .../ghidra/feature/fid/plugin/IngestTask.java | 45 ++- .../feature/fid/plugin/PopulateFidDialog.java | 34 ++- .../DefaultFidPopulateResultReporter.java | 3 + .../feature/fid/service/FidService.java | 40 +-- .../fid/service/FidServiceLibraryIngest.java | 58 ++-- .../ghidra/feature/fid/db/FidFilterTest.java | 106 +++++++ 24 files changed, 861 insertions(+), 215 deletions(-) create mode 100644 Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFilter.java create mode 100644 Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidProgramID.java create mode 100644 Ghidra/Features/FunctionID/src/test/java/ghidra/feature/fid/db/FidFilterTest.java diff --git a/Ghidra/Features/FunctionID/ghidra_scripts/CreateMultipleLibraries.java b/Ghidra/Features/FunctionID/ghidra_scripts/CreateMultipleLibraries.java index f26a48e342..0b97c7196b 100644 --- a/Ghidra/Features/FunctionID/ghidra_scripts/CreateMultipleLibraries.java +++ b/Ghidra/Features/FunctionID/ghidra_scripts/CreateMultipleLibraries.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +31,6 @@ import ghidra.feature.fid.service.*; import ghidra.feature.fid.service.FidPopulateResult.Disposition; import ghidra.framework.model.*; import ghidra.program.database.ProgramContentHandler; -import ghidra.program.model.lang.LanguageID; import ghidra.program.model.listing.*; import ghidra.program.model.mem.MemoryAccessException; import ghidra.util.Msg; @@ -57,7 +56,7 @@ public class CreateMultipleLibraries extends GhidraScript { private FileOutputStream outlog = null; private File commonSymbolsFile = null; private List commonSymbols = null; - private LanguageID languageID = null; + private String languageID = null; private MyFidPopulateResultReporter reporter = null; @@ -277,10 +276,11 @@ public class CreateMultipleLibraries extends GhidraScript { ArrayList programs = new ArrayList<>(); try { findPrograms(programs, folder); + FidFilter programFilter = new FidFilter(languageID, "", ""); FidPopulateResult result = service.createNewLibraryFromPrograms(fidDb, currentLibraryName, currentLibraryVersion, currentLibraryVariant, programs, null, - languageID, null, commonSymbols, TaskMonitor.DUMMY); + programFilter, null, commonSymbols, TaskMonitor.DUMMY); reporter.report(result); } catch (CancelledException e) { @@ -367,8 +367,7 @@ public class CreateMultipleLibraries extends GhidraScript { catch (CancelledException e) { commonSymbolsFile = null; // Common symbols file may be null } - String lang = askString("Enter LanguageID To Process", "Language ID: "); - languageID = new LanguageID(lang); + languageID = askString("Enter LanguageID To Process", "Language ID: "); parseSymbols(); reporter = new MyFidPopulateResultReporter(); diff --git a/Ghidra/Features/FunctionID/ghidra_scripts/FidStatistics.java b/Ghidra/Features/FunctionID/ghidra_scripts/FidStatistics.java index c00fab77b4..6f05008c2f 100644 --- a/Ghidra/Features/FunctionID/ghidra_scripts/FidStatistics.java +++ b/Ghidra/Features/FunctionID/ghidra_scripts/FidStatistics.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,12 +19,12 @@ import java.io.*; import java.util.*; import ghidra.app.script.GhidraScript; +import ghidra.feature.fid.db.FidProgramID; import ghidra.feature.fid.db.FidQueryService; import ghidra.feature.fid.service.*; import ghidra.framework.model.DomainFile; import ghidra.framework.model.DomainFolder; import ghidra.program.database.ProgramContentHandler; -import ghidra.program.model.lang.Language; import ghidra.program.model.listing.*; import ghidra.program.model.mem.MemoryAccessException; import ghidra.program.model.symbol.Symbol; @@ -564,7 +564,7 @@ public class FidStatistics extends GhidraScript { highFalsePositive = new FileWriter(highFile); FidQueryService queryService = null; - Language lastLanguage = null; + FidProgramID lastID = null; int maxPrograms = programList.size(); int counter = 0; try { @@ -572,12 +572,13 @@ public class FidStatistics extends GhidraScript { Program program = null; try { program = (Program) domainFile.getDomainObject(this, false, false, monitor); - if (queryService == null || !lastLanguage.equals(program.getLanguage())) { + FidProgramID currentID = new FidProgramID(program, false); + if (queryService == null || !lastID.equals(currentID)) { if (queryService != null) { queryService.close(); } - lastLanguage = program.getLanguage(); - queryService = service.openFidQueryService(lastLanguage, false); + lastID = currentID; + queryService = service.openFidQueryService(currentID, false); } processProgram(program, queryService); counter += 1; diff --git a/Ghidra/Features/FunctionID/ghidra_scripts/RemoveFunctions.java b/Ghidra/Features/FunctionID/ghidra_scripts/RemoveFunctions.java index fa6429b1d7..a3d30d9d5b 100644 --- a/Ghidra/Features/FunctionID/ghidra_scripts/RemoveFunctions.java +++ b/Ghidra/Features/FunctionID/ghidra_scripts/RemoveFunctions.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,8 +20,8 @@ import generic.stl.Pair; import ghidra.app.script.GhidraScript; import ghidra.feature.fid.db.*; import ghidra.feature.fid.service.FidService; -import ghidra.program.model.lang.*; -import ghidra.program.util.DefaultLanguageService; +import ghidra.program.model.lang.CompilerSpecID; +import ghidra.program.model.lang.LanguageID; import ghidra.util.exception.CancelledException; /** @@ -480,14 +480,14 @@ public class RemoveFunctions extends GhidraScript { } protected void runSearch() throws Exception { - LanguageService langService = DefaultLanguageService.getLanguageService(); FidService fidService = new FidService(); buildKnownHashes32(); LanguageID langId = new LanguageID("x86:LE:32:default"); - Language language = langService.getLanguage(langId); + CompilerSpecID compiler = new CompilerSpecID("windows"); println("Searching for x86:LE:32 hashes ..."); - FidQueryService fidQueryService = fidService.openFidQueryService(language, false); + FidQueryService fidQueryService = + fidService.openFidQueryService(new FidProgramID(langId, compiler, null), false); findMissingHashes(fidQueryService); REMOVE_HASHES.clear(); @@ -500,9 +500,9 @@ public class RemoveFunctions extends GhidraScript { buildKnownHashes64(); langId = new LanguageID("x86:LE:64:default"); - language = langService.getLanguage(langId); println("Searching for x86:LE:64 hashes ..."); - fidQueryService = fidService.openFidQueryService(language, false); + fidQueryService = + fidService.openFidQueryService(new FidProgramID(langId, compiler, null), false); findMissingHashes(fidQueryService); } diff --git a/Ghidra/Features/FunctionID/src/main/doc/fid.xml b/Ghidra/Features/FunctionID/src/main/doc/fid.xml index b6dfef9d7f..209e25de78 100644 --- a/Ghidra/Features/FunctionID/src/main/doc/fid.xml +++ b/Ghidra/Features/FunctionID/src/main/doc/fid.xml @@ -25,7 +25,9 @@ where the compiled form of the functions does not change. Because of the hashing functions remain identifiable even if the library is relocated during linking. Larger changes to the compilation process of the library however will likely prevent successful searches. -Function ID databases are necessarily targeted to a specific processor. +Function ID databases are necessarily targeted to a specific processor and optionally to specific +compilers or source languages. + Function ID generally runs as part of Auto Analysis but can @@ -202,7 +204,7 @@ presents some configurable options for the analyzer. - Instruction count threshold + Instruction Count Threshold This is the primary threshold a potential @@ -216,7 +218,7 @@ presents some configurable options for the analyzer. - Multiple match threshold + Multiple Match Threshold In general for a single function, if there are multiple potential matches @@ -228,7 +230,7 @@ presents some configurable options for the analyzer. - Always apply FID labels + Always Apply FID Labels If this toggle is on, the analyzer will report matches even if there is @@ -245,6 +247,17 @@ presents some configurable options for the analyzer. + + Ignore Database Filters + + + If this toggle is on, the analyzer will ignore any compiler or + source language filters built into a FID database. Matching will be + attempted when a database targets the same processor as the program, + even if it targets different compilers or source languages. + + + @@ -487,11 +500,39 @@ to populate the database. This is the required 4-field Ghidra Language ID (i.e. x86:LE:64:default) - specifying exactly what processor the new language will contain. While scanning during ingest, + specifying exactly what processor the new library will contain. While scanning during ingest, any program that does not match this program model will be automatically skipped. + + Compiler Specs + + + This is an optional comma separated list of compiler names. If at least one name is listed, + the new library will apply only to programs with a matching compiler specification. + Programs without a matching compiler specification will be filtered out during the + ingest process and, by default, cannot query against the library. + Each name listed must match a compiler specification associated with the selected Language ID. + If no names are listed, any compiler specification is considered matching for the + new library, and no programs will be filtered. + + + + + Source Languages + + + This is an optional comma separated list of names that are compared against a + program's "Source Languages" property. If at least one name is listed, + the new library will apply only to programs with a matching source language. + Programs that do not match will be filtered out during the + ingest process and, by default, cannot query against the library. + If no names are listed, any source language is considered matching for the + new library, and no programs will be filtered. + + + Common Symbol File diff --git a/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionID.html b/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionID.html index d03e7cdffa..6858662e9f 100644 --- a/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionID.html +++ b/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionID.html @@ -4,7 +4,7 @@ Function ID - + @@ -29,7 +29,9 @@ where the compiled form of the functions does not change. Because of the hashing functions remain identifiable even if the library is relocated during linking. Larger changes to the compilation process of the library however will likely prevent successful searches. -Function ID databases are necessarily targeted to a specific processor. +Function ID databases are necessarily targeted to a specific processor and optionally to specific +compilers or source languages. +

Function ID generally runs as part of Auto Analysis but can @@ -189,7 +191,7 @@ toggle whether the analyzer is active or not, and if presents some configurable options for the analyzer.

-
Instruction count threshold
+
Instruction Count Threshold

This is the primary threshold a potential match must exceed in order to be reported by @@ -199,7 +201,7 @@ presents some configurable options for the analyzer. the score counts the number of instructions in the function plus instructions in any matching parent or child.

-
Multiple match threshold
+
Multiple Match Threshold

In general for a single function, if there are multiple potential matches all with different names, this is a good sign that the matches are random, @@ -207,7 +209,7 @@ presents some configurable options for the analyzer. this threshold, the analyzer will report a Multiple Match. (See “Multiple Matches”)

-
Always apply FID labels
+
Always Apply FID Labels

If this toggle is on, the analyzer will report matches even if there is already an imported or user defined name for the function. @@ -217,6 +219,13 @@ presents some configurable options for the analyzer. The analyzer will only create bookmarks for matches if this toggle is on. This does not affect insertion of comments and symbols.

+
Ignore Database Filters
+

+ If this toggle is on, the analyzer will ignore any compiler or + source language filters built into a FID database. Matching will be + attempted when a database targets the same processor as the program, + even if it targets different compilers or source languages. +

diff --git a/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionIDDebug.html b/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionIDDebug.html index b7210dd796..5dfdd0bc9f 100644 --- a/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionIDDebug.html +++ b/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionIDDebug.html @@ -4,7 +4,7 @@ Function ID Debug Plug-in - + diff --git a/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionIDPlugin.html b/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionIDPlugin.html index 60d74bf16e..3980ade13c 100644 --- a/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionIDPlugin.html +++ b/Ghidra/Features/FunctionID/src/main/help/help/topics/FunctionID/FunctionIDPlugin.html @@ -4,7 +4,7 @@ Function ID Plug-in - + @@ -163,9 +163,29 @@ to populate the database.
Language

This is the required 4-field Ghidra Language ID (i.e. x86:LE:64:default) - specifying exactly what processor the new language will contain. While scanning during ingest, + specifying exactly what processor the new library will contain. While scanning during ingest, any program that does not match this program model will be automatically skipped.

+
Compiler Specs
+

+ This is an optional comma separated list of compiler names. If at least one name is listed, + the new library will apply only to programs with a matching compiler specification. + Programs without a matching compiler specification will be filtered out during the + ingest process and, by default, cannot query against the library. + Each name listed must match a compiler specification associated with the selected Language ID. + If no names are listed, any compiler specification is considered matching for the + new library, and no programs will be filtered. +

+
Source Languages
+

+ This is an optional comma separated list of names that are compared against a + program's "Source Languages" property. If at least one name is listed, + the new library will apply only to programs with a matching source language. + Programs that do not match will be filtered out during the + ingest process and, by default, cannot query against the library. + If no names are listed, any source language is considered matching for the + new library, and no programs will be filtered. +

Common Symbol File

This is an optional parameter that provides a list of common function symbols to diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/analyzer/FidAnalyzer.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/analyzer/FidAnalyzer.java index 82b92c58be..2e535b1de2 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/analyzer/FidAnalyzer.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/analyzer/FidAnalyzer.java @@ -20,6 +20,7 @@ import ghidra.app.plugin.core.analysis.AutoAnalysisManager; import ghidra.app.services.*; import ghidra.app.util.importer.MessageLog; import ghidra.feature.fid.cmd.ApplyFidEntriesCommand; +import ghidra.feature.fid.db.FidProgramID; import ghidra.feature.fid.service.FidService; import ghidra.framework.options.Options; import ghidra.program.model.address.AddressSetView; @@ -39,11 +40,16 @@ public class FidAnalyzer extends AbstractAnalyzer { private FidService service; // Options - public static final String OPTION_NAME_CREATE_BOOKMARKS = "Create Analysis Bookmarks"; + private static final String OPTION_NAME_CREATE_BOOKMARKS = "Create Analysis Bookmarks"; private static final String OPTION_DESCRIPTION_CREATE_BOOKMARKS = "If checked, an analysis bookmark will be created for each function which was matched " + "against one or more known library functions."; + private static final String IGNORE_FILTERS_OPTION_NAME = "Ignore Database Filters"; + private static final String IGNORE_FILTERS_OPTION_DESCRIPTION = "If checked, the " + + "analyzer will try to apply labels from any FID file even if the targeted compilers " + + "and source languages don't match the program. Only the FID file's language ID must match"; + public static final String APPLY_ALL_FID_LABELS_OPTION_NAME = "Always Apply FID Labels"; private static final String APPLY_ALL_FID_LABELS_OPTION_DESCRIPTION = "Enable this option to " + "always apply FID labels at functions regardless of existing labels at that function." + @@ -54,10 +60,12 @@ public class FidAnalyzer extends AbstractAnalyzer { // Default Option Values private static final boolean APPLY_ALL_FID_LABELS_DEFAULT = false; private static final boolean OPTION_DEFAULT_CREATE_BOOKMARKS_ENABLED = true; + private static final boolean IGNORE_FILTERS_DEFAULT = false; // Option Variables private boolean alwaysApplyFidLabels = APPLY_ALL_FID_LABELS_DEFAULT; private boolean createBookmarksEnabled = OPTION_DEFAULT_CREATE_BOOKMARKS_ENABLED; + private boolean ignoreFilters = IGNORE_FILTERS_DEFAULT; private static final String SCORE_THRESHOLD_OPTION_NAME = "Instruction Count Threshold"; @@ -130,7 +138,7 @@ public class FidAnalyzer extends AbstractAnalyzer { public boolean added(Program program, AddressSetView set, TaskMonitor monitor, MessageLog log) throws CancelledException { - if (!service.canProcess(program.getLanguage())) { + if (!service.canProcess(new FidProgramID(program, ignoreFilters))) { // This can now happen, since we no longer check in canAnalyze() Msg.debug(this, "No FID Libraries apply for language " + program.getLanguageID()); return false; @@ -142,7 +150,7 @@ public class FidAnalyzer extends AbstractAnalyzer { ApplyFidEntriesCommand cmd; cmd = new ApplyFidEntriesCommand(set, scoreThreshold, multiScoreThreshold, - alwaysApplyFidLabels, createBookmarksEnabled); + alwaysApplyFidLabels, createBookmarksEnabled, ignoreFilters); cmd.applyTo(program, monitor); // Name Change can change the nature of a function from a system @@ -181,6 +189,8 @@ public class FidAnalyzer extends AbstractAnalyzer { APPLY_ALL_FID_LABELS_OPTION_DESCRIPTION); options.registerOption(OPTION_NAME_CREATE_BOOKMARKS, createBookmarksEnabled, null, OPTION_DESCRIPTION_CREATE_BOOKMARKS); + options.registerOption(IGNORE_FILTERS_OPTION_NAME, ignoreFilters, null, + IGNORE_FILTERS_OPTION_DESCRIPTION); } @Override @@ -198,6 +208,8 @@ public class FidAnalyzer extends AbstractAnalyzer { options.getBoolean(APPLY_ALL_FID_LABELS_OPTION_NAME, APPLY_ALL_FID_LABELS_DEFAULT); createBookmarksEnabled = options.getBoolean(OPTION_NAME_CREATE_BOOKMARKS, createBookmarksEnabled); + ignoreFilters = + options.getBoolean(IGNORE_FILTERS_OPTION_NAME, ignoreFilters); } } diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/cmd/ApplyFidEntriesCommand.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/cmd/ApplyFidEntriesCommand.java index 645f37be2d..a7b200762c 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/cmd/ApplyFidEntriesCommand.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/cmd/ApplyFidEntriesCommand.java @@ -20,6 +20,7 @@ import java.util.*; import ghidra.app.cmd.label.SetLabelPrimaryCmd; import ghidra.app.util.demangler.DemangledObject; +import ghidra.feature.fid.db.FidProgramID; import ghidra.feature.fid.db.FidQueryService; import ghidra.feature.fid.service.*; import ghidra.framework.cmd.BackgroundCommand; @@ -46,25 +47,29 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { private float scoreThreshold; private float multiNameScoreThreshold; private boolean createBookmarksEnabled; + private boolean ignoreCompilerSpec; public ApplyFidEntriesCommand(AddressSetView set, float scoreThreshold, float multiThreshold, - boolean alwaysApplyFidLabels, boolean createBookmarksEnabled) { + boolean alwaysApplyFidLabels, boolean createBookmarksEnabled, + boolean ignoreCompilerSpec) { super("ApplyFidEntriesCommand", true, true, false); this.scoreThreshold = scoreThreshold; this.multiNameScoreThreshold = multiThreshold; this.alwaysApplyFidLabels = alwaysApplyFidLabels; this.createBookmarksEnabled = createBookmarksEnabled; + this.ignoreCompilerSpec = ignoreCompilerSpec; } @Override public boolean applyTo(Program program, TaskMonitor monitor) { FidService service = new FidService(); - if (!service.canProcess(program.getLanguage())) { + FidProgramID programID = new FidProgramID(program, ignoreCompilerSpec); + if (!service.canProcess(programID)) { return false; } try (FidQueryService fidQueryService = - service.openFidQueryService(program.getLanguage(), false)) { + service.openFidQueryService(programID, false)) { monitor.setMessage("FID Analysis"); List processProgram = diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidDB.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidDB.java index 363ba3fe3c..966509b279 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidDB.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidDB.java @@ -25,7 +25,6 @@ import db.DBRecord; import ghidra.feature.fid.hash.FidHashQuad; import ghidra.framework.store.db.PackedDBHandle; import ghidra.framework.store.db.PackedDatabase; -import ghidra.program.model.lang.CompilerSpecID; import ghidra.program.model.lang.LanguageID; import ghidra.util.Msg; import ghidra.util.ReadOnlyException; @@ -463,18 +462,19 @@ public class FidDB implements Closeable { * @param languageID the language id * @param languageVersion the language version * @param languageMinorVersion the language minor version - * @param compilerSpecID the compiler spec id + * @param compilerSpecs the allowed compiler specs, as a comma separated list of names + * @param sourceLanguages the allowed source languages, as a comma separated list of names * @return the newly created library record */ public LibraryRecord createNewLibrary(String libraryFamilyName, String libraryVersion, String libraryVariant, String ghidraVersion, LanguageID languageID, int languageVersion, - int languageMinorVersion, CompilerSpecID compilerSpecID) { + int languageMinorVersion, String compilerSpecs, String sourceLanguages) { try { checkUpdateAllowed(); DBRecord record = librariesTable.createLibrary(libraryFamilyName, libraryVersion, libraryVariant, ghidraVersion, languageID, languageVersion, languageMinorVersion, - compilerSpecID); + compilerSpecs, sourceLanguages); return new LibraryRecord(record); } catch (ReadOnlyException e) { diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFile.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFile.java index ca555e72a1..744fc73650 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFile.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFile.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,12 +17,9 @@ package ghidra.feature.fid.db; import java.io.File; import java.io.IOException; -import java.util.*; import org.apache.commons.io.FilenameUtils; -import ghidra.program.model.lang.*; -import ghidra.program.util.DefaultLanguageService; import ghidra.util.Msg; import ghidra.util.exception.VersionException; @@ -49,8 +46,8 @@ public class FidFile implements Comparable { private final boolean isInstalled; private boolean isActive = true; private FidFileManager fidFileManager; - private Set supportedLanguages; private FidDB openUpdateableFidDB; + private FidFilter supportedLanguages; FidFile(FidFileManager fidFileManager, File file, boolean isInstalled) { this.fidFileManager = fidFileManager; @@ -104,7 +101,7 @@ public class FidFile implements Comparable { openUpdateableFidDB = fidDB; } if (supportedLanguages == null) { - supportedLanguages = getSupportedLanguages(fidDB); + supportedLanguages = new FidFilter(fidDB); } return fidDB; } @@ -180,45 +177,36 @@ public class FidFile implements Comparable { } /** - * Tests if the Fid database for this FidFile supports the given language. - * @param language the language to test. - * @return true if this Fid Database supports the given language. + * Tests if the database for this FidFile supports the given language, compiler, and source. + * LanguageDescription maps to the set of CompilerSpecID that are supported for that language. + * If there is no matching LanguageDescription in the map, the Program is not supported. + * If the mapped Set is empty, then all specs are supported for the language. + * If the mapped Set is not empty but there is no matching CompilerSpecID, + * then the Program is not supported. + * If the compilerSpec parameter is null, then the user wants to apply this + * file even if the Program's compiler spec doesn't match. + * If a set of supported source languages is present, then at least one provided source + * language must match or the Program is not supported. + * If the sourceSet parameter is null, then the user wants to apply this file even if + * the Program's source languages don't match. + * @param programID the program features to test + * @return true if this Fid Database supports the given Program. */ - public boolean canProcessLanguage(Language language) { + public boolean canProcess(FidProgramID programID) { if (supportedLanguages == null) { supportedLanguages = getSupportedLanguages(); } - return supportedLanguages.contains(language.getLanguageDescription()); + return supportedLanguages.test(programID); } - private Set getSupportedLanguages(FidDB fidDB) { - - Set languages = new TreeSet<>(new ProcessorSizeComparator()); - LanguageService languageService = DefaultLanguageService.getLanguageService(); - - List allLibraries = fidDB.getAllLibraries(); - for (LibraryRecord libraryRecord : allLibraries) { - LanguageID ghidraLanguageID = libraryRecord.getGhidraLanguageID(); - try { - LanguageDescription languageDescription = - languageService.getLanguageDescription(ghidraLanguageID); - languages.add(languageDescription); - } - catch (LanguageNotFoundException e) { - // ignore language - } - } - return languages; - } - - private Set getSupportedLanguages() { + private FidFilter getSupportedLanguages() { if (supportedLanguages != null) { return supportedLanguages; } if (isValidFile() && supportedLanguages != null) { return supportedLanguages; } - supportedLanguages = new TreeSet<>(new ProcessorSizeComparator()); + supportedLanguages = new FidFilter(); return supportedLanguages; } @@ -227,27 +215,4 @@ public class FidFile implements Comparable { openUpdateableFidDB = null; } } - - /** - * Comparator for deciding if a target language "matches" an architecture for a library - * We want processor family, endianness and "size" to match, but variant can be different - */ - private static class ProcessorSizeComparator implements Comparator { - - @Override - public int compare(LanguageDescription o1, LanguageDescription o2) { - int res = o1.getProcessor().compareTo(o2.getProcessor()); - if (res != 0) { - return res; - } - if (o1.getSize() != o2.getSize()) { - return (o1.getSize() < o2.getSize()) ? -1 : 1; - } - if (o1.getInstructionEndian() != o2.getInstructionEndian()) { - return o1.getInstructionEndian().isBigEndian() ? -1 : 1; - } - return 0; - } - - } } diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFileManager.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFileManager.java index f989de0220..a19222844e 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFileManager.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFileManager.java @@ -26,7 +26,6 @@ import generic.jar.ResourceFile; import generic.util.Path; import ghidra.framework.Application; import ghidra.framework.preferences.Preferences; -import ghidra.program.model.lang.Language; import ghidra.util.Msg; import ghidra.util.datastruct.WeakDataStructureFactory; import ghidra.util.datastruct.WeakSet; @@ -183,19 +182,22 @@ public class FidFileManager { } /** - * Opens all the Fid Databases applicable for the given language and returns a FidQueryService - * which is a convenience for querying multiple databases at the same time. - * @param language the language of the programs to be queried. + * Opens all the Fid Databases applicable for the given Language and CompilerSpec, + * and returns a FidQueryService as a convenience for querying multiple FID files + * at the same time. The CompilerSpec can be null to indicate the service should not + * filter out results whose compiler spec doesn't match. + * + * @param programID the properties of the program to be queried * @param openForUpdate if true, all non-installation databases will be open for update; * otherwise they will be read-only. * @return a FidQueryService which is a convenience for querying multiple databases at the same time. * @throws VersionException if any of the fidFiles have a database Schema that is not the current version. * @throws IOException if a general I/O error occurs. */ - public FidQueryService openFidQueryService(Language language, boolean openForUpdate) + public FidQueryService openFidQueryService(FidProgramID programID, boolean openForUpdate) throws VersionException, IOException { loadFidFiles(); - return new FidQueryService(fidFiles, language, openForUpdate); + return new FidQueryService(fidFiles, programID, openForUpdate); } /** @@ -225,14 +227,15 @@ public class FidFileManager { } /** - * Returns true if any FidFile database known to the application can support the given language. - * @param language the language to test. - * @return true if any FidFile database known to the application can support the given language. + * Returns true if any FidFile database known to the application can support + * the given language and compiler. + * @param programID the properties of the program we want to query against + * @return true if any known FidFile database can support the given language and compiler. */ - public boolean canQuery(Language language) { + public boolean canQuery(FidProgramID programID) { loadFidFiles(); for (FidFile file : fidFiles) { - if (file.isActive() && file.canProcessLanguage(language)) { + if (file.isActive() && file.canProcess(programID)) { return true; } } diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFilter.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFilter.java new file mode 100644 index 0000000000..6a4137c0e5 --- /dev/null +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidFilter.java @@ -0,0 +1,269 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.feature.fid.db; + +import java.util.*; +import java.util.stream.Collectors; + +import ghidra.app.util.sourcelanguage.SourceLanguageID; +import ghidra.program.model.lang.CompilerSpecID; +import ghidra.program.model.lang.LanguageID; +import ghidra.program.model.listing.Program; + +/** + * A filter on programs that are either being ingested into or queried against + * a single FID database file. + */ +public class FidFilter { + private TreeMap> languages; + private Set sources; + + /** + * Create a set of CompilerSpecID from a list of names. + * A null or empty string means create a set that matches all compiler specs, + * which is returned as an empty set. + * @param rawString is a comma separated list of compiler spec names + * @return the set of CompilerSpecID + */ + private static Set buildCompilerSpecIDSet(String rawString) { + if (rawString == null) { + return new HashSet<>(); + } + String[] ids = rawString.split("[ ,]+"); + Set idSet = Arrays.stream(ids) + .filter(s -> !s.isEmpty()) + .map(CompilerSpecID::new) + .collect(Collectors.toSet()); + return idSet; + } + + /** + * Create a set of SourceLanguageID from a list of names. + * A null or empty string means create a set that matches all source languages, + * which is returned null. + * @param rawString is a comma separated list of source language names + * @return the set of SourceLanguageID + */ + private static Set buildSourceLanguageIDSet(String rawString) { + if (rawString == null) { + return null; + } + String[] ids = rawString.split("[ ,]+"); + Set idSet = Arrays.stream(ids) + .filter(s -> !s.isEmpty()) + .map(SourceLanguageID::new) + .collect(Collectors.toSet()); + if (idSet.isEmpty()) { + return null; + } + return idSet; + } + + /** + * Add the program properties used to create the given library to this filter. + * @param record is the given library + */ + private void addLibrary(LibraryRecord record) { + Set sourceSet = + buildSourceLanguageIDSet(record.getGhidraSourceLanguageID()); + if (sources != null) { + if (sourceSet != null) { + sources.addAll(sourceSet); + } + else { + sources = null; // All source languages are supported + } + } + LanguageID languageID = record.getGhidraLanguageID(); + Set compilerSet = + buildCompilerSpecIDSet(record.getGhidraCompilerSpecID()); + Set origSet = languages.get(languageID); + if (origSet != null) { + if (!origSet.isEmpty()) { // Empty indicates all specs allowed (dont modify) + if (compilerSet == null) { // NULL indicates library allows all specs + origSet.clear(); + } + else { + origSet.addAll(compilerSet); + } + } + } + else { + if (compilerSet == null) { // NULL indicates library allows all specs + compilerSet = new HashSet<>(); // Empty indicates all specs allowed + } + languages.put(languageID, compilerSet); + } + } + + /** + * Construct a filter that matches no programs + */ + public FidFilter() { + languages = new TreeMap<>(new ProcessorSizeComparator()); + sources = null; + } + + /** + * Construct a filter that matches programs described by the given FID database + * @param fidDB is the database + */ + public FidFilter(FidDB fidDB) { + languages = new TreeMap<>(new ProcessorSizeComparator()); + sources = new HashSet<>(); + List allLibraries = fidDB.getAllLibraries(); + for (LibraryRecord libraryRecord : allLibraries) { + addLibrary(libraryRecord); + } + } + + /** + * Construct a filter from ID strings. + * A null compilerSpecs means that all compiler specs will pass. + * A null sourceLanguages means that all source languages will pass. + * @param langID is the language ID as a string + * @param compilerSpecs is a comma separated list of compiler specs (or null) + * @param sourceLanguages is a comma separated list of source languages (or null) + */ + public FidFilter(String langID, String compilerSpecs, String sourceLanguages) { + languages = new TreeMap<>(new ProcessorSizeComparator()); + LanguageID languageID = new LanguageID(langID); + Set compilerSet = buildCompilerSpecIDSet(compilerSpecs); + languages.put(languageID, compilerSet); + sources = buildSourceLanguageIDSet(sourceLanguages); + } + + /** + * Test if a program with specific properties passes this filter + * @param programID the program properties + * @return true if the program passes this filter + */ + public boolean test(FidProgramID programID) { + if (programID.language == null) { + return true; // Match everything + } + if (sources != null && programID.sources != null) { + boolean match = false; + for (SourceLanguageID id : programID.sources) { + if (sources.contains(id)) { + match = true; + break; + } + } + if (!match) { // There must be at least one match + return false; + } + } + Set specsSet = languages.get(programID.language); + if (specsSet == null) { + return false; + } + if (specsSet.isEmpty()) { + return true; // An empty set here means all compiler specs are supported + } + if (programID.compiler == null) { + return true; // Apply this file even if compiler spec doesn't match + } + return specsSet.contains(programID.compiler); + } + + /** + * Test if a program passes this filter + * @param program the program + * @return true if the program passes this filter + */ + public boolean test(Program program) { + return test(new FidProgramID(program, false)); + } + + /** + * If this filters on a single LanguageID, return it. Return null otherwise. + * @return the single LanguageID or null + */ + public LanguageID getLanguageID() { + if (languages.size() != 1) { + return null; + } + return languages.firstKey(); + } + + /** + * Build a string of comma separated compiler spec names associated with the given language + * @param lang is the given language + * @return the string + */ + public String getCompilerSpecString(LanguageID lang) { + String compilerString = ""; + Set compilerSet = languages.get(lang); + if (compilerSet != null && !compilerSet.isEmpty()) { + compilerString = compilerSet.stream() + .sorted() + .map(CompilerSpecID::getIdAsString) + .collect(Collectors.joining(",")); + } + return compilerString; + } + + /** + * Get set of compiler specs that pass for the given language + * @param lang is the given language + * @return the set or null if the language doesn't pass this filter + */ + public Set getCompilerSpecs(LanguageID lang) { + return languages.get(lang); + } + + /** + * Build a string of comma separated source language names associated with this + * @return the string + */ + public String getSourceLanguageString() { + String sourceString = ""; + if (sources != null && !sources.isEmpty()) { + sourceString = sources.stream() + .sorted() + .map(SourceLanguageID::getIdAsString) + .collect(Collectors.joining(",")); + } + return sourceString; + } + + /** + * Comparator for deciding if a target language "matches" an architecture for a library + * We want processor family, endianness and "size" to match, but variant can be different + */ + private static class ProcessorSizeComparator implements Comparator { + + @Override + public int compare(LanguageID o1, LanguageID o2) { + int o1pos = o1.getIdAsString().lastIndexOf(':'); + int o2pos = o2.getIdAsString().lastIndexOf(':'); + if (o1pos >= 0 && o2pos >= 0) { + String o1Front = o1.getIdAsString().substring(0, o1pos); + String o2Front = o2.getIdAsString().substring(0, o2pos); + return o1Front.compareTo(o2Front); + } + if (o1pos >= 0) { + return 1; + } + if (o2pos >= 0) { + return -1; + } + return o1.getIdAsString().compareTo(o2.getIdAsString()); + } + + } +} diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidProgramID.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidProgramID.java new file mode 100644 index 0000000000..af1d29eb92 --- /dev/null +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidProgramID.java @@ -0,0 +1,125 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.feature.fid.db; + +import java.util.Set; + +import ghidra.app.util.sourcelanguage.SourceLanguageID; +import ghidra.program.model.lang.CompilerSpecID; +import ghidra.program.model.lang.LanguageID; +import ghidra.program.model.listing.Program; + +/** + * A set of program properties that a FID database filters on + */ +public class FidProgramID { + LanguageID language; // Ghidra ID of the program (null matches any program) + CompilerSpecID compiler; // Compiler spec of the program (null matches any spec) + Set sources; // Source languages of the program (null matches all languages) + + /** + * Construct a program id that will match against all databases + */ + public FidProgramID() { + language = null; + compiler = null; + sources = null; + } + + /** + * Construct a program id for a specific program. The caller can optionally request + * that compiler spec and source language be ignored when querying or ingesting into + * a FID database. + * @param program is the specific program + * @param ignoreCompilerSpec is true to ignore compiler spec and source language + */ + public FidProgramID(Program program, boolean ignoreCompilerSpec) { + language = program.getLanguageID(); + if (ignoreCompilerSpec) { + compiler = null; + sources = null; + } + else { + compiler = program.getCompilerSpec().getCompilerSpecID(); + sources = program.getSourceLanguageIDs(); + } + } + + /** + * Construct id that matches a specific LanguageID and compiler spec + * @param lang is the LanguageID to match + * @param comp is the compiler spec to match (may be null) + * @param src is the set of source languages to match (may be null) + */ + public FidProgramID(LanguageID lang, CompilerSpecID comp, Set src) { + language = lang; + compiler = comp; + sources = src; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof FidProgramID)) { + return false; + } + FidProgramID otherID = (FidProgramID) obj; + if (language != null) { + if (!language.equals(otherID.language)) { + return false; + } + } + else if (otherID.language != null) { + return false; + } + if (compiler != null) { + if (!compiler.equals(otherID.compiler)) { + return false; + } + } + else if (otherID.compiler != null) { + return false; + } + if (sources != null) { + return sources.equals(otherID.sources); + } + else if (otherID.sources != null) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int res = 0; + if (language != null) { + res += language.hashCode(); + } + if (compiler != null) { + res += compiler.hashCode(); + } + if (sources != null) { + res += sources.hashCode(); + } + return res; + } + +} diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidQueryService.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidQueryService.java index 215462468f..b6634bf12c 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidQueryService.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/FidQueryService.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.*; import ghidra.feature.fid.hash.FidHashQuad; -import ghidra.program.model.lang.Language; import ghidra.util.exception.VersionException; /** @@ -32,10 +31,20 @@ public class FidQueryService implements Closeable { List fidDbList = new ArrayList<>(); List listeners = new ArrayList<>(); - FidQueryService(Set fidFiles, Language language, boolean openForUpdate) + /** + * Construct a FID query service. A set of FID files is provided for the + * service to query against, but any file that doesn't match the given + * program properties is filtered out. + * @param fidFiles is the set of files the service could query against + * @param programID is the program properties to filter on + * @param openForUpdate is true to indicate that FID files should be opened for later updates + * @throws VersionException if a database schema version of one of the files isn't current + * @throws IOException for an I/O error accessing the FID files + */ + FidQueryService(Set fidFiles, FidProgramID programID, boolean openForUpdate) throws VersionException, IOException { for (FidFile fidFile : fidFiles) { - if (fidFile.isActive() && (language == null || fidFile.canProcessLanguage(language))) { + if (fidFile.isActive() && fidFile.canProcess(programID)) { // NOTE: assumes fidFiles have been pre-checked for version compatibility fidDbList.add(fidFile.getFidDB(openForUpdate)); } diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/LibrariesTable.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/LibrariesTable.java index 45fdfb57f3..05430756ba 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/LibrariesTable.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/LibrariesTable.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,15 +19,21 @@ import java.io.IOException; import java.util.*; import db.*; -import ghidra.program.model.lang.CompilerSpecID; import ghidra.program.model.lang.LanguageID; import ghidra.util.UniversalIdGenerator; import ghidra.util.exception.VersionException; /** - * The libraries table for FID. Note that all entries associated with a single library must have the same - * LanguageID and CompilerSpecID. If supporting multiple architectures, simply create multiple libraries - * in the same database file. + * The libraries table for FID. Libraries are filtered primarily by LanguageID, but can optionally + * be filtered by compiler spec IDs and/or source language IDs. Libraries are currently filtered + * at the database level, so all libraries in one database should have the same IDs. + * + * Compiler spec IDs and source language IDs are both stored in the metadata column + * {@link #LIBRARY_METADATA_COL} as a string of comma separated compiler spec IDs, followed by a ':', + * followed by comma separated source language IDs. For backward compatibility with the + * previous database format, the column may not have a ':' in it, in which case, the string is + * interpreted as a comma separated list of compiler spec IDs only. Either list may be empty + * meaning a filter of that ID type is not applied. */ public class LibrariesTable { static final String LIBRARIES_TABLE = "Libraries Table"; @@ -46,7 +52,7 @@ public class LibrariesTable { static final int GHIDRA_LANGUAGE_ID_COL = 4; static final int GHIDRA_LANGUAGE_VERSION_COL = 5; static final int GHIDRA_LANGUAGE_MINOR_VERSION_COL = 6; - static final int GHIDRA_COMPILER_SPEC_ID_COL = 7; + static final int LIBRARY_METADATA_COL = 7; // @formatter:off static final Schema SCHEMA = new Schema(VERSION, "Library ID", new Field[] { @@ -56,7 +62,7 @@ public class LibrariesTable { }, new String[] { "Library Family Name", "Library Version", "Library Variant", "Ghidra Version", "Ghidra Language ID", "Ghidra Language Version", "Ghidra Language Minor Version", - "Ghidra Compiler Spec ID" + "Library Metadata" }); // @formatter:on @@ -67,7 +73,6 @@ public class LibrariesTable { /** * Creates or attaches a libraries table. * @param handle database handle - * @param create whether to create or just attach * @throws IOException if create fails * @throws VersionException if the saved database version is incompatible with this software */ @@ -106,13 +111,15 @@ public class LibrariesTable { * @param languageID the LanguageID of the language in this library * @param languageVersion the version of the language * @param languageMinorVersion the minor version of the language - * @param compilerSpecID the CompilerSpecID in this library + * @param compilerSpecs the allowed specs in this library + * @param sourceLanguages the allow source languages * @return the new library record * @throws IOException if the database create fails */ public DBRecord createLibrary(String libraryFamilyName, String libraryVersion, String libraryVariant, String ghidraVersion, LanguageID languageID, int languageVersion, - int languageMinorVersion, CompilerSpecID compilerSpecID) throws IOException { + int languageMinorVersion, String compilerSpecs, String sourceLanguages) + throws IOException { DBRecord record = SCHEMA.createRecord(UniversalIdGenerator.nextID().getValue()); record.setString(LIBRARY_FAMILY_NAME_COL, libraryFamilyName); record.setString(LIBRARY_VERSION_COL, libraryVersion); @@ -121,7 +128,7 @@ public class LibrariesTable { record.setString(GHIDRA_LANGUAGE_ID_COL, languageID.getIdAsString()); record.setIntValue(GHIDRA_LANGUAGE_VERSION_COL, languageVersion); record.setIntValue(GHIDRA_LANGUAGE_MINOR_VERSION_COL, languageMinorVersion); - record.setString(GHIDRA_COMPILER_SPEC_ID_COL, compilerSpecID.getIdAsString()); + record.setString(LIBRARY_METADATA_COL, compilerSpecs + ':' + sourceLanguages); table.putRecord(record); return record; } @@ -151,7 +158,7 @@ public class LibrariesTable { * @param version is the optional version string * @param variant is the optional variant string * @return matching list of libraries - * @throws IOException + * @throws IOException for problems accessing the database */ public List getLibrariesByName(String name, String version, String variant) throws IOException { diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/LibraryRecord.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/LibraryRecord.java index c6706350df..5df0cec080 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/LibraryRecord.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/db/LibraryRecord.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,9 +16,9 @@ package ghidra.feature.fid.db; import static ghidra.feature.fid.db.LibrariesTable.*; -import ghidra.program.model.lang.CompilerSpecID; -import ghidra.program.model.lang.LanguageID; + import db.DBRecord; +import ghidra.program.model.lang.LanguageID; /** * Represents a library record in the FID database. @@ -105,11 +105,46 @@ public class LibraryRecord { } /** - * Returns the Ghidra CompilerSpecID (used to create the library). - * @return the Ghidra CompilerSpecID + * Returns a list of CompilerSpecIDs (used to create the library) + * as a string of comma separated names. + * A null value indicates that all CompilerSpecIDs are allowed. + * @return a list of CompilerSpecIDs or null */ - public CompilerSpecID getGhidraCompilerSpecID() { - return new CompilerSpecID(record.getString(GHIDRA_COMPILER_SPEC_ID_COL)); + public String getGhidraCompilerSpecID() { + String rawString = record.getString(LIBRARY_METADATA_COL); + if (rawString == null) { + return null; + } + int pos = rawString.indexOf(':'); + if (pos >= 0) { + rawString = rawString.substring(0, pos); + } + if (rawString.length() == 0) { + return null; + } + return rawString; + } + + /** + * Returns a list of SourceLanguageIDs (used to create the library) + * as a string of comma separated names. + * A null value means that all SourceLanguageIDs are allowed. + * @return a list of SourceLanguageIDs or null + */ + public String getGhidraSourceLanguageID() { + String rawString = record.getString(LIBRARY_METADATA_COL); + if (rawString == null) { + return null; + } + int pos = rawString.indexOf(':'); + if (pos < 0) { + return null; + } + rawString = rawString.substring(pos + 1); + if (rawString.length() == 0) { + return null; + } + return rawString; } /** diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/debug/FidSearchDebugDialog.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/debug/FidSearchDebugDialog.java index 3c99b8eb8c..3077a1479f 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/debug/FidSearchDebugDialog.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/debug/FidSearchDebugDialog.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,8 +23,7 @@ import docking.DialogComponentProvider; import docking.widgets.label.GDLabel; import generic.theme.GThemeDefaults.Ids.Fonts; import generic.theme.Gui; -import ghidra.feature.fid.db.FidFileManager; -import ghidra.feature.fid.db.FidQueryService; +import ghidra.feature.fid.db.*; import ghidra.feature.fid.plugin.FidPlugin; import ghidra.feature.fid.service.FidService; import ghidra.util.HelpLocation; @@ -50,7 +49,8 @@ public class FidSearchDebugDialog extends DialogComponentProvider { this.service = service; addDismissButton(); addWorkPanel(buildPanel()); - fidQueryService = FidFileManager.getInstance().openFidQueryService(null, true); + fidQueryService = + FidFileManager.getInstance().openFidQueryService(new FidProgramID(), true); setRememberSize(false); setHelpLocation(new HelpLocation(FidPlugin.FID_HELP, "debugsearch")); } diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/plugin/IngestTask.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/plugin/IngestTask.java index c3a0eeb9c7..ecd55b3abe 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/plugin/IngestTask.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/plugin/IngestTask.java @@ -23,8 +23,9 @@ import ghidra.feature.fid.service.*; import ghidra.framework.model.DomainFile; import ghidra.framework.model.DomainFolder; import ghidra.program.database.ProgramContentHandler; -import ghidra.program.model.lang.LanguageID; +import ghidra.program.model.lang.*; import ghidra.program.model.mem.MemoryAccessException; +import ghidra.program.util.DefaultLanguageService; import ghidra.util.Msg; import ghidra.util.exception.CancelledException; import ghidra.util.exception.VersionException; @@ -41,15 +42,17 @@ public class IngestTask extends Task { protected String libraryFamilyName; protected String libraryVersion; protected String libraryVariant; - private LanguageID languageId; + private FidFilter programFilter; private File commonSymbolsFile; private FidService fidService; private FidPopulateResultReporter reporter; public IngestTask(String title, FidFile fidFile, LibraryRecord libraryRecord, DomainFolder folder, String libraryFamilyName, String libraryVersion, - String libraryVariant, String languageId, File commonSymbolsFile, FidService fidService, - FidPopulateResultReporter reporter) { + String libraryVariant, FidFilter programFilter, File commonSymbolsFile, + FidService fidService, FidPopulateResultReporter reporter) + throws IllegalArgumentException, LanguageNotFoundException, + CompilerSpecNotFoundException { super(title, true, false, false, false); this.fidFile = fidFile; this.libraryRecord = libraryRecord; @@ -60,7 +63,37 @@ public class IngestTask extends Task { this.commonSymbolsFile = commonSymbolsFile; this.fidService = fidService; this.reporter = reporter; - this.languageId = new LanguageID(languageId); + this.programFilter = programFilter; + checkArgumentValidity(); + } + + private void checkArgumentValidity() throws IllegalArgumentException, LanguageNotFoundException, + CompilerSpecNotFoundException { + if (libraryFamilyName == null || libraryFamilyName.isBlank()) { + throw new IllegalArgumentException("Library Family Name is empty"); + } + if (libraryVersion == null || libraryVersion.isBlank()) { + throw new IllegalArgumentException("Library Version is empty"); + } + if (libraryVariant == null || libraryVariant.isBlank()) { + throw new IllegalArgumentException("Library Variant is empty"); + } + LanguageID languageId = programFilter.getLanguageID(); + if (languageId == null) { + throw new IllegalArgumentException("Language is empty"); + } + LanguageService languageService = DefaultLanguageService.getLanguageService(); + LanguageDescription language = languageService.getLanguageDescription(languageId); + Set compilerSpecs = programFilter.getCompilerSpecs(languageId); + if (compilerSpecs != null) { + for (CompilerSpecID id : compilerSpecs) { + language.getCompilerSpecDescriptionByID(id); // Verify spec is present + } + } + if (commonSymbolsFile != null && !commonSymbolsFile.isFile()) { + throw new IllegalArgumentException( + "Common Symbols File: " + commonSymbolsFile.getName() + " does not exist"); + } } @Override @@ -93,7 +126,7 @@ public class IngestTask extends Task { monitor.setMessage("Populating library..."); FidPopulateResult result = fidService.createNewLibraryFromPrograms(fidDb, - libraryFamilyName, libraryVersion, libraryVariant, programs, null, languageId, + libraryFamilyName, libraryVersion, libraryVariant, programs, null, programFilter, libraryRecord == null ? null : Arrays.asList(libraryRecord), commonSymbols, monitor); reporter.report(result); diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/plugin/PopulateFidDialog.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/plugin/PopulateFidDialog.java index af624e64c0..f248c15075 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/plugin/PopulateFidDialog.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/plugin/PopulateFidDialog.java @@ -27,6 +27,7 @@ import java.util.List; import javax.swing.*; import docking.DialogComponentProvider; +import docking.widgets.OkDialog; import docking.widgets.button.BrowseButton; import docking.widgets.combobox.GComboBox; import docking.widgets.filechooser.GhidraFileChooser; @@ -62,6 +63,8 @@ public class PopulateFidDialog extends DialogComponentProvider { private JTextField variantTextField; private FidService fidService; private JTextField languageIdField; + private JTextField compilerSpecField; + private JTextField sourceLanguageField; private JTextField symbolsFileTextField; protected PopulateFidDialog(PluginTool tool, FidService fidService) { @@ -85,12 +88,23 @@ public class PopulateFidDialog extends DialogComponentProvider { String libraryVersion = versionTextField.getText().trim(); String libraryVariant = variantTextField.getText().trim(); DomainFolder folder = getDomainFolder(); - String languageFilter = languageIdField.getText().trim(); + String languageID = languageIdField.getText().trim(); + String compilerSpecs = compilerSpecField.getText().trim(); + String sourceLanguages = sourceLanguageField.getText().trim(); File commonSymbolsFile = getCommonSymbolsFile(); - Task task = new IngestTask("Populate Library Task", fidFile, libraryRecord, folder, - libraryFamilyName, libraryVersion, libraryVariant, languageFilter, commonSymbolsFile, - fidService, new DefaultFidPopulateResultReporter()); + Task task; + try { + FidFilter programFilter = new FidFilter(languageID, compilerSpecs, sourceLanguages); + + task = new IngestTask("Populate Library Task", fidFile, libraryRecord, folder, + libraryFamilyName, libraryVersion, libraryVariant, programFilter, commonSymbolsFile, + fidService, new DefaultFidPopulateResultReporter()); + } + catch (Exception ex) { + OkDialog.showError("Illegal Argument", ex.getMessage()); + return; // Let the user continue editing the parameters + } close(); tool.execute(task); } @@ -145,6 +159,16 @@ public class PopulateFidDialog extends DialogComponentProvider { panel.add(new GLabel("Language: ", SwingConstants.RIGHT)); panel.add(buildLanguageField()); + panel.add(new GLabel("Compiler Specs: ", SwingConstants.RIGHT)); + compilerSpecField = new JTextField(); + compilerSpecField.getDocument().addUndoableEditListener(e -> updateOkEnablement()); + panel.add(compilerSpecField); + + panel.add(new GLabel("Source Languages: ", SwingConstants.RIGHT)); + sourceLanguageField = new JTextField(); + sourceLanguageField.getDocument().addUndoableEditListener(e -> updateOkEnablement()); + panel.add(sourceLanguageField); + panel.add(new GLabel("Common Symbols File: ", SwingConstants.RIGHT)); panel.add(buildSymbolsFileField(), jLabel); panel.getAccessibleContext().setAccessibleName("Populate Fid"); @@ -292,7 +316,7 @@ public class PopulateFidDialog extends DialogComponentProvider { return false; } String symbolsFilePath = symbolsFileTextField.getText().trim(); - if (!symbolsFilePath.isEmpty() && !(new File(symbolsFilePath).exists())) { + if (!symbolsFilePath.isEmpty() && !(new File(symbolsFilePath).isFile())) { return false; } return true; diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/DefaultFidPopulateResultReporter.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/DefaultFidPopulateResultReporter.java index fbcd50ca63..f4f17bbab8 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/DefaultFidPopulateResultReporter.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/DefaultFidPopulateResultReporter.java @@ -18,6 +18,7 @@ package ghidra.feature.fid.service; import java.util.Map.Entry; import docking.DockingWindowManager; +import docking.widgets.OkDialog; import ghidra.feature.fid.plugin.TextAreaDialog; import ghidra.feature.fid.service.FidPopulateResult.Disposition; @@ -30,6 +31,8 @@ public class DefaultFidPopulateResultReporter implements FidPopulateResultReport @Override public void report(FidPopulateResult result) { if (result == null) { + OkDialog.showError("No library created", + "No programs matched the language and compiler spec requirement"); return; } diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidService.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidService.java index 0cf56125df..208a6280ac 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidService.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidService.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,7 @@ import generic.stl.Pair; import ghidra.feature.fid.db.*; import ghidra.feature.fid.hash.*; import ghidra.framework.model.DomainFile; -import ghidra.program.model.lang.*; +import ghidra.program.model.lang.Processor; import ghidra.program.model.listing.*; import ghidra.program.model.mem.MemoryAccessException; import ghidra.util.classfinder.ClassSearcher; @@ -178,7 +178,7 @@ public class FidService { * @param libraryVariant the library variant * @param programDomainFiles the domain files to use when populating the library * @param functionFilter a filter to possibly reject functions from the library - * @param languageId the ghidra language id to filter on, or null + * @param programFilter a filter on which programs to ingest * @param linkLibraries libraries to search for (internally) unresolved symbols * @param commonSymbols is a list of symbols for which relationships are not generated * @param monitor a task monitor @@ -190,12 +190,13 @@ public class FidService { public FidPopulateResult createNewLibraryFromPrograms(FidDB fidDb, String libraryFamilyName, String libraryVersion, String libraryVariant, List programDomainFiles, - Predicate> functionFilter, LanguageID languageId, - List linkLibraries, List commonSymbols, TaskMonitor monitor) + Predicate> functionFilter, FidFilter programFilter, + List linkLibraries, + List commonSymbols, TaskMonitor monitor) throws MemoryAccessException, VersionException, CancelledException, IllegalStateException, IOException { FidServiceLibraryIngest ingest = new FidServiceLibraryIngest(fidDb, this, libraryFamilyName, - libraryVersion, libraryVariant, programDomainFiles, functionFilter, languageId, + libraryVersion, libraryVariant, programDomainFiles, functionFilter, programFilter, linkLibraries, monitor); ingest.markCommonChildReferences(commonSymbols); return ingest.create(); @@ -290,29 +291,30 @@ public class FidService { } /** - * Returns true if at least one FidLibraryDatabases can process programs with the given language - * @param language the language to test Fid Databases for + * Returns true if at least one FidLibraryDatabases can process programs with + * the given language, compiler, and source language + * @param programID properties of the program to test Fid Databases for * @return true if at least one FidLibraryDatabases can process programs with the given language */ - public boolean canProcess(Language language) { - return fidFileManager.canQuery(language); + public boolean canProcess(FidProgramID programID) { + return fidFileManager.canQuery(programID); } /** - * Creates a new FidQueryService that can facilitate performing a query over multiple - * Fid databases. This causes the appropriate databases to be opened, and therefore, the - * caller of this method is responsible for closing the FidQueryService when done with it. - * @param language the language that will be queried against + * Creates a new FidQueryServices and opens the multiple FID databases that match the given + * Language and CompilerSpec. + * The caller of this method is responsible for closing the FidQueryService when done with it. + * @param programID properties of the program that will be queried against * @param openForUpdate if true, the databases will be opened for read/write. Otherwise * it will be opened for reading only. * @return a new FidQueryService that allows querying across all appropriate fid databases for the given language. - * @throws VersionException - * @throws IOException + * @throws VersionException if any database Schema does not match the current version. + * @throws IOException if a general I/O error occurs. */ - public FidQueryService openFidQueryService(Language language, boolean openForUpdate) + public FidQueryService openFidQueryService(FidProgramID programID, boolean openForUpdate) throws VersionException, IOException { - return fidFileManager.openFidQueryService(language, openForUpdate); + return fidFileManager.openFidQueryService(programID, openForUpdate); } } diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidServiceLibraryIngest.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidServiceLibraryIngest.java index 001ecc9ad3..2861eb5198 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidServiceLibraryIngest.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidServiceLibraryIngest.java @@ -28,7 +28,8 @@ import ghidra.feature.fid.service.FidPopulateResult.Disposition; import ghidra.framework.Application; import ghidra.framework.model.DomainFile; import ghidra.program.model.address.*; -import ghidra.program.model.lang.*; +import ghidra.program.model.lang.Language; +import ghidra.program.model.lang.LanguageID; import ghidra.program.model.listing.*; import ghidra.program.model.mem.*; import ghidra.program.model.symbol.*; @@ -47,12 +48,11 @@ class FidServiceLibraryIngest { private String libraryVariant; // Variant string private List programFiles; private Predicate> functionFilter; - private LanguageID languageId; // Language for everything in this library + private FidFilter programFilter; // Filter on programs to include in library private List linkLibraries; private TaskMonitor monitor; private LibraryRecord library = null; // Database record of the library we are creating - private CompilerSpec compilerSpec = null; private Map> unresolvedSymbols = new HashMap<>(); private TreeSet globalUniqueFunction = new TreeSet<>(); private FidPopulateResult result = null; @@ -172,14 +172,15 @@ class FidServiceLibraryIngest { * @param libraryVariant the library variant * @param programFiles the list of program files * @param functionFilter the function filter - * @param languageId the Ghidra language id to filter programs by + * @param programFilter the program filter * @param linkLibraries the list of libraries to use for unresolved symbols * @param monitor a task monitor */ public FidServiceLibraryIngest(FidDB fidDb, FidService service, String libraryFamilyName, String libraryVersion, String libraryVariant, List programFiles, - Predicate> functionFilter, LanguageID languageId, - List linkLibraries, TaskMonitor monitor) { + Predicate> functionFilter, FidFilter programFilter, + List linkLibraries, + TaskMonitor monitor) { this.fidDb = fidDb; this.service = service; this.libraryFamilyName = libraryFamilyName; @@ -187,19 +188,19 @@ class FidServiceLibraryIngest { this.libraryVariant = libraryVariant; this.programFiles = programFiles; this.functionFilter = functionFilter; - this.languageId = languageId; + this.programFilter = programFilter; this.linkLibraries = linkLibraries; this.monitor = monitor; - if (languageId == null) { - throw new IllegalArgumentException("LanugageID can't be null"); // null used to be allowed, so add special check + if (programFilter.getLanguageID() == null) { + throw new IllegalArgumentException( + "FidServiceLibraryIngest: Program filter must specify exactly one LanguageID"); } } /** * Mark a set of function symbols as "very common" so a match relationship won't be generated with * functions that call it. - * - * @param symbols The symbols + * @param symbols is a list of the symbol names to mark */ public void markCommonChildReferences(List symbols) { if (symbols == null) { @@ -218,6 +219,9 @@ class FidServiceLibraryIngest { monitor.setMessage("Populating library from programs..."); monitor.initialize(programFiles.size()); Object consumer = new Object(); + LanguageID languageID = programFilter.getLanguageID(); + String compilerSpecs = programFilter.getCompilerSpecString(languageID); + String sourceLanguages = programFilter.getSourceLanguageString(); for (DomainFile programFile : programFiles) { monitor.checkCancelled(); Program program = null; @@ -225,18 +229,16 @@ class FidServiceLibraryIngest { program = (Program) programFile.getDomainObject(consumer, false, false, TaskMonitor.DUMMY); monitor.incrementProgress(1); - if (!checkLanguageCompilerSpec(program)) { + if (!programFilter.test(program)) { continue; } - languageId = program.getLanguageID(); - compilerSpec = program.getCompilerSpec(); if (library == null) { Language language = program.getLanguage(); library = fidDb.createNewLibrary(libraryFamilyName, libraryVersion, libraryVariant, - Application.getApplicationVersion(), languageId, language.getVersion(), - language.getMinorVersion(), compilerSpec.getCompilerSpecID()); + Application.getApplicationVersion(), languageID, language.getVersion(), + language.getMinorVersion(), compilerSpecs, sourceLanguages); result = new FidPopulateResult(library); } @@ -590,30 +592,6 @@ class FidServiceLibraryIngest { return !list.isEmpty(); } - /** - * Make sure all programs have the same language and compiler spec, - * otherwise throw and exception or return false based on failOnNewLanguage - * @param program the program - * @return true if the program passes the filter - */ - private boolean checkLanguageCompilerSpec(Program program) { - if (!languageId.equals(program.getLanguageID())) { - Msg.error(this, - "Program " + program.getName() + " does not have language " + languageId); - return false; - } - if (compilerSpec != null) { - if (!compilerSpec.getCompilerSpecID() - .equals(program.getCompilerSpec().getCompilerSpecID())) { - throw new IllegalArgumentException( - "Program " + program.getName() + " has different compiler spec (" + - program.getCompilerSpec().getCompilerSpecID() + - ") than already established (" + compilerSpec.getCompilerSpecID() + ")"); - } - } - return true; - } - /** * Returns whether a function is external. * @param function the function diff --git a/Ghidra/Features/FunctionID/src/test/java/ghidra/feature/fid/db/FidFilterTest.java b/Ghidra/Features/FunctionID/src/test/java/ghidra/feature/fid/db/FidFilterTest.java new file mode 100644 index 0000000000..e778fc7d04 --- /dev/null +++ b/Ghidra/Features/FunctionID/src/test/java/ghidra/feature/fid/db/FidFilterTest.java @@ -0,0 +1,106 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.feature.fid.db; + +import static org.junit.Assert.*; + +import java.util.HashSet; +import java.util.Set; + +import org.junit.Before; +import org.junit.Test; + +import generic.test.AbstractGenericTest; +import ghidra.app.util.sourcelanguage.SourceLanguageID; +import ghidra.program.model.lang.CompilerSpecID; +import ghidra.program.model.lang.LanguageID; + +public class FidFilterTest extends AbstractGenericTest { + + private LanguageID x64, arm64; + private CompilerSpecID winSpec, gccSpec; + private Set sourceSet; + private Set multiSourceSet; + private FidProgramID prog1, prog2, prog3, prog4; + + @Before + public void setup() { + x64 = new LanguageID("x86:LE:64:default"); + arm64 = new LanguageID("AARCH64:LE:64:v8A"); + winSpec = new CompilerSpecID("windows"); + gccSpec = new CompilerSpecID("gcc"); + sourceSet = new HashSet<>(); + sourceSet.add(new SourceLanguageID("rust")); + multiSourceSet = new HashSet<>(); + multiSourceSet.add(new SourceLanguageID("rust")); + multiSourceSet.add(new SourceLanguageID("swift")); + + prog1 = new FidProgramID(x64, gccSpec, multiSourceSet); + prog2 = new FidProgramID(x64, winSpec, null); + prog3 = new FidProgramID(arm64, winSpec, sourceSet); + prog4 = new FidProgramID(x64, winSpec, sourceSet); + } + + @Test + public void testFidProgramIDEquality() { + assertFalse(prog1.equals(prog2)); + assertFalse(prog3.equals(prog4)); + FidProgramID newProg = new FidProgramID(x64, winSpec, null); + assertTrue(prog2.equals(newProg)); + FidProgramID nullProg = new FidProgramID(); + assertFalse(nullProg.equals(prog4)); + } + + @Test + public void testFilter() { + FidFilter filter1 = new FidFilter("x86:LE:64:default", "gcc", "rust"); + assertTrue(filter1.test(prog1)); + assertFalse(filter1.test(prog2)); + assertFalse(filter1.test(prog3)); + assertFalse(filter1.test(prog4)); + FidFilter filter2 = new FidFilter("x86:LE:64:variant", null, "rust"); + assertTrue(filter2.test(prog1)); + assertTrue(filter2.test(prog2)); + assertFalse(filter2.test(prog3)); + assertTrue(filter2.test(prog4)); + FidFilter filter3 = new FidFilter("x86:LE:64:default", "other,gcc", "rust,other"); + assertTrue(filter3.test(prog1)); + assertFalse(filter3.test(prog2)); + assertFalse(filter3.test(prog3)); + assertFalse(filter3.test(prog4)); + FidFilter filter4 = new FidFilter("AARCH64:LE:64:variant", "windows,other", null); + assertFalse(filter4.test(prog1)); + assertFalse(filter4.test(prog2)); + assertTrue(filter4.test(prog3)); + assertFalse(filter4.test(prog4)); + } + + @Test + public void testNoneFilter() { + FidFilter filter = new FidFilter(); + assertFalse(filter.test(prog1)); + assertFalse(filter.test(prog2)); + assertFalse(filter.test(prog3)); + assertFalse(filter.test(prog4)); + } + + @Test + public void testAnyProgram() { + FidProgramID prog = new FidProgramID(); + FidFilter filter = new FidFilter("x86:LE:64:default", "other,gcc", "rust,other"); + assertTrue(filter.test(prog)); + } +}