GP-1688 - Fixed broken MS demangler option to toggle function signature

application; added option to not apply function signature calling
conventions

Ghidra/Features/Base/src/main/help/help/topics/AutoAnalysisPlugin/AutoAnalysis.htm

@@ -398,7 +398,15 @@
 				</TD>
 				<TD>
 				Apply any recovered function signature type information
-				 in addition to the function name
+				 in addition to the function name.
+				</TD>				
+			</TR>
+			<TR>
+				<TD>Apply Function Calling Convention
+				</TD>
+				<TD>
+				Apply any recovered function calling convention information.  This option is 
+				ignored if the <B>Apply Function Signatures</B> option is <CODE>false</CODE>.
 				</TD>				
 			</TR>
 			<TR>

Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/AbstractDemanglerAnalyzer.java

@@ -207,6 +207,7 @@ public abstract class AbstractDemanglerAnalyzer extends AbstractAnalyzer {
 		//       analysis options change
 		DemanglerOptions options = new DemanglerOptions();
 		options.setApplySignature(true);
+		options.setApplyCallingConvention(true);
 		options.setDoDisassembly(true);
 		options.setDemangleOnlyKnownPatterns(false);
 		return options;

Ghidra/Features/Base/src/main/java/ghidra/app/util/demangler/DemangledFunction.java

@@ -434,7 +434,8 @@ public class DemangledFunction extends DemangledObject {
 			return true;
 		}
 
-		Structure classStructure = maybeUpdateCallingConventionAndCreateClass(program, function);
+		Structure classStructure =
+			maybeUpdateCallingConventionAndCreateClass(program, function, options);
 
 		FunctionDefinitionDataType signature = new FunctionDefinitionDataType(function, true);
 
@@ -598,9 +599,9 @@ public class DemangledFunction extends DemangledObject {
 	}
 
 	private Structure maybeUpdateCallingConventionAndCreateClass(Program program,
-			Function function) {
+			Function function, DemanglerOptions options) {
 
-		String convention = validateCallingConvention(program, function);
+		String convention = validateCallingConvention(program, function, options);
 		if (convention == null) {
 			if (!isThisCall(function)) {
 				return null;
@@ -619,7 +620,12 @@ public class DemangledFunction extends DemangledObject {
 		return null;
 	}
 
-	private String validateCallingConvention(Program program, Function function) {
+	private String validateCallingConvention(Program program, Function function,
+			DemanglerOptions options) {
+
+		if (!options.applyCallingConvention()) {
+			return null;
+		}
 
 		if (callingConvention == null) {
 			return null;

Ghidra/Features/Base/src/main/java/ghidra/app/util/demangler/DemanglerOptions.java

@@ -20,6 +20,7 @@ package ghidra.app.util.demangler;
  */
 public class DemanglerOptions {
 
+	private boolean applyCallingConvention = true;
 	private boolean applySignature = true;
 	private boolean doDisassembly = true;
 	private boolean demangleOnlyKnownPatterns = true;
@@ -62,6 +63,24 @@ public class DemanglerOptions {
 		return doDisassembly;
 	}
 
+	/**
+	 * Checks if the apply function signature calling convention option is currently set
+	 *
+	 * @return true if set to apply calling conventions
+	 */
+	public boolean applyCallingConvention() {
+		return applyCallingConvention;
+	}
+
+	/**
+	 * Set the option to apply function signature calling conventions
+	 *
+	 * @param applyCallingConvention true to apply calling conventions
+	 */
+	public void setApplyCallingConvention(boolean applyCallingConvention) {
+		this.applyCallingConvention = applyCallingConvention;
+	}
+
 	/**
 	 * Sets the option to perform disassembly for known data structures (like functions) when
 	 * demangling

Ghidra/Features/GnuDemangler/src/main/java/ghidra/app/plugin/core/analysis/GnuDemanglerAnalyzer.java

@@ -48,6 +48,11 @@ public class GnuDemanglerAnalyzer extends AbstractDemanglerAnalyzer {
 	private static final String OPTION_DESCRIPTION_APPLY_SIGNATURE =
 		"Apply any recovered function signature, in addition to the function name";
 
+	private static final String OPTION_NAME_APPLY_CALLING_CONVENTION =
+		"Apply Function Calling Conventions";
+	private static final String OPTION_DESCRIPTION_APPLY_CALLING_CONVENTION =
+		"Apply any recovered function signature calling convention";
+
 	static final String OPTION_NAME_USE_DEPRECATED_DEMANGLER = "Use Deprecated Demangler";
 	private static final String OPTION_DESCRIPTION_DEPRECATED_DEMANGLER =
 		"Signals to use the deprecated demangler when the modern demangler cannot demangle a " +
@@ -57,7 +62,8 @@ public class GnuDemanglerAnalyzer extends AbstractDemanglerAnalyzer {
 	private static final String OPTION_DESCRIPTION_DEMANGLER_FORMAT =
 		"The demangling format to use";
 
-	private boolean doSignatureEnabled = true;
+	private boolean applyFunctionSignature = true;
+	private boolean applyCallingConvention = true;
 	private boolean demangleOnlyKnownPatterns = false;
 	private GnuDemanglerFormat demanglerFormat = GnuDemanglerFormat.AUTO;
 	private boolean useDeprecatedDemangler = false;
@@ -78,9 +84,12 @@ public class GnuDemanglerAnalyzer extends AbstractDemanglerAnalyzer {
 	public void registerOptions(Options options, Program program) {
 
 		HelpLocation help = new HelpLocation("AutoAnalysisPlugin", "Demangler_Analyzer");
-		options.registerOption(OPTION_NAME_APPLY_SIGNATURE, doSignatureEnabled, help,
+		options.registerOption(OPTION_NAME_APPLY_SIGNATURE, applyFunctionSignature, help,
 			OPTION_DESCRIPTION_APPLY_SIGNATURE);
 
+		options.registerOption(OPTION_NAME_APPLY_CALLING_CONVENTION, applyCallingConvention, help,
+			OPTION_DESCRIPTION_APPLY_CALLING_CONVENTION);
+
 		options.registerOption(OPTION_NAME_DEMANGLE_USE_KNOWN_PATTERNS, demangleOnlyKnownPatterns,
 			help, OPTION_DESCRIPTION_USE_KNOWN_PATTERNS);
 
@@ -106,7 +115,10 @@ public class GnuDemanglerAnalyzer extends AbstractDemanglerAnalyzer {
 
 	@Override
 	public void optionsChanged(Options options, Program program) {
-		doSignatureEnabled = options.getBoolean(OPTION_NAME_APPLY_SIGNATURE, doSignatureEnabled);
+		applyFunctionSignature =
+			options.getBoolean(OPTION_NAME_APPLY_SIGNATURE, applyFunctionSignature);
+		applyCallingConvention =
+			options.getBoolean(OPTION_NAME_APPLY_CALLING_CONVENTION, applyCallingConvention);
 		demangleOnlyKnownPatterns =
 			options.getBoolean(OPTION_NAME_DEMANGLE_USE_KNOWN_PATTERNS, demangleOnlyKnownPatterns);
 		demanglerFormat = options.getEnum(OPTION_NAME_DEMANGLER_FORMAT, GnuDemanglerFormat.AUTO);
@@ -119,7 +131,8 @@ public class GnuDemanglerAnalyzer extends AbstractDemanglerAnalyzer {
 		GnuDemanglerOptions options =
 			new GnuDemanglerOptions(demanglerFormat, useDeprecatedDemangler);
 		options.setDoDisassembly(true);
-		options.setApplySignature(doSignatureEnabled);
+		options.setApplySignature(applyFunctionSignature);
+		options.setApplyCallingConvention(applyCallingConvention);
 		options.setDemangleOnlyKnownPatterns(demangleOnlyKnownPatterns);
 		return options;
 	}

Ghidra/Features/MicrosoftDemangler/src/main/java/ghidra/app/plugin/core/analysis/MicrosoftDemanglerAnalyzer.java

@@ -31,10 +31,17 @@ public class MicrosoftDemanglerAnalyzer extends AbstractDemanglerAnalyzer {
 		"After a function is created, this analyzer will attempt to demangle " +
 			"the name and apply datatypes to parameters.";
 
-	private final static String OPTION_NAME_APPLY_SIGNATURE = "Apply Function Signatures";
+	public static final String OPTION_NAME_APPLY_SIGNATURE = "Apply Function Signatures";
 	private static final String OPTION_DESCRIPTION_APPLY_SIGNATURE =
 		"Apply any recovered function signature, in addition to the function name";
+
+	public static final String OPTION_NAME_APPLY_CALLING_CONVENTION =
+		"Apply Function Calling Conventions";
+	private static final String OPTION_DESCRIPTION_APPLY_CALLING_CONVENTION =
+		"Apply any recovered function signature calling convention";
+
 	private boolean applyFunctionSignature = true;
+	private boolean applyCallingConvention = true;
 	private MicrosoftDemangler demangler = new MicrosoftDemangler();
 
 	public MicrosoftDemanglerAnalyzer() {
@@ -51,6 +58,9 @@ public class MicrosoftDemanglerAnalyzer extends AbstractDemanglerAnalyzer {
 	public void registerOptions(Options options, Program program) {
 		options.registerOption(OPTION_NAME_APPLY_SIGNATURE, applyFunctionSignature, null,
 			OPTION_DESCRIPTION_APPLY_SIGNATURE);
+
+		options.registerOption(OPTION_NAME_APPLY_CALLING_CONVENTION, applyCallingConvention, null,
+			OPTION_DESCRIPTION_APPLY_CALLING_CONVENTION);
 	}
 
 	@Override
@@ -59,6 +69,14 @@ public class MicrosoftDemanglerAnalyzer extends AbstractDemanglerAnalyzer {
 			options.getBoolean(OPTION_NAME_APPLY_SIGNATURE, applyFunctionSignature);
 	}
 
+	@Override
+	protected DemanglerOptions getOptions() {
+		DemanglerOptions options = new DemanglerOptions();
+		options.setApplySignature(applyFunctionSignature);
+		options.setApplyCallingConvention(applyCallingConvention);
+		return options;
+	}
+
 	@Override
 	protected DemangledObject doDemangle(String mangled, DemanglerOptions options, MessageLog log)
 			throws DemangledException {

Ghidra/Features/MicrosoftDemangler/src/test/java/ghidra/app/plugin/core/analysis/MicrosoftDemanglerAnalyzerTest.java

@@ -0,0 +1,140 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.app.plugin.core.analysis;
+
+import static org.junit.Assert.*;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import ghidra.app.cmd.label.AddLabelCmd;
+import ghidra.app.util.importer.MessageLog;
+import ghidra.framework.options.Options;
+import ghidra.program.database.ProgramBuilder;
+import ghidra.program.database.ProgramDB;
+import ghidra.program.model.address.Address;
+import ghidra.program.model.listing.*;
+import ghidra.program.model.symbol.SourceType;
+import ghidra.test.AbstractGhidraHeadedIntegrationTest;
+import ghidra.test.ToyProgramBuilder;
+import ghidra.util.Msg;
+import ghidra.util.task.TaskMonitor;
+
+public class MicrosoftDemanglerAnalyzerTest extends AbstractGhidraHeadedIntegrationTest {
+
+	private ProgramDB program;
+	private MicrosoftDemanglerAnalyzer analyzer = new MicrosoftDemanglerAnalyzer();
+
+	private MessageLog log = new MessageLog() {
+
+		// overridden to prevent stack traces from appearing in the console
+		@Override
+		public void appendException(Throwable t) {
+			appendMsg(t.toString());
+		}
+	};
+
+	@Before
+	public void setUp() throws Exception {
+
+		ProgramBuilder builder = new ToyProgramBuilder("test", true);
+		builder.createMemory(".text", "0x0100", 0x100);
+		program = builder.getProgram();
+		registerOptions();
+	}
+
+	@Override
+	protected void testFailed(Throwable e) {
+		Msg.error(this, "Test failed - analysis log:\n" + log);
+	}
+
+	@Test
+	public void testOptions__ApplyFunctionSignature() throws Exception {
+
+		String mangled = "?InvokeHelperV@COleDispatchDriver@@QAEXJGGPAXPBEPAD@Z";
+
+		Address addr = addr("0x110");
+		createSymbol(addr, mangled);
+
+		setOption(MicrosoftDemanglerAnalyzer.OPTION_NAME_APPLY_SIGNATURE, true);
+
+		analyze();
+
+		FunctionManager fm = program.getFunctionManager();
+		Function function = fm.getFunctionAt(addr);
+		assertNotNull(function);
+		assertTrue("Funciton signature not applied", function.getParameterCount() > 0);
+	}
+
+	@Test
+	public void testOptions__DoNotApplyFunctionSignature() throws Exception {
+
+		String mangled = "?InvokeHelperV@COleDispatchDriver@@QAEXJGGPAXPBEPAD@Z";
+
+		Address addr = addr("0x110");
+		createSymbol(addr, mangled);
+
+		setOption(MicrosoftDemanglerAnalyzer.OPTION_NAME_APPLY_SIGNATURE, false);
+
+		analyze();
+
+		FunctionManager fm = program.getFunctionManager();
+		Function function = fm.getFunctionAt(addr);
+		assertNotNull(function);
+		assertEquals("undefined InvokeHelperV(void)", function.getSignature().toString());
+	}
+
+//==================================================================================================
+// Private Methods
+//==================================================================================================
+
+	private void analyze() {
+		tx(program, () -> analyzer.added(program, program.getMemory(), TaskMonitor.DUMMY, log));
+	}
+
+	private void setOption(String optionName, boolean doUse) {
+
+		String fullOptionName = analyzer.getName() + Options.DELIMITER_STRING + optionName;
+		Options options = program.getOptions("Analyzers");
+
+		for (String name : options.getOptionNames()) {
+			if (name.equals(fullOptionName)) {
+				tx(program, () -> options.setBoolean(optionName, doUse));
+
+				// we must call this manually, since we are not using a tool
+				analyzer.optionsChanged(options, program);
+				return;
+			}
+		}
+
+		fail("Could not find option '" + optionName + "'");
+	}
+
+	private void registerOptions() {
+		Options options = program.getOptions(Program.ANALYSIS_PROPERTIES);
+		Options analyzerOptions = options.getOptions(analyzer.getName());
+		analyzer.registerOptions(analyzerOptions, program);
+	}
+
+	private void createSymbol(Address addr, String mangled) {
+		AddLabelCmd cmd = new AddLabelCmd(addr, mangled, SourceType.ANALYSIS);
+		applyCmd(program, cmd);
+	}
+
+	private Address addr(String addr) {
+		return program.getAddressFactory().getAddress(addr);
+	}
+}

Ghidra/Test/IntegrationTest/src/test.slow/java/ghidra/app/util/demangler/DemangledFunctionTest.java

@@ -26,6 +26,7 @@ import ghidra.program.database.ProgramDB;
 import ghidra.program.model.address.Address;
 import ghidra.program.model.address.AddressSet;
 import ghidra.program.model.data.VoidDataType;
+import ghidra.program.model.lang.CompilerSpec;
 import ghidra.program.model.listing.*;
 import ghidra.program.model.symbol.*;
 import ghidra.test.AbstractGhidraHeadlessIntegrationTest;
@@ -371,6 +372,25 @@ public class DemangledFunctionTest extends AbstractGhidraHeadlessIntegrationTest
 		assertEquals("__gthread_active_p()", ns.getName(false));
 	}
 
+	@Test
+	public void testApply_Function_DoNotApplyCallingConvention() throws Exception {
+
+		String mangled = "?CloseM@CRegKeyM@ATL@@QAEJXZ";
+		DemangledObject demangled = DemanglerUtil.demangle(mangled);
+		assertTrue(demangled instanceof DemangledFunction);
+
+		DemangledFunction demangledFunction = (DemangledFunction) demangled;
+		demangledFunction.setCallingConvention(CompilerSpec.CALLING_CONVENTION_stdcall);
+
+		Address addr = addr("0x0101");
+		DemanglerOptions options = new DemanglerOptions();
+		options.setApplyCallingConvention(false);
+		assertTrue(demangled.applyTo(program, addr, options, TaskMonitor.DUMMY));
+
+		Function function = assertFunction("CloseM", addr);
+		assertEquals("unknown", function.getCallingConventionName());
+	}
+
 	private void assertNoBookmarkAt(Address addr) {
 		BookmarkManager bm = program.getBookmarkManager();
 		Bookmark[] bookmarks = bm.getBookmarks(addr);