Merge remote-tracking branch 'origin/GP-972_ghidra007_ImproveGccClassRecovery--SQUASHED' into Ghidra_10.0

Ghidra/Features/Base/ghidra_scripts/GccRttiAnalysisScript.java

@@ -680,7 +680,6 @@ public class GccRttiAnalysisScript extends GhidraScript {
 
 	/**
 	 * Method to process the primary vtable for each "vtable" label
-	 * @return the vftable Address in the vtable
 	 * @throws Exception if Data cannot be created
 	 */
 	private void processVtables() throws Exception {

Ghidra/Features/Decompiler/ghidra_scripts/ApplyClassFunctionDefinitionUpdatesScript.java

@@ -13,16 +13,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-// Script to apply any changes the user has made to class virtual function definitions, ie ones they 
-// have edited in the data type manager. To run the script, put the cursor on any member of the 
+// Script to apply any changes the user has made to recovered class virtual function definitions 
+// edited in the data type manager. To run the script, put the cursor on any member of the 
 // desired class in the listing then run the script. For each function definition in the given class 
 // that differs from the associated function signature in the listing, the script will update the 
-// listing function signatures of any related virtual vunctions (ie parents and children) and update 
-// related data types such as function definitions of the given class and related classes and also 
-// field names in related vftable structures.
+// listing function signatures of any related virtual functions belonging to parent and children 
+// classes. It will also update related data types including function definitions and vftable structures.
 // Note: The script will not work if the vftable structures were not originally applied to 
 // the vftables using the RecoverClassesFromRTTIScript. 
-// At some point, the Ghidra API will be updated to do this automatically instead of needing the script to do so. 
+// At some point, the Ghidra API will be updated to do this automatically instead of needing the 
+// script to do so. 
 //@category C++
 
 import java.util.List;

Ghidra/Features/Decompiler/ghidra_scripts/ApplyClassFunctionSignatureUpdatesScript.java

@@ -13,16 +13,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-// Script to apply any changes the user has made to class virtual function signatures, ie ones they 
-// have edited in the listing. To run the script, put the cursor on any member of the desired class in
+// Script to apply any changes the user has made to recovered class virtual function signatures 
+// edited in the listing. To run the script, put the cursor on any member of the desired class in
 // the listing then run the script. For each function signature in the given class that differs from
 // the associated function definition in the data type manager, the script will update the listing 
-// function signatures of any related virtual vunctions (ie parents and children) and update related 
-// data types such as function definitions of the given class and related classes and also field names 
-// in related vftable structures. 
+// function signatures of any related virtual functions belonging to parents and children classes.
+// It will also update related data types including function definitions and vftable structures. 
 // Note: The script will not work if the vftable structures were not originally applied to 
 // the vftables using the RecoverClassesFromRTTIScript.
-// At some point, the Ghidra API will be updated to do this automatically instead of needing the script to do so. 
+// At some point, the Ghidra API will be updated to do this automatically instead of needing the 
+// script to do so. 
 //@category C++
 
 import java.util.List;

Ghidra/Features/Decompiler/ghidra_scripts/RecoverClassesFromRTTIScript.java

@@ -35,18 +35,20 @@
 // this script and default vfunctions named by this script are likely to change in the future 
 // once an official design for Object Oriented representation is determined.  
 // NOTE: Windows class recovery is more complete and tested than gcc class recovery, which is still 
-// in early stages of development. Gcc class data types have not been recovered yet but if the program
-// has DWARF, there will be some amount of data recovered by the DWARF analyzer in the DWARF data folder.
+// in early stages of development. Gcc class data types are only recovered for classes without multiple or
+// virtual inheritance but if the program contains DWARF, there will be some amount of data recovered 
+// by the DWARF analyzer.
 // NOTE: For likely the best results, run this script on freshly analyzed programs. No testing has been 
 // done on user marked-up programs. 
 // NOTE: After running this script if you edit function signatures in the listing for a particular
-// class and wish to update the corresponding class data (function definition data types, vftable 
+// class and wish to update the corresponding class data function definition data types (vftable 
 // structure field names, ...) then you can run the ApplyClassFunctionSignatureUpdatesScript.java 
-// to have it do so for you.
+// to have it do so for you. See that script's description for more info.
 // Conversely, if you update a particular class's function definitions in the data type manager and  
 // wish to have related function signatures in the listing updated, as well as other data types that 
-// are related, then run the ApplyClassFunctionDefinitionsUpdatesScript.java to do so. At some point, 
-// the Ghidra API will be updated to do this all automatically instead of needing the scripts to do so. 
+// are related, then run the ApplyClassFunctionDefinitionsUpdatesScript.java to do so. See that script's
+// description for more info. At some point, the Ghidra API will be updated to do the updates 
+// automatically instead of needing the mentioned scripts to do so. 
 
 //@category C++
 
@@ -152,6 +154,7 @@ public class RecoverClassesFromRTTIScript extends GhidraScript {
 
 
 		if (isWindows()) {
+
 			isPDBLoaded = isPDBLoadedInProgram();
 			nameVfunctions = !isPDBLoaded;
 			recoverClassesFromRTTI = new RTTIWindowsClassRecoverer(currentProgram,
@@ -159,7 +162,7 @@ public class RecoverClassesFromRTTIScript extends GhidraScript {
 				USE_SHORT_TEMPLATE_NAMES_IN_STRUCTURE_FIELDS, nameVfunctions, isPDBLoaded, monitor);
 		}
 		else if (isGcc()) {
-			// for now assume gcc has named vfunctions until a way to check is developed
+
 			nameVfunctions = true;
 			recoverClassesFromRTTI = new RTTIGccClassRecoverer(currentProgram, currentLocation,
 				state.getTool(), this, BOOKMARK_FOUND_FUNCTIONS,
@@ -176,7 +179,6 @@ public class RecoverClassesFromRTTIScript extends GhidraScript {
 			return;
 		}
 
-		// possibly more picky subtype than just checking windows/gcc
 		if (!recoverClassesFromRTTI.isValidProgramType()) {
 			println("This script will not work on this program type");
 			return;
@@ -465,9 +467,41 @@ public class RecoverClassesFromRTTIScript extends GhidraScript {
 	 */
 	private boolean isGcc() {
 
-		isGcc =
+		boolean isELF = currentProgram.getExecutableFormat().contains("ELF");
+		if (!isELF) {
+			return false;
+		}
+
+		boolean isCompilerSpecGcc =
 			currentProgram.getCompilerSpec().getCompilerSpecID().getIdAsString().equalsIgnoreCase(
 				"gcc");
+		if (isCompilerSpecGcc) {
+			return true;
+		}
+
+		MemoryBlock commentBlock = currentProgram.getMemory().getBlock(".comment");
+		if (commentBlock == null) {
+			return false;
+		}
+
+		if (!commentBlock.isLoaded()) {
+			return false;
+		}
+
+
+		// check memory bytes in block for GCC: bytes
+		byte[] gccBytes = { (byte) 0x47, (byte) 0x43, (byte) 0x43, (byte) 0x3a };
+		byte[] maskBytes = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff };
+
+		Address found = currentProgram.getMemory().findBytes(commentBlock.getStart(),
+				commentBlock.getEnd(), gccBytes, maskBytes, true, monitor);
+		if (found == null) {
+			isGcc = false;
+		}
+		else {
+			isGcc = true;
+		}
+
 		return isGcc;
 	}
 

Ghidra/Features/Decompiler/ghidra_scripts/classrecovery/RTTIWindowsClassRecoverer.java

@@ -485,7 +485,7 @@ public class RTTIWindowsClassRecoverer extends RTTIClassRecoverer {
 		for (int i = 0; i < numBaseClasses; i++) {
 
 			monitor.checkCanceled();
-
+			//TODO: extraUtils.getReferencedAddress(address, getIboIf64bit);
 			Address baseClassDescriptorAddress = getReferencedAddress(address.add(i * 4));
 
 			Data baseClassDescriptor = extraUtils.getDataAt(baseClassDescriptorAddress);
@@ -572,6 +572,7 @@ public class RTTIWindowsClassRecoverer extends RTTIClassRecoverer {
 	private Address createClassHierarchyDescriptor(Address address, Namespace classNamespace)
 			throws CancelledException, MemoryAccessException, InvalidInputException, Exception {
 
+		//TODO: extraUtils.getReferencedAddress(address, getIboIf64bit);
 		Address classHierarchyDescriptorAddress = getReferencedAddress(address);
 
 		Data classHierarchyStructure = extraUtils.getDataAt(classHierarchyDescriptorAddress);
@@ -652,6 +653,7 @@ public class RTTIWindowsClassRecoverer extends RTTIClassRecoverer {
 
 			int numBaseClasses = extraUtils.getInt(classHierarchyDescriptorAddress.add(8));
 
+			//TODO: extraUtils.getReferencedAddress(address, getIboIf64bit);
 			Address baseClassArrayAddress =
 				getReferencedAddress(classHierarchyDescriptorAddress.add(12));
 
@@ -1295,8 +1297,7 @@ public class RTTIWindowsClassRecoverer extends RTTIClassRecoverer {
 	 */
 	private void setClassInheritanceType(RecoveredClass recoveredClass, int inheritanceType) {
 
-		// TODO: update the single virtual case from here too - may need to update class
-		// TODO: ?? add multi-repeate base inh flag? do I care other than to give info to user?
+		// TODO: add multi-repeated base inh flag? 
 
 		if ((inheritanceType & CHD_MULTINH) == 0) {
 			recoveredClass.setHasSingleInheritance(true);
@@ -1305,11 +1306,11 @@ public class RTTIWindowsClassRecoverer extends RTTIClassRecoverer {
 			if ((inheritanceType & CHD_VIRTINH) == 0) {
 				recoveredClass.setInheritsVirtualAncestor(false);
 			}
-//			else {
-//			//  TODO: might have to update the single virt inh here
-//				println("Flag indicates single inheritance virtual ancestor for class " +
-//					recoveredClass.getName());
-//			}
+			// Flag indicates single inheritance virtual ancestor for class " +
+			//	recoveredClass.getName());
+			else {
+				recoveredClass.setInheritsVirtualAncestor(true);
+			}
 		}
 		else {
 			recoveredClass.setHasSingleInheritance(false);
@@ -1317,10 +1318,11 @@ public class RTTIWindowsClassRecoverer extends RTTIClassRecoverer {
 			if ((inheritanceType & CHD_VIRTINH) == 0) {
 				recoveredClass.setHasMultipleVirtualInheritance(false);
 			}
+
+			// Flag indicates multiple inheritance virtual ancestor for class " +
+			// recoveredClass.getName());
 			else {
 				recoveredClass.setHasMultipleVirtualInheritance(true);
-//				println("Flag indicates multiple inheritance virtual ancestor for class " +
-//					recoveredClass.getName());
 			}
 		}
 		//TODO: CHD_AMBIGUOUS = 0x00000004;

Ghidra/Features/Decompiler/ghidra_scripts/classrecovery/RecoveredClassUtils.java

@@ -41,6 +41,7 @@ import ghidra.app.plugin.core.decompile.actions.FillOutStructureCmd;
 import ghidra.app.plugin.core.decompile.actions.FillOutStructureCmd.OffsetPcodeOpPair;
 import ghidra.app.plugin.core.navigation.locationreferences.LocationReference;
 import ghidra.app.plugin.core.navigation.locationreferences.ReferenceUtils;
+import ghidra.app.util.NamespaceUtils;
 import ghidra.framework.plugintool.PluginTool;
 import ghidra.program.database.data.DataTypeUtilities;
 import ghidra.program.flatapi.FlatProgramAPI;
@@ -85,6 +86,8 @@ public class RecoveredClassUtils {
 
 	private static int MIN_OPERATOR_NEW_REFS = 10;
 
+	private static final boolean DEBUG = false;
+
 	private Map<Address, RecoveredClass> vftableToClassMap = new HashMap<Address, RecoveredClass>();
 
 	// map from vftable references to the vftables they point to
@@ -1012,7 +1015,9 @@ public class RecoveredClassUtils {
 
 		List<RecoveredClass> functionClasses = getClasses(function);
 		if (functionClasses == null) {
-			Msg.debug(this, "no function to class map for " + function.getEntryPoint());
+			if (DEBUG) {
+				Msg.debug(this, "no function to class map for " + function.getEntryPoint());
+			}
 			return true;
 		}
 		Iterator<RecoveredClass> functionClassesIterator = functionClasses.iterator();
@@ -1098,8 +1103,10 @@ public class RecoveredClassUtils {
 
 		List<Address> vftableReferenceList = getVftableReferences(function);
 		if (vftableReferenceList == null) {
-			Msg.debug(this, "In update maps: function to class map doesn't exist for " +
-				function.getEntryPoint().toString());
+			if (DEBUG) {
+				Msg.debug(this, "In update maps: function to class map doesn't exist for " +
+					function.getEntryPoint().toString());
+			}
 			return;
 		}
 		Collections.sort(vftableReferenceList);
@@ -1113,10 +1120,11 @@ public class RecoveredClassUtils {
 		RecoveredClass vftableClass = getVftableClass(vftableAddress);
 		if (!vftableClass.equals(recoveredClass)) {
 
-			Msg.debug(this,
-				"updating struct for " + recoveredClass.getName() +
+			if (DEBUG) {
+				Msg.debug(this, "updating struct for " + recoveredClass.getName() +
 					" but first vftable in function " + function.getEntryPoint().toString() +
 					" is in class " + vftableClass.getName());
+			}
 
 			return;
 		}
@@ -1196,8 +1204,10 @@ public class RecoveredClassUtils {
 				return;
 			}
 
-			Msg.debug(this, "Could not find variable pointing to vftable in " +
-				function.getEntryPoint().toString());
+			if (DEBUG) {
+				Msg.debug(this, "Could not find variable pointing to vftable in " +
+					function.getEntryPoint().toString());
+			}
 
 		}
 
@@ -2046,8 +2056,10 @@ public class RecoveredClassUtils {
 		}
 
 		if (vftableAddresses.size() != classOffsetToVftableMap.size()) {
-			Msg.debug(this, recoveredClass.getName() + " has " + vftableAddresses.size() +
-				" vftables but " + classOffsetToVftableMap.size() + " offset to vftable maps");
+			if (DEBUG) {
+				Msg.debug(this, recoveredClass.getName() + " has " + vftableAddresses.size() +
+					" vftables but " + classOffsetToVftableMap.size() + " offset to vftable maps");
+			}
 		}
 
 		List<Integer> offsetList = new ArrayList<Integer>(classOffsetToVftableMap.keySet());
@@ -2717,20 +2729,19 @@ public class RecoveredClassUtils {
 			// Get class name from class vftable is in
 			Namespace vftableNamespace = vftableSymbol.getParentNamespace();
 			if (vftableNamespace.equals(globalNamespace)) {
-				Msg.debug(this,
-					"vftable is in the global namespace, ie not in a class namespace, so cannot process");
+				if (DEBUG) {
+					Msg.debug(this,
+						"vftable is in the global namespace, ie not in a class namespace, so cannot process");
+				}
 				continue;
 			}
 
-			SymbolType namespaceType = vftableNamespace.getSymbol().getSymbolType();
-			if (namespaceType != SymbolType.CLASS) {
-				// if it is a namespace but not a class we need to promote it to a class namespace
-				if (namespaceType == SymbolType.NAMESPACE) {
-
-					//vftableNamespace = promoteToClassNamespace(vftableNamespace);
-
-					// else just leave the old one as a namepace
-
+			// promote any non-class namespaces in the vftableNamespace path to class namespaces
+			boolean success = promoteNamespaces(vftableNamespace.getSymbol());
+			if (!success) {
+				if (DEBUG) {
+					Msg.debug(this, "Unable to promote all non-class namespaces for " +
+						vftableNamespace.getName(true));
 				}
 			}
 
@@ -2808,6 +2819,60 @@ public class RecoveredClassUtils {
 		return recoveredClasses;
 	}
 
+	private boolean promoteNamespaces(Symbol symbol) throws CancelledException {
+
+		Namespace namespace = symbol.getParentNamespace();
+		while (!namespace.isGlobal()) {
+
+			monitor.checkCanceled();
+			SymbolType namespaceType = namespace.getSymbol().getSymbolType();
+			if (namespaceType != SymbolType.CLASS) {
+				// if it is a namespace but not a class we need to promote it to a class namespace
+				if (namespaceType == SymbolType.NAMESPACE) {
+					namespace = promoteToClassNamespace(namespace);
+					if (namespace == null) {
+						return false;
+					}
+					if (DEBUG) {
+						Msg.debug(this,
+							"Promoted namespace " + namespace.getName() + " to a class namespace");
+					}
+				}
+			}
+			else {
+				namespace = namespace.getParentNamespace();
+			}
+		}
+		return true;
+	}
+
+	/**
+	 * Method to promote the namespace is a class namespace. 
+	 * @return true if namespace is (now) a class namespace or false if it could not be promoted.
+	 */
+	private Namespace promoteToClassNamespace(Namespace namespace) {
+
+		try {
+			Namespace newClass = NamespaceUtils.convertNamespaceToClass(namespace);
+
+			SymbolType symbolType = newClass.getSymbol().getSymbolType();
+			if (symbolType == SymbolType.CLASS) {
+				return newClass;
+			}
+			if (DEBUG) {
+				Msg.debug(this,
+					"Could not promote " + namespace.getName() + " to a class namespace");
+			}
+			return null;
+		}
+		catch (InvalidInputException e) {
+
+			Msg.debug(this, "Could not promote " + namespace.getName() +
+				" to a class namespace because " + e.getMessage());
+			return null;
+		}
+	}
+
 	/**
 	 * Method to create mapping to possible constructor/destructor functions
 	 * @param referencesToVftable list of references to a particular vftable
@@ -3397,8 +3462,10 @@ public class RecoveredClassUtils {
 		if (symbolsByNameAtAddress.size() == 0) {
 			AddLabelCmd lcmd = new AddLabelCmd(address, name, namespace, SourceType.ANALYSIS);
 			if (!lcmd.applyTo(program)) {
-				Msg.debug(this,
-					"ERROR: Could not add new symbol " + name + " to " + address.toString());
+				if (DEBUG) {
+					Msg.debug(this,
+						"ERROR: Could not add new symbol " + name + " to " + address.toString());
+				}
 			}
 		}
 		//put the same name one in the namespace
@@ -3491,8 +3558,10 @@ public class RecoveredClassUtils {
 			AddLabelCmd lcmd =
 				new AddLabelCmd(function.getEntryPoint(), name, namespace, SourceType.ANALYSIS);
 			if (!lcmd.applyTo(program)) {
-				Msg.debug(this, "ERROR: Could not add new function label " + name + " to " +
-					function.getEntryPoint().toString());
+				if (DEBUG) {
+					Msg.debug(this, "ERROR: Could not add new function label " + name + " to " +
+						function.getEntryPoint().toString());
+				}
 				return;
 			}
 
@@ -3501,8 +3570,10 @@ public class RecoveredClassUtils {
 				SetLabelPrimaryCmd scmd =
 					new SetLabelPrimaryCmd(function.getEntryPoint(), name, namespace);
 				if (!scmd.applyTo(program)) {
-					Msg.debug(this, "ERROR: Could not make function label " + name +
-						" primary at " + function.getEntryPoint().toString());
+					if (DEBUG) {
+						Msg.debug(this, "ERROR: Could not make function label " + name +
+							" primary at " + function.getEntryPoint().toString());
+					}
 				}
 			}
 		}
@@ -3756,8 +3827,10 @@ public class RecoveredClassUtils {
 					}
 
 					else {
-						Msg.debug(this, "ERROR: " + function.getEntryPoint().toString() +
-							" Could not replace parameter " + i + " with undefined pointer.");
+						if (DEBUG) {
+							Msg.debug(this, "ERROR: " + function.getEntryPoint().toString() +
+								" Could not replace parameter " + i + " with undefined pointer.");
+						}
 					}
 				}
 			}
@@ -4818,9 +4891,11 @@ public class RecoveredClassUtils {
 
 			//TODO: remove after testing
 			if (!classVftableRef.equals(otherWayRef)) {
-				Msg.debug(this, recoveredClass.getName() + " function " +
-					destructorFunction.getEntryPoint().toString() + " first ref: " +
-					classVftableRef.toString() + " other way ref: " + otherWayRef.toString());
+				if (DEBUG) {
+					Msg.debug(this, recoveredClass.getName() + " function " +
+						destructorFunction.getEntryPoint().toString() + " first ref: " +
+						classVftableRef.toString() + " other way ref: " + otherWayRef.toString());
+				}
 			}
 
 			String markupString = classNamespace.getName(true) + "::~" + className;
@@ -4857,10 +4932,12 @@ public class RecoveredClassUtils {
 			}
 			//TODO: remove after testing
 			if (!classVftableRef.equals(otherWayRef)) {
+				if (DEBUG) {
 				Msg.debug(this,
 					recoveredClass.getName() + " function " +
 					functionContainingInline.getEntryPoint().toString() + " first ref: " +
 					classVftableRef.toString() + " other way ref: " + otherWayRef.toString());
+				}
 			}
 
 			String markupString = "inlined constructor or destructor (approx location) for " +
@@ -5070,11 +5147,12 @@ public class RecoveredClassUtils {
 			return;
 		}
 		if (!vftableReference.equals(otherWayRef)) {
-			Msg.debug(this,
-				recoveredClass.getName() + " function " +
+			if (DEBUG) {
+				Msg.debug(this, recoveredClass.getName() + " function " +
 					virtualFunction.getEntryPoint().toString() + " first ref: " +
 					vftableReference.toString() + " other way ref (with ances): " +
 					otherWayRef.toString());
+			}
 		}
 
 		List<Function> possibleParentDestructors = getPossibleParentDestructors(virtualFunction);
@@ -5108,7 +5186,9 @@ public class RecoveredClassUtils {
 
 				recoveredClass.addDeletingDestructor(virtualFunction);
 				if (recoveredClass.getDestructorList().contains(virtualFunction)) {
-					Msg.debug(this, "Already created vfunction as a destructor");
+					if (DEBUG) {
+						Msg.debug(this, "Already created vfunction as a destructor");
+					}
 				}
 				recoveredClass.removeFromConstructorDestructorList(virtualFunction);
 				recoveredClass.removeIndeterminateConstructorOrDestructor(virtualFunction);
@@ -6053,8 +6133,10 @@ public class RecoveredClassUtils {
 		Function operatorDeleteFunction =
 			findOperatorDeleteUsingKnownDeletingDestructors(recoveredClasses);
 		if (operatorDeleteFunction == null) {
-			Msg.debug(this,
-				"Could not find operator delete function. Cannot process more deleting destructors.");
+			if (DEBUG) {
+				Msg.debug(this,
+					"Could not find operator delete function. Cannot process more deleting destructors.");
+			}
 			return;
 		}
 
@@ -6645,7 +6727,9 @@ public class RecoveredClassUtils {
 				}
 				// if they ever don't match return 
 				else if (!possiblePureCall.equals(sameFunction)) {
-					Msg.debug(this, "Could not identify pure call. ");
+					if (DEBUG) {
+						Msg.debug(this, "Could not identify pure call. ");
+					}
 					return;
 				}
 			}