diff --git a/Ghidra/Features/PDB/src/main/java/ghidra/app/util/pdb/pdbapplicator/CppCompositeType.java b/Ghidra/Features/PDB/src/main/java/ghidra/app/util/pdb/pdbapplicator/CppCompositeType.java index 2c25ba618c..1fc61e94e3 100644 --- a/Ghidra/Features/PDB/src/main/java/ghidra/app/util/pdb/pdbapplicator/CppCompositeType.java +++ b/Ghidra/Features/PDB/src/main/java/ghidra/app/util/pdb/pdbapplicator/CppCompositeType.java @@ -27,6 +27,7 @@ import ghidra.app.util.pdb.classtype.*; import ghidra.program.model.data.*; import ghidra.program.model.gclass.ClassID; import ghidra.program.model.gclass.ClassUtils; +import ghidra.util.InvalidNameException; import ghidra.util.Msg; import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskMonitor; @@ -43,6 +44,11 @@ public class CppCompositeType { private static final String VIRTUAL_BASE_SPECULATIVE_COMMENT = "Virtual Base - Speculative Placement"; + private static final boolean CREATE_BASE_NAMES = + Boolean.getBoolean("ghidra.pdb.createBaseNames"); + + private static final boolean CREATE_MY_DATA = Boolean.getBoolean("ghidra.pdb.createMyData"); + private boolean isFinal; private ClassKey classKey; private String className; // String for now. @@ -1230,7 +1236,7 @@ public class CppCompositeType { */ private void createClassLayout(MsVxtManager vxtManager, ObjectOrientedClassLayout layoutOptions, TaskMonitor monitor) throws CancelledException, PdbException { - List selfBaseMembers = getSelfBaseClassMembers(); + List selfBaseMembers = getSelfBaseClassMembers(monitor); mainVft = getMainVft(vxtManager); if (mainVft != null) { updateMainVft(); @@ -1249,8 +1255,8 @@ public class CppCompositeType { selfBaseMembers, msg -> Msg.warn(this, msg), monitor)) { clearComponents(composite); } - ClassPdbMember directClassPdbMember = - new ClassPdbMember("", selfBaseType, false, 0, SELF_BASE_COMMENT); + ClassPdbMember directClassPdbMember = new ClassPdbMember(getBaseClassName(selfBaseType), + selfBaseType, false, 0, SELF_BASE_COMMENT); mainVbt = getMainVbt(vxtManager); if (mainVbt != null) { @@ -1286,7 +1292,34 @@ public class CppCompositeType { sourceHierarchy = determineBaseSourceOrder(); composite.setDescription(sourceHierarchy); + } + /** + * Temporary debug method + * @param applicator the applicator + * @param structure the input structure + * @return the flattened structure or null if could not or didn't need to be flattened + */ + public static Structure createFlattenedTemp(DefaultPdbApplicator applicator, + Structure structure) { + Structure f = ClassUtils.getReplacementType(structure); + if (f == null) { + return null; + } +// CategoryPath p = ClassUtils.getClassInternalsPath(f); + CategoryPath p = f.getCategoryPath(); + if (f instanceof StructureDataType s) { + String n = structure.getName() + "_TEMP_Flattened"; + try { + s.setName(n); + s.setCategoryPath(p); + return s; + } + catch (InvalidNameException e) { + // + } + } + return null; } // Taken from PdbUtil without change. Would have had to change access on class PdbUtil and @@ -1316,7 +1349,8 @@ public class CppCompositeType { * regular members * @return the members */ - private List getSelfBaseClassMembers() { + private List getSelfBaseClassMembers(TaskMonitor monitor) + throws CancelledException { // Using TreeMap to get base classes and vxtptrs in the correct order. None of these // should have the same offset unless there are zero-sized base classes in play. Found // examples, however where some "empty" base classes were given unique offsets (e.g., 12, @@ -1370,8 +1404,8 @@ public class CppCompositeType { String comment = BASE_COMMENT; Composite baseDataType = base.getSelfBaseDataType(); // This does not have attributes like "Member" does (consider changes?) - ClassPdbMember classPdbMember = - new ClassPdbMember("", baseDataType, false, offset, comment); + ClassPdbMember classPdbMember = new ClassPdbMember(getBaseClassName(baseDataType), + baseDataType, false, offset, comment); map.put(new OffsetOrdinal(offset, ordinal++), classPdbMember); } hasZeroBaseSize = hasZeroParentBaseSize; @@ -1402,14 +1436,7 @@ public class CppCompositeType { List members = new ArrayList<>(map.values()); int lastOffset = members.isEmpty() ? -1 : members.getLast().getOffset(); - List standardMembers = new ArrayList<>(); - for (Member member : layoutMembers) { - ClassPdbMember classPdbMember = - new ClassPdbMember(member.getName(), member.getDataType(), - member.isFlexibleArray(), member.getOffset(), member.getComment()); - standardMembers.add(classPdbMember); - //members.add(classPdbMember); - } + List standardMembers = getStandardMembers(monitor); int firstStandardOffset = standardMembers.isEmpty() ? lastOffset + 1 : standardMembers.getFirst().getOffset(); @@ -1456,7 +1483,8 @@ public class CppCompositeType { Composite baseDataType = base.getSelfBaseDataType(); // This does not have attributes ClassPdbMember classPdbMember = - new ClassPdbMember("", baseDataType, false, offset.intValue(), comment); + new ClassPdbMember(getBaseClassName(baseDataType), baseDataType, false, + offset.intValue(), comment); map.put(offset, classPdbMember); accumulatedComment = ""; } @@ -1472,6 +1500,56 @@ public class CppCompositeType { return map; } + private String getBaseClassName(Composite baseDataType) { + if (!CREATE_BASE_NAMES) { + return ""; + } + return baseDataType.getName() + "_base"; + } + + private List getStandardMembers(TaskMonitor monitor) + throws CancelledException { + List members = new ArrayList<>(); + if (layoutMembers.isEmpty()) { + return members; + } + if (CREATE_MY_DATA) { + int minOffset = Integer.MAX_VALUE; + for (Member member : layoutMembers) { + minOffset = Integer.min(minOffset, member.getOffset()); + } + for (Member member : layoutMembers) { + // subtracts minOffset + ClassPdbMember classPdbMember = + new ClassPdbMember(member.getName(), member.getDataType(), + member.isFlexibleArray(), member.getOffset() - minOffset, + member.getComment()); + members.add(classPdbMember); + } + DataTypePath selfBasePath = createSelfBaseCategoryPath(this); // use same path as self + String dataName = composite.getName() + "_data"; + Composite data = new StructureDataType(selfBasePath.getCategoryPath(), dataName, 0, + composite.getDataTypeManager()); + data.setDescription("Data of " + selfBasePath.getDataTypeName()); + if (!DefaultCompositeMember.applyDataTypeMembers(data, false, false, 0, + members, msg -> Msg.warn(this, msg), monitor)) { + clearComponents(composite); + } + members.clear(); + members.add(new ClassPdbMember(dataName, data, false, minOffset, "")); + } + else { + // does not subtract minOffset + for (Member member : layoutMembers) { + ClassPdbMember classPdbMember = + new ClassPdbMember(member.getName(), member.getDataType(), + member.isFlexibleArray(), member.getOffset(), member.getComment()); + members.add(classPdbMember); + } + } + return members; + } + /** * Finds all virtual base and virtual function pointers in the hierarchy of this class's * self base. diff --git a/Ghidra/Features/PDB/src/main/java/ghidra/app/util/pdb/pdbapplicator/DefaultPdbApplicator.java b/Ghidra/Features/PDB/src/main/java/ghidra/app/util/pdb/pdbapplicator/DefaultPdbApplicator.java index 955ccd6f5b..702cf1fbf8 100644 --- a/Ghidra/Features/PDB/src/main/java/ghidra/app/util/pdb/pdbapplicator/DefaultPdbApplicator.java +++ b/Ghidra/Features/PDB/src/main/java/ghidra/app/util/pdb/pdbapplicator/DefaultPdbApplicator.java @@ -81,6 +81,9 @@ public class DefaultPdbApplicator implements PdbApplicator { private static final String THUNK_NAME_PREFIX = "[thunk]:"; + private static final boolean CREATE_FLATTENED_CLASSES = + Boolean.getBoolean("ghidra.pdb.createFlattenedClasses"); + //============================================================================================== private static final String PDB_ANALYSIS_LOOKUP_STATE = "PDB_UNIVERSAL_ANALYSIS_STATE"; @@ -373,6 +376,7 @@ public class DefaultPdbApplicator implements PdbApplicator { processTypes(); processSymbols(); vxtManager.createTables(dataTypeManager, ClearDataMode.CLEAR_ALL_CONFLICT_DATA); + doTempResearch(); break; default: throw new PdbException("PDB: Invalid Application Control: " + @@ -381,6 +385,20 @@ public class DefaultPdbApplicator implements PdbApplicator { Msg.info(this, "PDB Types and Main Symbols Processing Terminated Normally"); } + private void doTempResearch() { + if (!CREATE_FLATTENED_CLASSES) { + return; + } + for (CppCompositeType cppType : classTypeByMsTypeNum.values()) { + if (cppType.getComposite() instanceof Structure s) { + Structure x = CppCompositeType.createFlattenedTemp(this, s); + if (x != null) { + resolve(x); + } + } + } + } + private void doDisassemblyWork() throws PdbException, CancelledException { if (program != null) { disassembleFunctions(); diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/gclass/ClassUtils.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/gclass/ClassUtils.java index c156cf8dc3..93e63d8627 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/gclass/ClassUtils.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/gclass/ClassUtils.java @@ -15,11 +15,19 @@ */ package ghidra.program.model.gclass; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + import ghidra.app.util.SymbolPath; import ghidra.program.model.data.*; /** * Utility class for Class-related software modeling. + *

+ * This class is experimental and subject to unannounced changes, including changes to processing + * philosophies and removal of methods */ public class ClassUtils { @@ -303,4 +311,262 @@ public class ClassUtils { return new CategoryPath(category, symbolPath.getName()); } + /** + * Finds and returns list of replacement pointer types for the specified owner class structure + * @param dtm the data type manager + * @param type the class structure type + * @return the map of offset to owner replacement types + */ + public static Map getReplacementPointers(DataTypeManager dtm, + Structure type) { + CategoryPath path = getClassPath(type); + Map results = new HashMap<>(); + Category category = dtm.getCategory(path); + if (category == null) { + return results; + } + for (DataType dt : category.getDataTypes()) { + if (!(dt instanceof Structure struct)) { + continue; + } + Long offset = + ClassUtils.validateVtableDescriptionOffsetTag(struct.getDescription()); + if (offset == null) { + continue; + } + Pointer vxtptr = new PointerDataType(struct); + results.put(offset, vxtptr); + } + return results; + } + + /** + * Record containing a name and a pointer data type + * @param name the name + * @param pointer the pointer type + */ + public static record NameAndPointer(String name, DataType pointer) {} + + /** + * Tries to provide an appropriate data type replacement for special components, particularly + * for class objects such as virtual function table and virtual base table pointers within + * a flattened class structure + * @param component the component to be checked + * @param accumulatedOffset the accumulated offset of the component due to flattening + * @param ownerVxtptrs the map of offset to owner vxtptr types + * @return the replacement data type or the original type if there is no replacement needed + */ + public static NameAndPointer getReplacementType(DataTypeComponent component, + long accumulatedOffset, Map ownerVxtptrs) { + if (!hasReplaceAttribute(component)) { + return null; + } + String fieldName = component.getFieldName(); + Pointer vxtptr = ownerVxtptrs.get(accumulatedOffset); + if (vxtptr == null) { + return null; + } + DataType dt = vxtptr.getDataType(); + String dtName = dt.getName(); // We are not using the full path name + String newFieldName; + if (dtName.startsWith(ClassUtils.VTABLE)) { + if (!ClassUtils.VTPTR.equals(fieldName)) { + return null; + } + newFieldName = fieldName + dtName.substring(VTABLE.length()); //crash if not more char + } + else if (dtName.startsWith(ClassUtils.VBTABLE)) { + if (!ClassUtils.VBPTR.equals(fieldName)) { + return null; + } + newFieldName = fieldName + dtName.substring(VFTABLE.length()); //crash if not more char + } + else if (dtName.startsWith(ClassUtils.VFTABLE)) { + if (!ClassUtils.VFPTR.equals(fieldName)) { + return null; + } + newFieldName = fieldName + dtName.substring(VBTABLE.length()); //crash if not more char + } + else { + return null; + } + return new NameAndPointer(newFieldName, vxtptr); + } + + /** + * Tries to provide an appropriate data type replacement for special components, particularly + * for class objects such as virtual function table and virtual base table pointers within + * a flattened class structure. The {@code structure} argument becomes the return type if + * {@code enabled} is {@code false}, if the argument structure does not have class + * attributes, or if there is no suitable replacement for it + * @param structure the structure to process + * @param enabled {@code false} will immediately return the argument type + * @return the replacement data type or null if could not or did not need to be replaced + */ + public static Structure getReplacementType(Structure structure, boolean enabled) { + if (!enabled) { + return structure; + } + if (!hasClassAttribute(structure)) { + return structure; + } + Structure replacement = getReplacementType(structure); + return replacement == null ? structure : replacement; + } + + /** + * Tries to provide an appropriate data type replacement for special components, particularly + * for class objects such as virtual function table and virtual base table pointers within + * a flattened class structure + * @param structure the structure to process + * @return the replacement data type or null if could not or did not need to be replaced + */ + public static Structure getReplacementType(Structure structure) { + DataTypeManager dtm = structure.getDataTypeManager(); + Map vxtptrs = ClassUtils.getReplacementPointers(dtm, structure); + StructureDataType newStruct = new StructureDataType(structure.getCategoryPath(), + structure.getName(), 0, structure.getDataTypeManager()); + newStruct.setPackingEnabled(false); + // Future: consider whether we need to strip the class attribute from the description + // of the resultant type. Decompiler might still want/need it; but we probably don't + // want it if it allows us to do replacement again (unless it doesn't really do + // anything on another pass of replacement). This comment is really for while we are + // using the description field to hold an attribute; this comment can be deleted once + // we are not using the field for holding an attribute. + newStruct.setDescription(structure.getDescription()); + try { + if (processComponents(structure, newStruct, 0, vxtptrs)) { + newStruct.setLength(structure.getLength()); + // The original structure should be packed, so we can use its alignment + // as the alignment of our flattened structure. We do not want to turn on + // packing for the flattened structure unless we supply appropriate padding + newStruct.align(structure.getAlignment()); + return newStruct; + } + } + catch (InvalidDataTypeException e) { + // squelch + } + return null; + } + + /** + * Tries to provide an appropriate data type replacement for special components, particularly + * for class objects such as virtual function table and virtual base table pointers within + * a flattened class structure + * @param type the structure to process + * @param newType the new structure being created + * @param baseOffset the accumulated offset of the component due to flattening + * @param ownerVxtptrs the map of offset to owner vxtptr types + * @return {@code true} if successful + * @throws InvalidDataTypeException upon error + */ + private static boolean processComponents(Structure type, StructureDataType newType, + int baseOffset, Map ownerVxtptrs) throws InvalidDataTypeException { + DataTypeComponent[] comps = type.getDefinedComponents(); + boolean mod = false; + for (DataTypeComponent comp : comps) { + int accumulatedOffset = baseOffset + comp.getOffset(); + if ((comp.getDataType() instanceof Structure struct && hasFlattenAttribute(comp))) { + processComponents(struct, newType, accumulatedOffset, ownerVxtptrs); + mod = true; + continue; + } + if (comp.getLength() == 0) { + continue; + } + if (comp instanceof BitFieldDataType bfComp) { + DataTypeComponent bfdtc = newType.insertBitFieldAt(accumulatedOffset, + bfComp.getBaseTypeSize(), bfComp.getBitOffset(), comp.getDataType(), + bfComp.getBitSize(), comp.getFieldName(), comp.getComment()); + if (bfdtc.getOffset() != accumulatedOffset) { + throw new InvalidDataTypeException(); + } + continue; + } + ClassUtils.NameAndPointer nap = + ClassUtils.getReplacementType(comp, accumulatedOffset, ownerVxtptrs); + String fieldName; + DataType fieldType; + if (nap == null) { + fieldName = comp.getFieldName(); + fieldType = comp.getDataType(); + } + else { + fieldName = nap.name(); + fieldType = nap.pointer(); + mod = true; + } + if (fieldName == null || fieldName.length() == 0) { + fieldName = comp.getDefaultFieldName(); + } + DataTypeComponent dtc = + newType.insertAtOffset(accumulatedOffset, fieldType, fieldType.getLength(), + fieldName, comp.getComment()); + if (dtc.getOffset() != accumulatedOffset) { + throw new InvalidDataTypeException(); + } + } + return mod; + } + + /** + * This method returns true if the argument structure has a class attribute + * @param structure the structure under question + * @return {@code true} if has a class attribute + */ + public static boolean hasClassAttribute(Structure structure) { + // Future: Check attribute on structure + String description = structure.getDescription(); + if (StringUtils.isEmpty(description)) { + return false; + } + return true; // true for now... later do the next line + //return description.contains("{{class}}"); + } + + /** + * We hope to have the ability to set and use a "flatten" attribute on the component of + * the structure + *

This method is temporary for investigations... should rely on a real component attribute + * @param component the member to check + * @return {@code true} if has flatten attribute + */ + private static boolean hasFlattenAttribute(DataTypeComponent component) { + // Future: Check ComponentMutationEnum/Mode + String comment = component.getComment(); + if (comment == null) { + return false; + } + if (comment.startsWith("Base") || comment.startsWith("Self Base") || + comment.startsWith("Virtual Base")) { + return true; + } + return comment.contains("{{flatten}}"); + } + + /** + * We hope to have the ability to set and use a "replace" attribute on the component of + * the structure + *

This method is temporary for investigations... should rely on a real component attribute + * @param component the member to check + * @return {@code true} if has replace attribute + */ + private static boolean hasReplaceAttribute(DataTypeComponent component) { + // Future: Check ComponentMutationEnum/Mode + DataType fieldType = component.getDataType(); + if (!(fieldType instanceof Pointer ptr) || ptr.getDataType() != null) { + return false; + } + String fieldName = component.getFieldName(); + if (ClassUtils.VFPTR.equals(fieldName) || ClassUtils.VBPTR.equals(fieldName)) { + return true; + } + String comment = component.getComment(); + if (comment == null) { + return false; + } + return comment.contains("{{replace}}"); + } + } diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PcodeDataTypeManager.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PcodeDataTypeManager.java index e026957f81..629c44ff5b 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PcodeDataTypeManager.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PcodeDataTypeManager.java @@ -28,6 +28,7 @@ import ghidra.program.database.data.PointerTypedefInspector; import ghidra.program.model.address.AddressSpace; import ghidra.program.model.data.*; import ghidra.program.model.data.Enum; +import ghidra.program.model.gclass.ClassUtils; import ghidra.program.model.lang.CompilerSpec; import ghidra.program.model.lang.DecompilerLanguage; import ghidra.program.model.listing.Program; @@ -38,7 +39,7 @@ import ghidra.xml.XmlParseException; /** * * Class for marshaling DataType objects to and from the Decompiler. - * + * */ public class PcodeDataTypeManager { @@ -119,6 +120,17 @@ public class PcodeDataTypeManager { private TypeMap byteMap; private int pointerWordSize; // Wordsize to assign to all pointer datatypes + // If we continue down this path, the following will probably get replaced with a tool option + // and might eventually be eliminated, such that we always do or never do type replacement + // in this manner. + private static final boolean TYPE_REPLACEMENT_ENABLED = + Boolean.getBoolean("ghidra.decompiler.typeReplacement"); + + /** + * Constructor + * @param prog the program for the p-code data type maanger + * @param simplifier the name transformer to be used + */ public PcodeDataTypeManager(Program prog, NameTransformer simplifier) { program = prog; @@ -134,14 +146,26 @@ public class PcodeDataTypeManager { pointerWordSize = ((SleighLanguage) prog.getLanguage()).getDefaultPointerWordSize(); } + /** + * Returns the program associated with this PcodeDataTypeManager + * @return the program + */ public Program getProgram() { return program; } + /** + * Returns the name transformer + * @return the name transformer + */ public NameTransformer getNameTransformer() { return nameTransformer; } + /** + * Sets the name transformer + * @param newTransformer the name transformer to set to this manager + */ public void setNameTransformer(NameTransformer newTransformer) { nameTransformer = newTransformer; } @@ -184,7 +208,7 @@ public class PcodeDataTypeManager { * Decode a data-type from the stream * @param decoder is the stream decoder * @return the decoded data-type object - * @throws DecoderException for invalid encodings + * @throws DecoderException for invalid encodings */ public DataType decodeDataType(Decoder decoder) throws DecoderException { int el = decoder.openElement(); @@ -484,6 +508,7 @@ public class PcodeDataTypeManager { encoder.writeString(ATTRIB_METATYPE, "struct"); encoder.writeSignedInteger(ATTRIB_SIZE, sz); encoder.writeSignedInteger(ATTRIB_ALIGNMENT, type.getAlignment()); + type = ClassUtils.getReplacementType(type, TYPE_REPLACEMENT_ENABLED); DataTypeComponent[] comps = type.getDefinedComponents(); for (DataTypeComponent comp : comps) { if (comp.getLength() == 0) { @@ -1082,7 +1107,7 @@ public class PcodeDataTypeManager { /** * Encode information for a data-type to the stream - * + * * @param encoder is the stream encoder * @param type is the data-type to encode * @param size is the size of the data-type @@ -1159,7 +1184,7 @@ public class PcodeDataTypeManager { /** * Build the list of core data-types. Data-types that are always available to the Decompiler * and are associated with a (metatype,size) pair. - * + * */ private void generateCoreTypes() { voidDt = new VoidDataType(progDataTypes);