Merge remote-tracking branch 'origin/GP-5025_CParser_HashtagError' into

Ghidra_11.3 (Closes #7001)
This commit is contained in:
Ryan Kurtz
2025-01-27 06:04:36 -05:00
18 changed files with 1144 additions and 293 deletions
@@ -80,7 +80,6 @@ isa_availability.h
mbdata.h
msdos.h
mtdll.h
nlsdownlevel.h
nlsint.h
nmmintrin.h
oscalls.h
@@ -137,6 +136,9 @@ mmdeviceapi.h
mprapi.h
msctfmonitorapi.h
ndfapi.h
winsock2.h
ws2tcpip.h
iphlpapi.h
netioapi.h
npapi.h
nspapi.h
@@ -171,7 +173,6 @@ wpapi.h
wpcapi.h
wscapi.h
wsdapi.h
wspiapi.h
rpcproxy.h
-D_M_IX86=300
@@ -80,7 +80,6 @@ isa_availability.h
mbdata.h
msdos.h
mtdll.h
nlsdownlevel.h
nlsint.h
nmmintrin.h
oscalls.h
@@ -137,6 +136,9 @@ mmdeviceapi.h
mprapi.h
msctfmonitorapi.h
ndfapi.h
winsock2.h
ws2tcpip.h
iphlpapi.h
netioapi.h
npapi.h
nspapi.h
@@ -171,12 +173,11 @@ wpapi.h
wpcapi.h
wscapi.h
wsdapi.h
wspiapi.h
rpcproxy.h
-D_M_IX86=300
-D_MSC_VER=1200
-D_INTEGRAL_MAX_BITS=64
-DNTDDI_VERSION=0x09000000
-DWINVER=0x0900
-D_AMD64_
-D_M_AMD64
@@ -204,10 +205,12 @@ rpcproxy.h
-v0
-D__inner_checkReturn=""
/data/HeaderFiles/VC/VS12/include
/data/HeaderFiles/VC/VS12/src
/data/HeaderFiles/VC/VS12/include
/data/HeaderFiles/VC/SDK/v7.1A/Include
x86:LE:64:default
windows
@@ -115,8 +115,6 @@ xenroll.h
af_irda.h
in6addr.h
mstcpip.h
ws2def.h
winsock.h
winsock2.h
nsemail.h
nspapi.h
@@ -2,7 +2,7 @@ sys/types.h
types.h
stddef.h
stddef.h
openssl/opensslconf-x86_64.h
openssl/opensslconf.h
openssl/bn.h
openssl/ssl.h
openssl/asn1_mac.h
@@ -107,7 +107,6 @@ stdint.h
stdio.h
stdlib.h
string.h
tgmath.h
time.h
wchar.h
wctype.h
@@ -220,6 +219,7 @@ arpa/telnet.h
arpa/tftp.h
-D_X86_
-D__i386__
-D__STDC__
-D_GNU_SOURCE
-D__WORDSIZE=32
@@ -236,6 +236,7 @@ arpa/tftp.h
/data/HeaderFiles/linux/include
/data/HeaderFiles/linux/include/sys
/data/HeaderFiles/linux/gcc/include
/data/HeaderFiles/linux/include/openssl
/data/HeaderFiles/linux/x86_64-redhat-linux5E/include
/data/HeaderFiles/linux/x86_64-redhat-linux5E/include/sys
@@ -2,7 +2,7 @@ sys/types.h
types.h
stddef.h
stddef.h
openssl/opensslconf-x86_64.h
openssl/opensslconf.h
openssl/bn.h
openssl/ssl.h
openssl/asn1_mac.h
@@ -107,7 +107,6 @@ stdint.h
stdio.h
stdlib.h
string.h
tgmath.h
time.h
wchar.h
wctype.h
@@ -220,6 +219,7 @@ arpa/telnet.h
arpa/tftp.h
-D_X86_
-D__x86_64__
-D__STDC__
-D_GNU_SOURCE
-D__WORDSIZE=64
@@ -236,6 +236,7 @@ arpa/tftp.h
/data/HeaderFiles/linux/include
/data/HeaderFiles/linux/include/sys
/data/HeaderFiles/linux/gcc/include
/data/HeaderFiles/linux/include/openssl
/data/HeaderFiles/linux/x86_64-redhat-linux5E/include
/data/HeaderFiles/linux/x86_64-redhat-linux5E/include/sys
@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -22,10 +22,11 @@
import java.io.File;
import java.io.PrintWriter;
import java.util.Iterator;
import java.util.*;
import ghidra.app.script.GhidraScript;
import ghidra.program.model.data.*;
import ghidra.program.model.data.Enum;
import ghidra.program.model.data.StandAloneDataTypeManager.ArchiveWarning;
import ghidra.util.UniversalID;
@@ -162,7 +163,22 @@ public class CompareGDTs extends GhidraScript {
return dtmArchive.findDataTypeForID(universalID);
}
}
return dtmArchive.getDataType(dataType.getCategoryPath(), dataType.getName());
// find by exact path
DataType fdt = dtmArchive.getDataType(dataType.getCategoryPath(), dataType.getName());
if (fdt != null) {
return fdt;
}
// find by name
List<DataType> list = new ArrayList<DataType>();
dtmArchive.findDataTypes(dataType.getName(), list );
for (DataType dtc : list) {
if (dataType.getCategoryPath().getPath().toLowerCase().equals(dtc.getCategoryPath().getPath().toLowerCase())) {
return dtc;
}
}
return null;
}
private long outputWhereTypesDiffer(FileDataTypeManager dtmArchive1,
@@ -247,6 +263,10 @@ public class CompareGDTs extends GhidraScript {
Class<?> dtClass = dataType.getClass();
Class<?> sameNamedDtClass = matchingDataType.getClass();
if (dtClass == sameNamedDtClass) {
if (dataType instanceof Enum && (((Enum) dataType).getCount()==1)) {
// don't check single entry enums. Size will vary, and they are extracted defines
return checkEnum((Enum) dataType, (Enum) matchingDataType);
}
if (!dataType.isEquivalent(matchingDataType)) {
String message =
dataType.getPathName() + " (" + dtClass.getSimpleName() + ")";
@@ -258,6 +278,22 @@ public class CompareGDTs extends GhidraScript {
return false;
}
private boolean checkEnum(Enum e1, Enum e2) {
if (e1.getCount() != e2.getCount()) {
return true;
}
// Check that the name is the same
if (! e1.getNames()[0].equals(e2.getNames()[0])) {
return true;
}
// Check the value is the same
if (e1.getValues()[0] != e2.getValues()[0]) {
return true;
}
return false;
}
private boolean outputIfDifferentSizes(DataType dataType, FileDataTypeManager dtmArchive) {
if (!checkPointers && dataType instanceof Pointer) {
@@ -273,6 +309,10 @@ public class CompareGDTs extends GhidraScript {
Class<?> dtClass = dataType.getClass();
Class<?> sameNamedDtClass = matchingDataType.getClass();
if (dtClass == sameNamedDtClass) {
if (dataType instanceof Enum && (((Enum) dataType).getCount()==1)) {
// don't check single entry enums. Size will vary, and they are extracted defines
return checkEnum((Enum) dataType, (Enum) matchingDataType);
}
if (dataType.getLength() != matchingDataType.getLength()) {
String message = dataType.getPathName() + " (" + dtClass.getSimpleName() +
") " + dataType.getLength() + " != " + matchingDataType.getLength();
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,138 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//
// Parses UEFI header file gdt archives from
// git clone https://github.com/tianocore/edk2
//
// To replace existing header files and have the data type ID's synchronized
//
// Must run SynchronizeGDTCategoryPaths.java script with old and replacement GDT
// archive to synchronize upper/lower case paths
/// (only on windows archives)
//
// Then Run DataTypeArchiveTransformer in eclipse to synchronize old data types ID's
// if an existing .gdt file is being replaced
//
//@category Data Types
import java.io.File;
import java.io.IOException;
import ghidra.app.script.GhidraScript;
import ghidra.app.util.cparser.C.CParserUtils;
import ghidra.app.util.cparser.C.CParserUtils.CParseResults;
import ghidra.app.util.cparser.C.ParseException;
import ghidra.program.model.data.DataTypeManager;
import ghidra.program.model.data.FileDataTypeManager;
import ghidra.util.Msg;
public class CreateUEFIGDTArchivesScript extends GhidraScript {
private File outputDirectory;
private static String headerFilePath = "/data/HeaderFiles/git/edk2";
@Override
protected void run() throws Exception {
outputDirectory = askDirectory("Select Directory for GDT files", "Select GDT Output Dir");
parseUEFIHeaders("X64", "x86:LE:64:default", "windows");
parseUEFIHeaders("Ia32", "x86:LE:32:default", "windows");
parseUEFIHeaders("AArch64", "AARCH64:LE:64:v8A", "windows");
parseUEFIHeaders("Arm", "ARM:LE:32:v8", "default");
parseUEFIHeaders("RiscV64", "RISCV:LE:64:RV64G", "gcc");
parseUEFIHeaders("LoongArch64", "Loongarch:LE:64:lp64d", "default");
}
private void parseHeaderFilesToGDT(File outputDir, String gdtName, String languageID, String compiler,
String[] filenames, String includePaths[], String[] args)
throws ParseException, ghidra.app.util.cparser.CPP.ParseException, IOException {
DataTypeManager openTypes[] = null;
parseHeaderFilesToGDT(openTypes, outputDir, gdtName, languageID, compiler, filenames, includePaths, args);
}
private void parseHeaderFilesToGDT(DataTypeManager openTypes[], File outputDir, String gdtName, String languageID, String compiler,
String[] filenames, String[] includePaths, String[] args)
throws ParseException, ghidra.app.util.cparser.CPP.ParseException, IOException {
String dataTypeFile = outputDir + File.separator + gdtName + ".gdt";
File f = getArchiveFile(dataTypeFile);
FileDataTypeManager dtMgr = CParserUtils.parseHeaderFiles(openTypes, filenames,
includePaths, args, f.getAbsolutePath(), languageID, compiler, monitor);
dtMgr.save();
dtMgr.close();
}
/**
* Turn string into a file, delete old archive if it exists
*
* @param dataTypeFile
*
* @return file
*/
private File getArchiveFile(String dataTypeFile) {
File f = new File(dataTypeFile);
if (f.exists()) {
f.delete();
}
String lockFile = dataTypeFile + ".ulock";
File lf = new File(lockFile);
if (lf.exists()) {
lf.delete();
}
return f;
}
public void parseUEFIHeaders(String name, String languageID, String compiler) throws Exception {
String filenames[] = {
"ProcessorBind.h",
"Uefi/UefiBaseType.h",
"Uefi/UefiSpec.h",
"PiDxe.h",
"PiMm.h",
"PiPei.h",
"PiSmm.h",
"Library/DxeCoreEntryPoint.h",
"Library/PeiCoreEntryPoint.h",
"Library/PeimEntryPoint.h",
"Library/StandaloneMmDriverEntryPoint.h",
"Library/UefiApplicationEntryPoint.h",
"Library/UefiDriverEntryPoint.h",
headerFilePath+"/MdePkg/Include/Pi/",
headerFilePath+"/MdePkg/Include/Ppi/",
headerFilePath+"/MdePkg/Include/Protocol/",
headerFilePath+"/MdePkg/Include/IndustryStandard/",
};
String includePaths[] = {
headerFilePath+"/MdePkg/Include/"+name,
headerFilePath+"/MdePkg/Include",
};
String args[] = {
};
parseHeaderFilesToGDT(outputDirectory, "uefi_"+name, languageID, compiler, filenames, includePaths, args);
}
}
@@ -260,20 +260,22 @@ public class CParserPlugin extends ProgramPlugin {
}
protected String getFormattedParseMessage(String errMsg) {
String message = "";
String message = "<html>";
if (errMsg != null) {
message += errMsg + "\n\n";
message = HTMLUtilities.toHTML(message);
}
String msg = (results == null ? null : results.cParseMessages());
if (msg != null && msg.length() != 0) {
message += "CParser Messages:\n" + msg + "\n\n";
message += "<b>CParser Messages:</b><br>\n" + HTMLUtilities.toHTML(msg) + "<br>\n<br>\n";
}
msg = (results == null ? null : results.cppParseMessages());
if (msg != null && msg.length() != 0) {
message += "PreProcessor Messages:\n" + msg;
message += "<b>PreProcessor Messages:</b><br>\n" + HTMLUtilities.toHTML(msg);
}
return message;
@@ -45,7 +45,7 @@ public class CParserUtils {
message += errMsg + "\n\n";
}
String msg = cppParseMessages;
String msg = cParseMessages;
if (msg != null && msg.length() != 0) {
message += "CParser Messages:\n" + msg + "\n\n";
}
@@ -539,7 +539,7 @@ public class CParserUtils {
cppMessages = cpp.getParseMessages();
if (!parseSucceeded) {
return new CParseResults(cpp, "", cppMessages, false);
return new CParseResults(cpp, cppMessages, "", false);
}
// process all the defines and add any that are integer values into
@@ -567,8 +567,8 @@ public class CParserUtils {
parserMessages = cParser.getParseMessages();
}
}
return new CParseResults(cpp, parserMessages, cppMessages, cparseSucceeded);
return new CParseResults(cpp, cppMessages, parserMessages, cparseSucceeded);
}
private static String parseFile(String filename, TaskMonitor monitor, PreProcessor cpp)
@@ -388,6 +388,7 @@ public class DefineTable {
int replacedSubpieceLen = currKey.length();
if (argv == null && sublist.contains(currKey)) {
if (!initialList) {
// stop recursion of replacement
System.err.println("DONT Replace " + currKey + " in: " + buf);
}
return -1;
@@ -2029,13 +2029,13 @@ void StaticAssert() : {
Token t, message=null;
}
{
( ( t=<STATICASSERT> ) "(" expr=ConstantExpression() [ "," message=<STRING_LITERAL> ] ")" )
( ( t=<STATICASSERT> ) "(" expr=ConstantExpression() [ "," (message=<STRING_LITERAL> | ("#" <IDENTIFIER>) )+ ] ")" )
{
if (expr != null) {
Integer evalue = getConstantValue(expr,1);
if (evalue == 0) {
String smessage = (message == null ? "" : message.image);
addNearParseMessage("Static_Asssert has failed " + " \"" + smessage + "\"");
addNearParseMessage("Warning: Static_Assert possibly failed " + " \"" + smessage + "\"");
}
}
}
File diff suppressed because it is too large Load Diff
@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -246,9 +246,9 @@ public class CParserTest extends AbstractGhidraHeadlessIntegrationTest {
assertTrue("Duplicate ENUM message missing", parseMessages.contains("duplicate enum value: options_enum : PLUS_SET : 16"));
assertTrue("Duplicate ENUM message missing", parseMessages.contains("Static_Asssert has failed \"\"math fail!\"\""));
assertTrue("Static assert fail missing", parseMessages.contains("Static_Assert possibly failed \"\"math fail!\"\""));
assertTrue("Duplicate ENUM message missing", parseMessages.contains("Static_Asssert has failed \"\"1 + 1 == 3, fail!\"\""));
assertTrue("Static assert fail missing", parseMessages.contains("Static_Assert possibly failed \"\"1 + 1 == 3, fail!\"\""));
dt = dtMgr.getDataType(new CategoryPath("/"), "_IO_FILE_complete");
Structure sldt = (Structure) dt;
@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -91,7 +91,7 @@ public class PreProcessorTest extends AbstractGenericTest {
System.out.println(parser.getParseMessages());
// Uncomment to print out parse results
// System.err.println(baos.toString());
//System.err.println(baos.toString());
dtMgr = new StandAloneDataTypeManager("parsed");
parser.getDefinitions().populateDefineEquates(null, dtMgr);
@@ -129,13 +129,29 @@ public class PreProcessorTest extends AbstractGenericTest {
"\" fourth line\")") != -1);
assertTrue("multi line #pragma failed ", results
.indexOf("#pragma multiple lines pragma") != -1);
.indexOf("#pragma multiple lines pragma") != -1);
assertTrue("#pragma with comment failed ", results
.indexOf("#pragma no comment here") != -1);
.indexOf("#pragma no comment here") != -1);
assertTrue("#pragma with EOL comment failed ", results
.indexOf("#pragma with no EOL comment here") != -1);
.indexOf("#pragma with no EOL comment here") != -1);
}
@Test
public void testCommenting() throws Exception {
String results = baos.toString("ASCII");
assertTrue("IntShouldBeCommented", results
.indexOf("///- int IntShouldBeCommented;") != -1);
assertTrue("PragmaShouldBeCommented", results
.indexOf("///- #pragma PragmaShouldBeCommented") != -1);
assertTrue("IntShouldNotBeCommented", results
.indexOf("int IntShouldBeCommented;") != -1);
assertTrue("PragmaShouldNotBeCommented", results
.indexOf("#pragma PragmaShouldNotBeCommented") != -1);
}
@Test
@@ -1235,6 +1235,8 @@ int check_assert(void)
int x;
static_assert(sizeof(int) > sizeof(char));
_Static_assert(sizeof(char) <= 1, "message " #Name " is broken " #Up );
}
struct statcheck {
@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,6 +19,34 @@
#undef a
#endif
#ifdef __WCHAR_MAX__
# define __WCHAR_MAX __WCHAR_MAX__
#elif L'\0' - 1 > 0
# define __WCHAR_MAX (0xffffffffu + L'\0')
#else
# define __WCHAR_MAX (0x7fffffff + L'\0')
#endif
/* test for comment parsing */
# define AComment(a) B(a) /**/
# define BComment(a) C(a) /***/
# define CComment /****/
# ifndef FOO
#define DidFOO "true" /* test comment */
# endif
#ifdef NOFOO
int IntShouldBeCommented;
#pragma PragmaShouldBeCommented
#endif
#ifndef NOFOO
int IntShouldNotBeCommented;
#pragma PragmaShouldNotBeCommented;
#endif
/* definition coming from -D, should evaluate to true */
#if FROM_ARG_VALUE
@@ -171,10 +199,10 @@ int foo;
#define TWOFISH 2
#if (ONEFISH + TWOFISH + REDFISH + BLUEFISH) > 2
#error "Too many fish"
#define TOO_MANY_FISH 0
int TooManyFish;
#else
#error "Too few fish"
int NotEnoughFish;
#endif
@@ -442,6 +470,23 @@ ldp LDP((
_Pragma("clang diagnostic push") \
_Pragma("clang diagnostic ignored \"-Wmismatched-tags\"")
/**
** Multi line false ifdef with Quoted string
*/
#define __USE_GNU "1"
#ifdef __USE_GNU
extern int pthread_yield (void) __THROW;
# ifdef __REDIRECT_NTH
extern int __REDIRECT_NTH (pthread_yield, (void), sched_yield)
__attribute_deprecated_msg__ ("\
pthread_yield is deprecated, use sched_yield instead");
# else
# define pthread_yield sched_yield
# endif
#endif
/**
** Protected from macro expansion
**/
@@ -536,5 +581,32 @@ D = DUAL_MULTILINE(2, "Caution: First line"
" second line"
" third line"
" fourth line")
//
// check calculations
//
#define NUMBER1 0x10000000
#define NUMBER2 0x10000001
#define NUMBER3 0xF0000002
#if (NUMBER2 <= NUMBER1) || (NUMBER3 <= NUMBER2)
#error new number must be greater than the old one
#endif
#if (NUMBER1 & 0xE0000000)
#error NUMBER1 & 0xE0000000 should be false
#endif
#if (NUMBER2 & 0xE0000000)
#error NUMBER2 & 0xE0000000 should be false
#endif
#if !(NUMBER3 & 0xE0000000)
#error NUMBER3 & 0xE0000000 should be true
#endif
#if ((NUMBER2 & 0xE0000000) || (NUMBER2 & 0xE0000000)) || !(NUMBER3 & 0xE0000000)
#error ((NUMBER2 & 0xE0000000) || (NUMBER2 & 0xE0000000)) || !(NUMBER3 & 0xE0000000) should be false
#endif
theEnd();
@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,15 +17,15 @@
#define INCLUDE1
#pragma pack(push,1)
#else if !defined(INCLUDE2)
#elif !defined(INCLUDE2)
#define INCLUDE2
#pragma pack(push, 2)
#else if !defined(INCLUDE3)
#elif !defined(INCLUDE3)
#define INCLUDE3
#pragma pack(push, 4)
#else if !defined(INCLUDE4)
#elif !defined(INCLUDE4)
#define INCLUDE4
#pragma pack(push, 8)