diff --git a/Ghidra/Extensions/MachineLearning/Module.manifest b/Ghidra/Extensions/MachineLearning/Module.manifest new file mode 100644 index 0000000000..b819a8578c --- /dev/null +++ b/Ghidra/Extensions/MachineLearning/Module.manifest @@ -0,0 +1,10 @@ +MODULE FILE LICENSE: lib/olcut-config-protobuf-5.2.0.jar BSD-2-ORACLE +MODULE FILE LICENSE: lib/olcut-core-5.2.0.jar BSD-2-ORACLE +MODULE FILE LICENSE: lib/protobuf-java-3.17.3.jar BSD-3-GOOGLE +MODULE FILE LICENSE: lib/tribuo-classification-core-4.2.0.jar Apache License 2.0 +MODULE FILE LICENSE: lib/tribuo-classification-tree-4.2.0.jar Apache License 2.0 +MODULE FILE LICENSE: lib/tribuo-common-tree-4.2.0.jar Apache License 2.0 +MODULE FILE LICENSE: lib/tribuo-core-4.2.0.jar Apache License 2.0 +MODULE FILE LICENSE: lib/tribuo-data-4.2.0.jar Apache License 2.0 +MODULE FILE LICENSE: lib/tribuo-math-4.2.0.jar Apache License 2.0 +MODULE FILE LICENSE: lib/tribuo-util-onnx-4.2.0.jar Apache License 2.0 diff --git a/Ghidra/Extensions/MachineLearning/build.gradle b/Ghidra/Extensions/MachineLearning/build.gradle new file mode 100644 index 0000000000..0305c9117d --- /dev/null +++ b/Ghidra/Extensions/MachineLearning/build.gradle @@ -0,0 +1,46 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +apply from: "$rootProject.projectDir/gradle/distributableGhidraExtension.gradle" +apply from: "$rootProject.projectDir/gradle/javaProject.gradle" +apply from: "$rootProject.projectDir/gradle/javaTestProject.gradle" +apply from: "$rootProject.projectDir/gradle/helpProject.gradle" + +apply plugin: 'eclipse' +eclipse.project.name = 'Xtra MachineLearning' + +dependencies { + api project(':Base') + helpPath project(path: ":Base", configuration: 'helpPath') + + api "com.oracle.labs.olcut:olcut-config-protobuf:5.2.0" //{exclude group: "com.google.protobuf", module: "protobuf-java"} + api ("com.oracle.labs.olcut:olcut-core:5.2.0") {exclude group: "org.jline"} + api "com.google.protobuf:protobuf-java:3.17.3" //only needed for running junits + api "org.tribuo:tribuo-classification-core:4.2.0" + api "org.tribuo:tribuo-classification-tree:4.2.0" + api "org.tribuo:tribuo-common-tree:4.2.0" + api 'org.tribuo:tribuo-core:4.2.0' + api ("org.tribuo:tribuo-data:4.2.0") {exclude group: "com.opencsv"} + api "org.tribuo:tribuo-math:4.2.0" + api ("org.tribuo:tribuo-util-onnx:4.2.0") //{exclude group: "com.google.protobuf", module: "protobuf-java"} + + testImplementation project(path: ':SoftwareModeling', configuration: 'testArtifacts') + +} + + + + + diff --git a/Ghidra/Extensions/MachineLearning/certification.manifest b/Ghidra/Extensions/MachineLearning/certification.manifest new file mode 100644 index 0000000000..a5eedd190b --- /dev/null +++ b/Ghidra/Extensions/MachineLearning/certification.manifest @@ -0,0 +1,10 @@ +##VERSION: 2.0 +##MODULE IP: Apache License 2.0 +##MODULE IP: BSD-2-ORACLE +##MODULE IP: BSD-3-GOOGLE +Module.manifest||GHIDRA||||END| +extension.properties||GHIDRA||||END| +lib/README.txt||GHIDRA||||END| +src/main/help/help/TOC_Source.xml||GHIDRA||||END| +src/main/help/help/topics/RandomForestFunctionFinderPlugin/RandomForestFunctionFinderPlugin.htm||GHIDRA||||END| +src/main/resources/images/README.txt||GHIDRA||||END| diff --git a/Ghidra/Extensions/MachineLearning/developer_scripts/DumpCalls.java b/Ghidra/Extensions/MachineLearning/developer_scripts/DumpCalls.java new file mode 100644 index 0000000000..a589914918 --- /dev/null +++ b/Ghidra/Extensions/MachineLearning/developer_scripts/DumpCalls.java @@ -0,0 +1,58 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +//Writes a list of the addresses of all call sites to a file. +//@category machineLearning + +import java.io.*; + +import ghidra.app.script.GhidraScript; +import ghidra.program.model.listing.*; +import ghidra.program.model.pcode.PcodeOp; + +public class DumpCalls extends GhidraScript { + + private static final String DATA_DIR = "/local/calls"; + + @Override + protected void run() throws Exception { + File outFile = new File(DATA_DIR + File.separator + currentProgram.getName() + "_calls"); + FileWriter fWriter = new FileWriter(outFile); + BufferedWriter bWriter = new BufferedWriter(fWriter); + InstructionIterator fIter = currentProgram.getListing().getInstructions(true); + int numCalls = 0; + int numInstructions = 0; + while (fIter.hasNext()) { + Instruction inst = fIter.next(); + if (inst.getPcode() == null || inst.getPcode().length == 0) { + continue; + } + numInstructions++; + for (int i = 0; i < inst.getPcode().length; i++) { + PcodeOp pCode = inst.getPcode()[i]; + int opCode = pCode.getOpcode(); + if (opCode == PcodeOp.CALL || opCode == PcodeOp.CALLIND) { + //printf("Inst: %s at %s\n", inst.toString(), inst.getAddress()); + numCalls++; + bWriter.write(inst.getAddress().toString() + "\n"); + } + } + } + printf("total num calls: %d\n", numCalls); + printf("total num instructions: %d\n", numInstructions); + bWriter.close(); + } + +} diff --git a/Ghidra/Extensions/MachineLearning/developer_scripts/DumpFunctionStarts.java b/Ghidra/Extensions/MachineLearning/developer_scripts/DumpFunctionStarts.java new file mode 100644 index 0000000000..db733e6beb --- /dev/null +++ b/Ghidra/Extensions/MachineLearning/developer_scripts/DumpFunctionStarts.java @@ -0,0 +1,47 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +//Writes a list of the addresses of all function starts and their sizes to a file +//@category machineLearning + +import java.io.*; + +import ghidra.app.script.GhidraScript; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionIterator; + +public class DumpFunctionStarts extends GhidraScript { + + private static final String DATA_DIR = "/local/funcstarts/stripped"; + + @Override + protected void run() throws Exception { + File outFile = + new File(DATA_DIR + File.separator + currentProgram.getName() + "_stripped_funcs"); + FileWriter fWriter = new FileWriter(outFile); + BufferedWriter bWriter = new BufferedWriter(fWriter); + FunctionIterator fIter = currentProgram.getFunctionManager().getFunctions(true); + while (fIter.hasNext()) { + Function func = fIter.next(); + if (func.isExternal()) { + continue; + } + long size = func.getBody().getNumAddresses(); + bWriter.write(func.getEntryPoint().toString() + "," + size + "\n"); + } + bWriter.close(); + } + +} diff --git a/Ghidra/Extensions/MachineLearning/developer_scripts/ExampleTribuoRunner.java b/Ghidra/Extensions/MachineLearning/developer_scripts/ExampleTribuoRunner.java new file mode 100644 index 0000000000..e2bca66cd6 --- /dev/null +++ b/Ghidra/Extensions/MachineLearning/developer_scripts/ExampleTribuoRunner.java @@ -0,0 +1,74 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.nio.file.Paths; + +import org.tribuo.*; +import org.tribuo.classification.Label; +import org.tribuo.classification.LabelFactory; +import org.tribuo.classification.dtree.CARTClassificationTrainer; +import org.tribuo.classification.ensemble.VotingCombiner; +import org.tribuo.classification.evaluation.LabelEvaluation; +import org.tribuo.classification.evaluation.LabelEvaluator; +import org.tribuo.common.tree.RandomForestTrainer; +import org.tribuo.data.csv.CSVLoader; +import org.tribuo.ensemble.EnsembleModel; +import org.tribuo.evaluation.TrainTestSplitter; + +public class ExampleTribuoRunner { + + public static void main(String args[]) throws IOException { + + var irisHeaders = + new String[] { "sepalLength", "sepalWidth", "petalLength", "petalWidth", "species" }; + DataSource