Merge remote-tracking branch 'origin/patch'

Ghidra/Extensions/MachineLearning/src/main/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTask.java

@@ -277,15 +277,18 @@ public class RandomForestTrainingTask extends Task {
 		monitor.setMessage(
 			"Selecting " + numEntries * factor + " random addresses within function interiors");
 		start = System.nanoTime();
-		AddressSetView randomFuncInteriors =
-			RandomSubsetUtils.randomSubset(selectableInteriors, numEntries * factor, monitor);
+		long numInteriors = numEntries * factor;
+
+		AddressSetView randomFuncInteriors = numInteriors < selectableInteriors.getNumAddresses()
+				? RandomSubsetUtils.randomSubset(selectableInteriors, numInteriors, monitor)
+				: selectableInteriors;
 		end = System.nanoTime();
 		Msg.info(this, String.format("factor: %d elapsed selecting random interiors: %g seconds",
 			factor, (end - start) / NANOSECONDS_PER_SECOND));
 		trainingNegative = trainingNegative.union(randomFuncInteriors);
 		if (trainingNegative.isEmpty()) {
 			Msg.showError(this, null, "Data Gathering Error",
-				"No function interiors in training set");
+				"No non-starts in training set for sampling factor " + factor);
 			return null;
 		}
 		if (trainingPositive.intersects(trainingNegative)) {

Ghidra/Extensions/MachineLearning/src/test.slow/java/ghidra/machinelearning/functionfinding/RandomForestTrainingTaskTest.java

@@ -358,4 +358,33 @@ public class RandomForestTrainingTaskTest extends AbstractProgramBasedTest {
 		assertTrue(data.getTestNegative().contains(definedData));
 	}
 
+	@Test
+	public void testExhaustingFunctionInteriors() throws CancelledException {
+		params = new FunctionStartRFParams(program);
+		params.setMaxStarts(5);
+		int tooBig = 10;
+		Address begin = program.getSymbolTable().getSymbols("entry").next().getAddress();
+		AddressSet entries = new AddressSet();
+		for (int i = 0; i < 10; ++i) {
+			entries.add(begin.add(i));
+		}
+		AddressSet interiors = new AddressSet();
+		for (int i = 10; i < 25; ++i) {
+			interiors.add(begin.add(i));
+		}
+		AddressSet definedData = new AddressSet();
+		for (int i = 25; i < 30; ++i) {
+			definedData.add(begin.add(i));
+		}
+		RandomForestTrainingTask task = new RandomForestTrainingTask(program, params, null,
+			RandomForestFunctionFinderPlugin.TEST_SET_MAX_SIZE_DEFAULT);
+		TrainingAndTestData data =
+			task.getTrainingAndTestData(entries, interiors, definedData, tooBig, TaskMonitor.DUMMY);
+		assertTrue(data.getTrainingPositive().getNumAddresses() == 5);
+		assertTrue(data.getTestPositive().getNumAddresses() == 5);
+		assertTrue(data.getTestPositive().union(data.getTrainingPositive()).equals(entries));
+		assertTrue(data.getTrainingNegative().equals(interiors));
+		assertTrue(data.getTestNegative().equals(definedData));
+	}
+
 }

Ghidra/Processors/8051/certification.manifest

@@ -1,19 +1,19 @@
 ##VERSION: 2.0
 Module.manifest||GHIDRA||||END|
 data/languages/80251.cspec||GHIDRA||||END|
-data/languages/80251.pspec||GHIDRA||reviewed||END|
+data/languages/80251.pspec||GHIDRA||||END|
 data/languages/80251.sinc||GHIDRA||||END|
 data/languages/80251.slaspec||GHIDRA||||END|
 data/languages/80390.cspec||GHIDRA||||END|
 data/languages/80390.slaspec||GHIDRA||||END|
 data/languages/8051.cspec||GHIDRA||||END|
 data/languages/8051.ldefs||GHIDRA||||END|
-data/languages/8051.pspec||GHIDRA||reviewed||END|
+data/languages/8051.pspec||GHIDRA||||END|
 data/languages/8051.slaspec||GHIDRA||||END|
 data/languages/8051_archimedes.cspec||GHIDRA||||END|
 data/languages/8051_main.sinc||GHIDRA||||END|
 data/languages/mx51.cspec||GHIDRA||||END|
-data/languages/mx51.pspec||GHIDRA||reviewed||END|
+data/languages/mx51.pspec||GHIDRA||||END|
 data/languages/mx51.sinc||GHIDRA||||END|
 data/languages/mx51.slaspec||GHIDRA||||END|
 data/languages/old/8051v1.lang||GHIDRA||reviewed||END|

Ghidra/Processors/8051/data/languages/80251.pspec

@@ -6,6 +6,7 @@
   
   <volatile outputop="write_volatile" inputop="read_volatile">
     <range space="SFR" first="0x0" last="0x1FF"/>
+    <range space="BITS" first="0x400" last="0x800"/>
   </volatile>
   
   <default_symbols>

Ghidra/Processors/8051/data/languages/8051.pspec

@@ -6,6 +6,7 @@
   
   <volatile outputop="write_volatile" inputop="read_volatile">
     <range space="SFR" first="0x0" last="0xFF"/>
+    <range space="BITS" first="0x80" last="0xFF"/>
   </volatile>
   
   <default_symbols>

Ghidra/Processors/8051/data/languages/mx51.pspec

@@ -6,6 +6,7 @@
   
   <volatile outputop="write_volatile" inputop="read_volatile">
     <range space="SFR" first="0x0" last="0xFF"/>
+    <range space="BITS" first="0x80" last="0xFF"/>
   </volatile>
   
   <default_symbols>
@@ -47,10 +48,10 @@
     <symbol name="BANK3_R7" address="RAM:7f001f"/>
   
     <symbol name="P0" address="SFR:80"/>
-    		<symbol name="SP" address="SFR:81"/>
-		    <symbol name="DPL" address="SFR:82"/>
-		    <symbol name="DPH" address="SFR:83"/>
-		    <symbol name="DPXL" address="SFR:84"/>
+    <symbol name="SP" address="SFR:81"/>
+    <symbol name="DPL" address="SFR:82"/>
+    <symbol name="DPH" address="SFR:83"/>
+    <symbol name="DPXL" address="SFR:84"/>
     <symbol name="PCON" address="SFR:87"/>
     <symbol name="TCON" address="SFR:88"/>
     <symbol name="TMOD" address="SFR:89"/>
@@ -274,10 +275,10 @@
     <symbol name="P0.1" address="BITS:81"/>
     <symbol name="P0.2" address="BITS:82"/>
     <symbol name="P0.3" address="BITS:83"/>
-    <symbol name="P0.4" address="BITS:84"/>
-    <symbol name="P0.5" address="BITS:85"/>
-    <symbol name="P0.6" address="BITS:86"/>
-    <symbol name="P0.7" address="BITS:87"/>
+    <symbol name="P0.4" address="BITS:84/>
+    <symbol name="P0.5" address="BITS:85/>
+    <symbol name="P0.6" address="BITS:86/>
+    <symbol name="P0.7" address="BITS:87/>
     <symbol name="IT0" address="BITS:88"/>
     <symbol name="IE0" address="BITS:89"/>
     <symbol name="IT1" address="BITS:8a"/>

Ghidra/Processors/RISCV/data/languages/riscv.reg.sinc

@@ -709,7 +709,10 @@ define token instr (32)
   op2030=(20,30)
   op2031=(20,31)
   sop2031=(20,31) signed
+  op2121=(21,21)
   op2130=(21,30)
+  op2222=(22,22)
+  op2323=(23,23)
   op2324=(23,24)
   op2424=(24,24)
   op2427=(24,27)
@@ -753,8 +756,10 @@ define token cinstr (16)
   cr0206=(2,6)
   cfr0206=(2,6)
   cop0212=(2,12)
+  cop0303=(3,3)
   cop0304=(3,4)
   cop0305=(3,5)
+  cop0404=(4,4)
   cop0406=(4,6)
   cop0505=(5,5)
   cop0506=(5,6)
@@ -773,6 +778,7 @@ define token cinstr (16)
   cfr0711=(7,11)
   cop0712=(7,12)
   cop0808=(8,8)
+  cop0909=(9,9)
   cop0910=(9,10)
   cop0912=(9,12)
   cop1010=(10,10)

Ghidra/Processors/RISCV/data/languages/riscv.rv32i.sinc

@@ -123,11 +123,18 @@
 }
 
 # jal d,a 0000006f 0000007f JSR (0, 0) 
+# call if RA set, destreg == RA
+:jal ra,immUJ is immUJ & ra & r0711=1 & op0001=0x3 & op0204=0x3 & op0506=0x3
+{
+	ra = inst_next;
+	call immUJ;
+}
+
+# goto if RA not set, destreg != RA
 :jal rd,immUJ is immUJ & rd & op0001=0x3 & op0204=0x3 & op0506=0x3
 {
-	#TODO  RAS: push inst_next if rd=x1/x5
 	rd = inst_next;
-	call immUJ;
+	goto immUJ;
 }
 
 # j a 0000006f 00000fff BRANCH|ALIAS (0, 0)
@@ -136,14 +143,21 @@
 	goto immUJ;
 }
 
-
 # jalr d,s,j 00000067 0000707f JSR (0, 0)
+# call if RA set, destreg == RA
+:jalr ra,rs1,immI is rs1 & immI & ra & r0711=1 & op0001=0x3 & op0204=0x1 & op0506=0x3 & funct3=0x0
+{
+	local ea:$(XLEN) = (rs1 + immI) & ~1;
+	ra = inst_next;
+	call [ea];
+}
+
+# goto if RA not set, destreg != RA
 :jalr rd,rs1,immI is rs1 & immI & rd & op0001=0x3 & op0204=0x1 & op0506=0x3 & funct3=0x0
 {
-	#TODO  RAS: push inst_next/pop based on rd=x1/x5 or rs1=x1/x5
 	local ea:$(XLEN) = (rs1 + immI) & ~1;
 	rd = inst_next;
-	call [ea];
+	goto [ea];
 }
 
 # jr o(s) 00000067 00007fff BRANCH|ALIAS (0, 0)

Ghidra/Processors/RISCV/data/languages/riscv.rvc.sinc

@@ -355,11 +355,13 @@
 	*[ram]:4 ea = crs2:4;
 }
 
-# c.unimp  00000000 0000ffff SIMPLE (0, 0) 
-:c.unimp  is cop0001=0x0 & cop1315=0x0 & cop0212=0x0
-{
-	trap();
-}
+# c.unimp  00000000 0000ffff SIMPLE (0, 0)
+# would be better not to decode as it is used as padding
+#
+# :c.unimp  is cop0001=0x0 & cop1315=0x0 & cop0212=0x0
+#{
+#	trap();
+#}
 
 # c.xor Cs,Ct 00008c21 0000fc63 SIMPLE (0, 0) 
 :c.xor cr0709s,cr0204s is cr0204s & cr0709s & cop0001=0x1 & cop1315=0x4 & cop0506=0x1 & cop1012=0x3

Ghidra/Processors/RISCV/data/languages/riscv.table.sinc

@@ -203,21 +203,25 @@ cbimm: reloc is scop1212 & cop1011 & cop0506 & cop0304 & cop0202 [ reloc = inst_
 # used for goto
 cjimm: reloc is scop1212 & cop1111 & cop0910 & cop0808 & cop0707 & cop0606 & cop0305 & cop0202 [ reloc = inst_start + ((scop1212 << 11) | (cop1111 << 4) | (cop0910 << 8) | (cop0808 << 10) | (cop0707 << 6) | (cop0606 << 7) | (cop0305 << 1) | (cop0202 << 5)); ] { export *[ram]:$(XLEN) reloc; }
 
+
+nzuimm5: is cop0606=1 | cop0505=1 | cop0404=1 | cop0303 = 1 | cop0202=1 {}
+nzuimm6: is cop1212=1 | cop0606=1 | cop0505=1 | cop0404=1 | cop0303 = 1 | cop0202=1 {}
+
 @if ADDRSIZE == "32"
-#TODO  nonzero, would like to have cop0205>0
-c6imm: uimm is cop1212=0 & cop0206 [ uimm = (cop0206 + 0); ] { local tmp:$(XLEN) = uimm; export tmp; }
+c6imm: uimm is cop1212=0 & cop0206 & nzuimm5 [ uimm = (cop0206 + 0); ] { local tmp:$(XLEN) = uimm; export tmp; }
 @elif ADDRSIZE == "64"
-#TODO  nonzero, would like to have cop0205>0 | cop0206>0
-c6imm: uimm is cop1212 & cop0206 [ uimm = (cop1212 << 5) | (cop0206); ] { local tmp:$(XLEN) = uimm; export tmp; }
+c6imm: uimm is cop1212 & cop0206 & nzuimm6 [ uimm = (cop1212 << 5) | (cop0206); ] { local tmp:$(XLEN) = uimm; export tmp; }
 @elif ADDRSIZE == "128"
 c6imm: uimm is cop1212 & cop0206 [ uimm = (cop1212 << 5) | (cop0206); ] { local tmp:$(XLEN) = uimm + (64 * (uimm == 0)); export tmp; }
 @endif
 
-cbigimm: uimm is cop1212 & scop1212 & cop0206 [ uimm = (cop1212 << 5) | (cop0206); ] { local tmp:$(XLEN) = (scop1212 << 17) | (cop0206 << 12); export tmp; }
+cbigimm: uimm is cop1212 & scop1212 & cop0206 & nzuimm6 [ uimm = (cop1212 << 5) | (cop0206); ] { local tmp:$(XLEN) = (scop1212 << 17) | (cop0206 << 12); export tmp; }
 
-caddi4spnimm: uimm is cop1112 & cop0710 & cop0606 & cop0505 [ uimm = (cop0710 << 6) | (cop1112 << 4) | (cop0505 << 3) | (cop0606 << 2); ] { local tmp:$(XLEN) = uimm; export tmp; }
+nzcaddi4: is cop1212=1 | cop1111=1 | cop1010=1 | cop0909=1 | cop0808=1 | cop0707=1 | cop0606=1 | cop0505=1 {}
 
-caddi16spimm: imm is scop1212 & cop0606 & cop0505 & cop0304 & cop0202 [ imm = (scop1212 << 9) | (cop0304 << 7) | (cop0505 << 6) | (cop0202 << 5) | (cop0606 << 4); ] { local tmp:$(XLEN) = imm; export tmp; }
+caddi4spnimm: uimm is nzcaddi4 & cop1112 & cop0710 & cop0606 & cop0505 [ uimm = (cop0710 << 6) | (cop1112 << 4) | (cop0505 << 3) | (cop0606 << 2); ] { local tmp:$(XLEN) = uimm; export tmp; }
+
+caddi16spimm: imm is scop1212 & cop0606 & cop0505 & cop0304 & cop0202 & nzuimm6 [ imm = (scop1212 << 9) | (cop0304 << 7) | (cop0505 << 6) | (cop0202 << 5) | (cop0606 << 4); ] { local tmp:$(XLEN) = imm; export tmp; }
 
 
 clwimm: uimm is cop1012 & cop0606 & cop0505 [ uimm = (cop1012 << 3) | (cop0606 << 2) | (cop0505 << 6); ] { local tmp:$(XLEN) = uimm; export tmp; }

Ghidra/Processors/RISCV/data/languages/riscv32.cspec

@@ -69,20 +69,20 @@
       </output>
       <unaffected>
         <register name="sp"/>
-	<register name="gp"/>
-	<register name="tp"/>
+        <register name="gp"/>
+        <register name="tp"/>
         <register name="s0"/>
-      	<register name="s1"/>
-	<register name="s2"/>
-	<register name="s3"/>
-	<register name="s4"/>
-	<register name="s5"/>
-	<register name="s6"/>
-	<register name="s7"/>
-	<register name="s8"/>
-	<register name="s9"/>
-	<register name="s10"/>
-	<register name="s11"/>
+        <register name="s1"/>
+        <register name="s2"/>
+        <register name="s3"/>
+        <register name="s4"/>
+        <register name="s5"/>
+        <register name="s6"/>
+        <register name="s7"/>
+        <register name="s8"/>
+        <register name="s9"/>
+        <register name="s10"/>
+        <register name="s11"/>
       </unaffected>
     </prototype>
   </default_proto>

Ghidra/Processors/RISCV/data/languages/riscv64.cspec

@@ -69,20 +69,20 @@
       </output>
       <unaffected>
         <register name="sp"/>
-	<register name="gp"/>
-	<register name="tp"/>
+        <register name="gp"/>
+        <register name="tp"/>
         <register name="s0"/>
-      	<register name="s1"/>
-	<register name="s2"/>
-	<register name="s3"/>
-	<register name="s4"/>
-	<register name="s5"/>
-	<register name="s6"/>
-	<register name="s7"/>
-	<register name="s8"/>
-	<register name="s9"/>
-	<register name="s10"/>
-	<register name="s11"/>
+        <register name="s1"/>
+        <register name="s2"/>
+        <register name="s3"/>
+        <register name="s4"/>
+        <register name="s5"/>
+        <register name="s6"/>
+        <register name="s7"/>
+        <register name="s8"/>
+        <register name="s9"/>
+        <register name="s10"/>
+        <register name="s11"/>
       </unaffected>
     </prototype>
   </default_proto>

Ghidra/Processors/RISCV/data/patterns/riscv_gc_patterns.xml

@@ -1,52 +1,45 @@
 <patternlist>
-  <patternpairs totalbits="32" postbits="0">
+
+  <patternpairs totalbits="27" postbits="11">
+    <!--  Higher confidence patterns, after a return and more defined bits -->
     <prepatterns>
-      <!--  76543210 54321098 32109876 10987654 -->
-      <data>10000010 10000000</data> <!-- ret -->
-      <data>......01 101.....</data> <!-- c.j imm -->
+      <data>10000010 10000000</data>                    <!-- ret -->
+      <data>10000010 10000000 00000000 00000000 </data> <!-- ret; padding -->
     </prepatterns>
     <postpatterns>
-      <!--  76543210 54321098 32109876 10987654 -->
-      <data>0.....01 01110001</data> <!-- c.addi16sp sp,-imm -->
-      <data>0.....01 00010001</data> <!-- c.addi sp,-imm -->
-      <funcstart/>
-    </postpatterns>
-  </patternpairs>
-    <patternpairs totalbits="48" postbits="0">
-    <prepatterns>
-      <!--  76543210 54321098 32109876 10987654 -->
-      <data>10000010 10000000</data> <!-- ret -->
-      <data>......01 101.....</data> <!-- c.j imm -->
-    </prepatterns>
-    <postpatterns>
-      <!--  76543210 54321098 32109876 10987654 -->
+      <data>0.....01 01110001</data>                   <!-- c.addi16sp sp,-imm -->
+      <data>0.....01 00010001</data>                   <!-- c.addi sp,-imm -->
       <data>00010011 00000001 ....0001 1.......</data> <!--  addi sp,sp,-imm -->
-      <data>.0010111 ........ ........ ........</data> <!--  auipc rd,imm-->
       <funcstart/>
     </postpatterns>
   </patternpairs>
-  <patternpairs totalbits="48" postbits="0">
+  
+  <patternpairs totalbits="16" postbits="11">
+    <!--  Medium confidence, more bits, but prepattern are jumps, not return -->
     <prepatterns>
-      <!--  76543210 54321098 32109876 10987654 -->
-      <data>01101111 ....0000 ........ ........</data> <!-- j imm -->
+      <data>01101111 ....0000 ........ ........</data>  <!-- j imm -->
+      <data>......01 101..... </data>                   <!-- c.j imm -->
+      <data>......01 101..... 00000000 00000000 </data> <!-- c.j imm; padding -->
     </prepatterns>
     <postpatterns>
-      <!--  76543210 54321098 32109876 10987654 -->
-      <data>0.....01 01110001</data> <!-- c.addi16sp sp,-imm -->
-      <data>0.....01 00010001</data> <!-- c.addi sp,-imm -->
-      <funcstart/>
-    </postpatterns>
-  </patternpairs>
-    <patternpairs totalbits="64" postbits="0">
-    <prepatterns>
-      <!--  76543210 54321098 32109876 10987654 -->
-      <data>01101111 ....0000 ........ ........</data> <!-- j imm -->
-    </prepatterns>
-    <postpatterns>
-      <!--  76543210 54321098 32109876 10987654 -->
+      <data>0.....01 01110001</data>                   <!-- c.addi16sp sp,-imm -->
+      <data>0.....01 00010001</data>                   <!-- c.addi sp,-imm -->
       <data>00010011 00000001 ....0001 1.......</data> <!--  addi sp,sp,-imm -->
-      <data>.0010111 ........ ........ ........</data> <!--  auipc rd,imm-->
-      <funcstart/>
+      <possiblefuncstart/>
     </postpatterns>
   </patternpairs>
-</patternlist>
+  
+  <patternpairs totalbits="20" postbits="7">
+    <!--  Lower confidence, too few bits on start pattern -->
+    <prepatterns>
+      <data>10000010 10000000</data>                    <!-- ret -->
+      <data>10000010 10000000 00000000 00000000 </data> <!-- ret; padding -->
+      <data>......01 101..... 00000000 00000000 </data> <!-- c.j imm; padding -->
+      <data>01101111 ....0000 ........ ........</data>  <!-- j imm -->
+    </prepatterns>
+    <postpatterns>
+      <data>.0010111 ........ ........ ........</data> <!--  auipc rd,imm-->
+      <possiblefuncstart/>
+    </postpatterns>
+  </patternpairs>
+</patternlist>