diff --git a/gradle/root/test.gradle b/gradle/root/test.gradle index d4a8e05237..214fd9a6bd 100644 --- a/gradle/root/test.gradle +++ b/gradle/root/test.gradle @@ -367,7 +367,7 @@ def createTestTask(Project subproject, String testType, String bucketName, int t group "test" testClassesDirs = files subproject.sourceSets["$testType"].output.classesDirs classpath = subproject.sourceSets["$testType"].runtimeClasspath - + maxParallelForks = numMaxParallelForks initTestJVM(t, testRootDirName) @@ -411,17 +411,13 @@ def createTestTask(Project subproject, String testType, String bucketName, int t *********************************************************************************/ configure(subprojects.findAll {parallelMode == true}) { subproject -> afterEvaluate { - // "subprojects { afterEvaluate { evaluate()" - // forces evaluation of subproject configuration. Needed for inheriting excludes - // from non-parallel counterpart. -// subproject.evaluate() if (!shouldSkipTestTaskCreation(subproject)) { logger.info("parallelCombinedTestReport: Creating 'test' tasks for " + subproject.name + " subproject.") - Map testMap = getTestsForSubProject(subproject.sourceSets.test.java) + Map testMap = getTestsForSubProject(subproject.sourceSets.test.java) - for (Map.Entry classMap : testMap.entrySet()) { + for (Map.Entry classMap : testMap.entrySet()) { String bucketName = classMap.getKey(); @@ -429,12 +425,10 @@ configure(subprojects.findAll {parallelMode == true}) { subproject -> int taskNameCounter = 1 // task suffix int numMaxParallelForks = 40 // unit tests are fast; 40 seems to be a reasonable number - Map tests = classMap.getValue(); - def sorted = tests.sort { a, b -> b.value <=> a.value }; - List classesList = new ArrayList(sorted.keySet()); + List tests = classMap.getValue(); - while (classesListPosition < classesList.size()) { - createTestTask(subproject, "test", bucketName, taskNameCounter, classesList, classesListPosition, numMaxParallelForks) + while (classesListPosition < tests.size()) { + createTestTask(subproject, "test", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks) classesListPosition+=numMaxParallelForks taskNameCounter+=1; // "test_1_appConfig", "test_2_appConfig, etc. } @@ -444,9 +438,9 @@ configure(subprojects.findAll {parallelMode == true}) { subproject -> if (!shouldSkipIntegrationTestTaskCreation(subproject)) { logger.info("parallelCombinedTestReport: Creating 'integrationTest' tasks for " + subproject.name + " subproject.") - Map testMap = getTestsForSubProject(subproject.sourceSets.integrationTest.java) + Map testMap = getTestsForSubProject(subproject.sourceSets.integrationTest.java) - for (Map.Entry classMap : testMap.entrySet()) { + for (Map.Entry classMap : testMap.entrySet()) { String bucketName = classMap.getKey(); @@ -458,12 +452,10 @@ configure(subprojects.findAll {parallelMode == true}) { subproject -> // 20 seems like a good balance of throughput vs resource usage for ghidratest server. int numMaxParallelForks = 20 - Map tests = classMap.getValue(); - def sorted = tests.sort { a, b -> b.value <=> a.value }; - List classesList = new ArrayList(sorted.keySet()); - - while (classesListPosition < classesList.size()) { - createTestTask(subproject, "integrationTest", bucketName, taskNameCounter, classesList, classesListPosition, numMaxParallelForks) + List tests = classMap.getValue(); + + while (classesListPosition < tests.size()) { + createTestTask(subproject, "integrationTest", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks) classesListPosition+=numMaxParallelForks taskNameCounter+=1; // "integrationTest_1_appConfig", "integrationTest_2_appConfig, etc. } diff --git a/gradle/support/app_config_breakout.txt b/gradle/support/app_config_breakout.txt index 1199881418..ad13021655 100644 --- a/gradle/support/app_config_breakout.txt +++ b/gradle/support/app_config_breakout.txt @@ -82,6 +82,7 @@ AbstractToolSavingTest AbstractVersionControlActionTest AbstractVTCorrelatorTest AbstractVTMarkupItemTest +CallTreePluginTest DiffTestAdapter DWARFTestBase AbstractSelfSimilarCorrelatorTest diff --git a/gradle/support/testUtils.gradle b/gradle/support/testUtils.gradle index 45dad3f85f..ff2009dd0d 100644 --- a/gradle/support/testUtils.gradle +++ b/gradle/support/testUtils.gradle @@ -4,9 +4,10 @@ import java.lang.reflect.Constructor; import java.lang.*; import java.io.*; -// This is a map of configuration type names (integration vs. non-integration vs. docking etc..) -// to tests (test name, duration) -ext.testReport = null; +ext.integrationConfigs = new ArrayList<>(); +ext.dockingConfigs = new ArrayList<>(); +ext.appConfigs = new ArrayList<>(); +ext.ghidraConfigs = new ArrayList<>(); /* * Checks if html test report for an individual test class has a valid name. @@ -15,190 +16,6 @@ boolean hasValidTestReportClassName(String name) { return name.endsWith("Test.html") && !name.contains("Suite") } -/* - * Returns duration for a test class report. - */ -long getDurationFromTestReportClass(String fileContents, String fileName) { - /* The duration for the entire test class appears in the test report as (multiline): - *
- *
0s
- * The duration value may appear in the format of: 1m2s, 1m2.3s, 3.4s - */ - Pattern p = Pattern.compile("(?<=id=\"duration\">[\r\n])(.*?)(?= 0 - - long durationInMillis - - // Parse out the duration - if (duration.contains("m") && duration.contains("s")) { // has minute and seconds - int minutes = Integer.parseInt(duration.substring(0, duration.indexOf("m"))) - double seconds = Double.parseDouble(duration.substring(duration.indexOf("m") + 1 - , duration.length()-1)) - durationInMillis = (minutes * 60 * 1000) + (seconds * 1000) - } else if (!duration.contains("m") && duration.contains("s")) { // has only seconds - double seconds = Double.parseDouble(duration.substring(0, duration.length()-1)) - durationInMillis = (seconds * 1000) - } else { // unknown format - assert false : "getDurationFromTestReportClass: Unknown duration format in $fileName. 'duration' value is $duration" - } - logger.debug("getDurationFromTestReportClass: durationInMillis = '"+ durationInMillis - +"' parsed from duration = '" + duration + "' in $fileName") - return durationInMillis -} - -/* - * Creates a map of tests to their durations, organized by the type of - * application configuration they require. - * - * When creating groups of tests to run, we have to ensure that we not only - * group them by duration (to make the parallelization more efficient) but also - * by the type of application config they require (to avoid a catastrophic test - * failure). - * - * This timing information is gleaned by parsing the html results of a previous - * test run. - * - * The application config information is contained in the resource file - * app_config_breakout.txt. - * - * eg: GhidraAppConfiguration -> DiffTestTypeAdapter, 0.135s - */ -def Map> getTestReport() { - - // If we have already created the test report, do not waste time creating - // it again. Just return it. - if (project.testReport != null) { - return project.testReport; - } - - logger.debug("getTestReport: Populating 'testReport' using '$testTimeParserInputDir'") - - testReport = new HashMap(); - - List integrationConfigs = new ArrayList<>(); - List dockingConfigs = new ArrayList<>(); - List appConfigs = new ArrayList<>(); - List ghidraConfigs = new ArrayList<>(); - parseApplicationConfigs(dockingConfigs, integrationConfigs, appConfigs, ghidraConfigs); - - File classesReportDir = new File(testTimeParserInputDir) - if(!classesReportDir.exists()) { - logger.info("getTestReport: The path '$testTimeParserInputDir' does not exist on the file system." + - " Returning empty testReport map.") - return Collections.emptyMap(); - } - - // These are the configuration 'buckets' that each test class will be dumped - // into. Only tests in the same bucket will be run together. - Map dockingBucket = new HashMap(); - Map integrationBucket = new HashMap(); - Map appBucket = new HashMap(); - Map ghidraBucket = new HashMap(); - Map unknownBucket = new HashMap(); - - int excludedHtmlFiles = 0 // counter - int totalHtmlFiles = 0 - String excludedHtmlFileNames = "" // for log.info summary message - - classesReportDir.eachFileRecurse (FileType.FILES) { file -> - - totalHtmlFiles++ - // Only read html file for a Test and not a test Suite - if(hasValidTestReportClassName(file.name)) { - String fileContents = file.text - /* The fully qualified class name appears in the test report as: - *

Class ghidra.app.plugin.assembler.sleigh.BuilderTest

- */ - String fqNameFromTestReport = fileContents.find("(?<=

Class\\s).*?(?=

)") - int nameIndex = fqNameFromTestReport.lastIndexOf('.') - String shortName = fqNameFromTestReport.substring(nameIndex+1); - long durationInMillis = getDurationFromTestReportClass(fileContents, file.name) - - - File rootDir = project.rootDir.getParentFile(); - File foundFile; - fileTree(rootDir.getAbsolutePath()).visit { FileVisitDetails details -> - if (details.getName().contains(shortName + ".java")) { - foundFile = details.getFile(); - } - } - - if (!foundFile.exists()) { - // throw error - } - - String javaFileContents = foundFile.text; - - if (javaFileContents.contains(shortName)) { - - // Match the word right after "extends", if there is one - Pattern p = Pattern.compile("extends\\W+(\\w+)"); - Matcher m = p.matcher(javaFileContents); - String extendsClass = ""; - while (m.find()) { - extendsClass = m.group(1); - break; - } - - if (extendsClass.isEmpty()) { - unknownBucket.put(fqNameFromTestReport, durationInMillis); - } - else { - if (integrationConfigs.contains(extendsClass)) { - integrationBucket.put(fqNameFromTestReport, durationInMillis); - } - else if (dockingConfigs.contains(extendsClass)) { - dockingBucket.put(fqNameFromTestReport, durationInMillis); - } - else if (appConfigs.contains(extendsClass)) { - appBucket.put(fqNameFromTestReport, durationInMillis); - } - else if (ghidraConfigs.contains(extendsClass)) { - ghidraBucket.put(fqNameFromTestReport, durationInMillis); - } - else { - unknownBucket.put(fqNameFromTestReport, durationInMillis); - } - } - } - - testReport.put("integration", integrationBucket); - testReport.put("docking", dockingBucket); - testReport.put("app", appBucket); - testReport.put("ghidra", ghidraBucket); - testReport.put("unknown", unknownBucket); - - logger.debug("getTestReport: Added to testReport: class name = '" - + fqNameFromTestReport + "' and durationInMillis = '"+ durationInMillis - +"' from " + file.name) - } - else { - logger.debug("getTestReport: Excluding " + file.name + " from test report parsing.") - excludedHtmlFileNames += file.name + ", " - excludedHtmlFiles++ - } - } - - int processedFiles = 0; - for (Map.Entry entry : testReport.entrySet()) { - Map testMap = entry.getValue(); - processedFiles += testMap.size(); - } - assert totalHtmlFiles != 0 : "getTestReport: Did not parse any valid html files in $testTimeParserInputDir. Directory might be empty" - assert totalHtmlFiles == (processedFiles + excludedHtmlFiles) : "Not all html files processed." - logger.info("getTestReport:\n" + - "\tIncluded " + testReport.size() + " and excluded " + excludedHtmlFiles - + " html files out of " + totalHtmlFiles + " in Junit test report.\n" - + "\tExcluded html file names are: " + excludedHtmlFileNames + "\n" - + "\tParsed test report located at " + testTimeParserInputDir) - - return project.testReport -} - /** * Parses the file containing the mapping of test classes to application configs and assigns those * classes to the appropriate lists. @@ -299,12 +116,11 @@ String constructFullyQualifiedClassName(String fileContents, String fileName) { return packageName + "." + fileName.replace(".java","") } -/* Creates a list of test classes, sorted by duration, for a subproject. - * First parses JUnit test report located at 'testTimeParserInputDir' for . - * Then traverses a test sourceSet for a subproject for a test to include and assigns a duration value. - * Returns a sorted list of test classes for the sourceSet parameter. +/* + * Creates a map of config types to the test classes for that type. This should be + * used to creates sets of tests that can be run in parallel. */ -def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet) { +def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet) { def testsForSubProject = new HashMap(); @@ -317,9 +133,18 @@ def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet logger.debug("getTestsForSubProject: Found " + sourceDirectorySet.files.size() + " file(s) in source set to process.") - Map testReports = getTestReport(); - assert (testReports != null) : "getTestsForSubProject: testReport should not be null" - + // Read in the config file that indicates which base test classes are associated with + // which application configs. This is not a comprehensive list of all test classes + // in Ghidra - it's the list of all classes that are extended in Ghidra. + parseApplicationConfigs(dockingConfigs, integrationConfigs, appConfigs, ghidraConfigs); + + // "Buckets" that delineate which test classes should be run together. + List dockingBucket = new ArrayList(); + List integrationBucket = new ArrayList(); + List appBucket = new ArrayList(); + List ghidraBucket = new ArrayList(); + List unknownBucket = new ArrayList(); + for (File file : sourceDirectorySet.getFiles()) { logger.debug("getTestsForSubProject: Found file in sourceSet = " + file.name) @@ -331,7 +156,7 @@ def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet } String fileContents = file.text - + // Must not have a Category annotation if (hasCategoryExcludes(fileContents)) { logger.debug("getTestsForSubProject: Found category exclude for '" @@ -339,75 +164,68 @@ def Map getTestsForSubProject(SourceDirectorySet sourceDirectorySet excludedClassFilesCategory++ continue } - - String fqName = constructFullyQualifiedClassName( fileContents, file.name) - boolean foundTest = false; - for (Map.Entry entry : testReports.entrySet()) { - String configName = entry.getKey(); - Map tests = entry.getValue(); + // Get any extending class so we can see what bucket it belongs to. Do this + // by grabbing the next word after "extends". + Pattern p = Pattern.compile("extends\\W+(\\w+)"); + Matcher m = p.matcher(fileContents); + String extendsClass = ""; + while (m.find()) { + extendsClass = m.group(1); + break; + } + + // Get full package name of the class. + Pattern p2 = Pattern.compile("package\\s+([a-zA_Z_][\\.\\w]*);"); + Matcher m2 = p2.matcher(fileContents); + String packageName = ""; + while (m2.find()) { + packageName = m2.group(1); + break; + } + + // Construct a var of the form ".". This will + // be stored in the appropriate bucket and be used when creating test + // tasks later on. + String className = packageName + "." + file.name + className = className.replace(".java", "") - if (tests.containsKey(fqName)) { - foundTest = true; - if (!testsForSubProject.containsKey(configName)) { - Map configToTestMap = new LinkedHashMap<>(); - testsForSubProject.put(configName, configToTestMap); - } - - Map subTests = testsForSubProject.get(configName); - - long duration = tests.get(fqName); - - if (duration > 0) { - subTests.put(fqName,duration); - logger.debug("getTestsForSubProject: Adding '" + fqName + "'") - includedClassFilesInTestReport++ - } - else { - logger.debug("getTestsForSubProject: Excluding '" + fqName - + "' because duration from test report is " + duration - + "ms. Probably because all test methods are @Ignore'd." ) - excludedClassAllTestsIgnored++ - } + if (extendsClass.isEmpty()) { + unknownBucket.add(className); + } + else { + if (integrationConfigs.contains(extendsClass)) { + integrationBucket.add(className); + } + else if (dockingConfigs.contains(extendsClass)) { + dockingBucket.add(className); + } + else if (appConfigs.contains(extendsClass)) { + appBucket.add(className); + } + else if (ghidraConfigs.contains(extendsClass)) { + ghidraBucket.add(className); + } + else { + unknownBucket.add(className); } } - if (!foundTest) { - // Don't know what this test is so put it in the "unknown" bucket - if (!testsForSubProject.containsKey("unknown")) { - Map configToTestMap = new LinkedHashMap<>(); - testsForSubProject.put("unknown", configToTestMap); - } - - Map subTests = testsForSubProject.get("unknown"); - - logger.debug("getTestsForSubProject: Found test class not in test report." - + " Bumping to front of tasks '" + fqName + "'") - subTests.put(fqName, 3600000) // cheap way to bump to front of (eventually) sorted list - includedClassFilesNotInTestReport++ - } } - - // Sort by duration - for (Map.Entry entry : testsForSubProject.entrySet()) { - Map testMap = entry.getValue(); - testMap.sort { a, b -> b.value <=> a.value } - } - -/* logger.info ("getTestsForSubProject:\n" - + "\tIncluding " + includedClassFilesInTestReport + " test classes for this sourceSet because they are in the test report.\n" - + "\tIncluding/bumping " + includedClassFilesNotInTestReport + " not in test report.\n" - + "\tExcluding "+ excludedClassFilesBadName +" based on name not ending in 'Test' or contains 'Abstract' or 'Suite', " + excludedClassFilesCategory - + " based on '@Category, " + excludedClassAllTestsIgnored + " because duration = 0ms.\n" - + "\tReturning sorted list of size "+ sorted.size() + " out of " + sourceDirectorySet.files.size() - + " total files found in sourceSet.") -*/ - - int filesProcessed = includedClassFilesNotInTestReport + includedClassFilesInTestReport + - excludedClassFilesBadName + excludedClassFilesCategory + excludedClassAllTestsIgnored - - assert sourceDirectorySet.files.size() == filesProcessed : "getTestsForSubProject did not process every file in sourceSet" + + Map testBuckets = new HashMap(); + testBuckets.put("docking", dockingBucket) + testBuckets.put("integration", integrationBucket) + testBuckets.put("app", appBucket) + testBuckets.put("ghidra", ghidraBucket) + testBuckets.put("unknown", unknownBucket) + + logger.debug("integration bucket: " + integrationBucket) + logger.debug("docking bucket: " + dockingBucket) + logger.debug("app bucket: " + appBucket) + logger.debug("ghidra bucket: " + ghidraBucket) + logger.debug("unknown bucket: " + unknownBucket) - return testsForSubProject; + return testBuckets; } /*********************************************************************************