Extend ART/D8 testing with a subsequent R8/debug step, also:
add CompilationMode support for TestConditions
Bug:
Change-Id: Ic8af625d6df971914a2f5d8452889d53b9ab1469
diff --git a/src/test/java/com/android/tools/r8/JctfTestSpecifications.java b/src/test/java/com/android/tools/r8/JctfTestSpecifications.java
index 49c0837..03eddc7 100644
--- a/src/test/java/com/android/tools/r8/JctfTestSpecifications.java
+++ b/src/test/java/com/android/tools/r8/JctfTestSpecifications.java
@@ -5,8 +5,8 @@
package com.android.tools.r8;
import static com.android.tools.r8.TestCondition.D8_COMPILER;
-import static com.android.tools.r8.TestCondition.R8_COMPILER;
import static com.android.tools.r8.TestCondition.R8DEBUG_AFTER_D8_COMPILER;
+import static com.android.tools.r8.TestCondition.R8_COMPILER;
import static com.android.tools.r8.TestCondition.any;
import static com.android.tools.r8.TestCondition.match;
import static com.android.tools.r8.TestCondition.runtimes;
@@ -4808,32 +4808,37 @@
.put("lang.RuntimePermission.Class.RuntimePermission_class_A13", any())
.build(); // end of timeoutsWithArt
- private static final boolean testMatch(Multimap<String, TestCondition> testConditions,
+ private static final boolean testMatch(
+ Multimap<String, TestCondition> testConditions,
String name,
- CompilerUnderTest compilerUnderTest, DexVm dexVm) {
+ CompilerUnderTest compilerUnderTest,
+ DexVm dexVm,
+ CompilationMode compilationMode) {
Collection<TestCondition> entries = testConditions.get(name);
for (TestCondition entry : entries) {
- if (entry.test(DexTool.NONE, compilerUnderTest, dexVm)) {
+ if (entry.test(DexTool.NONE, compilerUnderTest, dexVm, compilationMode)) {
return true;
}
}
return false;
}
- public static final Outcome getExpectedOutcome(String name,
+ public static final Outcome getExpectedOutcome(
+ String name,
CompilerUnderTest compilerUnderTest,
- DexVm dexVm) {
+ DexVm dexVm,
+ CompilationMode compilationMode) {
Outcome outcome = null;
- if (testMatch(failuresToTriage, name, compilerUnderTest, dexVm)) {
+ if (testMatch(failuresToTriage, name, compilerUnderTest, dexVm, compilationMode)) {
outcome = Outcome.FAILS_WITH_ART;
}
- if (testMatch(timeoutsWithArt, name, compilerUnderTest, dexVm)) {
+ if (testMatch(timeoutsWithArt, name, compilerUnderTest, dexVm, compilationMode)) {
assert outcome == null;
outcome = Outcome.TIMEOUTS_WITH_ART;
}
- if (testMatch(flakyWithArt, name, compilerUnderTest, dexVm)) {
+ if (testMatch(flakyWithArt, name, compilerUnderTest, dexVm, compilationMode)) {
assert outcome == null;
outcome = Outcome.FLAKY_WITH_ART;
}
diff --git a/src/test/java/com/android/tools/r8/R8RunArtTestsTest.java b/src/test/java/com/android/tools/r8/R8RunArtTestsTest.java
index 5c5975c..8f1dadd 100644
--- a/src/test/java/com/android/tools/r8/R8RunArtTestsTest.java
+++ b/src/test/java/com/android/tools/r8/R8RunArtTestsTest.java
@@ -459,7 +459,9 @@
// large array as still live and the subsequent allocations will fail to reach the desired
// size before an out-of-memory error occurs. See:
// tests/art/{dx,jack}/104-growth-limit/src/Main.java:40
- .put("104-growth-limit", TestCondition.match(TestCondition.R8_COMPILER))
+ .put(
+ "104-growth-limit",
+ TestCondition.match(TestCondition.R8_COMPILER, TestCondition.RELEASE_MODE))
.put(
"461-get-reference-vreg",
TestCondition.match(
@@ -790,28 +792,23 @@
}
}
- private static Map<SpecificationKey, TestSpecification> AVAILABLE_TESTS_MAP_R8 =
- data(CompilerUnderTest.R8);
-
- private static Map<SpecificationKey, TestSpecification> AVAILABLE_TESTS_MAP_D8 =
- data(CompilerUnderTest.D8);
-
-
- private static Set<String> collectTestsMatchingConditions(DexTool dexTool,
- CompilerUnderTest compilerUnderTest, DexVm dexVm,
+ private static Set<String> collectTestsMatchingConditions(
+ DexTool dexTool,
+ CompilerUnderTest compilerUnderTest,
+ DexVm dexVm,
+ CompilationMode mode,
Multimap<String, TestCondition> testConditionsMap) {
Set<String> set = Sets.newHashSet();
for (Map.Entry<String, TestCondition> kv : testConditionsMap.entries()) {
- if (kv.getValue().test(dexTool, compilerUnderTest, dexVm)) {
+ if (kv.getValue().test(dexTool, compilerUnderTest, dexVm, mode)) {
set.add(kv.getKey());
}
}
return set;
}
-
- private static Map<SpecificationKey, TestSpecification> data(
- CompilerUnderTest compilerUnderTest) {
+ private static Map<SpecificationKey, TestSpecification> get_tests_map(
+ CompilerUnderTest compilerUnderTest, CompilationMode compilationMode) {
File artTestDir = new File(ART_TESTS_DIR);
if (!artTestDir.exists()) {
// Don't run any tests if the directory does not exist.
@@ -834,8 +831,9 @@
DexVm dexVm = ToolHelper.getDexVm();
for (DexTool dexTool : DexTool.values()) {
// Collect the tests failing code generation.
- Set<String> failsWithCompiler = collectTestsMatchingConditions(dexTool, compilerUnderTest,
- dexVm, failingWithCompiler);
+ Set<String> failsWithCompiler =
+ collectTestsMatchingConditions(
+ dexTool, compilerUnderTest, dexVm, compilationMode, failingWithCompiler);
// Collect tests that has no input:
if (dexTool == DexTool.NONE) {
@@ -843,15 +841,18 @@
}
// Collect the test that we should skip in this configuration.
- skipArt.addAll(collectTestsMatchingConditions(dexTool, compilerUnderTest,
- dexVm, timeoutOrSkipRunWithArt));
+ skipArt.addAll(
+ collectTestsMatchingConditions(
+ dexTool, compilerUnderTest, dexVm, compilationMode, timeoutOrSkipRunWithArt));
// Collect the tests failing to run in Art (we still run R8/D8 on these).
- Set<String> failsWithArt = collectTestsMatchingConditions(dexTool, compilerUnderTest, dexVm,
- failingRunWithArt);
+ Set<String> failsWithArt =
+ collectTestsMatchingConditions(
+ dexTool, compilerUnderTest, dexVm, compilationMode, failingRunWithArt);
{
- Set<String> tmpSet = collectTestsMatchingConditions(dexTool, compilerUnderTest, dexVm,
- expectedToFailRunWithArt);
+ Set<String> tmpSet =
+ collectTestsMatchingConditions(
+ dexTool, compilerUnderTest, dexVm, compilationMode, expectedToFailRunWithArt);
failsWithArt.addAll(tmpSet);
}
@@ -863,14 +864,17 @@
}
// Collect the tests failing with output differences in Art.
- Set<String> failsRunWithArtOutput = collectTestsMatchingConditions(dexTool, compilerUnderTest,
- dexVm, failingRunWithArtOutput);
- Set<String> expectedToFailWithCompilerSet = collectTestsMatchingConditions(dexTool,
- compilerUnderTest, dexVm, expectedToFailWithCompiler);
+ Set<String> failsRunWithArtOutput =
+ collectTestsMatchingConditions(
+ dexTool, compilerUnderTest, dexVm, compilationMode, failingRunWithArtOutput);
+ Set<String> expectedToFailWithCompilerSet =
+ collectTestsMatchingConditions(
+ dexTool, compilerUnderTest, dexVm, compilationMode, expectedToFailWithCompiler);
// Collect the tests where the original works in Art and the R8/D8 generated output does not.
- Set<String> failsRunWithArtOriginalOnly = collectTestsMatchingConditions(dexTool,
- compilerUnderTest, dexVm, failingRunWithArtOriginalOnly);
+ Set<String> failsRunWithArtOriginalOnly =
+ collectTestsMatchingConditions(
+ dexTool, compilerUnderTest, dexVm, compilationMode, failingRunWithArtOriginalOnly);
File compilerTestDir = artTestDir.toPath().resolve(dexToolDirectory(dexTool)).toFile();
File[] testDirs = compilerTestDir.listFiles();
@@ -895,6 +899,22 @@
return data;
}
+ private static CompilationMode defaultCompilationMode(CompilerUnderTest compilerUnderTest) {
+ CompilationMode compilationMode = null;
+ switch (compilerUnderTest) {
+ case R8:
+ compilationMode = CompilationMode.RELEASE;
+ break;
+ case D8:
+ case R8DEBUG_AFTER_D8:
+ compilationMode = CompilationMode.DEBUG;
+ break;
+ default:
+ throw new RuntimeException("Unreachable.");
+ }
+ return compilationMode;
+ }
+
private static String dexToolDirectory(DexTool tool) {
// DexTool.NONE uses class files in the dx directory.
return tool == DexTool.JACK ? "jack" : "dx";
@@ -936,12 +956,6 @@
}
private void executeCompilerUnderTest(
- CompilerUnderTest compilerUnderTest, Collection<String> fileNames, String resultPath)
- throws IOException, ProguardRuleParserException, ExecutionException, CompilationException {
- executeCompilerUnderTest(compilerUnderTest, fileNames, resultPath, null, null);
- }
-
- private void executeCompilerUnderTest(
CompilerUnderTest compilerUnderTest,
Collection<String> fileNames,
String resultPath,
@@ -957,43 +971,48 @@
CompilationMode mode,
String keepRulesFile)
throws IOException, ProguardRuleParserException, ExecutionException, CompilationException {
+ assert mode != null;
switch (compilerUnderTest) {
- case D8: {
- assert keepRulesFile == null : "Keep-rules file specified for D8.";
- D8Command.Builder builder =
- D8Command.builder()
- .setMode(mode == null ? CompilationMode.DEBUG : mode)
- .addProgramFiles(ListUtils.map(fileNames, Paths::get));
- Integer minSdkVersion = needMinSdkVersion.get(name);
- if (minSdkVersion != null) {
- builder.setMinApiLevel(minSdkVersion);
- }
- D8Output output = D8.run(builder.build());
- output.write(Paths.get(resultPath));
- break;
- }
- case R8: {
- R8Command.Builder builder =
- R8Command.builder()
- .setMode(mode == null ? CompilationMode.RELEASE : mode)
- .setOutputPath(Paths.get(resultPath))
- .addProgramFiles(ListUtils.map(fileNames, Paths::get))
- .setIgnoreMissingClasses(true);
- Integer minSdkVersion = needMinSdkVersion.get(name);
- if (minSdkVersion != null) {
- builder.setMinApiLevel(minSdkVersion);
- }
- if (keepRulesFile != null) {
- builder.addProguardConfigurationFiles(Paths.get(keepRulesFile));
- }
- // Add internal flags for testing purposes.
- ToolHelper.runR8(builder.build(), options -> {
- if (enableInterfaceMethodDesugaring.contains(name)) {
- options.interfaceMethodDesugaring = OffOrAuto.Auto;
+ case D8:
+ {
+ assert keepRulesFile == null : "Keep-rules file specified for D8.";
+ D8Command.Builder builder =
+ D8Command.builder()
+ .setMode(mode)
+ .addProgramFiles(ListUtils.map(fileNames, Paths::get));
+ Integer minSdkVersion = needMinSdkVersion.get(name);
+ if (minSdkVersion != null) {
+ builder.setMinApiLevel(minSdkVersion);
}
- });
- break;
- }
+ D8Output output = D8.run(builder.build());
+ output.write(Paths.get(resultPath));
+ break;
+ }
+ case R8:
+ {
+ R8Command.Builder builder =
+ R8Command.builder()
+ .setMode(mode)
+ .setOutputPath(Paths.get(resultPath))
+ .addProgramFiles(ListUtils.map(fileNames, Paths::get))
+ .setIgnoreMissingClasses(true);
+ Integer minSdkVersion = needMinSdkVersion.get(name);
+ if (minSdkVersion != null) {
+ builder.setMinApiLevel(minSdkVersion);
+ }
+ if (keepRulesFile != null) {
+ builder.addProguardConfigurationFiles(Paths.get(keepRulesFile));
+ }
+ // Add internal flags for testing purposes.
+ ToolHelper.runR8(
+ builder.build(),
+ options -> {
+ if (enableInterfaceMethodDesugaring.contains(name)) {
+ options.interfaceMethodDesugaring = OffOrAuto.Auto;
+ }
+ });
+ break;
+ }
default:
assert false : compilerUnderTest;
}
@@ -1003,7 +1022,7 @@
return builder.setMinification(false);
}
- private static final boolean isAuxClassFile(String fileName, String auxClassFileBase) {
+ private static boolean isAuxClassFile(String fileName, String auxClassFileBase) {
return fileName.endsWith(".class")
&& (fileName.startsWith(auxClassFileBase + "$")
|| fileName.startsWith(auxClassFileBase + "_"));
@@ -1060,11 +1079,13 @@
compilerUnderTest == CompilerUnderTest.R8DEBUG_AFTER_D8
? CompilerUnderTest.D8
: compilerUnderTest;
+ CompilationMode compilationMode = defaultCompilationMode(compilerUnderTest);
File resultDir = temp.newFolder(firstCompilerUnderTest.toString().toLowerCase() + "-output");
- JctfTestSpecifications.Outcome expectedOutcome = JctfTestSpecifications
- .getExpectedOutcome(name, firstCompilerUnderTest, dexVm);
+ JctfTestSpecifications.Outcome expectedOutcome =
+ JctfTestSpecifications.getExpectedOutcome(
+ name, firstCompilerUnderTest, dexVm, compilationMode);
TestSpecification specification = new TestSpecification(name, DexTool.NONE, resultDir,
expectedOutcome == JctfTestSpecifications.Outcome.TIMEOUTS_WITH_ART
|| expectedOutcome == JctfTestSpecifications.Outcome.FLAKY_WITH_ART,
@@ -1133,11 +1154,12 @@
fileNames.add(f.getCanonicalPath());
}
- runJctfTestDoRunOnArt(fileNames,
+ runJctfTestDoRunOnArt(
+ fileNames,
specification,
firstCompilerUnderTest,
fullClassName,
- null,
+ compilationMode,
dexVm,
resultDir);
@@ -1149,8 +1171,10 @@
.map(Path::toString)
.collect(Collectors.toList());
File r8ResultDir = temp.newFolder("r8-output");
- expectedOutcome = JctfTestSpecifications
- .getExpectedOutcome(name, CompilerUnderTest.R8DEBUG_AFTER_D8, dexVm);
+ compilationMode = CompilationMode.DEBUG;
+ expectedOutcome =
+ JctfTestSpecifications.getExpectedOutcome(
+ name, CompilerUnderTest.R8DEBUG_AFTER_D8, dexVm, compilationMode);
specification = new TestSpecification(name, DexTool.DX, r8ResultDir,
expectedOutcome == JctfTestSpecifications.Outcome.TIMEOUTS_WITH_ART
|| expectedOutcome == JctfTestSpecifications.Outcome.FLAKY_WITH_ART,
@@ -1163,7 +1187,7 @@
specification,
CompilerUnderTest.R8,
fullClassName,
- CompilationMode.DEBUG,
+ compilationMode,
dexVm,
r8ResultDir);
}
@@ -1234,16 +1258,16 @@
protected void runArtTest(DexVm version, CompilerUnderTest compilerUnderTest)
throws Throwable {
- Map<SpecificationKey, TestSpecification> specificationMap = null;
+ CompilerUnderTest firstCompilerUnderTest =
+ compilerUnderTest == CompilerUnderTest.R8DEBUG_AFTER_D8
+ ? CompilerUnderTest.D8
+ : compilerUnderTest;
- if (compilerUnderTest == CompilerUnderTest.R8) {
- specificationMap = AVAILABLE_TESTS_MAP_R8;
- } else {
- assert (compilerUnderTest == CompilerUnderTest.D8);
- specificationMap = AVAILABLE_TESTS_MAP_D8;
- }
+ CompilationMode compilationMode = defaultCompilationMode(compilerUnderTest);
- TestSpecification specification = specificationMap.get(new SpecificationKey(name, toolchain));
+ TestSpecification specification =
+ get_tests_map(firstCompilerUnderTest, compilationMode)
+ .get(new SpecificationKey(name, toolchain));
if (specification == null) {
throw new RuntimeException("Test " + name + " has no specification for toolchain"
@@ -1280,12 +1304,52 @@
for (File file : inputFiles) {
fileNames.add(file.getCanonicalPath());
}
- File resultDir = temp.getRoot();
+ File resultDir = temp.newFolder(firstCompilerUnderTest.toString().toLowerCase() + "-output");
+
+ runArtTestDoRunOnArt(
+ version, firstCompilerUnderTest, specification, fileNames, resultDir, compilationMode);
+
+ if (compilerUnderTest == CompilerUnderTest.R8DEBUG_AFTER_D8) {
+ compilationMode = CompilationMode.DEBUG;
+ specification =
+ get_tests_map(CompilerUnderTest.R8DEBUG_AFTER_D8, compilationMode)
+ .get(new SpecificationKey(name, DexTool.DX));
+
+ if (specification == null) {
+ throw new RuntimeException(
+ "Test " + name + " has no specification for toolchain" + toolchain + ".");
+ }
+
+ if (specification.skipTest) {
+ return;
+ }
+
+ fileNames.clear();
+ for (File file : resultDir.listFiles((File file) -> file.getName().endsWith(".dex"))) {
+ fileNames.add(file.getCanonicalPath());
+ }
+
+ resultDir = temp.newFolder("r8-output");
+
+ runArtTestDoRunOnArt(
+ version, CompilerUnderTest.R8, specification, fileNames, resultDir, compilationMode);
+ }
+ }
+
+ private void runArtTestDoRunOnArt(
+ DexVm version,
+ CompilerUnderTest compilerUnderTest,
+ TestSpecification specification,
+ List<String> fileNames,
+ File resultDir,
+ CompilationMode compilationMode)
+ throws Throwable {
if (specification.expectedToFailWithX8) {
thrown.expect(CompilationError.class);
try {
- executeCompilerUnderTest(compilerUnderTest, fileNames, resultDir.getCanonicalPath());
+ executeCompilerUnderTest(
+ compilerUnderTest, fileNames, resultDir.getCanonicalPath(), compilationMode, null);
} catch (CompilationException e) {
throw new CompilationError(e.getMessage(), e);
} catch (ExecutionException e) {
@@ -1295,11 +1359,13 @@
return;
} else if (specification.failsWithX8) {
thrown.expect(Throwable.class);
- executeCompilerUnderTest(compilerUnderTest, fileNames, resultDir.getCanonicalPath());
+ executeCompilerUnderTest(
+ compilerUnderTest, fileNames, resultDir.getCanonicalPath(), compilationMode);
System.err.println("Should have failed R8/D8 compilation with an exception.");
return;
} else {
- executeCompilerUnderTest(compilerUnderTest, fileNames, resultDir.getCanonicalPath());
+ executeCompilerUnderTest(
+ compilerUnderTest, fileNames, resultDir.getCanonicalPath(), compilationMode);
}
if (!specification.skipArt && ToolHelper.artSupported()) {
diff --git a/src/test/java/com/android/tools/r8/TestCondition.java b/src/test/java/com/android/tools/r8/TestCondition.java
index 5ffd779..eabc95e 100644
--- a/src/test/java/com/android/tools/r8/TestCondition.java
+++ b/src/test/java/com/android/tools/r8/TestCondition.java
@@ -38,27 +38,46 @@
}
}
+ static class CompilationModeSet {
+
+ final EnumSet<CompilationMode> set;
+
+ public CompilationModeSet(EnumSet<CompilationMode> set) {
+ this.set = set;
+ }
+ }
+
public static final CompilerSet D8_COMPILER = compilers(CompilerUnderTest.D8);
public static final CompilerSet R8_COMPILER = compilers(CompilerUnderTest.R8);
public static final CompilerSet R8DEBUG_AFTER_D8_COMPILER =
compilers(CompilerUnderTest.R8DEBUG_AFTER_D8);
+ public static final CompilationModeSet DEBUG_MODE =
+ new CompilationModeSet(EnumSet.of(CompilationMode.DEBUG));
+ public static final CompilationModeSet RELEASE_MODE =
+ new CompilationModeSet(EnumSet.of(CompilationMode.RELEASE));
+
private static final ToolSet ANY_TOOL = new ToolSet(EnumSet.allOf(DexTool.class));
private static final CompilerSet ANY_COMPILER =
new CompilerSet(EnumSet.allOf(CompilerUnderTest.class));
private static final RuntimeSet ANY_RUNTIME = new RuntimeSet(EnumSet.allOf(DexVm.class));
+ private static final CompilationModeSet ANY_MODE =
+ new CompilationModeSet(EnumSet.allOf(CompilationMode.class));
private final EnumSet<DexTool> dexTools;
private final EnumSet<CompilerUnderTest> compilers;
private final EnumSet<DexVm> dexVms;
+ private final EnumSet<CompilationMode> compilationModes;
public TestCondition(
EnumSet<DexTool> dexTools,
EnumSet<CompilerUnderTest> compilers,
- EnumSet<DexVm> dexVms) {
+ EnumSet<DexVm> dexVms,
+ EnumSet<CompilationMode> compilationModes) {
this.dexTools = dexTools;
this.compilers = compilers;
this.dexVms = dexVms;
+ this.compilationModes = compilationModes;
}
public static ToolSet tools(DexTool... tools) {
@@ -76,8 +95,16 @@
return new RuntimeSet(EnumSet.copyOf(Arrays.asList(runtimes)));
}
+ public static TestCondition match(
+ ToolSet tools,
+ CompilerSet compilers,
+ RuntimeSet runtimes,
+ CompilationModeSet compilationModes) {
+ return new TestCondition(tools.set, compilers.set, runtimes.set, compilationModes.set);
+ }
+
public static TestCondition match(ToolSet tools, CompilerSet compilers, RuntimeSet runtimes) {
- return new TestCondition(tools.set, compilers.set, runtimes.set);
+ return match(tools, compilers, runtimes, TestCondition.ANY_MODE);
}
public static TestCondition any() {
@@ -100,6 +127,10 @@
return match(TestCondition.ANY_TOOL, compilers, TestCondition.ANY_RUNTIME);
}
+ public static TestCondition match(CompilerSet compilers, CompilationModeSet compilationModes) {
+ return match(TestCondition.ANY_TOOL, compilers, TestCondition.ANY_RUNTIME, compilationModes);
+ }
+
public static TestCondition match(CompilerSet compilers, RuntimeSet runtimes) {
return match(TestCondition.ANY_TOOL, compilers, runtimes);
}
@@ -108,7 +139,11 @@
return match(TestCondition.ANY_TOOL, TestCondition.ANY_COMPILER, runtimes);
}
- public boolean test(DexTool dexTool, CompilerUnderTest compilerUnderTest, DexVm dexVm) {
+ public boolean test(
+ DexTool dexTool,
+ CompilerUnderTest compilerUnderTest,
+ DexVm dexVm,
+ CompilationMode compilationMode) {
// R8DEBUG_AFTER_D8 will be set in the R8 phase of the D8-then-R8 tests. So R8DEBUG_AFTER_D8
// must match both with plain R8 and itself.
boolean compilerMatches = compilers.contains(compilerUnderTest)
@@ -116,6 +151,7 @@
&& compilers.contains(CompilerUnderTest.R8));
return dexTools.contains(dexTool)
&& compilerMatches
- && dexVms.contains(dexVm);
+ && dexVms.contains(dexVm)
+ && compilationModes.contains(compilationMode);
}
}
diff --git a/tools/create_art_tests.py b/tools/create_art_tests.py
index 3a83cbc..69d2be7 100755
--- a/tools/create_art_tests.py
+++ b/tools/create_art_tests.py
@@ -63,9 +63,13 @@
for dir in dirs:
class_name = "Art" + dir.replace("-", "_") + "Test"
for tool in TOOLS:
+ if tool == "d8" and toolchain == "none":
+ tool_enum = 'R8DEBUG_AFTER_D8'
+ else:
+ tool_enum = upper(tool)
contents = TEMPLATE.substitute(
name=dir,
- compilerUnderTestEnum=upper(tool),
+ compilerUnderTestEnum=tool_enum,
compilerUnderTest=tool,
testGeneratingToolchain=toolchain,
testGeneratingToolchainEnum=upper(toolchain),
