Fix assertion failure on D8+Nests+enum

- Add enum tests for D8/R8 + Nests
- Add test to compile R8_11 with D8
- Remove dead code NestReportType
- Make flags more consistent with shouldDesugarNests()

Bug:136692900
Change-Id: I9f839b4fde37a54ed348f547a219a6401e8fcd0f
diff --git a/src/main/java/com/android/tools/r8/R8.java b/src/main/java/com/android/tools/r8/R8.java
index f18c30f..de7b945 100644
--- a/src/main/java/com/android/tools/r8/R8.java
+++ b/src/main/java/com/android/tools/r8/R8.java
@@ -445,8 +445,7 @@
 
       AppView<AppInfoWithLiveness> appViewWithLiveness = appView.withLiveness();
       appView.setGraphLense(new MemberRebindingAnalysis(appViewWithLiveness).run());
-      if (options.testing.enableForceNestBasedAccessDesugaringForTest
-          || (options.enableNestBasedAccessDesugaring && !options.canUseNestBasedAccess())) {
+      if (options.shouldDesugarNests()) {
         timing.begin("NestBasedAccessDesugaring");
         R8NestBasedAccessDesugaring analyzer = new R8NestBasedAccessDesugaring(appViewWithLiveness);
         boolean changed =
diff --git a/src/main/java/com/android/tools/r8/ir/conversion/IRConverter.java b/src/main/java/com/android/tools/r8/ir/conversion/IRConverter.java
index 4cf4d05..e26790e 100644
--- a/src/main/java/com/android/tools/r8/ir/conversion/IRConverter.java
+++ b/src/main/java/com/android/tools/r8/ir/conversion/IRConverter.java
@@ -262,7 +262,7 @@
       this.uninstantiatedTypeOptimization = null;
       this.typeChecker = null;
       this.d8NestBasedAccessDesugaring =
-          options.enableNestBasedAccessDesugaring ? new D8NestBasedAccessDesugaring(appView) : null;
+          options.shouldDesugarNests() ? new D8NestBasedAccessDesugaring(appView) : null;
     }
     this.deadCodeRemover = new DeadCodeRemover(appView, codeRewriter);
     this.idempotentFunctionCallCanonicalizer = new IdempotentFunctionCallCanonicalizer(appView);
diff --git a/src/main/java/com/android/tools/r8/ir/desugar/D8NestBasedAccessDesugaring.java b/src/main/java/com/android/tools/r8/ir/desugar/D8NestBasedAccessDesugaring.java
index 0f3944d..b159ef7 100644
--- a/src/main/java/com/android/tools/r8/ir/desugar/D8NestBasedAccessDesugaring.java
+++ b/src/main/java/com/android/tools/r8/ir/desugar/D8NestBasedAccessDesugaring.java
@@ -65,7 +65,8 @@
         if (instruction.isInvokeMethod() && !instruction.isInvokeSuper()) {
           InvokeMethod invokeMethod = instruction.asInvokeMethod();
           DexMethod methodCalled = invokeMethod.getInvokedMethod();
-          DexEncodedMethod encodedMethodCalled = appView.definitionFor(methodCalled);
+          DexEncodedMethod encodedMethodCalled =
+              methodCalled.holder.isClassType() ? appView.definitionFor(methodCalled) : null;
           if (encodedMethodCalled != null
               && invokeRequiresRewriting(encodedMethodCalled, currentClass)) {
             DexMethod bridge = ensureInvokeBridge(encodedMethodCalled);
diff --git a/src/main/java/com/android/tools/r8/utils/InternalOptions.java b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
index 029c308..b3236c4 100644
--- a/src/main/java/com/android/tools/r8/utils/InternalOptions.java
+++ b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
@@ -299,6 +299,13 @@
     }
   }
 
+  public boolean shouldDesugarNests() {
+    if (testing.enableForceNestBasedAccessDesugaringForTest) {
+      return true;
+    }
+    return enableNestBasedAccessDesugaring && !canUseNestBasedAccess();
+  }
+
   public Set<String> extensiveLoggingFilter = getExtensiveLoggingFilter();
   public Set<String> extensiveFieldMinifierLoggingFilter = getExtensiveFieldMinifierLoggingFilter();
   public Set<String> extensiveInterfaceMethodMinifierLoggingFilter =
@@ -581,12 +588,6 @@
   /** A set of dexitems we have reported missing to dedupe warnings. */
   private final Set<DexItem> reportedMissingForDesugaring = Sets.newConcurrentHashSet();
 
-  public enum NestReportType {
-    FATAL_ERROR,
-    DESUGAR_WARNING,
-    WARNING
-  }
-
   public void errorMissingClassMissingNestHost(DexClass compiledClass) {
     throw reporter.fatalError(messageErrorMissingNestHost(compiledClass));
   }
diff --git a/src/test/examplesJava11/nesthostexample/NestHostExample.java b/src/test/examplesJava11/nesthostexample/NestHostExample.java
index 3aa3546..ee5c2ce 100644
--- a/src/test/examplesJava11/nesthostexample/NestHostExample.java
+++ b/src/test/examplesJava11/nesthostexample/NestHostExample.java
@@ -36,6 +36,12 @@
     return new StaticNestMemberInner(field);
   }
 
+  public enum ExampleEnumCompilation {
+    CASE1,
+    CASE2,
+    CASE3;
+  }
+
   public NestMemberInner createNestMemberInner(String field) {
     return new NestMemberInner(field);
   }
@@ -312,5 +318,7 @@
     System.out.println(o2.accessPrivateInterface(i1));
     System.out.println(o4.accessPrivateInterface(i1));
     System.out.println(i1.accessPrivateInterface(i1));
+
+    System.out.println(ExampleEnumCompilation.values().length);
   }
 }
diff --git a/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/Java11D8CompilationTest.java b/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/Java11D8CompilationTest.java
new file mode 100644
index 0000000..975f240
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/Java11D8CompilationTest.java
@@ -0,0 +1,27 @@
+// Copyright (c) 2019, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.desugar.nestaccesscontrol;
+
+import static junit.framework.TestCase.assertTrue;
+
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.ToolHelper;
+import com.android.tools.r8.utils.codeinspector.CodeInspector;
+import org.junit.Test;
+
+public class Java11D8CompilationTest extends TestBase {
+
+  private static void assertNoNests(CodeInspector inspector) {
+    assertTrue(inspector.allClasses().stream().noneMatch(subj -> subj.getDexClass().isInANest()));
+  }
+
+  @Test
+  public void testR8CompiledWithD8() throws Exception {
+    testForD8()
+        .addProgramFiles(ToolHelper.R8_WITH_RELOCATED_DEPS_JAR_11)
+        .compile()
+        .inspect(Java11D8CompilationTest::assertNoNests);
+  }
+}
diff --git a/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/Java11BootstrapTest.java b/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/Java11R8BootstrapTest.java
similarity index 97%
rename from src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/Java11BootstrapTest.java
rename to src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/Java11R8BootstrapTest.java
index 730fd28..073eda1 100644
--- a/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/Java11BootstrapTest.java
+++ b/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/Java11R8BootstrapTest.java
@@ -33,7 +33,7 @@
  * and identical.
  */
 @RunWith(Parameterized.class)
-public class Java11BootstrapTest extends TestBase {
+public class Java11R8BootstrapTest extends TestBase {
 
   private static final Path MAIN_KEEP = Paths.get("src/main/keep.txt");
   private static final String[] HELLO_KEEP = {
@@ -43,7 +43,7 @@
   private static Path r8Lib11NoDesugar;
   private static Path r8Lib11Desugar;
 
-  public Java11BootstrapTest(TestParameters parameters) {
+  public Java11R8BootstrapTest(TestParameters parameters) {
     this.parameters = parameters;
   }
 
diff --git a/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/NestAccessControlTestUtils.java b/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/NestAccessControlTestUtils.java
index 77ae469..2838b43 100644
--- a/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/NestAccessControlTestUtils.java
+++ b/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/NestAccessControlTestUtils.java
@@ -65,7 +65,8 @@
           "NestHostExample$NestMemberInner$NestMemberInnerInner",
           "NestHostExample$StaticNestMemberInner",
           "NestHostExample$StaticNestMemberInner$StaticNestMemberInnerInner",
-          "NestHostExample$StaticNestInterfaceInner");
+          "NestHostExample$StaticNestInterfaceInner",
+          "NestHostExample$ExampleEnumCompilation");
   public static final int NUMBER_OF_TEST_CLASSES = CLASS_NAMES.size();
 
   // The following map use ids, i.e., strings which represents respectively
@@ -142,7 +143,8 @@
                   "staticInterfaceMethodstaticStaticInterfaceMethod",
                   "staticInterfaceMethodstaticStaticInterfaceMethod",
                   "staticInterfaceMethodstaticStaticInterfaceMethod",
-                  "staticInterfaceMethodstaticStaticInterfaceMethod"))
+                  "staticInterfaceMethodstaticStaticInterfaceMethod",
+                  "3"))
           .put(
               "pvtCallInlined",
               StringUtils.lines(