Add a -D flag for forcing nest based access desugaring

Flag -Dcom.android.tools.r8.forceNestDesugaring will override
-Dcom.android.tools.r8.emitNestAnnotationsInDex, and in the future
also override native Art nest based access based on min API level
and desugar nest based access.

Fixes: b/311336264
Change-Id: I3a093ee0feb7cba937eb85418ddb3fe1a6f9269a
diff --git a/src/main/java/com/android/tools/r8/dex/ApplicationWriter.java b/src/main/java/com/android/tools/r8/dex/ApplicationWriter.java
index e6c4db1..105ea1c 100644
--- a/src/main/java/com/android/tools/r8/dex/ApplicationWriter.java
+++ b/src/main/java/com/android/tools/r8/dex/ApplicationWriter.java
@@ -836,7 +836,7 @@
               options.itemFactory));
     }
 
-    if (options.emitNestAnnotationsInDex) {
+    if (options.canUseNestBasedAccess()) {
       if (clazz.isNestHost()) {
         annotations.add(
             DexAnnotation.createNestMembersAnnotation(
diff --git a/src/main/java/com/android/tools/r8/ir/code/InvokeMethodWithReceiver.java b/src/main/java/com/android/tools/r8/ir/code/InvokeMethodWithReceiver.java
index edc96f6..3b3b87f 100644
--- a/src/main/java/com/android/tools/r8/ir/code/InvokeMethodWithReceiver.java
+++ b/src/main/java/com/android/tools/r8/ir/code/InvokeMethodWithReceiver.java
@@ -93,7 +93,7 @@
   }
 
   protected boolean isPrivateNestMethodInvoke(DexBuilder builder) {
-    if (!builder.getOptions().emitNestAnnotationsInDex) {
+    if (!builder.getOptions().canUseNestBasedAccess()) {
       return false;
     }
     DexProgramClass holder = builder.getProgramMethod().getHolder();
diff --git a/src/main/java/com/android/tools/r8/ir/conversion/IRToDexFinalizer.java b/src/main/java/com/android/tools/r8/ir/conversion/IRToDexFinalizer.java
index 87f9a4b..c5e44d0 100644
--- a/src/main/java/com/android/tools/r8/ir/conversion/IRToDexFinalizer.java
+++ b/src/main/java/com/android/tools/r8/ir/conversion/IRToDexFinalizer.java
@@ -33,7 +33,7 @@
   @Override
   public DexCode finalizeCode(
       IRCode code, BytecodeMetadataProvider bytecodeMetadataProvider, Timing timing) {
-    if (options.emitNestAnnotationsInDex) {
+    if (options.canUseNestBasedAccess()) {
       D8NestBasedAccessDesugaring.checkAndFailOnIncompleteNests(appView);
     }
     DexEncodedMethod method = code.method();
diff --git a/src/main/java/com/android/tools/r8/ir/desugar/CfInstructionDesugaringCollection.java b/src/main/java/com/android/tools/r8/ir/desugar/CfInstructionDesugaringCollection.java
index bcace75..a2dd4ef 100644
--- a/src/main/java/com/android/tools/r8/ir/desugar/CfInstructionDesugaringCollection.java
+++ b/src/main/java/com/android/tools/r8/ir/desugar/CfInstructionDesugaringCollection.java
@@ -7,6 +7,7 @@
 import com.android.tools.r8.androidapi.AndroidApiLevelCompute;
 import com.android.tools.r8.cf.code.CfInstruction;
 import com.android.tools.r8.contexts.CompilationContext.MethodProcessingContext;
+import com.android.tools.r8.errors.CompilationError;
 import com.android.tools.r8.graph.AppView;
 import com.android.tools.r8.graph.ProgramMethod;
 import com.android.tools.r8.ir.desugar.desugaredlibrary.apiconversion.DesugaredLibraryAPIConverter;
@@ -29,6 +30,10 @@
 
   public static CfInstructionDesugaringCollection create(
       AppView<?> appView, AndroidApiLevelCompute apiLevelCompute) {
+    if (appView.options().desugarState.isOff() && appView.options().forceNestDesugaring) {
+      throw new CompilationError(
+          "Cannot combine -Dcom.android.tools.r8.forceNestDesugaring with desugaring turned off");
+    }
     if (appView.options().desugarState.isOn()) {
       return new NonEmptyCfInstructionDesugaringCollection(appView, apiLevelCompute);
     }
diff --git a/src/main/java/com/android/tools/r8/utils/InternalOptions.java b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
index 0e73e2a..fdc76a4 100644
--- a/src/main/java/com/android/tools/r8/utils/InternalOptions.java
+++ b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
@@ -490,6 +490,9 @@
   // Flag to allow nest annotations in DEX. See b/231930852 for context.
   public boolean emitNestAnnotationsInDex =
       System.getProperty("com.android.tools.r8.emitNestAnnotationsInDex") != null;
+  // Flag to allow force nest desugaring, even if natively supported on the chosen API level.
+  public boolean forceNestDesugaring =
+      System.getProperty("com.android.tools.r8.forceNestDesugaring") != null;
 
   // TODO(b/293591931): Remove this flag.
   // Flag to allow permitted subclasses annotations in DEX. See b/231930852 for context.
@@ -2676,7 +2679,7 @@
   }
 
   public boolean canUseNestBasedAccess() {
-    return hasFeaturePresentFrom(null) || emitNestAnnotationsInDex;
+    return (hasFeaturePresentFrom(null) || emitNestAnnotationsInDex) && !forceNestDesugaring;
   }
 
   public boolean canUseRecords() {
diff --git a/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/NestAttributesNotInDexWithForceNestDesugaringTest.java b/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/NestAttributesNotInDexWithForceNestDesugaringTest.java
new file mode 100644
index 0000000..86eff4e
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/NestAttributesNotInDexWithForceNestDesugaringTest.java
@@ -0,0 +1,127 @@
+// Copyright (c) 2023, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.desugar.nestaccesscontrol;
+
+import static com.android.tools.r8.utils.codeinspector.Matchers.isPresentIf;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assume.assumeTrue;
+
+import com.android.tools.r8.CompilationFailedException;
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.desugar.nestaccesscontrol.NestAttributesNotInDexWithForceNestDesugaringTest.Host.Member1;
+import com.android.tools.r8.desugar.nestaccesscontrol.NestAttributesNotInDexWithForceNestDesugaringTest.Host.Member2;
+import com.android.tools.r8.graph.MethodAccessFlags;
+import com.android.tools.r8.synthesis.SyntheticItemsTestUtils;
+import com.android.tools.r8.transformers.ClassFileTransformer;
+import com.android.tools.r8.transformers.ClassFileTransformer.MethodPredicate;
+import com.android.tools.r8.utils.BooleanUtils;
+import com.google.common.collect.ImmutableList;
+import java.util.Collection;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class NestAttributesNotInDexWithForceNestDesugaringTest extends NestAttributesInDexTestBase {
+
+  @Parameter(0)
+  public TestParameters parameters;
+
+  @Parameter(1)
+  public boolean forceNestDesugaring;
+
+  @Parameters(name = "{0}, forceNestDesugaring: {1}")
+  public static List<Object[]> data() {
+    return buildParameters(
+        getTestParameters().withDexRuntimesAndAllApiLevels().build(), BooleanUtils.values());
+  }
+
+  @Test
+  public void testD8() throws Exception {
+    testForD8()
+        .addProgramClassFileData(getTransformedClasses())
+        .setMinApi(parameters)
+        .addOptionsModification(options -> options.emitNestAnnotationsInDex = true)
+        .addOptionsModification(options -> options.forceNestDesugaring = forceNestDesugaring)
+        .compile()
+        .inspect(
+            inspector -> {
+              assertThat(
+                  inspector
+                      .clazz(Member1.class)
+                      .uniqueMethodWithOriginalName(
+                          SyntheticItemsTestUtils.syntheticNestInstanceMethodAccessor(
+                                  Member1.class.getDeclaredMethod("m1"))
+                              .getMethodName()),
+                  isPresentIf(forceNestDesugaring));
+              assertThat(
+                  inspector
+                      .clazz(Member2.class)
+                      .uniqueMethodWithOriginalName(
+                          SyntheticItemsTestUtils.syntheticNestStaticMethodAccessor(
+                                  Member2.class.getDeclaredMethod("m2"))
+                              .getMethodName()),
+                  isPresentIf(forceNestDesugaring));
+            });
+  }
+
+  @Test
+  public void testD8NoDesugar() {
+    assumeTrue(forceNestDesugaring);
+    assertThrows(
+        CompilationFailedException.class,
+        () ->
+            testForD8(parameters.getBackend())
+                .addProgramClassFileData(getTransformedClasses())
+                .disableDesugaring()
+                .setMinApi(parameters)
+                .addOptionsModification(options -> options.emitNestAnnotationsInDex = true)
+                .addOptionsModification(
+                    options -> options.forceNestDesugaring = forceNestDesugaring)
+                .compile());
+  }
+
+  public Collection<byte[]> getTransformedClasses() throws Exception {
+    return ImmutableList.of(
+        withNest(Host.class)
+            .setAccessFlags(MethodPredicate.onName("h"), MethodAccessFlags::setPrivate)
+            .transform(),
+        withNest(Member1.class)
+            .setAccessFlags(MethodPredicate.onName("m1"), MethodAccessFlags::setPrivate)
+            .transform(),
+        withNest(Member2.class)
+            .setAccessFlags(MethodPredicate.onName("m2"), MethodAccessFlags::setPrivate)
+            .transform());
+  }
+
+  private ClassFileTransformer withNest(Class<?> clazz) throws Exception {
+    return transformer(clazz).setNest(Host.class, Member1.class, Member2.class);
+  }
+
+  static class TestClass {
+
+    public static void main(String[] args) {}
+  }
+
+  static class Host {
+    static class Member1 {
+      void m1() { // Will be private.
+        Member2.m2();
+      }
+    }
+
+    static class Member2 {
+      static void m2() { // Will be private.
+      }
+    }
+
+    void h() { // Will be private.
+      new Member1().m1();
+    }
+  }
+}