Fail compilation when a nest member is on library

Bug: b/343641859
Change-Id: Icdcc343442959cab7d82e27f76c0ba6b141a7925
diff --git a/src/main/java/com/android/tools/r8/ir/desugar/nest/D8NestBasedAccessDesugaring.java b/src/main/java/com/android/tools/r8/ir/desugar/nest/D8NestBasedAccessDesugaring.java
index 4fdeb0f..a4465a9 100644
--- a/src/main/java/com/android/tools/r8/ir/desugar/nest/D8NestBasedAccessDesugaring.java
+++ b/src/main/java/com/android/tools/r8/ir/desugar/nest/D8NestBasedAccessDesugaring.java
@@ -39,11 +39,18 @@
     forEachNest(
         nest -> {
           if (nest.hasMissingMembers()) {
-            throw appView.options().errorMissingNestMember(nest);
+            throw appView.options().fatalErrorMissingNestMember(nest);
+          }
+          if (nest.hasHostOrLibraryMember()) {
+            appView.options().errorMissingNestMember(nest);
           }
           DexClass hostClass = nest.getHostClass();
+          if (hostClass.isLibraryClass()) {
+            return;
+          }
           for (DexClass memberClass : nest.getMembers()) {
-            if (hostClass.isProgramClass() || memberClass.isProgramClass()) {
+            if (!memberClass.isLibraryClass()
+                && (hostClass.isProgramClass() || memberClass.isProgramClass())) {
               appView.appInfo().reportDependencyEdge(hostClass, memberClass);
               appView.appInfo().reportDependencyEdge(memberClass, hostClass);
             }
@@ -58,7 +65,10 @@
     forEachNest(
         nest -> {
           if (nest.hasMissingMembers()) {
-            throw appView.options().errorMissingNestMember(nest);
+            throw appView.options().fatalErrorMissingNestMember(nest);
+          }
+          if (nest.hasHostOrLibraryMember()) {
+            appView.options().errorMissingNestMember(nest);
           }
         },
         classWithoutHost -> {
diff --git a/src/main/java/com/android/tools/r8/ir/desugar/nest/Nest.java b/src/main/java/com/android/tools/r8/ir/desugar/nest/Nest.java
index 979eb9c..3e49a4a 100644
--- a/src/main/java/com/android/tools/r8/ir/desugar/nest/Nest.java
+++ b/src/main/java/com/android/tools/r8/ir/desugar/nest/Nest.java
@@ -73,8 +73,8 @@
     return missingMembers;
   }
 
-  public boolean hasLibraryMember() {
-    return Iterables.any(members, DexClass::isLibraryClass);
+  public boolean hasHostOrLibraryMember() {
+    return hostClass.isLibraryClass() || Iterables.any(members, DexClass::isLibraryClass);
   }
 
   public boolean hasMissingMembers() {
diff --git a/src/main/java/com/android/tools/r8/ir/desugar/nest/NestBasedAccessDesugaring.java b/src/main/java/com/android/tools/r8/ir/desugar/nest/NestBasedAccessDesugaring.java
index 127dff5..10f3501 100644
--- a/src/main/java/com/android/tools/r8/ir/desugar/nest/NestBasedAccessDesugaring.java
+++ b/src/main/java/com/android/tools/r8/ir/desugar/nest/NestBasedAccessDesugaring.java
@@ -407,7 +407,7 @@
   RuntimeException reportIncompleteNest(LibraryMember<?, ?> member) {
     Nest nest = Nest.create(appView, member.getHolder());
     assert nest != null : "Should be a compilation error if missing nest host on library class.";
-    throw appView.options().errorMissingNestMember(nest);
+    throw appView.options().fatalErrorMissingNestMember(nest);
   }
 
   DexMethod getFieldAccessBridgeReference(DexClassAndField field, boolean isGet) {
diff --git a/src/main/java/com/android/tools/r8/utils/InternalOptions.java b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
index 7a109a2..f565875 100644
--- a/src/main/java/com/android/tools/r8/utils/InternalOptions.java
+++ b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
@@ -1303,10 +1303,17 @@
         + " to be on program or class path.";
   }
 
-  public RuntimeException errorMissingNestMember(Nest nest) {
-    throw reporter.fatalError(
-        new IncompleteNestNestDesugarDiagnosic(
-            nest.getHostClass().getOrigin(), Position.UNKNOWN, messageErrorIncompleteNest(nest)));
+  public RuntimeException fatalErrorMissingNestMember(Nest nest) {
+    throw reporter.fatalError(createIncompleteNestNestDesugarDiagnosic(nest));
+  }
+
+  public void errorMissingNestMember(Nest nest) {
+    reporter.error(createIncompleteNestNestDesugarDiagnosic(nest));
+  }
+
+  public IncompleteNestNestDesugarDiagnosic createIncompleteNestNestDesugarDiagnosic(Nest nest) {
+    return new IncompleteNestNestDesugarDiagnosic(
+        nest.getHostClass().getOrigin(), Position.UNKNOWN, messageErrorIncompleteNest(nest));
   }
 
   private static String messageErrorIncompleteNest(Nest nest) {
diff --git a/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/NestOnProgramAndLibraryPathTest.java b/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/NestOnProgramAndLibraryPathTest.java
new file mode 100644
index 0000000..74c344e
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/desugar/nestaccesscontrol/NestOnProgramAndLibraryPathTest.java
@@ -0,0 +1,95 @@
+// Copyright (c) 2024, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.desugar.nestaccesscontrol;
+
+import static com.android.tools.r8.desugar.nestaccesscontrol.NestAccessControlTestUtils.CLASSES_PATH;
+import static com.android.tools.r8.desugar.nestaccesscontrol.NestAccessControlTestUtils.CLASS_NAMES;
+import static com.android.tools.r8.desugar.nestaccesscontrol.NestAccessControlTestUtils.JAR;
+import static com.android.tools.r8.utils.FileUtils.CLASS_EXTENSION;
+import static java.util.stream.Collectors.toList;
+import static org.hamcrest.core.StringContains.containsString;
+import static org.hamcrest.core.StringEndsWith.endsWith;
+import static org.junit.Assert.assertThrows;
+
+import com.android.tools.r8.CompilationFailedException;
+import com.android.tools.r8.D8TestCompileResult;
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.TestParametersCollection;
+import java.nio.file.Path;
+import java.util.List;
+import org.hamcrest.Matcher;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class NestOnProgramAndLibraryPathTest extends TestBase {
+
+  public NestOnProgramAndLibraryPathTest(TestParameters parameters) {
+    this.parameters = parameters;
+  }
+
+  private final TestParameters parameters;
+
+  @Parameters(name = "{0}")
+  public static TestParametersCollection data() {
+    return getTestParameters().withDexRuntimes().withAllApiLevels().build();
+  }
+
+  @Test
+  public void testD8MethodBridges() {
+    Assume.assumeTrue(parameters.isDexRuntime());
+    // 1 inner class.
+    assertThrows(
+        CompilationFailedException.class,
+        () ->
+            compileClassesWithD8ProgramClassesMatching(
+                containsString("BasicNestHostWithInnerClassMethods$BasicNestedClass")));
+    // Outer class.
+    assertThrows(
+        CompilationFailedException.class,
+        () ->
+            compileClassesWithD8ProgramClassesMatching(
+                endsWith("BasicNestHostWithInnerClassMethods")));
+    // 2 inner classes.
+    assertThrows(
+        CompilationFailedException.class,
+        () ->
+            compileClassesWithD8ProgramClassesMatching(
+                containsString("NestHostExample$StaticNestMemberInner")));
+  }
+
+  @Test
+  public void testD8ConstructorBridges() {
+    Assume.assumeTrue(parameters.isDexRuntime());
+    assertThrows(
+        CompilationFailedException.class,
+        () ->
+            compileClassesWithD8ProgramClassesMatching(
+                containsString("BasicNestHostWithInnerClassConstructors$BasicNestedClass")));
+    assertThrows(
+        CompilationFailedException.class,
+        () ->
+            compileClassesWithD8ProgramClassesMatching(
+                endsWith("BasicNestHostWithInnerClassConstructors")));
+  }
+
+  private D8TestCompileResult compileClassesWithD8ProgramClassesMatching(Matcher<String> matcher)
+      throws Exception {
+    List<Path> matchingClasses =
+        CLASS_NAMES.stream()
+            .filter(matcher::matches)
+            .map(name -> CLASSES_PATH.resolve(name + CLASS_EXTENSION))
+            .collect(toList());
+    return testForD8()
+        .setMinApi(parameters)
+        .addProgramFiles(matchingClasses)
+        .addLibraryFiles(JAR)
+        .compile();
+  }
+}