Make LibraryAnalyzer robust towards EnclosingMethod/InnerClasses

Bug: b/491380621
Change-Id: I2053d6549ae1dfe7d9a04fef47df6895a051d491
diff --git a/src/libanalyzer/java/com/android/tools/r8/libanalyzer/LibraryAnalyzer.java b/src/libanalyzer/java/com/android/tools/r8/libanalyzer/LibraryAnalyzer.java
index 8d1af82..dd3184f 100644
--- a/src/libanalyzer/java/com/android/tools/r8/libanalyzer/LibraryAnalyzer.java
+++ b/src/libanalyzer/java/com/android/tools/r8/libanalyzer/LibraryAnalyzer.java
@@ -76,10 +76,13 @@
 
   public static void run(LibraryAnalyzerCommand command) throws CompilationFailedException {
     LibraryAnalyzerOptions options = command.getInternalOptions();
-    run(
-        command,
-        ThreadUtils.getExecutorService(options.threadCount, options.getThreadingModule()),
-        options);
+    ExecutorService executorService =
+        ThreadUtils.getExecutorService(options.threadCount, options.getThreadingModule());
+    try {
+      run(command, executorService, options);
+    } finally {
+      executorService.shutdown();
+    }
   }
 
   public static void run(LibraryAnalyzerCommand command, ExecutorService executorService)
@@ -121,7 +124,10 @@
             .setProgramConsumer(sizeConsumer);
     configure(commandBuilder);
     try {
-      D8.run(commandBuilder.build(), executorService);
+      D8.LibraryAnalyzerEntryPoint.run(
+          commandBuilder.build(),
+          executorService,
+          d8Options -> d8Options.libraryAnalyzerSubCompilation = true);
     } catch (CompilationFailedException e) {
       reporter.warning(new ExceptionDiagnostic(e));
       reporter.clearAbort();
@@ -144,6 +150,7 @@
           commandBuilder.build(),
           executorService,
           r8Options -> {
+            r8Options.libraryAnalyzerSubCompilation = true;
             r8Options.ignoreUnusedProguardRules = true;
             if (options.blastRadiusOutputPath != null) {
               r8Options.getBlastRadiusOptions().outputPath =
diff --git a/src/main/java/com/android/tools/r8/D8.java b/src/main/java/com/android/tools/r8/D8.java
index 4f38638..3e7e4f6 100644
--- a/src/main/java/com/android/tools/r8/D8.java
+++ b/src/main/java/com/android/tools/r8/D8.java
@@ -481,4 +481,19 @@
     @Override
     public void finished(DiagnosticsHandler handler) {}
   }
+
+  public static class LibraryAnalyzerEntryPoint {
+
+    public static void run(
+        D8Command command,
+        ExecutorService executorService,
+        Consumer<InternalOptions> optionsModification)
+        throws CompilationFailedException {
+      AndroidApp app = command.getInputApp();
+      InternalOptions options = command.getInternalOptions();
+      optionsModification.accept(options);
+      ExceptionUtils.withR8CompilationHandler(
+          options.reporter, () -> runInternal(app, options, executorService));
+    }
+  }
 }
diff --git a/src/main/java/com/android/tools/r8/R8.java b/src/main/java/com/android/tools/r8/R8.java
index ef64db4..af91e5c 100644
--- a/src/main/java/com/android/tools/r8/R8.java
+++ b/src/main/java/com/android/tools/r8/R8.java
@@ -1405,14 +1405,7 @@
       InternalOptions options = command.getInternalOptions();
       optionsModification.accept(options);
       ExceptionUtils.withR8CompilationHandler(
-          options.reporter,
-          () -> {
-            try {
-              runInternal(app, options, executorService);
-            } finally {
-              executorService.shutdown();
-            }
-          });
+          options.reporter, () -> runInternal(app, options, executorService));
     }
   }
 
diff --git a/src/main/java/com/android/tools/r8/shaking/AnnotationRemover.java b/src/main/java/com/android/tools/r8/shaking/AnnotationRemover.java
index 123a01b..0acd18d 100644
--- a/src/main/java/com/android/tools/r8/shaking/AnnotationRemover.java
+++ b/src/main/java/com/android/tools/r8/shaking/AnnotationRemover.java
@@ -183,7 +183,7 @@
   }
 
   public AnnotationRemover ensureValid() {
-    keep.ensureValid(appView.options().forceProguardCompatibility);
+    keep.ensureValid(appView.options());
     return this;
   }
 
diff --git a/src/main/java/com/android/tools/r8/shaking/ProguardKeepAttributes.java b/src/main/java/com/android/tools/r8/shaking/ProguardKeepAttributes.java
index 015bc50..7a43c71 100644
--- a/src/main/java/com/android/tools/r8/shaking/ProguardKeepAttributes.java
+++ b/src/main/java/com/android/tools/r8/shaking/ProguardKeepAttributes.java
@@ -4,6 +4,7 @@
 package com.android.tools.r8.shaking;
 
 import com.android.tools.r8.errors.CompilationError;
+import com.android.tools.r8.utils.InternalOptions;
 import com.android.tools.r8.utils.ListUtils;
 import java.util.ArrayList;
 import java.util.List;
@@ -173,11 +174,13 @@
     lineNumberTable = update(lineNumberTable, LINE_NUMBER_TABLE, patterns);
   }
 
-  public void ensureValid(boolean forceProguardCompatibility) {
-    if (forceProguardCompatibility && innerClasses != enclosingMethod) {
-      // If only one is true set both to true in Proguard compatibility mode.
-      enclosingMethod = true;
-      innerClasses = true;
+  public void ensureValid(InternalOptions options) {
+    if (options.forceProguardCompatibility || options.libraryAnalyzerSubCompilation) {
+      if (innerClasses != enclosingMethod) {
+        // If only one is true set both to true in Proguard compatibility mode.
+        enclosingMethod = true;
+        innerClasses = true;
+      }
     }
     if (innerClasses && !enclosingMethod) {
       throw new CompilationError("Attribute InnerClasses requires EnclosingMethod attribute. "
diff --git a/src/main/java/com/android/tools/r8/utils/InternalOptions.java b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
index 82983a6..76dc157 100644
--- a/src/main/java/com/android/tools/r8/utils/InternalOptions.java
+++ b/src/main/java/com/android/tools/r8/utils/InternalOptions.java
@@ -1117,6 +1117,7 @@
   public R8PartialCompilationConfiguration partialCompilationConfiguration =
       R8PartialCompilationConfiguration.disabledConfiguration();
   public R8PartialSubCompilationConfiguration partialSubCompilationConfiguration = null;
+  public boolean libraryAnalyzerSubCompilation;
   public final TestingOptions testing = new TestingOptions();
 
   public List<ProguardConfigurationRule> mainDexKeepRules = ImmutableList.of();
diff --git a/src/test/java/com/android/tools/r8/libanalyzer/LibraryAnalyzerEnclosingMethodAndInnerClassesTest.java b/src/test/java/com/android/tools/r8/libanalyzer/LibraryAnalyzerEnclosingMethodAndInnerClassesTest.java
new file mode 100644
index 0000000..6898843
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/libanalyzer/LibraryAnalyzerEnclosingMethodAndInnerClassesTest.java
@@ -0,0 +1,77 @@
+// Copyright (c) 2026, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+package com.android.tools.r8.libanalyzer;
+
+import static com.android.tools.r8.utils.codeinspector.AssertUtils.assertFailsCompilation;
+
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.utils.AndroidApiLevel;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class LibraryAnalyzerEnclosingMethodAndInnerClassesTest extends TestBase {
+
+  @Parameter(0)
+  public TestParameters parameters;
+
+  @Parameters(name = "{0}")
+  public static TestParametersCollection data() {
+    return getTestParameters().withNoneRuntime().build();
+  }
+
+  @Test
+  public void testR8EnclosingMethod() throws Exception {
+    assertFailsCompilation(
+        () ->
+            testForR8(Backend.DEX)
+                .addProgramClasses(Main.class)
+                .addKeepRules("-keep class *", "-keepattributes EnclosingMethod")
+                .setMinApi(AndroidApiLevel.getDefault())
+                .compile());
+  }
+
+  @Test
+  public void testR8InnerClasses() throws Exception {
+    assertFailsCompilation(
+        () ->
+            testForR8(Backend.DEX)
+                .addProgramClasses(Main.class)
+                .addKeepRules("-keep class *", "-keepattributes InnerClasses")
+                .setMinApi(AndroidApiLevel.getDefault())
+                .compile());
+  }
+
+  @Test
+  public void testLibraryAnalyzerEnclosingMethod() throws Exception {
+    testForLibraryAnalyzer()
+        .addProgramClasses(Main.class)
+        .addDefaultLibrary()
+        .addKeepRules("-keep class *", "-keepattributes EnclosingMethod")
+        .setAar()
+        .setMinApi(AndroidApiLevel.getDefault())
+        .compile();
+  }
+
+  @Test
+  public void testLibraryAnalyzerInnerClasses() throws Exception {
+    testForLibraryAnalyzer()
+        .addProgramClasses(Main.class)
+        .addDefaultLibrary()
+        .addKeepRules("-keep class *", "-keepattributes InnerClasses")
+        .setAar()
+        .setMinApi(AndroidApiLevel.getDefault())
+        .compile();
+  }
+
+  static class Main {
+
+    public static void main(String[] args) {}
+  }
+}