Account for cross-inline and anonymous or local classes

Bug: 157544211
Change-Id: I05e791cb2c65b895adac2833fbc1a148dec3db45
diff --git a/src/main/java/com/android/tools/r8/kotlin/KotlinFunctionInfo.java b/src/main/java/com/android/tools/r8/kotlin/KotlinFunctionInfo.java
index 341d7b0..c5be533 100644
--- a/src/main/java/com/android/tools/r8/kotlin/KotlinFunctionInfo.java
+++ b/src/main/java/com/android/tools/r8/kotlin/KotlinFunctionInfo.java
@@ -37,6 +37,8 @@
   private final KotlinJvmMethodSignatureInfo signature;
   // Information about the lambdaClassOrigin.
   private final KotlinTypeReference lambdaClassOrigin;
+  // A value describing if any of the parameters are crossinline.
+  private final boolean crossInlineParameter;
 
   private KotlinFunctionInfo(
       int flags,
@@ -46,7 +48,8 @@
       List<KotlinValueParameterInfo> valueParameters,
       List<KotlinTypeParameterInfo> typeParameters,
       KotlinJvmMethodSignatureInfo signature,
-      KotlinTypeReference lambdaClassOrigin) {
+      KotlinTypeReference lambdaClassOrigin,
+      boolean crossInlineParameter) {
     this.flags = flags;
     this.name = name;
     this.returnType = returnType;
@@ -55,19 +58,34 @@
     this.typeParameters = typeParameters;
     this.signature = signature;
     this.lambdaClassOrigin = lambdaClassOrigin;
+    this.crossInlineParameter = crossInlineParameter;
+  }
+
+  public boolean hasCrossInlineParameter() {
+    return crossInlineParameter;
   }
 
   static KotlinFunctionInfo create(
       KmFunction kmFunction, DexItemFactory factory, Reporter reporter) {
+    boolean isCrossInline = false;
+    List<KotlinValueParameterInfo> valueParameters =
+        KotlinValueParameterInfo.create(kmFunction.getValueParameters(), factory, reporter);
+    for (KotlinValueParameterInfo valueParameter : valueParameters) {
+      if (valueParameter.isCrossInline()) {
+        isCrossInline = true;
+        break;
+      }
+    }
     return new KotlinFunctionInfo(
         kmFunction.getFlags(),
         kmFunction.getName(),
         KotlinTypeInfo.create(kmFunction.getReturnType(), factory, reporter),
         KotlinTypeInfo.create(kmFunction.getReceiverParameterType(), factory, reporter),
-        KotlinValueParameterInfo.create(kmFunction.getValueParameters(), factory, reporter),
+        valueParameters,
         KotlinTypeParameterInfo.create(kmFunction.getTypeParameters(), factory, reporter),
         KotlinJvmMethodSignatureInfo.create(JvmExtensionsKt.getSignature(kmFunction), factory),
-        getlambdaClassOrigin(kmFunction, factory));
+        getlambdaClassOrigin(kmFunction, factory),
+        isCrossInline);
   }
 
   private static KotlinTypeReference getlambdaClassOrigin(
diff --git a/src/main/java/com/android/tools/r8/kotlin/KotlinMetadataEnqueuerExtension.java b/src/main/java/com/android/tools/r8/kotlin/KotlinMetadataEnqueuerExtension.java
index 52f4f33..c326f61 100644
--- a/src/main/java/com/android/tools/r8/kotlin/KotlinMetadataEnqueuerExtension.java
+++ b/src/main/java/com/android/tools/r8/kotlin/KotlinMetadataEnqueuerExtension.java
@@ -17,6 +17,7 @@
 import com.android.tools.r8.graph.DexMethod;
 import com.android.tools.r8.graph.DexProgramClass;
 import com.android.tools.r8.graph.DexType;
+import com.android.tools.r8.graph.EnclosingMethodAttribute;
 import com.android.tools.r8.graph.analysis.EnqueuerAnalysis;
 import com.android.tools.r8.shaking.Enqueuer;
 import com.google.common.collect.Sets;
@@ -45,6 +46,7 @@
             || enqueuer.isPinned(kotlinMetadataType);
     if (enqueuer.getMode().isInitialTreeShaking()) {
       Set<DexMethod> keepByteCodeFunctions = Sets.newIdentityHashSet();
+      Set<DexProgramClass> localOrAnonymousClasses = Sets.newIdentityHashSet();
       enqueuer.forAllLiveClasses(
           clazz -> {
             boolean onlyProcessLambdas = !keepMetadata || !enqueuer.isPinned(clazz.type);
@@ -57,8 +59,28 @@
                     appView.options().reporter,
                     onlyProcessLambdas,
                     method -> keepByteCodeFunctions.add(method.method)));
+            if (clazz.getEnclosingMethod() != null
+                && clazz.getEnclosingMethod().getEnclosingMethod() != null) {
+              localOrAnonymousClasses.add(clazz);
+            }
           });
       appView.setCfByteCodePassThrough(keepByteCodeFunctions);
+      for (DexProgramClass localOrAnonymousClass : localOrAnonymousClasses) {
+        EnclosingMethodAttribute enclosingAttribute = localOrAnonymousClass.getEnclosingMethod();
+        DexClass holder =
+            definitionSupplier.definitionForHolder(enclosingAttribute.getEnclosingMethod());
+        if (holder == null) {
+          continue;
+        }
+        DexEncodedMethod method = holder.lookupMethod(enclosingAttribute.getEnclosingMethod());
+        // If we cannot lookup the method, the conservative choice is keep the byte code.
+        if (method == null
+            || (method.getKotlinMemberInfo().isFunction()
+                && method.getKotlinMemberInfo().asFunction().hasCrossInlineParameter())) {
+          localOrAnonymousClass.forEachProgramMethod(
+              m -> keepByteCodeFunctions.add(m.getReference()));
+        }
+      }
     } else {
       assert verifyKotlinMetadataModeledForAllClasses(enqueuer, keepMetadata);
     }
diff --git a/src/main/java/com/android/tools/r8/kotlin/KotlinValueParameterInfo.java b/src/main/java/com/android/tools/r8/kotlin/KotlinValueParameterInfo.java
index bd7daac..83dd5d2 100644
--- a/src/main/java/com/android/tools/r8/kotlin/KotlinValueParameterInfo.java
+++ b/src/main/java/com/android/tools/r8/kotlin/KotlinValueParameterInfo.java
@@ -16,6 +16,7 @@
 import kotlinx.metadata.KmType;
 import kotlinx.metadata.KmValueParameter;
 import kotlinx.metadata.KmValueParameterVisitor;
+import kotlinx.metadata.internal.metadata.deserialization.Flags;
 
 // Provides access to Kotlin information about value parameter.
 class KotlinValueParameterInfo implements EnqueuerMetadataTraceable {
@@ -37,6 +38,10 @@
     this.varargElementType = varargElementType;
   }
 
+  boolean isCrossInline() {
+    return Flags.IS_CROSSINLINE.get(flags);
+  }
+
   static KotlinValueParameterInfo create(
       KmValueParameter kmValueParameter, DexItemFactory factory, Reporter reporter) {
     if (kmValueParameter == null) {
diff --git a/src/test/java/com/android/tools/r8/kotlin/coroutines/KotlinxCoroutinesTestRunner.java b/src/test/java/com/android/tools/r8/kotlin/coroutines/KotlinxCoroutinesTestRunner.java
index 7925c58..2da6a3d 100644
--- a/src/test/java/com/android/tools/r8/kotlin/coroutines/KotlinxCoroutinesTestRunner.java
+++ b/src/test/java/com/android/tools/r8/kotlin/coroutines/KotlinxCoroutinesTestRunner.java
@@ -5,8 +5,6 @@
 package com.android.tools.r8.kotlin.coroutines;
 
 import static com.android.tools.r8.KotlinCompilerTool.KOTLINC;
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 
 import com.android.tools.r8.KotlinTestBase;
@@ -65,17 +63,7 @@
 
   @Test
   public void runKotlinxCoroutinesTests_smoke() throws Exception {
-    Path testJar =
-        kotlinc(KOTLINC, targetVersion)
-            .addArguments(
-                "-Xuse-experimental=kotlinx.coroutines.InternalCoroutinesApi",
-                "-Xuse-experimental=kotlinx.coroutines.ObsoleteCoroutinesApi",
-                "-Xuse-experimental=kotlinx.coroutines.ExperimentalCoroutinesApi")
-            .addClasspathFiles(DEPENDENCIES)
-            .addClasspathFiles(BASE_LIBRARY)
-            .addSourceFiles(TEST_SOURCES)
-            .compile();
-    runTestsInJar(testJar, BASE_LIBRARY);
+    runTestsInJar(compileTestSources(BASE_LIBRARY), BASE_LIBRARY);
   }
 
   @Test
@@ -93,26 +81,26 @@
                 "-dontwarn org.junit.rules.TestRule")
             .compile()
             .writeToZip();
-    ProcessResult kotlincResult =
-        kotlinc(KOTLINC, targetVersion)
-            .addArguments(
-                "-Xuse-experimental=kotlinx.coroutines.InternalCoroutinesApi",
-                "-Xuse-experimental=kotlinx.coroutines.ObsoleteCoroutinesApi",
-                "-Xuse-experimental=kotlinx.coroutines.ExperimentalCoroutinesApi")
-            .addClasspathFiles(DEPENDENCIES)
-            .addClasspathFiles(baseJar)
-            .addSourceFiles(TEST_SOURCES)
-            .setOutputPath(temp.newFolder().toPath())
-            .compileRaw();
-    assertEquals(1, kotlincResult.exitCode);
-    assertThat(
-        kotlincResult.stderr,
-        containsString("Couldn't inline method call 'CoroutineExceptionHandler'"));
+    compileTestSources(baseJar);
+    // TODO(b/157977713): We should be able to run tests.
+    // runTestsInJar(testJar, baseJar);
   }
 
-  private void runTestsInJar(Path testJar, Path deps) throws Exception {
+  private Path compileTestSources(Path baseJar) throws Exception {
+    return kotlinc(KOTLINC, targetVersion)
+        .addArguments(
+            "-Xuse-experimental=kotlinx.coroutines.InternalCoroutinesApi",
+            "-Xuse-experimental=kotlinx.coroutines.ObsoleteCoroutinesApi",
+            "-Xuse-experimental=kotlinx.coroutines.ExperimentalCoroutinesApi")
+        .addClasspathFiles(DEPENDENCIES)
+        .addClasspathFiles(baseJar)
+        .addSourceFiles(TEST_SOURCES)
+        .compile();
+  }
+
+  private void runTestsInJar(Path testJar, Path baseJar) throws Exception {
     List<Path> dependencies = new ArrayList<>(DEPENDENCIES);
-    dependencies.add(deps);
+    dependencies.add(baseJar);
     dependencies.add(testJar);
     ZipUtils.iter(
         testJar.toString(),
diff --git a/src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineFunctionTest.java b/src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineAnonFunctionTest.java
similarity index 72%
copy from src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineFunctionTest.java
copy to src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineAnonFunctionTest.java
index e181e44..c6dd4fd 100644
--- a/src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineFunctionTest.java
+++ b/src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineAnonFunctionTest.java
@@ -5,15 +5,10 @@
 package com.android.tools.r8.kotlin.metadata;
 
 import static com.android.tools.r8.KotlinCompilerTool.KOTLINC;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.StringContains.containsString;
-import static org.junit.Assert.assertEquals;
 
 import com.android.tools.r8.TestParameters;
 import com.android.tools.r8.ToolHelper;
 import com.android.tools.r8.ToolHelper.KotlinTargetVersion;
-import com.android.tools.r8.ToolHelper.ProcessResult;
-import com.android.tools.r8.shaking.ProguardKeepAttributes;
 import com.android.tools.r8.utils.DescriptorUtils;
 import com.android.tools.r8.utils.StringUtils;
 import java.nio.file.Path;
@@ -26,11 +21,11 @@
 import org.junit.runners.Parameterized;
 
 @RunWith(Parameterized.class)
-public class MetadataRewriteCrossinlineFunctionTest extends KotlinMetadataTestBase {
+public class MetadataRewriteCrossinlineAnonFunctionTest extends KotlinMetadataTestBase {
 
   private final String EXPECTED = StringUtils.lines("foo");
-  private static final String PKG_LIB = PKG + ".crossinline_lib";
-  private static final String PKG_APP = PKG + ".crossinline_app";
+  private static final String PKG_LIB = PKG + ".crossinline_anon_lib";
+  private static final String PKG_APP = PKG + ".crossinline_anon_app";
 
   @Parameterized.Parameters(name = "{0} target: {1}")
   public static Collection<Object[]> data() {
@@ -38,7 +33,7 @@
         getTestParameters().withCfRuntimes().build(), KotlinTargetVersion.values());
   }
 
-  public MetadataRewriteCrossinlineFunctionTest(
+  public MetadataRewriteCrossinlineAnonFunctionTest(
       TestParameters parameters, KotlinTargetVersion targetVersion) {
     super(targetVersion);
     this.parameters = parameters;
@@ -80,26 +75,22 @@
   public void testMetadataForLib() throws Exception {
     Path libJar =
         testForR8(parameters.getBackend())
-            .addClasspathFiles(ToolHelper.getKotlinStdlibJar())
             .addProgramFiles(libJars.get(targetVersion))
             // Allow renaming A to ensure that we rename in the flexible upper bound type.
             .addKeepAllClassesRule()
-            .addKeepAttributes(
-                ProguardKeepAttributes.RUNTIME_VISIBLE_ANNOTATIONS,
-                ProguardKeepAttributes.SIGNATURE,
-                ProguardKeepAttributes.INNER_CLASSES,
-                ProguardKeepAttributes.ENCLOSING_METHOD)
+            .addKeepAllAttributes()
             .compile()
             .writeToZip();
-    ProcessResult mainResult =
+    Path output =
         kotlinc(parameters.getRuntime().asCf(), KOTLINC, targetVersion)
             .addClasspathFiles(libJar)
             .addSourceFiles(
                 getKotlinFileInTest(DescriptorUtils.getBinaryNameFromJavaType(PKG_APP), "main"))
-            .setOutputPath(temp.newFolder().toPath())
-            .compileRaw();
-    // TODO(b/157544211): Should compile.
-    assertEquals(1, mainResult.exitCode);
-    assertThat(mainResult.stderr, containsString("Couldn't inline method call 'Handler'"));
+            .compile();
+    testForJvm()
+        .addRunClasspathFiles(ToolHelper.getKotlinStdlibJar(), libJar)
+        .addProgramFiles(output)
+        .run(parameters.getRuntime(), PKG_APP + ".MainKt")
+        .assertSuccessWithOutput(EXPECTED);
   }
 }
diff --git a/src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineFunctionTest.java b/src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineConcreteFunctionTest.java
similarity index 73%
rename from src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineFunctionTest.java
rename to src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineConcreteFunctionTest.java
index e181e44..b96a52c 100644
--- a/src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineFunctionTest.java
+++ b/src/test/java/com/android/tools/r8/kotlin/metadata/MetadataRewriteCrossinlineConcreteFunctionTest.java
@@ -5,15 +5,10 @@
 package com.android.tools.r8.kotlin.metadata;
 
 import static com.android.tools.r8.KotlinCompilerTool.KOTLINC;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.StringContains.containsString;
-import static org.junit.Assert.assertEquals;
 
 import com.android.tools.r8.TestParameters;
 import com.android.tools.r8.ToolHelper;
 import com.android.tools.r8.ToolHelper.KotlinTargetVersion;
-import com.android.tools.r8.ToolHelper.ProcessResult;
-import com.android.tools.r8.shaking.ProguardKeepAttributes;
 import com.android.tools.r8.utils.DescriptorUtils;
 import com.android.tools.r8.utils.StringUtils;
 import java.nio.file.Path;
@@ -26,11 +21,11 @@
 import org.junit.runners.Parameterized;
 
 @RunWith(Parameterized.class)
-public class MetadataRewriteCrossinlineFunctionTest extends KotlinMetadataTestBase {
+public class MetadataRewriteCrossinlineConcreteFunctionTest extends KotlinMetadataTestBase {
 
   private final String EXPECTED = StringUtils.lines("foo");
-  private static final String PKG_LIB = PKG + ".crossinline_lib";
-  private static final String PKG_APP = PKG + ".crossinline_app";
+  private static final String PKG_LIB = PKG + ".crossinline_concrete_lib";
+  private static final String PKG_APP = PKG + ".crossinline_concrete_app";
 
   @Parameterized.Parameters(name = "{0} target: {1}")
   public static Collection<Object[]> data() {
@@ -38,7 +33,7 @@
         getTestParameters().withCfRuntimes().build(), KotlinTargetVersion.values());
   }
 
-  public MetadataRewriteCrossinlineFunctionTest(
+  public MetadataRewriteCrossinlineConcreteFunctionTest(
       TestParameters parameters, KotlinTargetVersion targetVersion) {
     super(targetVersion);
     this.parameters = parameters;
@@ -80,26 +75,23 @@
   public void testMetadataForLib() throws Exception {
     Path libJar =
         testForR8(parameters.getBackend())
-            .addClasspathFiles(ToolHelper.getKotlinStdlibJar())
             .addProgramFiles(libJars.get(targetVersion))
             // Allow renaming A to ensure that we rename in the flexible upper bound type.
             .addKeepAllClassesRule()
-            .addKeepAttributes(
-                ProguardKeepAttributes.RUNTIME_VISIBLE_ANNOTATIONS,
-                ProguardKeepAttributes.SIGNATURE,
-                ProguardKeepAttributes.INNER_CLASSES,
-                ProguardKeepAttributes.ENCLOSING_METHOD)
+            .addKeepAllAttributes()
             .compile()
             .writeToZip();
-    ProcessResult mainResult =
+    Path output =
         kotlinc(parameters.getRuntime().asCf(), KOTLINC, targetVersion)
             .addClasspathFiles(libJar)
             .addSourceFiles(
                 getKotlinFileInTest(DescriptorUtils.getBinaryNameFromJavaType(PKG_APP), "main"))
             .setOutputPath(temp.newFolder().toPath())
-            .compileRaw();
-    // TODO(b/157544211): Should compile.
-    assertEquals(1, mainResult.exitCode);
-    assertThat(mainResult.stderr, containsString("Couldn't inline method call 'Handler'"));
+            .compile();
+    testForJvm()
+        .addRunClasspathFiles(ToolHelper.getKotlinStdlibJar(), libJar)
+        .addClasspath(output)
+        .run(parameters.getRuntime(), PKG_APP + ".MainKt")
+        .assertSuccessWithOutput(EXPECTED);
   }
 }
diff --git a/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_app/main.kt b/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_anon_app/main.kt
similarity index 65%
rename from src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_app/main.kt
rename to src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_anon_app/main.kt
index a2c30be..99a3882 100644
--- a/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_app/main.kt
+++ b/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_anon_app/main.kt
@@ -2,10 +2,10 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-package com.android.tools.r8.kotlin.metadata.crossinline_app
+package com.android.tools.r8.kotlin.metadata.crossinline_anon_app
 
-import com.android.tools.r8.kotlin.metadata.crossinline_lib.Context
-import com.android.tools.r8.kotlin.metadata.crossinline_lib.Handler
+import com.android.tools.r8.kotlin.metadata.crossinline_anon_lib.Context
+import com.android.tools.r8.kotlin.metadata.crossinline_anon_lib.Handler
 
 fun main() {
   Handler({ context, throwable ->
diff --git a/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_lib/lib.kt b/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_anon_lib/lib.kt
similarity index 89%
rename from src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_lib/lib.kt
rename to src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_anon_lib/lib.kt
index 8272c15..b820f4a 100644
--- a/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_lib/lib.kt
+++ b/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_anon_lib/lib.kt
@@ -2,7 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-package com.android.tools.r8.kotlin.metadata.crossinline_lib
+package com.android.tools.r8.kotlin.metadata.crossinline_anon_lib
 
 public interface Context {
 
diff --git a/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_app/main.kt b/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_concrete_app/main.kt
similarity index 64%
copy from src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_app/main.kt
copy to src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_concrete_app/main.kt
index a2c30be..4c5c3a7 100644
--- a/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_app/main.kt
+++ b/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_concrete_app/main.kt
@@ -2,10 +2,10 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-package com.android.tools.r8.kotlin.metadata.crossinline_app
+package com.android.tools.r8.kotlin.metadata.crossinline_concrete_app
 
-import com.android.tools.r8.kotlin.metadata.crossinline_lib.Context
-import com.android.tools.r8.kotlin.metadata.crossinline_lib.Handler
+import com.android.tools.r8.kotlin.metadata.crossinline_concrete_lib.Context
+import com.android.tools.r8.kotlin.metadata.crossinline_concrete_lib.Handler
 
 fun main() {
   Handler({ context, throwable ->
diff --git a/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_concrete_lib/lib.kt b/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_concrete_lib/lib.kt
new file mode 100644
index 0000000..cf5da42
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/kotlin/metadata/crossinline_concrete_lib/lib.kt
@@ -0,0 +1,25 @@
+// Copyright (c) 2020, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.kotlin.metadata.crossinline_concrete_lib
+
+public interface Context {
+
+}
+
+public inline fun Handler(crossinline handler: (Context, Throwable) -> Unit): Handler =
+  ConcreteClass().getHandler(handler)
+
+class ConcreteClass {
+
+  inline fun getHandler(crossinline handler: (Context, Throwable) -> Unit): Handler =
+    object : Handler {
+      override fun handle(context: Context, exception: Throwable) =
+        handler.invoke(context, exception)
+    }
+}
+
+public interface Handler {
+  fun handle(context: Context, exception: Throwable)
+}
\ No newline at end of file