Bail out of class inlining if returning receiver is unknown

Bug: 176381203
Change-Id: I01b52eb809afd201d201be52394fe68e76ed5191
diff --git a/src/main/java/com/android/tools/r8/ir/optimize/classinliner/InlineCandidateProcessor.java b/src/main/java/com/android/tools/r8/ir/optimize/classinliner/InlineCandidateProcessor.java
index c1fef74..a69bfd2 100644
--- a/src/main/java/com/android/tools/r8/ir/optimize/classinliner/InlineCandidateProcessor.java
+++ b/src/main/java/com/android/tools/r8/ir/optimize/classinliner/InlineCandidateProcessor.java
@@ -899,6 +899,12 @@
       return false;
     }
 
+    // We cannot guarantee the invoke returns the receiver or another instance and since the
+    // return value is used we have to bail out.
+    if (eligibility.returnsReceiver.isUnknown()) {
+      return false;
+    }
+
     // Add the out-value as a definite-alias if the invoke instruction is guaranteed to return the
     // receiver. Otherwise, the out-value may be an alias of the receiver, and it is added to the
     // may-alias set.
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/classinliner/ClassInlinerCheckCastWithUnknownReturnTest.java b/src/test/java/com/android/tools/r8/ir/optimize/classinliner/ClassInlinerCheckCastWithUnknownReturnTest.java
new file mode 100644
index 0000000..73b7f01
--- /dev/null
+++ b/src/test/java/com/android/tools/r8/ir/optimize/classinliner/ClassInlinerCheckCastWithUnknownReturnTest.java
@@ -0,0 +1,70 @@
+// Copyright (c) 2021, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.ir.optimize.classinliner;
+
+import static com.android.tools.r8.utils.codeinspector.Matchers.isPresent;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import com.android.tools.r8.TestBase;
+import com.android.tools.r8.TestParameters;
+import com.android.tools.r8.TestParametersCollection;
+import com.android.tools.r8.utils.codeinspector.ClassSubject;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+// This is a reproduction of b/176381203.
+@RunWith(Parameterized.class)
+public class ClassInlinerCheckCastWithUnknownReturnTest extends TestBase {
+
+  private final TestParameters parameters;
+
+  @Parameters(name = "{0}")
+  public static TestParametersCollection data() {
+    return getTestParameters().withAllRuntimesAndApiLevels().build();
+  }
+
+  public ClassInlinerCheckCastWithUnknownReturnTest(TestParameters parameters) {
+    this.parameters = parameters;
+  }
+
+  @Test()
+  public void testR8() throws Exception {
+    testForR8(parameters.getBackend())
+        .addInnerClasses(getClass())
+        .addKeepMainRule(Main.class)
+        .setMinApi(parameters.getApiLevel())
+        .run(parameters.getRuntime(), Main.class)
+        .assertFailureWithErrorThatThrows(ClassCastException.class)
+        .inspectFailure(
+            inspector -> {
+              ClassSubject aSubject = inspector.clazz(A.class);
+              assertThat(aSubject, isPresent());
+            });
+  }
+
+  public static class A {
+
+    public int number;
+
+    public Object abs() {
+      if (number == 0) {
+        return new Object();
+      }
+      return this;
+    }
+  }
+
+  public static class Main {
+
+    public static void main(String[] args) {
+      A a = new A();
+      a.number = args.length;
+      A returnedA = (A) (a.abs());
+      System.out.println("Hello World");
+    }
+  }
+}
diff --git a/src/test/java/com/android/tools/r8/ir/optimize/classinliner/ClassInlinerDirectWithUnknownReturnTest.java b/src/test/java/com/android/tools/r8/ir/optimize/classinliner/ClassInlinerDirectWithUnknownReturnTest.java
index 7998888..861dc71 100644
--- a/src/test/java/com/android/tools/r8/ir/optimize/classinliner/ClassInlinerDirectWithUnknownReturnTest.java
+++ b/src/test/java/com/android/tools/r8/ir/optimize/classinliner/ClassInlinerDirectWithUnknownReturnTest.java
@@ -4,20 +4,21 @@
 
 package com.android.tools.r8.ir.optimize.classinliner;
 
-import static com.android.tools.r8.DiagnosticsMatcher.diagnosticMessage;
-import static org.hamcrest.CoreMatchers.containsString;
+import static com.android.tools.r8.utils.codeinspector.Matchers.isPresent;
+import static org.hamcrest.MatcherAssert.assertThat;
 
-import com.android.tools.r8.CompilationFailedException;
 import com.android.tools.r8.TestBase;
 import com.android.tools.r8.TestParameters;
 import com.android.tools.r8.TestParametersCollection;
 import com.android.tools.r8.ir.optimize.Inliner.Reason;
+import com.android.tools.r8.utils.codeinspector.ClassSubject;
 import com.google.common.collect.ImmutableSet;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
+// This is a reproduction of b/176381203.
 @RunWith(Parameterized.class)
 public class ClassInlinerDirectWithUnknownReturnTest extends TestBase {
 
@@ -32,19 +33,22 @@
     this.parameters = parameters;
   }
 
-  @Test(expected = CompilationFailedException.class)
+  @Test()
   public void testR8() throws Exception {
     testForR8(parameters.getBackend())
         .addInnerClasses(getClass())
         .addKeepMainRule(Main.class)
         .setMinApi(parameters.getApiLevel())
         .addOptionsModification(
-            options -> options.testing.validInliningReasons = ImmutableSet.of(Reason.FORCE))
-        .compileWithExpectedDiagnostics(
-            diagnostics -> {
-              diagnostics.assertErrorsMatch(
-                  diagnosticMessage(
-                      containsString("Unexpected values live at entry to first block: [v1]")));
+            options -> {
+              options.testing.validInliningReasons = ImmutableSet.of(Reason.FORCE);
+            })
+        .run(parameters.getRuntime(), Main.class)
+        .assertSuccessWithOutputLines("Hello World 0")
+        .inspect(
+            inspector -> {
+              ClassSubject aSubject = inspector.clazz(A.class);
+              assertThat(aSubject, isPresent());
             });
   }