[Retrace] Prepare for filtering bytes before reaching ClassNameMapper
This is added before actual filtering to be certain that the reading
of bytes and creating strings in itself is competative with
bufferedReader.
Bug: b/226892337
Bug: b/201666766
Bug: b/226885646
Change-Id: I23a338beee56884a3342b5708ba2c4e03ad5caca
diff --git a/src/main/java/com/android/tools/r8/naming/ClassNameMapper.java b/src/main/java/com/android/tools/r8/naming/ClassNameMapper.java
index d67d177..7657adc 100644
--- a/src/main/java/com/android/tools/r8/naming/ClassNameMapper.java
+++ b/src/main/java/com/android/tools/r8/naming/ClassNameMapper.java
@@ -126,8 +126,8 @@
boolean allowEmptyMappedRanges,
boolean allowExperimentalMapping)
throws IOException {
- return mapperFromBufferedReader(
- CharSource.wrap(contents).openBufferedStream(),
+ return mapperFromLineReader(
+ LineReader.fromBufferedReader(CharSource.wrap(contents).openBufferedStream()),
diagnosticsHandler,
allowEmptyMappedRanges,
allowExperimentalMapping);
@@ -144,6 +144,19 @@
boolean allowEmptyMappedRanges,
boolean allowExperimentalMapping)
throws IOException {
+ return mapperFromLineReader(
+ LineReader.fromBufferedReader(reader),
+ diagnosticsHandler,
+ allowEmptyMappedRanges,
+ allowExperimentalMapping);
+ }
+
+ public static ClassNameMapper mapperFromLineReader(
+ LineReader reader,
+ DiagnosticsHandler diagnosticsHandler,
+ boolean allowEmptyMappedRanges,
+ boolean allowExperimentalMapping)
+ throws IOException {
try (ProguardMapReader proguardReader =
new ProguardMapReader(
reader,
@@ -156,8 +169,8 @@
}
}
- public static ClassNameMapper mapperFromBufferedReaderWithFiltering(
- BufferedReader reader,
+ public static ClassNameMapper mapperFromLineReaderWithFiltering(
+ LineReader reader,
DiagnosticsHandler diagnosticsHandler,
boolean allowEmptyMappedRanges,
boolean allowExperimentalMapping,
diff --git a/src/main/java/com/android/tools/r8/naming/LineReader.java b/src/main/java/com/android/tools/r8/naming/LineReader.java
new file mode 100644
index 0000000..a3f7cc3
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/naming/LineReader.java
@@ -0,0 +1,39 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.naming;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+
+/** This is an abstraction over BufferedReader */
+public interface LineReader {
+
+ String readLine() throws IOException;
+
+ void close() throws IOException;
+
+ static LineReader fromBufferedReader(BufferedReader bufferedReader) {
+ return new BufferedLineReader(bufferedReader);
+ }
+
+ class BufferedLineReader implements LineReader {
+
+ private final BufferedReader bufferedReader;
+
+ private BufferedLineReader(BufferedReader bufferedReader) {
+ this.bufferedReader = bufferedReader;
+ }
+
+ @Override
+ public String readLine() throws IOException {
+ return bufferedReader.readLine();
+ }
+
+ @Override
+ public void close() throws IOException {
+ bufferedReader.close();
+ }
+ }
+}
diff --git a/src/main/java/com/android/tools/r8/naming/ProguardMapReader.java b/src/main/java/com/android/tools/r8/naming/ProguardMapReader.java
index 3e6816b..bbea4c4 100644
--- a/src/main/java/com/android/tools/r8/naming/ProguardMapReader.java
+++ b/src/main/java/com/android/tools/r8/naming/ProguardMapReader.java
@@ -17,7 +17,6 @@
import com.android.tools.r8.utils.StringUtils;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
-import java.io.BufferedReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedList;
@@ -62,7 +61,7 @@
*/
public class ProguardMapReader implements AutoCloseable {
- private final BufferedReader reader;
+ private final LineReader reader;
private final JsonParser jsonParser = new JsonParser();
private final DiagnosticsHandler diagnosticsHandler;
private final boolean allowEmptyMappedRanges;
@@ -74,7 +73,7 @@
}
ProguardMapReader(
- BufferedReader reader,
+ LineReader reader,
DiagnosticsHandler diagnosticsHandler,
boolean allowEmptyMappedRanges,
boolean allowExperimentalMapping) {
diff --git a/src/main/java/com/android/tools/r8/naming/SeedMapper.java b/src/main/java/com/android/tools/r8/naming/SeedMapper.java
index 40fc12f..bf8044e 100644
--- a/src/main/java/com/android/tools/r8/naming/SeedMapper.java
+++ b/src/main/java/com/android/tools/r8/naming/SeedMapper.java
@@ -81,7 +81,9 @@
private static SeedMapper seedMapperFromInputStream(Reporter reporter, InputStream in)
throws IOException {
- BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
+ LineReader reader =
+ LineReader.fromBufferedReader(
+ new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)));
try (ProguardMapReader proguardReader = new ProguardMapReader(reader, reporter, false, false)) {
SeedMapper.Builder builder = SeedMapper.builder(reporter);
proguardReader.parse(builder);
diff --git a/src/main/java/com/android/tools/r8/retrace/ProguardMapProducer.java b/src/main/java/com/android/tools/r8/retrace/ProguardMapProducer.java
index c3f94be..f2b6783 100644
--- a/src/main/java/com/android/tools/r8/retrace/ProguardMapProducer.java
+++ b/src/main/java/com/android/tools/r8/retrace/ProguardMapProducer.java
@@ -6,28 +6,53 @@
import com.android.tools.r8.Keep;
import com.google.common.primitives.Bytes;
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
+import java.io.InputStream;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
import java.nio.file.Path;
/** Interface for producing a string format of a mapping file. */
@Keep
public interface ProguardMapProducer {
- Reader get() throws IOException;
+ InputStream get() throws IOException;
+
+ default boolean isFileBacked() {
+ return false;
+ }
+
+ default Path getPath() throws FileNotFoundException {
+ return null;
+ }
static ProguardMapProducer fromString(String proguardMapString) {
- return () -> new StringReader(proguardMapString);
+ return () -> new ByteArrayInputStream(proguardMapString.getBytes(StandardCharsets.UTF_8));
}
static ProguardMapProducer fromPath(Path path) {
- return () -> Files.newBufferedReader(path, StandardCharsets.UTF_8);
+ return new ProguardMapProducer() {
+ @Override
+ public InputStream get() throws IOException {
+ return new BufferedInputStream(new FileInputStream(path.toFile()));
+ }
+
+ @Override
+ public boolean isFileBacked() {
+ return true;
+ }
+
+ @Override
+ public Path getPath() {
+ return path;
+ }
+ };
}
static ProguardMapProducer fromBytes(byte[]... partitions) {
- return fromString(new String(Bytes.concat(partitions), StandardCharsets.UTF_8));
+ return () -> new ByteArrayInputStream(Bytes.concat(partitions));
}
}
diff --git a/src/main/java/com/android/tools/r8/retrace/Retrace.java b/src/main/java/com/android/tools/r8/retrace/Retrace.java
index 01dc051..82a04e6 100644
--- a/src/main/java/com/android/tools/r8/retrace/Retrace.java
+++ b/src/main/java/com/android/tools/r8/retrace/Retrace.java
@@ -33,9 +33,9 @@
import com.google.common.base.Equivalence.Wrapper;
import com.google.common.io.CharStreams;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
-import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -302,9 +302,10 @@
Timing timing = Timing.create("R8 retrace", command.printMemory());
RetraceOptions options = command.getOptions();
if (command.getOptions().isVerifyMappingFileHash()) {
- try (Reader reader = options.getProguardMapProducer().get()) {
+ try (InputStream reader = options.getProguardMapProducer().get()) {
VerifyMappingFileHashResult checkResult =
- ProguardMapChecker.validateProguardMapHash(CharStreams.toString(reader));
+ ProguardMapChecker.validateProguardMapHash(
+ CharStreams.toString(new InputStreamReader(reader, Charsets.UTF_8)));
if (checkResult.isError()) {
command
.getOptions()
diff --git a/src/main/java/com/android/tools/r8/retrace/internal/ProguardMapReaderWithFiltering.java b/src/main/java/com/android/tools/r8/retrace/internal/ProguardMapReaderWithFiltering.java
new file mode 100644
index 0000000..81fdd38
--- /dev/null
+++ b/src/main/java/com/android/tools/r8/retrace/internal/ProguardMapReaderWithFiltering.java
@@ -0,0 +1,213 @@
+// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+package com.android.tools.r8.retrace.internal;
+
+import static java.lang.Integer.MAX_VALUE;
+
+import com.android.tools.r8.naming.LineReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+public abstract class ProguardMapReaderWithFiltering implements LineReader {
+
+ private int startIndex = 0;
+ private int endIndex = 0;
+
+ public abstract byte[] read() throws IOException;
+
+ public abstract int getStartIndex();
+
+ public abstract int getEndIndex();
+
+ public abstract boolean exceedsBuffer();
+
+ public String readLine() throws IOException {
+ byte[] bytes = readLineFromMultipleReads();
+ if (bytes == null) {
+ return null;
+ }
+ return new String(bytes, startIndex, endIndex - startIndex, StandardCharsets.UTF_8);
+ }
+
+ private byte[] readLineFromMultipleReads() throws IOException {
+ startIndex = 0;
+ endIndex = 0;
+ byte[] currentReadBytes = null;
+ do {
+ byte[] readBytes = read();
+ if (readBytes == null) {
+ return currentReadBytes;
+ }
+ if (exceedsBuffer() || currentReadBytes != null) {
+ // We are building up a partial result where all bytes will be present in the
+ // currentReadBytes array.
+ int thisLength = getEndIndex() - getStartIndex();
+ int currentReadBytesLength = currentReadBytes == null ? 0 : currentReadBytes.length;
+ byte[] newReadBytes = new byte[thisLength + currentReadBytesLength];
+ if (currentReadBytes != null) {
+ System.arraycopy(currentReadBytes, 0, newReadBytes, 0, currentReadBytes.length);
+ }
+ System.arraycopy(
+ readBytes, getStartIndex(), newReadBytes, currentReadBytesLength, thisLength);
+ currentReadBytes = newReadBytes;
+ endIndex = newReadBytes.length;
+ } else {
+ currentReadBytes = readBytes;
+ startIndex = getStartIndex();
+ endIndex = getEndIndex();
+ }
+ } while (exceedsBuffer());
+ return currentReadBytes;
+ }
+
+ public static class ProguardMapReaderWithFilteringMappedBuffer
+ extends ProguardMapReaderWithFiltering {
+
+ private final int PAGE_SIZE = 1024 * 8;
+
+ private final FileChannel fileChannel;
+ private MappedByteBuffer mappedByteBuffer;
+ private final long channelSize;
+ private final byte[] buffer = new byte[PAGE_SIZE];
+
+ private int currentPosition = 0;
+ private int temporaryBufferPosition = 0;
+
+ public ProguardMapReaderWithFilteringMappedBuffer(Path mappingFile) throws IOException {
+ fileChannel = FileChannel.open(mappingFile, StandardOpenOption.READ);
+ channelSize = fileChannel.size();
+ readFromChannel();
+ }
+
+ private void readFromChannel() throws IOException {
+ mappedByteBuffer =
+ fileChannel.map(
+ MapMode.READ_ONLY,
+ currentPosition,
+ Math.min(channelSize - currentPosition, MAX_VALUE));
+ }
+
+ @Override
+ public byte[] read() throws IOException {
+ if (currentPosition >= channelSize) {
+ return null;
+ }
+ temporaryBufferPosition = 0;
+ while (currentPosition < channelSize) {
+ if (!mappedByteBuffer.hasRemaining()) {
+ readFromChannel();
+ }
+ byte readByte = readByte();
+ if (readByte == '\n') {
+ break;
+ }
+ buffer[temporaryBufferPosition++] = readByte;
+ if (temporaryBufferPosition == PAGE_SIZE) {
+ break;
+ }
+ }
+ return buffer;
+ }
+
+ @Override
+ public int getStartIndex() {
+ return 0;
+ }
+
+ @Override
+ public int getEndIndex() {
+ if (temporaryBufferPosition > 0 && buffer[temporaryBufferPosition - 1] == '\r') {
+ return temporaryBufferPosition - 1;
+ }
+ return temporaryBufferPosition;
+ }
+
+ @Override
+ public boolean exceedsBuffer() {
+ return temporaryBufferPosition == PAGE_SIZE;
+ }
+
+ private byte readByte() {
+ currentPosition += 1;
+ return mappedByteBuffer.get();
+ }
+
+ @Override
+ public void close() throws IOException {
+ fileChannel.close();
+ }
+ }
+
+ public static class ProguardMapReaderWithFilteringInputBuffer
+ extends ProguardMapReaderWithFiltering {
+
+ private final InputStream inputStream;
+
+ private final int BUFFER_SIZE = 1024 * 8;
+ private final byte[] buffer = new byte[BUFFER_SIZE];
+
+ private int bufferIndex = BUFFER_SIZE;
+ private int startIndex = 0;
+ private int endIndex = 0;
+ private int endReadIndex = 0;
+
+ public ProguardMapReaderWithFilteringInputBuffer(InputStream inputStream) {
+ this.inputStream = inputStream;
+ }
+
+ @Override
+ public void close() throws IOException {
+ inputStream.close();
+ }
+
+ @Override
+ public byte[] read() throws IOException {
+ if (bufferIndex >= endReadIndex) {
+ endReadIndex = inputStream.read(buffer);
+ if (endReadIndex == -1) {
+ return null;
+ }
+ bufferIndex = 0;
+ }
+ startIndex = bufferIndex;
+ boolean foundLineBreak = false;
+ for (endIndex = startIndex; endIndex < endReadIndex; endIndex++) {
+ if (buffer[endIndex] == '\n') {
+ foundLineBreak = true;
+ break;
+ }
+ }
+ bufferIndex = endIndex;
+ if (foundLineBreak) {
+ bufferIndex += 1;
+ }
+ return buffer;
+ }
+
+ @Override
+ public int getStartIndex() {
+ return startIndex;
+ }
+
+ @Override
+ public int getEndIndex() {
+ if (endIndex > 0 && buffer[endIndex - 1] == '\r') {
+ return endIndex - 1;
+ }
+ return endIndex;
+ }
+
+ @Override
+ public boolean exceedsBuffer() {
+ return endReadIndex == endIndex;
+ }
+ }
+}
diff --git a/src/main/java/com/android/tools/r8/retrace/internal/ProguardMappingProviderBuilderImpl.java b/src/main/java/com/android/tools/r8/retrace/internal/ProguardMappingProviderBuilderImpl.java
index e18adca..cb6991e 100644
--- a/src/main/java/com/android/tools/r8/retrace/internal/ProguardMappingProviderBuilderImpl.java
+++ b/src/main/java/com/android/tools/r8/retrace/internal/ProguardMappingProviderBuilderImpl.java
@@ -6,11 +6,13 @@
import com.android.tools.r8.DiagnosticsHandler;
import com.android.tools.r8.naming.ClassNameMapper;
+import com.android.tools.r8.naming.LineReader;
import com.android.tools.r8.references.ClassReference;
import com.android.tools.r8.retrace.InvalidMappingFileException;
import com.android.tools.r8.retrace.ProguardMapProducer;
import com.android.tools.r8.retrace.ProguardMappingProvider;
-import java.io.BufferedReader;
+import com.android.tools.r8.retrace.internal.ProguardMapReaderWithFiltering.ProguardMapReaderWithFilteringInputBuffer;
+import com.android.tools.r8.retrace.internal.ProguardMapReaderWithFiltering.ProguardMapReaderWithFilteringMappedBuffer;
import java.util.HashSet;
import java.util.Set;
@@ -62,23 +64,17 @@
@Override
public ProguardMappingProvider build() {
try {
- if (allowLookupAllClasses) {
- return new ProguardMappingProviderImpl(
- ClassNameMapper.mapperFromBufferedReader(
- new BufferedReader(proguardMapProducer.get()),
- diagnosticsHandler,
- true,
- allowExperimental));
- } else {
- return new ProguardMappingProviderImpl(
- ClassNameMapper.mapperFromBufferedReaderWithFiltering(
- new BufferedReader(proguardMapProducer.get()),
- diagnosticsHandler,
- true,
- allowExperimental,
- allowedLookup),
- allowedLookup);
- }
+ LineReader reader =
+ proguardMapProducer.isFileBacked()
+ ? new ProguardMapReaderWithFilteringMappedBuffer(proguardMapProducer.getPath())
+ : new ProguardMapReaderWithFilteringInputBuffer(proguardMapProducer.get());
+ return new ProguardMappingProviderImpl(
+ ClassNameMapper.mapperFromLineReaderWithFiltering(
+ reader,
+ diagnosticsHandler,
+ true,
+ allowExperimental,
+ allowLookupAllClasses ? null : allowedLookup));
} catch (Exception e) {
throw new InvalidMappingFileException(e);
}
diff --git a/src/test/java/com/android/tools/r8/retrace/RetraceCommandLineTests.java b/src/test/java/com/android/tools/r8/retrace/RetraceCommandLineTests.java
index d348b54..b1e37b4 100644
--- a/src/test/java/com/android/tools/r8/retrace/RetraceCommandLineTests.java
+++ b/src/test/java/com/android/tools/r8/retrace/RetraceCommandLineTests.java
@@ -126,6 +126,16 @@
}
@Test
+ public void testWindowsLineEndings() throws IOException {
+ ActualRetraceBotStackTrace stackTrace = new ActualRetraceBotStackTrace();
+ runTest(
+ stackTrace.mapping().replace("\n", "\r\n"),
+ StringUtils.joinLines(stackTrace.obfuscatedStackTrace()),
+ false,
+ StringUtils.joinLines(stackTrace.retracedStackTrace()) + StringUtils.LINE_SEPARATOR);
+ }
+
+ @Test
public void testRegularExpression() throws IOException {
ActualRetraceBotStackTrace stackTrace = new ActualRetraceBotStackTrace();
runTest(
