src/test/java/com/android/tools/r8/benchmarks/BenchmarkConfig.java

// Copyright (c) 2022, the R8 project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
package com.android.tools.r8.benchmarks;

import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;

public class BenchmarkConfig {

  public static void checkBenchmarkConsistency(BenchmarkConfig benchmark, BenchmarkConfig other) {
    if (benchmark.getTarget().equals(other.getTarget())) {
      throw new BenchmarkConfigError("Duplicate benchmark name and target: " + benchmark);
    }
    if (benchmark.isSingleBenchmark() != other.isSingleBenchmark()) {
      throw new BenchmarkConfigError(
          "Inconsistent single/group benchmark setup: " + benchmark + " and " + other);
    }
    Set<String> subNames =
        Sets.union(benchmark.getSubBenchmarks().keySet(), other.getSubBenchmarks().keySet());
    for (String subName : subNames) {
      if (!Objects.equals(
          benchmark.getSubBenchmarks().get(subName), other.getSubBenchmarks().get(subName))) {
        throw new BenchmarkConfigError(
            "Inconsistent metrics for sub-benchmark "
                + subName
                + " in benchmarks: "
                + benchmark
                + " and "
                + other);
      }
    }
    if (!benchmark.getSuite().equals(other.getSuite())) {
      throw new BenchmarkConfigError(
          "Inconsistent suite for benchmarks: " + benchmark + " and " + other);
    }
    if (benchmark.hasTimeWarmupRuns() != other.hasTimeWarmupRuns()) {
      throw new BenchmarkConfigError(
          "Inconsistent time-warmup for benchmarks: " + benchmark + " and " + other);
    }
  }

  public static Set<BenchmarkMetric> getCommonMetrics(List<BenchmarkConfig> variants) {
    return getConsistentRepresentative(variants).getMetrics();
  }

  public static BenchmarkSuite getCommonSuite(List<BenchmarkConfig> variants) {
    return getConsistentRepresentative(variants).getSuite();
  }

  public static boolean getCommonHasTimeWarmup(List<BenchmarkConfig> variants) {
    return getConsistentRepresentative(variants).hasTimeWarmupRuns();
  }

  public static Map<String, Set<BenchmarkMetric>> getCommonSubBenchmarks(
      List<BenchmarkConfig> variants) {
    return getConsistentRepresentative(variants).getSubBenchmarks();
  }

  // Use the largest configured timeout as the timeout for the full group.
  public static BenchmarkTimeout getCommonTimeout(List<BenchmarkConfig> variants) {
    BenchmarkTimeout timeout = null;
    for (BenchmarkConfig variant : variants) {
      BenchmarkTimeout variantTimeout = variant.getTimeout();
      if (timeout == null) {
        timeout = variantTimeout;
      } else if (variantTimeout != null && timeout.asSeconds() < variantTimeout.asSeconds()) {
        timeout = variantTimeout;
      }
    }
    return timeout;
  }

  private static BenchmarkConfig getConsistentRepresentative(List<BenchmarkConfig> variants) {
    if (variants.isEmpty()) {
      throw new BenchmarkConfigError("Unexpected attempt to check consistency of empty collection");
    }
    BenchmarkConfig representative = variants.get(0);
    for (int i = 1; i < variants.size(); i++) {
      checkBenchmarkConsistency(representative, variants.get(i));
    }
    return representative;
  }

  public static class Builder {

    private String name = null;
    private BenchmarkMethod method = null;
    private BenchmarkTarget target = null;
    private Set<BenchmarkMetric> metrics = new HashSet<>();
    private Map<String, Set<BenchmarkMetric>> subBenchmarks = new HashMap<>();
    private BenchmarkSuite suite = BenchmarkSuite.getDefault();
    private Collection<BenchmarkDependency> dependencies = new ArrayList<>();
    private int fromRevision = -1;
    private BenchmarkTimeout timeout = null;
    private boolean measureWarmup = false;

    private Builder() {}

    public BenchmarkConfig build() {
      if (name == null) {
        throw new BenchmarkConfigError("Benchmark name must be set");
      }
      if (method == null) {
        throw new BenchmarkConfigError("Benchmark method must be set");
      }
      if (target == null) {
        throw new BenchmarkConfigError("Benchmark target must be set");
      }
      if (suite == null) {
        throw new BenchmarkConfigError("Benchmark must have a suite");
      }
      if (fromRevision < 0) {
        throw new BenchmarkConfigError(
            "Benchmark must specify from which golem revision it is valid");
      }
      if (!metrics.isEmpty()) {
        if (subBenchmarks.containsKey(name)) {
          throw new BenchmarkConfigError(
              "Benchmark must not specify both direct metrics and a sub-benchmark of the same"
                  + " name");
        }
        subBenchmarks.put(name, ImmutableSet.copyOf(metrics));
      }
      if (subBenchmarks.isEmpty()) {
        throw new BenchmarkConfigError(
            "Benchmark must have at least one metric / sub-benchmark to measure");
      }
      if (measureWarmup) {
        for (Set<BenchmarkMetric> subMetrics : subBenchmarks.values()) {
          if (subMetrics.contains(BenchmarkMetric.StartupTime)) {
            throw new BenchmarkConfigError(
                "Benchmark cannot both measure warmup and set metric: "
                    + BenchmarkMetric.StartupTime);
          }
        }
      }
      return new BenchmarkConfig(
          name,
          method,
          target,
          subBenchmarks,
          suite,
          fromRevision,
          dependencies,
          timeout,
          measureWarmup);
    }

    public Builder setName(String name) {
      this.name = name;
      return this;
    }

    public Builder setTarget(BenchmarkTarget target) {
      this.target = target;
      return this;
    }

    public Builder setMethod(BenchmarkMethod method) {
      this.method = method;
      return this;
    }

    public Builder measureRunTime() {
      metrics.add(BenchmarkMetric.RunTimeRaw);
      return this;
    }

    public Builder measureCodeSize() {
      metrics.add(BenchmarkMetric.CodeSize);
      return this;
    }

    public Builder measureWarmup() {
      measureWarmup = true;
      return this;
    }

    public Builder addSubBenchmark(String name, BenchmarkMetric... metrics) {
      return addSubBenchmark(name, new HashSet<>(Arrays.asList(metrics)));
    }

    public Builder addSubBenchmark(String name, Set<BenchmarkMetric> metrics) {
      subBenchmarks.put(name, metrics);
      return this;
    }

    public Builder setSuite(BenchmarkSuite suite) {
      this.suite = suite;
      return this;
    }

    public Builder setFromRevision(int fromRevision) {
      this.fromRevision = fromRevision;
      return this;
    }

    public Builder addDependency(BenchmarkDependency dependency) {
      dependencies.add(dependency);
      return this;
    }

    public Builder setTimeout(long duration, TimeUnit unit) {
      timeout = new BenchmarkTimeout(duration, unit);
      return this;
    }
  }

  public static Builder builder() {
    return new Builder();
  }

  private final BenchmarkIdentifier id;
  private final BenchmarkMethod method;
  private final Map<String, Set<BenchmarkMetric>> benchmarks;
  private final BenchmarkSuite suite;
  private final Collection<BenchmarkDependency> dependencies;
  private final int fromRevision;
  private final BenchmarkTimeout timeout;
  private final boolean measureWarmup;

  private BenchmarkConfig(
      String name,
      BenchmarkMethod benchmarkMethod,
      BenchmarkTarget target,
      Map<String, Set<BenchmarkMetric>> benchmarks,
      BenchmarkSuite suite,
      int fromRevision,
      Collection<BenchmarkDependency> dependencies,
      BenchmarkTimeout timeout,
      boolean measureWarmup) {
    this.id = new BenchmarkIdentifier(name, target);
    this.method = benchmarkMethod;
    this.benchmarks = benchmarks;
    this.suite = suite;
    this.fromRevision = fromRevision;
    this.dependencies = dependencies;
    this.timeout = timeout;
    this.measureWarmup = measureWarmup;
  }

  public BenchmarkIdentifier getIdentifier() {
    return id;
  }

  public String getName() {
    return id.getName();
  }

  public String getDependencyDirectoryName() {
    return getName();
  }

  public BenchmarkTarget getTarget() {
    return id.getTarget();
  }

  public boolean isSingleBenchmark() {
    return benchmarks.size() == 1 && benchmarks.containsKey(getName());
  }

  public Map<String, Set<BenchmarkMetric>> getSubBenchmarks() {
    return benchmarks;
  }

  public Set<BenchmarkMetric> getMetrics() {
    if (!isSingleBenchmark()) {
      throw new BenchmarkConfigError("Attempt to get single metrics set from group benchmark");
    }
    return benchmarks.get(getName());
  }

  public BenchmarkSuite getSuite() {
    return suite;
  }

  public int getFromRevision() {
    return fromRevision;
  }

  public boolean hasTimeWarmupRuns() {
    return measureWarmup;
  }

  public Collection<BenchmarkDependency> getDependencies() {
    return dependencies;
  }

  public BenchmarkTimeout getTimeout() {
    return timeout;
  }

  public void run(BenchmarkEnvironment environment) throws Exception {
    method.run(environment);
  }

  @Override
  public String toString() {
    return id.getName() + "/" + id.getTarget().getIdentifierName();
  }
}