Add BenchmarkState analysis tools.

vizier/_src/benchmarks/analyzers/state_analyzer.py

@@ -0,0 +1,69 @@
+# Copyright 2023 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+"""Analyzers for BenchmarkStates for fast comparisons and statistics."""
+
+from typing import List
+
+from vizier._src.benchmarks.analyzers import convergence_curve
+from vizier._src.benchmarks.runners import benchmark_state
+
+
+class BenchmarkStateAnalyzer:
+  """Analyzer for BenchmarkStates."""
+
+  @classmethod
+  def to_curve(
+      cls,
+      states: List[benchmark_state.BenchmarkState],
+      flip_signs: bool = False,
+  ) -> convergence_curve.ConvergenceCurve:
+    """Generates a ConvergenceCurve from a batch of BenchmarkStates.
+
+    Each state in batch should represent the same study (different repeat).
+
+    Args:
+      states: List of BenchmarkStates.
+      flip_signs: If true, flip signs of curve when it is MINIMIZE metric.
+
+    Returns:
+      Convergence curve with batch size equal to length of states.
+
+    Raises:
+      ValueError: When problem statements are not the same or is multiobjective.
+    """
+    if states is None:
+      return ValueError('Empty States.')
+
+    problem_statement = states[0].experimenter.problem_statement()
+    if not problem_statement.is_single_objective:
+      raise ValueError('Multiobjective Conversion not supported yet.')
+
+    converter = convergence_curve.ConvergenceCurveConverter(
+        problem_statement.metric_information.item(), flip_signs=flip_signs
+    )
+    curves = []
+    for state in states:
+      if problem_statement != state.experimenter.problem_statement():
+        raise ValueError(
+            f'States must have same problem {problem_statement}'
+            f' and {state.experimenter.problem_statement()}'
+        )
+
+      state_trials = state.algorithm.supporter.GetTrials()
+      curve = converter.convert(state_trials)
+      curves.append(curve)
+    return convergence_curve.ConvergenceCurve.align_xs(curves)

vizier/_src/benchmarks/analyzers/state_analyzer_test.py

@@ -0,0 +1,57 @@
+# Copyright 2023 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from vizier import pyvizier as vz
+from vizier._src.algorithms.designers import random
+from vizier._src.benchmarks.analyzers import state_analyzer
+from vizier._src.benchmarks.experimenters import experimenter_factory
+from vizier._src.benchmarks.runners import benchmark_runner
+from vizier._src.benchmarks.runners import benchmark_state
+
+from absl.testing import absltest
+
+
+class StateAnalyzerTest(absltest.TestCase):
+
+  def test_curve_conversion(self):
+    dim = 10
+    experimenter = experimenter_factory.BBOBExperimenterFactory('Sphere', dim)()
+
+    def _designer_factory(config: vz.ProblemStatement, seed: int):
+      return random.RandomDesigner(config.search_space, seed=seed)
+
+    benchmark_state_factory = benchmark_state.DesignerBenchmarkStateFactory(
+        designer_factory=_designer_factory, experimenter=experimenter
+    )
+    num_trials = 20
+    runner = benchmark_runner.BenchmarkRunner(
+        benchmark_subroutines=[benchmark_runner.GenerateAndEvaluate()],
+        num_repeats=num_trials,
+    )
+
+    states = []
+    num_repeats = 3
+    for i in range(num_repeats):
+      bench_state = benchmark_state_factory(seed=i)
+      runner.run(bench_state)
+      states.append(bench_state)
+
+    curve = state_analyzer.BenchmarkStateAnalyzer.to_curve(states)
+    self.assertEqual(curve.ys.shape, (num_repeats, num_trials))
+
+
+if __name__ == '__main__':
+  absltest.main()