diff --git a/vizier/_src/benchmarks/analyzers/state_analyzer.py b/vizier/_src/benchmarks/analyzers/state_analyzer.py new file mode 100644 index 000000000..534ab5716 --- /dev/null +++ b/vizier/_src/benchmarks/analyzers/state_analyzer.py @@ -0,0 +1,69 @@ +# Copyright 2023 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +"""Analyzers for BenchmarkStates for fast comparisons and statistics.""" + +from typing import List + +from vizier._src.benchmarks.analyzers import convergence_curve +from vizier._src.benchmarks.runners import benchmark_state + + +class BenchmarkStateAnalyzer: + """Analyzer for BenchmarkStates.""" + + @classmethod + def to_curve( + cls, + states: List[benchmark_state.BenchmarkState], + flip_signs: bool = False, + ) -> convergence_curve.ConvergenceCurve: + """Generates a ConvergenceCurve from a batch of BenchmarkStates. + + Each state in batch should represent the same study (different repeat). + + Args: + states: List of BenchmarkStates. + flip_signs: If true, flip signs of curve when it is MINIMIZE metric. + + Returns: + Convergence curve with batch size equal to length of states. + + Raises: + ValueError: When problem statements are not the same or is multiobjective. + """ + if states is None: + return ValueError('Empty States.') + + problem_statement = states[0].experimenter.problem_statement() + if not problem_statement.is_single_objective: + raise ValueError('Multiobjective Conversion not supported yet.') + + converter = convergence_curve.ConvergenceCurveConverter( + problem_statement.metric_information.item(), flip_signs=flip_signs + ) + curves = [] + for state in states: + if problem_statement != state.experimenter.problem_statement(): + raise ValueError( + f'States must have same problem {problem_statement}' + f' and {state.experimenter.problem_statement()}' + ) + + state_trials = state.algorithm.supporter.GetTrials() + curve = converter.convert(state_trials) + curves.append(curve) + return convergence_curve.ConvergenceCurve.align_xs(curves) diff --git a/vizier/_src/benchmarks/analyzers/state_analyzer_test.py b/vizier/_src/benchmarks/analyzers/state_analyzer_test.py new file mode 100644 index 000000000..ca044152a --- /dev/null +++ b/vizier/_src/benchmarks/analyzers/state_analyzer_test.py @@ -0,0 +1,57 @@ +# Copyright 2023 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from vizier import pyvizier as vz +from vizier._src.algorithms.designers import random +from vizier._src.benchmarks.analyzers import state_analyzer +from vizier._src.benchmarks.experimenters import experimenter_factory +from vizier._src.benchmarks.runners import benchmark_runner +from vizier._src.benchmarks.runners import benchmark_state + +from absl.testing import absltest + + +class StateAnalyzerTest(absltest.TestCase): + + def test_curve_conversion(self): + dim = 10 + experimenter = experimenter_factory.BBOBExperimenterFactory('Sphere', dim)() + + def _designer_factory(config: vz.ProblemStatement, seed: int): + return random.RandomDesigner(config.search_space, seed=seed) + + benchmark_state_factory = benchmark_state.DesignerBenchmarkStateFactory( + designer_factory=_designer_factory, experimenter=experimenter + ) + num_trials = 20 + runner = benchmark_runner.BenchmarkRunner( + benchmark_subroutines=[benchmark_runner.GenerateAndEvaluate()], + num_repeats=num_trials, + ) + + states = [] + num_repeats = 3 + for i in range(num_repeats): + bench_state = benchmark_state_factory(seed=i) + runner.run(bench_state) + states.append(bench_state) + + curve = state_analyzer.BenchmarkStateAnalyzer.to_curve(states) + self.assertEqual(curve.ys.shape, (num_repeats, num_trials)) + + +if __name__ == '__main__': + absltest.main()