From 55500dd5bc4456517c7266e67e7a251e48a6ed5d Mon Sep 17 00:00:00 2001 From: Theresa Eimer Date: Mon, 30 Sep 2024 17:23:36 +0200 Subject: [PATCH] fix maximization logging --- Makefile | 15 ++- README.md | 13 +-- examples/configs/branin_carps_hebo.yaml | 4 +- examples/configs/branin_carps_smac.yaml | 4 +- examples/configs/branin_nevergrad.yaml | 4 +- examples/configs/branin_rs.yaml | 4 +- examples/configs/branin_smac.yaml | 4 +- examples/configs/mlp_carps_smac.yaml | 4 +- examples/configs/mlp_dehb.yaml | 4 +- examples/configs/mlp_hebo.yaml | 4 +- examples/configs/mlp_smac.yaml | 4 +- examples/configs/sac_pb2.yaml | 4 +- examples/configs/sac_pbt.yaml | 4 +- hydra_plugins/hyper_pbt/bg_pbt_utils.py | 9 +- hydra_plugins/hyper_pbt/pb2_utils.py | 9 +- .../hypersweeper/hypersweeper_sweeper.py | 10 +- pyproject.toml | 1 - tests/test_maximize.py | 31 +++++++ tests/test_pbt.py | 0 tests/test_smac.py | 93 ------------------- 20 files changed, 77 insertions(+), 148 deletions(-) create mode 100644 tests/test_maximize.py delete mode 100644 tests/test_pbt.py delete mode 100644 tests/test_smac.py diff --git a/Makefile b/Makefile index 7be65ea..648c81d 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,6 @@ clean-build: ## remove build artifacts rm -fr dist/ rm -fr .eggs/ find . -name '*.egg-info' -exec rm -fr {} + - find . -name '*.egg' -exec rm -f {} + clean-pyc: ## remove Python file artifacts find . -name '*.pyc' -exec rm -f {} + @@ -51,11 +50,11 @@ clean-docs: ## remove docs artifacts cd docs && make clean ruff: ## run ruff as a formatter - python -m ruff format hydra_plugins - python -m ruff check --silent --exit-zero --no-cache --fix hydra_plugins - python -m ruff check --exit-zero hydra_plugins + uvx ruff format hydra_plugins + uvx ruff check --silent --exit-zero --no-cache --fix hydra_plugins + uvx ruff check --exit-zero hydra_plugins isort: - python -m isort hydra_plugins tests + uvx isort hydra_plugins tests test: ## run tests quickly with the default Python python -m pytest tests @@ -99,13 +98,13 @@ dist: clean ## builds source and wheel package python setup.py bdist_wheel ls -l dist install: clean ## install the package to the active Python's site-packages - pip install -e . --config-settings editable_mode=compat + uv pip install -e . --config-settings editable_mode=compat install-dev: clean ## install the package to the active Python's site-packages - pip install -e ".[dev,examples,doc,all]" --config-settings editable_mode=compat + uv pip install -e ".[dev,examples,doc,all]" --config-settings editable_mode=compat check: - pre-commit run --all-files + uvx pre-commit run --all-files format: make ruff diff --git a/README.md b/README.md index 152b561..e1d861b 100644 --- a/README.md +++ b/README.md @@ -16,22 +16,15 @@ We recommend installing hypersweeper via a uv virtual environment: ```bash pip install uv -uv init -uv sync +uv venv --python 3.10 +make install ``` -For extra dependencies, add them to the sync command like this: +For extra dependencies, add them like this: ```bash uv sync --extra dev --extra carps ``` -Alternatively you can also install in a fresh conda environment: - -```bash -conda create -n hypersweeper python=3.10 -make install -``` - ## Basic Usage To use the sweeper, you need to specify a target function with a hydra interface (see our examples). diff --git a/examples/configs/branin_carps_hebo.yaml b/examples/configs/branin_carps_hebo.yaml index 4dce2ad..abdc364 100644 --- a/examples/configs/branin_carps_hebo.yaml +++ b/examples/configs/branin_carps_hebo.yaml @@ -16,6 +16,6 @@ hydra: task: ${carps_task} search_space: ${search_space} run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/branin_carps_hebo/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/branin_carps_hebo/ \ No newline at end of file diff --git a/examples/configs/branin_carps_smac.yaml b/examples/configs/branin_carps_smac.yaml index 6ba30d8..56787ef 100644 --- a/examples/configs/branin_carps_smac.yaml +++ b/examples/configs/branin_carps_smac.yaml @@ -40,6 +40,6 @@ hydra: _partial_: true eta: 3 run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/branin_carps_smac/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/branin_carps_smac/ \ No newline at end of file diff --git a/examples/configs/branin_nevergrad.yaml b/examples/configs/branin_nevergrad.yaml index e19903d..7f229be 100644 --- a/examples/configs/branin_nevergrad.yaml +++ b/examples/configs/branin_nevergrad.yaml @@ -17,6 +17,6 @@ hydra: budget: 100 search_space: ${search_space} run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/branin_nevergrad/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/branin_nevergrad/ \ No newline at end of file diff --git a/examples/configs/branin_rs.yaml b/examples/configs/branin_rs.yaml index 22111ed..5171f36 100644 --- a/examples/configs/branin_rs.yaml +++ b/examples/configs/branin_rs.yaml @@ -13,6 +13,6 @@ hydra: max_budget: 100 search_space: ${search_space} run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/branin_rs/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/branin_rs/ \ No newline at end of file diff --git a/examples/configs/branin_smac.yaml b/examples/configs/branin_smac.yaml index 5509953..b6bada0 100644 --- a/examples/configs/branin_smac.yaml +++ b/examples/configs/branin_smac.yaml @@ -24,6 +24,6 @@ hydra: output_directory: ${hydra.sweep.dir} search_space: ${search_space} run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/branin_smac/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/branin_smac/ \ No newline at end of file diff --git a/examples/configs/mlp_carps_smac.yaml b/examples/configs/mlp_carps_smac.yaml index 77ff558..9752cfd 100644 --- a/examples/configs/mlp_carps_smac.yaml +++ b/examples/configs/mlp_carps_smac.yaml @@ -35,6 +35,6 @@ hydra: eta: 3 search_space: ${search_space} run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/mlp_carps_smac/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/mlp_carps_smac/ \ No newline at end of file diff --git a/examples/configs/mlp_dehb.yaml b/examples/configs/mlp_dehb.yaml index 1eab3df..d681f0d 100644 --- a/examples/configs/mlp_dehb.yaml +++ b/examples/configs/mlp_dehb.yaml @@ -20,6 +20,6 @@ hydra: output_path: ${hydra.sweep.dir} search_space: ${search_space} run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/mlp_dehb/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/mlp_dehb/ \ No newline at end of file diff --git a/examples/configs/mlp_hebo.yaml b/examples/configs/mlp_hebo.yaml index 0bddc43..86c839c 100644 --- a/examples/configs/mlp_hebo.yaml +++ b/examples/configs/mlp_hebo.yaml @@ -11,6 +11,6 @@ hydra: n_trials: 10 search_space: ${search_space} run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/mlp_hebo/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/mlp_hebo/ \ No newline at end of file diff --git a/examples/configs/mlp_smac.yaml b/examples/configs/mlp_smac.yaml index f64ac0f..9641973 100644 --- a/examples/configs/mlp_smac.yaml +++ b/examples/configs/mlp_smac.yaml @@ -28,6 +28,6 @@ hydra: output_directory: ${hydra.sweep.dir} search_space: ${search_space} run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/mlp_smac/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/mlp_smac/ \ No newline at end of file diff --git a/examples/configs/sac_pb2.yaml b/examples/configs/sac_pb2.yaml index a99640b..52f0df7 100644 --- a/examples/configs/sac_pb2.yaml +++ b/examples/configs/sac_pb2.yaml @@ -20,6 +20,6 @@ hydra: load_tf: true search_space: ${search_space} run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/sac_pb2/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/sac_pb2/ \ No newline at end of file diff --git a/examples/configs/sac_pbt.yaml b/examples/configs/sac_pbt.yaml index 0e9a60c..42a8dac 100644 --- a/examples/configs/sac_pbt.yaml +++ b/examples/configs/sac_pbt.yaml @@ -19,6 +19,6 @@ hydra: load_tf: true search_space: ${search_space} run: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} + dir: ./tmp/sac_pbt/ sweep: - dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file + dir: ./tmp/sac_pbt/ \ No newline at end of file diff --git a/hydra_plugins/hyper_pbt/bg_pbt_utils.py b/hydra_plugins/hyper_pbt/bg_pbt_utils.py index 80ffd64..1a0f4f3 100644 --- a/hydra_plugins/hyper_pbt/bg_pbt_utils.py +++ b/hydra_plugins/hyper_pbt/bg_pbt_utils.py @@ -1,4 +1,5 @@ """All of this is copied/lightly adapted from the original BG-PBT code: https://github.com/xingchenwan/bgpbt.""" + from __future__ import annotations import logging @@ -167,15 +168,15 @@ def construct_bounding_box( weights = weights / np.prod(np.power(weights, 1.0 / len(weights))) lb, ub = np.zeros_like(x), np.ones_like(x) for i, _dim in enumerate(x): - if np.isnan(x[i]) or i >= len(cs): + if np.isnan(_dim) or i >= len(cs): lb[i], ub[i] = 0.0, 1.0 else: hp = cs[cs.get_hyperparameter_by_idx(i)] if type(hp) == CSH.CategoricalHyperparameter: lb[i], ub[i] = 0, len(hp.choices) else: - lb[i] = np.clip(x[i] - weights[i] * tr_length / 2.0, 0.0, 1.0) - ub[i] = np.clip(x[i] + weights[i] * tr_length / 2.0, 0.0, 1.0) + lb[i] = np.clip(_dim - weights[i] * tr_length / 2.0, 0.0, 1.0) + ub[i] = np.clip(_dim + weights[i] * tr_length / 2.0, 0.0, 1.0) if type(hp) in [ CSH.UniformIntegerHyperparameter, CSH.NormalIntegerHyperparameter, @@ -794,7 +795,7 @@ def rbf(d, ard): if exp == "rbf": k_cat = rbf(diff1, self.ard_num_dims is not None and self.ard_num_dims > 1) else: - raise ValueError("Exponentiation scheme %s is not recognised!" % exp) + raise ValueError(f"Exponentiation scheme {exp} is not recognised!") if diag: return torch.diag(k_cat).float() return k_cat.float() diff --git a/hydra_plugins/hyper_pbt/pb2_utils.py b/hydra_plugins/hyper_pbt/pb2_utils.py index 83d5f6a..fbda663 100644 --- a/hydra_plugins/hyper_pbt/pb2_utils.py +++ b/hydra_plugins/hyper_pbt/pb2_utils.py @@ -33,8 +33,7 @@ def __init__(self, input_dim, variance=1.0, lengthscale=1.0, epsilon=0.0, active def K(self, X, X2): """Compute the kernel.""" # time must be in the far left column - if self.epsilon > 0.5: # noqa: PLR2004 - self.epsilon = 0.5 + self.epsilon = min(self.epsilon, 0.5) if X2 is None: X2 = np.copy(X) T1 = X[:, 0].reshape(-1, 1) @@ -188,11 +187,9 @@ def K2(self, x, x2): def K(self, x, x2): """Compute the kernel.""" # clip epsilons - if self.epsilon_1 > 0.5: # noqa: PLR2004 - self.epsilon_1 = 0.5 + self.epsilon_1 = min(self.epsilon_1, 0.5) - if self.epsilon_2 > 0.5: # noqa: PLR2004 - self.epsilon_2 = 0.5 + self.epsilon_2 = min(self.epsilon_2, 0.5) # format data if x2 is None: diff --git a/hydra_plugins/hypersweeper/hypersweeper_sweeper.py b/hydra_plugins/hypersweeper/hypersweeper_sweeper.py index 9ef6a5c..02672a7 100644 --- a/hydra_plugins/hypersweeper/hypersweeper_sweeper.py +++ b/hydra_plugins/hypersweeper/hypersweeper_sweeper.py @@ -271,8 +271,6 @@ def run_configs(self, infos): for j in range(len(overrides)): performances.append(res[j].return_value) self.trials_run += 1 - if self.maximize: - performances = [-p for p in performances] return performances, costs def get_save_path(self, config_id, seed=None): @@ -308,7 +306,10 @@ def get_incumbent(self): Float Best performance value """ - best_current_id = np.argmin(self.history["performances"]) + if self.maximize: + best_current_id = np.argmax(self.history["performances"]) + else: + best_current_id = np.argmin(self.history["performances"]) inc_performance = self.history["performances"][best_current_id] inc_config = self.history["configs"][best_current_id] return inc_config, inc_performance @@ -381,7 +382,7 @@ def write_history(self): for i in range(len(configs)): current_config = configs[i] line = [] - for k in keywords[3:]: + for k in keywords[4:]: if k in current_config: line.append(current_config[k]) elif k in self.global_overrides: @@ -451,6 +452,7 @@ def run(self, verbose=False): if self.seeds and self.deterministic: seeds = np.zeros(len(performances)) for info, performance, cost in zip(infos, performances, costs, strict=True): + logged_performance = performance if not self.maximize else -performance value = Result(performance=logged_performance, cost=cost) self.optimizer.tell(info=info, value=value) self.record_iteration(performances, configs, budgets) diff --git a/pyproject.toml b/pyproject.toml index 50beb07..384f74b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,6 @@ changelog_start_rev = "1.0.0" [tool.ruff] target-version = "py310" line-length = 120 -show-source = true src = ["src", "tests", "examples"] lint.extend-safe-fixes = ["ALL"] diff --git a/tests/test_maximize.py b/tests/test_maximize.py new file mode 100644 index 0000000..885ad3e --- /dev/null +++ b/tests/test_maximize.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +"""Tests for `hypersweeper` package.""" +from __future__ import annotations +import json +import shutil +import subprocess +import pandas as pd +import numpy as np +from pathlib import Path + +def test_non_max_incumbent(): + subprocess.call(["python", "examples/branin.py", "--config-name=branin_rs", "-m"]) + assert Path("./tmp/branin_rs").exists(), "Run directory not created" + runhistory = pd.read_csv("./tmp/branin_rs/runhistory.csv") + with open(Path("./tmp/branin_rs/incumbent.json")) as f: + last_line = f.readlines()[-1] + incumbent = json.loads(last_line) + assert np.round(incumbent["score"], decimals=3) == np.round(runhistory["performance"].min(), decimals=3), "Incumbent is not the minimum score in the runhistory" + shutil.rmtree("./tmp") + +def test_max_incumbent(): + subprocess.call(["python", "examples/branin.py", "--config-name=branin_rs", "-m", "+hydra.sweeper.sweeper_kwargs.maximize=True"]) + assert Path("./tmp/branin_rs").exists(), "Run directory not created" + runhistory = pd.read_csv("./tmp/branin_rs/runhistory.csv") + with open(Path("./tmp/branin_rs/incumbent.json")) as f: + last_line = f.readlines()[-1] + incumbent = json.loads(last_line) + print(incumbent["score"], runhistory["performance"].max(), runhistory.performance.min()) + print(runhistory.values) + assert np.round(incumbent["score"], decimals=3) == np.round(runhistory["performance"].max(), decimals=3), "Incumbent is not the maximum score in the runhistory even though maximize is enabled" + shutil.rmtree("./tmp") \ No newline at end of file diff --git a/tests/test_pbt.py b/tests/test_pbt.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_smac.py b/tests/test_smac.py deleted file mode 100644 index 1127be2..0000000 --- a/tests/test_smac.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import annotations - -from ConfigSpace import Categorical, ConfigurationSpace, Float, Integer -from omegaconf import OmegaConf - -BBO_CONFIG = { - "smac_facade": OmegaConf.create( - { - "_target_": "smac.facade.blackbox_facade.BlackBoxFacade", - "_partial_": True, - "logging_level": 20, - } - ), - "scenario": { - "seed": 42, - "n_trials": 10, - "deterministic": True, - "n_workers": 4, - "name": "bb_test", - }, -} - -MF_CONFIG = { - "smac_facade": OmegaConf.create( - { - "_target_": "smac.facade.multi_fidelity_facade.MultiFidelityFacade", - "_partial_": True, - "logging_level": 20, - } - ), - "intensifier": OmegaConf.create( - { - "_target_": "smac.facade.multi_fidelity_facade.MultiFidelityFacade.get_intensifier", - "_partial_": True, - "eta": 3, - } - ), - "scenario": { - "seed": 42, - "n_trials": 10, - "deterministic": True, - "n_workers": 1, - "min_budget": 5, - "max_budget": 50, - "name": "mf_test", - }, -} - -DEFAULT_CONFIG_SPACE = ConfigurationSpace( - { - Float("a", bounds=[0.0, 1.0]), - Integer("b", bounds=[1, 10]), - Categorical("c", ["a", "b", "c"]), - } -) - -# ISSUE HERE: scenario saving doesn't work for some reason with this config. -# Not sure why and if this is a SMAC issue or not. - - -# class TestHyperSMAC: -# def setup(self, mf=False): -# configspace = DEFAULT_CONFIG_SPACE -# hyper_smac_args = deepcopy(BBO_CONFIG) if not mf else deepcopy(MF_CONFIG) -# hyper_smac_args["intensifier"] = instantiate(hyper_smac_args["intensifier"]) if mf else None -# hyper_smac_args["smac_facade"] = instantiate(hyper_smac_args["smac_facade"]) -# return make_smac(configspace, hyper_smac_args) - -# def test_init(self): -# hyper_smac = self.setup() -# assert hyper_smac.smac is not None, "SMAC is not initialized" - -# hyper_smac_mf = self.setup(mf=True) -# assert hyper_smac_mf.smac is not None, "SMAC is not initialized" - -# def test_ask(self): -# hyper_smac = self.setup(mf=True) -# info, _ = hyper_smac.ask() -# assert info.config is not None, "Configuration is not generated" -# assert isinstance(info, Info), "Return value is not an Info object" -# assert info.budget is not None, "Budget is not generated" -# assert info.load_path is None, "Load path is set" - -# def tell(self): -# hyper_smac = self.setup() -# info, _ = hyper_smac.ask() -# result = Result(0.0, 0.0) -# hyper_smac.tell(info, result) - -# hyper_smac = self.setup(mf=True) -# info, _ = hyper_smac.ask() -# result = Result(0.0, 0.0) -# hyper_smac.tell(info, result)