fix maximization logging

Makefile

@@ -33,7 +33,6 @@ clean-build: ## remove build artifacts
 	rm -fr dist/
 	rm -fr .eggs/
 	find . -name '*.egg-info' -exec rm -fr {} +
-	find . -name '*.egg' -exec rm -f {} +
 
 clean-pyc: ## remove Python file artifacts
 	find . -name '*.pyc' -exec rm -f {} +
@@ -51,11 +50,11 @@ clean-docs: ## remove docs artifacts
 	cd docs && make clean
 
 ruff: ## run ruff as a formatter
-	python -m ruff format hydra_plugins
-	python -m ruff check --silent --exit-zero --no-cache --fix hydra_plugins
-	python -m ruff check --exit-zero hydra_plugins
+	uvx ruff format hydra_plugins
+	uvx ruff check --silent --exit-zero --no-cache --fix hydra_plugins
+	uvx ruff check --exit-zero hydra_plugins
 isort:
-	python -m isort hydra_plugins tests
+	uvx isort hydra_plugins tests
 
 test: ## run tests quickly with the default Python
 	python -m pytest tests
@@ -99,13 +98,13 @@ dist: clean ## builds source and wheel package
 	python setup.py bdist_wheel
 	ls -l dist
 install: clean ## install the package to the active Python's site-packages
-	pip install -e . --config-settings editable_mode=compat
+	uv pip install -e . --config-settings editable_mode=compat
 
 install-dev: clean ## install the package to the active Python's site-packages
-	pip install -e ".[dev,examples,doc,all]" --config-settings editable_mode=compat
+	uv pip install -e ".[dev,examples,doc,all]" --config-settings editable_mode=compat
 
 check:
-	pre-commit run --all-files
+	uvx pre-commit run --all-files
 
 format:
 	make ruff

README.md

@@ -16,22 +16,15 @@ We recommend installing hypersweeper via a uv virtual environment:
 
 ```bash
 pip install uv
-uv init
-uv sync
+uv venv --python 3.10
+make install
 ```
 
-For extra dependencies, add them to the sync command like this:
+For extra dependencies, add them like this:
 ```bash
 uv sync --extra dev --extra carps
 ```
 
-Alternatively you can also install in a fresh conda environment:
-
-```bash
-conda create -n hypersweeper python=3.10
-make install
-```
-
 ## Basic Usage
 
 To use the sweeper, you need to specify a target function with a hydra interface (see our examples). 

examples/configs/branin_carps_hebo.yaml

@@ -16,6 +16,6 @@ hydra:
         task: ${carps_task}
     search_space: ${search_space}
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/branin_carps_hebo/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/branin_carps_hebo/

examples/configs/branin_carps_smac.yaml

@@ -40,6 +40,6 @@ hydra:
               _partial_: true
               eta: 3
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/branin_carps_smac/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/branin_carps_smac/

examples/configs/branin_nevergrad.yaml

@@ -17,6 +17,6 @@ hydra:
           budget: 100
     search_space: ${search_space}
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/branin_nevergrad/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/branin_nevergrad/

examples/configs/branin_rs.yaml

@@ -13,6 +13,6 @@ hydra:
       max_budget: 100
     search_space: ${search_space}
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/branin_rs/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/branin_rs/

examples/configs/branin_smac.yaml

@@ -24,6 +24,6 @@ hydra:
           output_directory: ${hydra.sweep.dir}
     search_space: ${search_space}
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/branin_smac/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/branin_smac/

examples/configs/mlp_carps_smac.yaml

@@ -35,6 +35,6 @@ hydra:
               eta: 3
     search_space: ${search_space}
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/mlp_carps_smac/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/mlp_carps_smac/

examples/configs/mlp_dehb.yaml

@@ -20,6 +20,6 @@ hydra:
           output_path: ${hydra.sweep.dir}
     search_space: ${search_space}
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/mlp_dehb/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/mlp_dehb/

examples/configs/mlp_hebo.yaml

@@ -11,6 +11,6 @@ hydra:
     n_trials: 10
     search_space: ${search_space}
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/mlp_hebo/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/mlp_hebo/

examples/configs/mlp_smac.yaml

@@ -28,6 +28,6 @@ hydra:
           output_directory: ${hydra.sweep.dir}
     search_space: ${search_space}
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/mlp_smac/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/mlp_smac/

examples/configs/sac_pb2.yaml

@@ -20,6 +20,6 @@ hydra:
       load_tf: true
     search_space: ${search_space}
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/sac_pb2/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/sac_pb2/

examples/configs/sac_pbt.yaml

@@ -19,6 +19,6 @@ hydra:
       load_tf: true
     search_space: ${search_space}
   run:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/sac_pbt/
   sweep:
-    dir: ./tmp/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: ./tmp/sac_pbt/

hydra_plugins/hyper_pbt/bg_pbt_utils.py

@@ -1,4 +1,5 @@
 """All of this is copied/lightly adapted from the original BG-PBT code: https://github.com/xingchenwan/bgpbt."""
+
 from __future__ import annotations
 
 import logging
@@ -167,15 +168,15 @@ def construct_bounding_box(
     weights = weights / np.prod(np.power(weights, 1.0 / len(weights)))
     lb, ub = np.zeros_like(x), np.ones_like(x)
     for i, _dim in enumerate(x):
-        if np.isnan(x[i]) or i >= len(cs):
+        if np.isnan(_dim) or i >= len(cs):
             lb[i], ub[i] = 0.0, 1.0
         else:
             hp = cs[cs.get_hyperparameter_by_idx(i)]
             if type(hp) == CSH.CategoricalHyperparameter:
                 lb[i], ub[i] = 0, len(hp.choices)
             else:
-                lb[i] = np.clip(x[i] - weights[i] * tr_length / 2.0, 0.0, 1.0)
-                ub[i] = np.clip(x[i] + weights[i] * tr_length / 2.0, 0.0, 1.0)
+                lb[i] = np.clip(_dim - weights[i] * tr_length / 2.0, 0.0, 1.0)
+                ub[i] = np.clip(_dim + weights[i] * tr_length / 2.0, 0.0, 1.0)
                 if type(hp) in [
                     CSH.UniformIntegerHyperparameter,
                     CSH.NormalIntegerHyperparameter,
@@ -794,7 +795,7 @@ def rbf(d, ard):
             if exp == "rbf":
                 k_cat = rbf(diff1, self.ard_num_dims is not None and self.ard_num_dims > 1)
             else:
-                raise ValueError("Exponentiation scheme %s is not recognised!" % exp)
+                raise ValueError(f"Exponentiation scheme {exp} is not recognised!")
             if diag:
                 return torch.diag(k_cat).float()
         return k_cat.float()

hydra_plugins/hyper_pbt/pb2_utils.py

@@ -33,8 +33,7 @@ def __init__(self, input_dim, variance=1.0, lengthscale=1.0, epsilon=0.0, active
     def K(self, X, X2):
         """Compute the kernel."""
         # time must be in the far left column
-        if self.epsilon > 0.5:  # noqa: PLR2004
-            self.epsilon = 0.5
+        self.epsilon = min(self.epsilon, 0.5)
         if X2 is None:
             X2 = np.copy(X)
         T1 = X[:, 0].reshape(-1, 1)
@@ -188,11 +187,9 @@ def K2(self, x, x2):
     def K(self, x, x2):
         """Compute the kernel."""
         # clip epsilons
-        if self.epsilon_1 > 0.5:  # noqa: PLR2004
-            self.epsilon_1 = 0.5
+        self.epsilon_1 = min(self.epsilon_1, 0.5)
 
-        if self.epsilon_2 > 0.5:  # noqa: PLR2004
-            self.epsilon_2 = 0.5
+        self.epsilon_2 = min(self.epsilon_2, 0.5)
 
         # format data
         if x2 is None:

hydra_plugins/hypersweeper/hypersweeper_sweeper.py

@@ -271,8 +271,6 @@ def run_configs(self, infos):
             for j in range(len(overrides)):
                 performances.append(res[j].return_value)
                 self.trials_run += 1
-        if self.maximize:
-            performances = [-p for p in performances]
         return performances, costs
 
     def get_save_path(self, config_id, seed=None):
@@ -308,7 +306,10 @@ def get_incumbent(self):
         Float
             Best performance value
         """
-        best_current_id = np.argmin(self.history["performances"])
+        if self.maximize:
+            best_current_id = np.argmax(self.history["performances"])
+        else:
+            best_current_id = np.argmin(self.history["performances"])
         inc_performance = self.history["performances"][best_current_id]
         inc_config = self.history["configs"][best_current_id]
         return inc_config, inc_performance
@@ -381,7 +382,7 @@ def write_history(self):
             for i in range(len(configs)):
                 current_config = configs[i]
                 line = []
-                for k in keywords[3:]:
+                for k in keywords[4:]:
                     if k in current_config:
                         line.append(current_config[k])
                     elif k in self.global_overrides:
@@ -451,6 +452,7 @@ def run(self, verbose=False):
             if self.seeds and self.deterministic:
                 seeds = np.zeros(len(performances))
             for info, performance, cost in zip(infos, performances, costs, strict=True):
+                logged_performance = performance if not self.maximize else -performance
                 value = Result(performance=logged_performance, cost=cost)
                 self.optimizer.tell(info=info, value=value)
             self.record_iteration(performances, configs, budgets)

pyproject.toml

@@ -97,7 +97,6 @@ changelog_start_rev = "1.0.0"
 [tool.ruff]
 target-version = "py310"
 line-length = 120
-show-source = true
 src = ["src", "tests", "examples"]
 lint.extend-safe-fixes = ["ALL"]
 

tests/test_maximize.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""Tests for `hypersweeper` package."""
+from __future__ import annotations
+import json
+import shutil
+import subprocess
+import pandas as pd
+import numpy as np
+from pathlib import Path
+
+def test_non_max_incumbent():
+    subprocess.call(["python", "examples/branin.py", "--config-name=branin_rs", "-m"])
+    assert Path("./tmp/branin_rs").exists(), "Run directory not created"
+    runhistory = pd.read_csv("./tmp/branin_rs/runhistory.csv")
+    with open(Path("./tmp/branin_rs/incumbent.json")) as f:
+        last_line = f.readlines()[-1]
+        incumbent = json.loads(last_line)
+    assert np.round(incumbent["score"], decimals=3) == np.round(runhistory["performance"].min(), decimals=3), "Incumbent is not the minimum score in the runhistory"
+    shutil.rmtree("./tmp")
+
+def test_max_incumbent():
+    subprocess.call(["python", "examples/branin.py", "--config-name=branin_rs", "-m", "+hydra.sweeper.sweeper_kwargs.maximize=True"])
+    assert Path("./tmp/branin_rs").exists(), "Run directory not created"
+    runhistory = pd.read_csv("./tmp/branin_rs/runhistory.csv")
+    with open(Path("./tmp/branin_rs/incumbent.json")) as f:
+        last_line = f.readlines()[-1]
+        incumbent = json.loads(last_line)
+    print(incumbent["score"], runhistory["performance"].max(), runhistory.performance.min())
+    print(runhistory.values)
+    assert np.round(incumbent["score"], decimals=3) == np.round(runhistory["performance"].max(), decimals=3), "Incumbent is not the maximum score in the runhistory even though maximize is enabled"
+    shutil.rmtree("./tmp")