From c5ad017489752c128d1a85346fa3273ad5a9ca5c Mon Sep 17 00:00:00 2001 From: tommoral Date: Thu, 16 Apr 2026 12:25:49 +0200 Subject: [PATCH 1/4] MNT update benchmark to compat with benchopt 1.9 and use callback API --- datasets/py_bench.py | 6 ++--- datasets/simulated.py | 18 ++++--------- objective.py | 17 +++++------- solvers/basinhopping.py | 10 +++++--- solvers/nevergrad.py | 56 +++++++++++++++++++++++++--------------- solvers/optuna.py | 57 +++++++++++++++++++++++++---------------- solvers/scipy.py | 16 +++++++----- 7 files changed, 100 insertions(+), 80 deletions(-) diff --git a/datasets/py_bench.py b/datasets/py_bench.py index 59cbda4..808ec2f 100644 --- a/datasets/py_bench.py +++ b/datasets/py_bench.py @@ -1,8 +1,6 @@ -from benchopt import safe_import_context from benchopt import BaseDataset -with safe_import_context() as import_ctx: - from PyBenchFCN import SingleObjectiveProblem as SOP +from PyBenchFCN import SingleObjectiveProblem as SOP class Dataset(BaseDataset): @@ -10,7 +8,7 @@ class Dataset(BaseDataset): name = "FCN" install_cmd = "conda" - requirements = ["pip:PyBenchFCN"] + requirements = ["pip::PyBenchFCN"] # List of parameters to generate the datasets. The benchmark will consider # the cross product for each key in the dictionary. diff --git a/datasets/simulated.py b/datasets/simulated.py index 8aeb681..4a7ee24 100644 --- a/datasets/simulated.py +++ b/datasets/simulated.py @@ -1,15 +1,11 @@ -from benchopt import safe_import_context from benchopt import BaseDataset -with safe_import_context() as import_ctx: - import numpy as np +import numpy as np class Dataset(BaseDataset): name = "simulated" - install_cmd = "conda" - requirements = ["numpy"] # List of parameters to generate the datasets. The benchmark will consider # the cross product for each key in the dictionary. @@ -17,12 +13,8 @@ class Dataset(BaseDataset): "dimension": [2, 10], } - def __init__(self, dimension=2, bounds=(-3, 3)): - self.function = lambda x: np.linalg.norm(x, 2) ** 2 - self.dimension = dimension - self.bounds = bounds - def get_data(self): - return dict(function=self.function, - dimension=self.dimension, - bounds=self.bounds) + return dict( + function=lambda x: np.linalg.norm(x, 2) ** 2, + dimension=self.dimension, bounds=(-3, 3) + ) diff --git a/objective.py b/objective.py index 9b6b54d..ccfa0ca 100644 --- a/objective.py +++ b/objective.py @@ -1,25 +1,22 @@ -from benchopt import BaseObjective, safe_import_context +from benchopt import BaseObjective -# Protect import to allow manipulating objective without importing library -# Useful for autocompletion and install commands -with safe_import_context() as import_ctx: - import numpy as np +import numpy as np class Objective(BaseObjective): - min_benchopt_version = "1.3" + min_benchopt_version = "1.9" name = "Zero-order test functions" - def get_one_solution(self): - # Return one solution. This should be compatible with 'self.compute'. - return np.zeros(self.dimension) + def get_one_result(self): + # Return one result for testing purpose. + return dict(x=np.zeros(self.dimension)) def set_data(self, function, dimension, bounds): self.function = function self.dimension = dimension self.bounds = bounds - def compute(self, x): + def evaluate_result(self, x): return self.function(x) def get_objective(self): diff --git a/solvers/basinhopping.py b/solvers/basinhopping.py index dbbed39..296c7d8 100644 --- a/solvers/basinhopping.py +++ b/solvers/basinhopping.py @@ -11,10 +11,9 @@ class Solver(BaseSolver): name = "basinhopping" install_cmd = "conda" - requirements = ["numpy", "scipy"] + requirements = ["scipy"] parameters = { "temperature": [1, 10], - "seed": [42], } def set_objective(self, function, dimension, bounds): @@ -24,7 +23,10 @@ def set_objective(self, function, dimension, bounds): def run(self, n_iter): f = self.function - rng = np.random.RandomState(self.seed) # fix seed + seed = self.get_seed( + use_repetition=True, use_dataset=True, use_solver=True + ) + rng = np.random.RandomState(seed) # fix seed x0 = rng.uniform(size=self.dimension, low=self.bounds[0], high=self.bounds[1]) @@ -35,4 +37,4 @@ def run(self, n_iter): self.xopt = result.x def get_result(self): - return self.xopt.flatten() + return dict(x=self.xopt.flatten()) diff --git a/solvers/nevergrad.py b/solvers/nevergrad.py index 80e211f..e87a3f7 100644 --- a/solvers/nevergrad.py +++ b/solvers/nevergrad.py @@ -1,8 +1,8 @@ -from benchopt import BaseSolver, safe_import_context +from benchopt import BaseSolver +from benchopt.stopping_criterion import SufficientProgressCriterion -with safe_import_context() as import_ctx: - import numpy as np - import nevergrad as ng +import numpy as np +import nevergrad as ng class Solver(BaseSolver): @@ -11,39 +11,55 @@ class Solver(BaseSolver): name = "nevergrad" install_cmd = "conda" - requirements = [ - "nevergrad", - ] + requirements = ["nevergrad"] parameters = { "solver": ["NGOpt", "RandomSearch", "ScrHammersleySearch", "TwoPointsDE", "CMA", "PSO"], - "seed": [42], } + stopping_criterion = SufficientProgressCriterion( + patience=3, strategy='callback' + ) + def set_objective(self, function, dimension, bounds): self.function = function self.dimension = dimension self.bounds = bounds - def run(self, n_iter): - rng = np.random.RandomState(self.seed) # fix seed + def run(self, cb): + f = self.function + self.xopt = None - if n_iter == 0: - x0 = rng.uniform(size=self.dimension, - low=self.bounds[0], - high=self.bounds[1]) - self.xopt = x0 - return + # Get a seed that varies across repetitions, datasets and solvers, + # to ensure a good coverage of the search space, while still being + # reproducible. + seed = self.get_seed( + use_repetition=True, use_dataset=True, use_solver=True + ) + rng = np.random.RandomState(seed) - f = self.function parametrization = ng.p.Array(shape=(self.dimension,)) parametrization.set_bounds(self.bounds[0], self.bounds[1]) parametrization.random_state = rng # fix seed optimizer = ng.optimizers.registry[self.solver]( - budget=n_iter, parametrization=parametrization, num_workers=1 + budget=1000, parametrization=parametrization, num_workers=1 ) + + def stop_criterion(optimizer): + if optimizer.num_tell == 0: + return False + + recommendation = optimizer.provide_recommendation() + if recommendation is not None: + self.xopt = np.asarray(recommendation.value).flatten() + return not cb() + + optimizer.register_callback("ask", ng.callbacks.EarlyStopping( + stop_criterion + )) + recommendation = optimizer.minimize(f) - self.xopt = np.array(recommendation.value) + self.xopt = np.asarray(recommendation.value).flatten() def get_result(self): - return self.xopt.flatten() + return dict(x=self.xopt) diff --git a/solvers/optuna.py b/solvers/optuna.py index 76e815b..2122bcc 100644 --- a/solvers/optuna.py +++ b/solvers/optuna.py @@ -1,10 +1,11 @@ -from benchopt import BaseSolver, safe_import_context +from benchopt import BaseSolver from benchopt.stopping_criterion import SufficientProgressCriterion -with safe_import_context() as import_ctx: - import numpy as np - import optuna - from optuna import samplers +import numpy as np +import optuna +from optuna import samplers +# Check that cmaes is installed +import cmaes # noqa: F401 class Solver(BaseSolver): @@ -13,27 +14,21 @@ class Solver(BaseSolver): name = "optuna" install_cmd = "conda" - requirements = [ - "optuna", - "cmaes", - "numpy", - ] + requirements = ["optuna", "cmaes"] parameters = { - "solver": ["cmaes", "TPE", "RandomSearch"], - "seed": [42], + "solver": ["cmaes", "TPE", "RandomSearch"] } stopping_criterion = SufficientProgressCriterion( - patience=3, strategy='iteration') + patience=3, strategy='callback' + ) def set_objective(self, function, dimension, bounds): self.function = function self.dimension = dimension self.bounds = bounds - def run(self, n_iter): - n_iter += 1 # no possible to call optuna with 0 trial - + def run(self, cb): def objective(trial): x = np.array([ trial.suggest_float(f'x_{k}', self.bounds[0], self.bounds[1]) @@ -41,7 +36,20 @@ def objective(trial): ]) return self.function(x) - seed = self.seed # to make results reproducible + class StopCallback: + + def __call__(self, study, trial): + # Call the callback after each function evaluation, and stop + # the optimization if the callback returns False. + if not cb(): + study.stop() + + # Get a seed that varies across repetitions, datasets and solvers, + # to ensure a good coverage of the search space, while still being + # reproducible. + seed = self.get_seed( + use_repetition=True, use_dataset=True, use_solver=True + ) if self.solver == "TPE": sampler = samplers.TPESampler(seed=seed, n_startup_trials=10) elif self.solver == "RandomSearch": @@ -50,11 +58,16 @@ def objective(trial): sampler = samplers.CmaEsSampler(seed=seed) else: raise NotImplementedError(f"Solver {self.solver} not implemented") - study = optuna.create_study(sampler=sampler, direction='minimize') + self.study_ = optuna.create_study( + sampler=sampler, direction='minimize' + ) optuna.logging.disable_default_handler() # limit verbosity - study.optimize(objective, n_trials=n_iter) - self.xopt = study.best_trial.params + self.study_.optimize( + objective, n_trials=1000, callbacks=[StopCallback()] + ) + self.xopt = self.study_.best_trial.params def get_result(self): - xopt = np.array([self.xopt[f'x_{k}'] for k in range(self.dimension)]) - return xopt + best_param = self.study_.best_trial.params + xopt = np.array([best_param[f'x_{k}'] for k in range(self.dimension)]) + return dict(x=xopt) diff --git a/solvers/scipy.py b/solvers/scipy.py index 4c0c958..91b5885 100644 --- a/solvers/scipy.py +++ b/solvers/scipy.py @@ -1,8 +1,7 @@ -from benchopt import BaseSolver, safe_import_context +from benchopt import BaseSolver -with safe_import_context() as import_ctx: - import numpy as np - from scipy.optimize import minimize +import numpy as np +from scipy.optimize import minimize class Solver(BaseSolver): @@ -14,7 +13,6 @@ class Solver(BaseSolver): requirements = ["numpy", "scipy"] parameters = { "solver": ["Nelder-Mead", "Powell", "BFGS"], - "seed": [42], } def set_objective(self, function, dimension, bounds): @@ -24,7 +22,11 @@ def set_objective(self, function, dimension, bounds): def run(self, n_iter): f = self.function - rng = np.random.RandomState(self.seed) # fix seed + + seed = self.get_seed( + use_repetition=True, use_dataset=True, use_solver=True + ) + rng = np.random.RandomState(seed) # fix seed x0 = rng.uniform(size=self.dimension, low=self.bounds[0], high=self.bounds[1]) @@ -42,4 +44,4 @@ def run(self, n_iter): self.xopt = result.x def get_result(self): - return self.xopt.flatten() + return dict(x=self.xopt.flatten()) From 0639fbbdb0d954b23e6556785873bd60f13d528c Mon Sep 17 00:00:00 2001 From: tommoral Date: Thu, 16 Apr 2026 12:30:55 +0200 Subject: [PATCH 2/4] CLN simplify callback --- solvers/nevergrad.py | 5 +---- solvers/optuna.py | 5 +---- solvers/scipy.py | 46 ++++++++++++++++++++++++++++++++------------ 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/solvers/nevergrad.py b/solvers/nevergrad.py index e87a3f7..3a937f3 100644 --- a/solvers/nevergrad.py +++ b/solvers/nevergrad.py @@ -1,5 +1,4 @@ from benchopt import BaseSolver -from benchopt.stopping_criterion import SufficientProgressCriterion import numpy as np import nevergrad as ng @@ -17,9 +16,7 @@ class Solver(BaseSolver): "TwoPointsDE", "CMA", "PSO"], } - stopping_criterion = SufficientProgressCriterion( - patience=3, strategy='callback' - ) + sampling_strategy = 'callback' def set_objective(self, function, dimension, bounds): self.function = function diff --git a/solvers/optuna.py b/solvers/optuna.py index 2122bcc..0b09953 100644 --- a/solvers/optuna.py +++ b/solvers/optuna.py @@ -1,5 +1,4 @@ from benchopt import BaseSolver -from benchopt.stopping_criterion import SufficientProgressCriterion import numpy as np import optuna @@ -19,9 +18,7 @@ class Solver(BaseSolver): "solver": ["cmaes", "TPE", "RandomSearch"] } - stopping_criterion = SufficientProgressCriterion( - patience=3, strategy='callback' - ) + sampling_strategy = 'callback' def set_objective(self, function, dimension, bounds): self.function = function diff --git a/solvers/scipy.py b/solvers/scipy.py index 91b5885..7975814 100644 --- a/solvers/scipy.py +++ b/solvers/scipy.py @@ -15,12 +15,14 @@ class Solver(BaseSolver): "solver": ["Nelder-Mead", "Powell", "BFGS"], } + sampling_strategy = 'callback' + def set_objective(self, function, dimension, bounds): self.function = function self.dimension = dimension self.bounds = bounds - def run(self, n_iter): + def run(self, cb): f = self.function seed = self.get_seed( @@ -30,18 +32,38 @@ def run(self, n_iter): x0 = rng.uniform(size=self.dimension, low=self.bounds[0], high=self.bounds[1]) + self.xopt = x0 + best_val = np.inf - if n_iter == 0: - self.xopt = x0 - return + class _StopScipy(Exception): + pass - result = minimize( - f, - x0=x0, - method=self.solver, - options={"maxiter": n_iter, "xatol": 1e-20, "fatol": 1e-20}, - ) - self.xopt = result.x + def objective(x): + nonlocal best_val + value = f(x) + if value < best_val: + best_val = value + self.xopt = np.asarray(x).flatten() + return value + + def scipy_callback(xk): + if not cb(): + raise _StopScipy() + + options = {} + if self.solver in ("Nelder-Mead", "Powell"): + options.update({"xatol": 1e-20, "fatol": 1e-20}) + + try: + minimize( + objective, + x0=x0, + method=self.solver, + callback=scipy_callback, + options=options, + ) + except _StopScipy: + pass def get_result(self): - return dict(x=self.xopt.flatten()) + return dict(x=self.xopt) From 70498d278de150cd651115d3a824b68005c5808a Mon Sep 17 00:00:00 2001 From: tommoral Date: Thu, 16 Apr 2026 13:32:20 +0200 Subject: [PATCH 3/4] MNT update run schedule --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f2a61a1..fc6fd45 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,8 +11,8 @@ on: branches: - main schedule: - # Run every day at 7:42am UTC. - - cron: '42 7 * * *' + # Run first day of the month + - cron: '42 7 1 * *' jobs: benchopt_dev: From a3c48de66fe28c1125e3d3b652b154ef347d266c Mon Sep 17 00:00:00 2001 From: tommoral Date: Thu, 16 Apr 2026 13:42:10 +0200 Subject: [PATCH 4/4] FIX don't use nevergrad from conda-forge which is outdated --- solvers/nevergrad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solvers/nevergrad.py b/solvers/nevergrad.py index 3a937f3..6754884 100644 --- a/solvers/nevergrad.py +++ b/solvers/nevergrad.py @@ -10,7 +10,7 @@ class Solver(BaseSolver): name = "nevergrad" install_cmd = "conda" - requirements = ["nevergrad"] + requirements = ["pip::nevergrad"] parameters = { "solver": ["NGOpt", "RandomSearch", "ScrHammersleySearch", "TwoPointsDE", "CMA", "PSO"],