diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f2a61a1..fc6fd45 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,8 +11,8 @@ on: branches: - main schedule: - # Run every day at 7:42am UTC. - - cron: '42 7 * * *' + # Run first day of the month + - cron: '42 7 1 * *' jobs: benchopt_dev: diff --git a/datasets/py_bench.py b/datasets/py_bench.py index 59cbda4..808ec2f 100644 --- a/datasets/py_bench.py +++ b/datasets/py_bench.py @@ -1,8 +1,6 @@ -from benchopt import safe_import_context from benchopt import BaseDataset -with safe_import_context() as import_ctx: - from PyBenchFCN import SingleObjectiveProblem as SOP +from PyBenchFCN import SingleObjectiveProblem as SOP class Dataset(BaseDataset): @@ -10,7 +8,7 @@ class Dataset(BaseDataset): name = "FCN" install_cmd = "conda" - requirements = ["pip:PyBenchFCN"] + requirements = ["pip::PyBenchFCN"] # List of parameters to generate the datasets. The benchmark will consider # the cross product for each key in the dictionary. diff --git a/datasets/simulated.py b/datasets/simulated.py index 8aeb681..4a7ee24 100644 --- a/datasets/simulated.py +++ b/datasets/simulated.py @@ -1,15 +1,11 @@ -from benchopt import safe_import_context from benchopt import BaseDataset -with safe_import_context() as import_ctx: - import numpy as np +import numpy as np class Dataset(BaseDataset): name = "simulated" - install_cmd = "conda" - requirements = ["numpy"] # List of parameters to generate the datasets. The benchmark will consider # the cross product for each key in the dictionary. @@ -17,12 +13,8 @@ class Dataset(BaseDataset): "dimension": [2, 10], } - def __init__(self, dimension=2, bounds=(-3, 3)): - self.function = lambda x: np.linalg.norm(x, 2) ** 2 - self.dimension = dimension - self.bounds = bounds - def get_data(self): - return dict(function=self.function, - dimension=self.dimension, - bounds=self.bounds) + return dict( + function=lambda x: np.linalg.norm(x, 2) ** 2, + dimension=self.dimension, bounds=(-3, 3) + ) diff --git a/objective.py b/objective.py index 9b6b54d..ccfa0ca 100644 --- a/objective.py +++ b/objective.py @@ -1,25 +1,22 @@ -from benchopt import BaseObjective, safe_import_context +from benchopt import BaseObjective -# Protect import to allow manipulating objective without importing library -# Useful for autocompletion and install commands -with safe_import_context() as import_ctx: - import numpy as np +import numpy as np class Objective(BaseObjective): - min_benchopt_version = "1.3" + min_benchopt_version = "1.9" name = "Zero-order test functions" - def get_one_solution(self): - # Return one solution. This should be compatible with 'self.compute'. - return np.zeros(self.dimension) + def get_one_result(self): + # Return one result for testing purpose. + return dict(x=np.zeros(self.dimension)) def set_data(self, function, dimension, bounds): self.function = function self.dimension = dimension self.bounds = bounds - def compute(self, x): + def evaluate_result(self, x): return self.function(x) def get_objective(self): diff --git a/solvers/basinhopping.py b/solvers/basinhopping.py index dbbed39..296c7d8 100644 --- a/solvers/basinhopping.py +++ b/solvers/basinhopping.py @@ -11,10 +11,9 @@ class Solver(BaseSolver): name = "basinhopping" install_cmd = "conda" - requirements = ["numpy", "scipy"] + requirements = ["scipy"] parameters = { "temperature": [1, 10], - "seed": [42], } def set_objective(self, function, dimension, bounds): @@ -24,7 +23,10 @@ def set_objective(self, function, dimension, bounds): def run(self, n_iter): f = self.function - rng = np.random.RandomState(self.seed) # fix seed + seed = self.get_seed( + use_repetition=True, use_dataset=True, use_solver=True + ) + rng = np.random.RandomState(seed) # fix seed x0 = rng.uniform(size=self.dimension, low=self.bounds[0], high=self.bounds[1]) @@ -35,4 +37,4 @@ def run(self, n_iter): self.xopt = result.x def get_result(self): - return self.xopt.flatten() + return dict(x=self.xopt.flatten()) diff --git a/solvers/nevergrad.py b/solvers/nevergrad.py index 80e211f..6754884 100644 --- a/solvers/nevergrad.py +++ b/solvers/nevergrad.py @@ -1,8 +1,7 @@ -from benchopt import BaseSolver, safe_import_context +from benchopt import BaseSolver -with safe_import_context() as import_ctx: - import numpy as np - import nevergrad as ng +import numpy as np +import nevergrad as ng class Solver(BaseSolver): @@ -11,39 +10,53 @@ class Solver(BaseSolver): name = "nevergrad" install_cmd = "conda" - requirements = [ - "nevergrad", - ] + requirements = ["pip::nevergrad"] parameters = { "solver": ["NGOpt", "RandomSearch", "ScrHammersleySearch", "TwoPointsDE", "CMA", "PSO"], - "seed": [42], } + sampling_strategy = 'callback' + def set_objective(self, function, dimension, bounds): self.function = function self.dimension = dimension self.bounds = bounds - def run(self, n_iter): - rng = np.random.RandomState(self.seed) # fix seed + def run(self, cb): + f = self.function + self.xopt = None - if n_iter == 0: - x0 = rng.uniform(size=self.dimension, - low=self.bounds[0], - high=self.bounds[1]) - self.xopt = x0 - return + # Get a seed that varies across repetitions, datasets and solvers, + # to ensure a good coverage of the search space, while still being + # reproducible. + seed = self.get_seed( + use_repetition=True, use_dataset=True, use_solver=True + ) + rng = np.random.RandomState(seed) - f = self.function parametrization = ng.p.Array(shape=(self.dimension,)) parametrization.set_bounds(self.bounds[0], self.bounds[1]) parametrization.random_state = rng # fix seed optimizer = ng.optimizers.registry[self.solver]( - budget=n_iter, parametrization=parametrization, num_workers=1 + budget=1000, parametrization=parametrization, num_workers=1 ) + + def stop_criterion(optimizer): + if optimizer.num_tell == 0: + return False + + recommendation = optimizer.provide_recommendation() + if recommendation is not None: + self.xopt = np.asarray(recommendation.value).flatten() + return not cb() + + optimizer.register_callback("ask", ng.callbacks.EarlyStopping( + stop_criterion + )) + recommendation = optimizer.minimize(f) - self.xopt = np.array(recommendation.value) + self.xopt = np.asarray(recommendation.value).flatten() def get_result(self): - return self.xopt.flatten() + return dict(x=self.xopt) diff --git a/solvers/optuna.py b/solvers/optuna.py index 76e815b..0b09953 100644 --- a/solvers/optuna.py +++ b/solvers/optuna.py @@ -1,10 +1,10 @@ -from benchopt import BaseSolver, safe_import_context -from benchopt.stopping_criterion import SufficientProgressCriterion +from benchopt import BaseSolver -with safe_import_context() as import_ctx: - import numpy as np - import optuna - from optuna import samplers +import numpy as np +import optuna +from optuna import samplers +# Check that cmaes is installed +import cmaes # noqa: F401 class Solver(BaseSolver): @@ -13,27 +13,19 @@ class Solver(BaseSolver): name = "optuna" install_cmd = "conda" - requirements = [ - "optuna", - "cmaes", - "numpy", - ] + requirements = ["optuna", "cmaes"] parameters = { - "solver": ["cmaes", "TPE", "RandomSearch"], - "seed": [42], + "solver": ["cmaes", "TPE", "RandomSearch"] } - stopping_criterion = SufficientProgressCriterion( - patience=3, strategy='iteration') + sampling_strategy = 'callback' def set_objective(self, function, dimension, bounds): self.function = function self.dimension = dimension self.bounds = bounds - def run(self, n_iter): - n_iter += 1 # no possible to call optuna with 0 trial - + def run(self, cb): def objective(trial): x = np.array([ trial.suggest_float(f'x_{k}', self.bounds[0], self.bounds[1]) @@ -41,7 +33,20 @@ def objective(trial): ]) return self.function(x) - seed = self.seed # to make results reproducible + class StopCallback: + + def __call__(self, study, trial): + # Call the callback after each function evaluation, and stop + # the optimization if the callback returns False. + if not cb(): + study.stop() + + # Get a seed that varies across repetitions, datasets and solvers, + # to ensure a good coverage of the search space, while still being + # reproducible. + seed = self.get_seed( + use_repetition=True, use_dataset=True, use_solver=True + ) if self.solver == "TPE": sampler = samplers.TPESampler(seed=seed, n_startup_trials=10) elif self.solver == "RandomSearch": @@ -50,11 +55,16 @@ def objective(trial): sampler = samplers.CmaEsSampler(seed=seed) else: raise NotImplementedError(f"Solver {self.solver} not implemented") - study = optuna.create_study(sampler=sampler, direction='minimize') + self.study_ = optuna.create_study( + sampler=sampler, direction='minimize' + ) optuna.logging.disable_default_handler() # limit verbosity - study.optimize(objective, n_trials=n_iter) - self.xopt = study.best_trial.params + self.study_.optimize( + objective, n_trials=1000, callbacks=[StopCallback()] + ) + self.xopt = self.study_.best_trial.params def get_result(self): - xopt = np.array([self.xopt[f'x_{k}'] for k in range(self.dimension)]) - return xopt + best_param = self.study_.best_trial.params + xopt = np.array([best_param[f'x_{k}'] for k in range(self.dimension)]) + return dict(x=xopt) diff --git a/solvers/scipy.py b/solvers/scipy.py index 4c0c958..7975814 100644 --- a/solvers/scipy.py +++ b/solvers/scipy.py @@ -1,8 +1,7 @@ -from benchopt import BaseSolver, safe_import_context +from benchopt import BaseSolver -with safe_import_context() as import_ctx: - import numpy as np - from scipy.optimize import minimize +import numpy as np +from scipy.optimize import minimize class Solver(BaseSolver): @@ -14,32 +13,57 @@ class Solver(BaseSolver): requirements = ["numpy", "scipy"] parameters = { "solver": ["Nelder-Mead", "Powell", "BFGS"], - "seed": [42], } + sampling_strategy = 'callback' + def set_objective(self, function, dimension, bounds): self.function = function self.dimension = dimension self.bounds = bounds - def run(self, n_iter): + def run(self, cb): f = self.function - rng = np.random.RandomState(self.seed) # fix seed + + seed = self.get_seed( + use_repetition=True, use_dataset=True, use_solver=True + ) + rng = np.random.RandomState(seed) # fix seed x0 = rng.uniform(size=self.dimension, low=self.bounds[0], high=self.bounds[1]) + self.xopt = x0 + best_val = np.inf - if n_iter == 0: - self.xopt = x0 - return + class _StopScipy(Exception): + pass - result = minimize( - f, - x0=x0, - method=self.solver, - options={"maxiter": n_iter, "xatol": 1e-20, "fatol": 1e-20}, - ) - self.xopt = result.x + def objective(x): + nonlocal best_val + value = f(x) + if value < best_val: + best_val = value + self.xopt = np.asarray(x).flatten() + return value + + def scipy_callback(xk): + if not cb(): + raise _StopScipy() + + options = {} + if self.solver in ("Nelder-Mead", "Powell"): + options.update({"xatol": 1e-20, "fatol": 1e-20}) + + try: + minimize( + objective, + x0=x0, + method=self.solver, + callback=scipy_callback, + options=options, + ) + except _StopScipy: + pass def get_result(self): - return self.xopt.flatten() + return dict(x=self.xopt)