From b959a6d9e304daf425b24b91155b64e06641f4a6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 11:56:18 +0000 Subject: [PATCH 1/9] Initial plan From e4d2e344c869e72a4bd5084dc710aa0114d39789 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 12:02:35 +0000 Subject: [PATCH 2/9] Improve error messages for dataset compatibility checks Co-authored-by: bruAristimunha <42702466+bruAristimunha@users.noreply.github.com> --- moabb/evaluations/base.py | 23 ++++++++++++++++++++++- moabb/evaluations/evaluations.py | 26 +++++++++++++++++++++++++- moabb/evaluations/splitters.py | 2 +- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/moabb/evaluations/base.py b/moabb/evaluations/base.py index a4926c4bc..242a14854 100644 --- a/moabb/evaluations/base.py +++ b/moabb/evaluations/base.py @@ -168,8 +168,11 @@ def __init__( ) rm.append(dataset) elif not valid_for_eval: + # Get specific reason for incompatibility + eval_type = self.__class__.__name__ + reason = self._get_incompatibility_reason(dataset) log.warning( - f"{dataset} not compatible with evaluation. " + f"{dataset} not compatible with {eval_type}: {reason}. " "Removing this dataset from the list." ) rm.append(dataset) @@ -324,6 +327,24 @@ def is_valid(self, dataset): The dataset to verify. """ + def _get_incompatibility_reason(self, dataset): + """Get a human-readable reason why dataset is incompatible. + + This method should be overridden by subclasses to provide + specific incompatibility reasons. + + Parameters + ---------- + dataset : dataset instance + The dataset to check. + + Returns + ------- + str + A human-readable reason for incompatibility. + """ + return "requirements not met" + def _grid_search(self, param_grid, name, grid_clf, inner_cv): extra_params = {} if param_grid is not None: diff --git a/moabb/evaluations/evaluations.py b/moabb/evaluations/evaluations.py index 674b3f7d0..b9a1e52e9 100644 --- a/moabb/evaluations/evaluations.py +++ b/moabb/evaluations/evaluations.py @@ -532,7 +532,10 @@ def evaluate( self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None ): if not self.is_valid(dataset): - raise AssertionError("Dataset is not appropriate for evaluation") + reason = self._get_incompatibility_reason(dataset) + raise AssertionError( + f"Dataset '{dataset.code}' is not appropriate for CrossSessionEvaluation: {reason}" + ) # Progressbar at subject level for subject in tqdm(dataset.subject_list, desc=f"{dataset.code}-CrossSession"): # check if we already have result for this subject/pipeline @@ -647,6 +650,16 @@ def evaluate( def is_valid(self, dataset): return dataset.n_sessions > 1 + def _get_incompatibility_reason(self, dataset): + """Get specific reason for dataset incompatibility.""" + n_sessions = dataset.n_sessions + if n_sessions <= 1: + return ( + f"dataset has only {n_sessions} session(s), " + "but CrossSessionEvaluation requires at least 2 sessions" + ) + return "requirements not met" + class CrossSubjectEvaluation(BaseEvaluation): """Cross-subject evaluation performance. @@ -855,3 +868,14 @@ def evaluate( def is_valid(self, dataset): return len(dataset.subject_list) > 1 + + def _get_incompatibility_reason(self, dataset): + """Get specific reason for dataset incompatibility.""" + n_subjects = len(dataset.subject_list) + if n_subjects <= 1: + return ( + f"dataset has only {n_subjects} subject(s), " + "but CrossSubjectEvaluation requires at least 2 subjects" + ) + return "requirements not met" + diff --git a/moabb/evaluations/splitters.py b/moabb/evaluations/splitters.py index d287833fe..3536c8604 100644 --- a/moabb/evaluations/splitters.py +++ b/moabb/evaluations/splitters.py @@ -256,7 +256,7 @@ def split(self, y, metadata): if len(sessions) <= 1: log.info( - f"Skipping subject {subject}: Only one session available" + f"Skipping subject {subject}: Only one session available. " f"Cross-session evaluation requires at least two sessions." ) continue # Skip subjects with only one session From 8fb220bd5ddb3ce2dc6c30f0ccc07c02d3c7a873 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 12:08:20 +0000 Subject: [PATCH 3/9] Add test for improved error messages Co-authored-by: bruAristimunha <42702466+bruAristimunha@users.noreply.github.com> --- moabb/tests/test_evaluations.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/moabb/tests/test_evaluations.py b/moabb/tests/test_evaluations.py index 6beb73edb..1901a36dc 100644 --- a/moabb/tests/test_evaluations.py +++ b/moabb/tests/test_evaluations.py @@ -382,6 +382,17 @@ def test_compatible_dataset(self): ds = FakeDataset(["left_hand", "right_hand"], n_sessions=2) assert self.eval.is_valid(dataset=ds) + def test_incompatibility_error_message(self): + """Test that incompatibility error message is clear and informative.""" + ds = FakeDataset(["left_hand", "right_hand"], n_sessions=1) + # Test that the error message includes the dataset code and reason + with pytest.raises(AssertionError) as exc_info: + list(self.eval.evaluate(ds, pipelines, None, None)) + error_msg = str(exc_info.value) + assert "CrossSessionEvaluation" in error_msg + assert "1 session" in error_msg + assert "requires at least 2 sessions" in error_msg + class UtilEvaluation: def test_save_model_cv(self): From 38d8b637def46f45b2fa2a8483f43d8d5e3cde46 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 12:11:23 +0000 Subject: [PATCH 4/9] Use self.__class__.__name__ for dynamic class names in error messages Co-authored-by: bruAristimunha <42702466+bruAristimunha@users.noreply.github.com> --- moabb/evaluations/evaluations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moabb/evaluations/evaluations.py b/moabb/evaluations/evaluations.py index b9a1e52e9..eb4213e50 100644 --- a/moabb/evaluations/evaluations.py +++ b/moabb/evaluations/evaluations.py @@ -534,7 +534,7 @@ def evaluate( if not self.is_valid(dataset): reason = self._get_incompatibility_reason(dataset) raise AssertionError( - f"Dataset '{dataset.code}' is not appropriate for CrossSessionEvaluation: {reason}" + f"Dataset '{dataset.code}' is not appropriate for {self.__class__.__name__}: {reason}" ) # Progressbar at subject level for subject in tqdm(dataset.subject_list, desc=f"{dataset.code}-CrossSession"): @@ -656,7 +656,7 @@ def _get_incompatibility_reason(self, dataset): if n_sessions <= 1: return ( f"dataset has only {n_sessions} session(s), " - "but CrossSessionEvaluation requires at least 2 sessions" + f"but {self.__class__.__name__} requires at least 2 sessions" ) return "requirements not met" @@ -875,7 +875,7 @@ def _get_incompatibility_reason(self, dataset): if n_subjects <= 1: return ( f"dataset has only {n_subjects} subject(s), " - "but CrossSubjectEvaluation requires at least 2 subjects" + f"but {self.__class__.__name__} requires at least 2 subjects" ) return "requirements not met" From 6a5eedd3a94f1dd5f32d20e51531aa7334cd6ed5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 12:14:23 +0000 Subject: [PATCH 5/9] Update CrossSubjectEvaluation error message for consistency Co-authored-by: bruAristimunha <42702466+bruAristimunha@users.noreply.github.com> --- moabb/evaluations/evaluations.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/moabb/evaluations/evaluations.py b/moabb/evaluations/evaluations.py index eb4213e50..714287aa0 100644 --- a/moabb/evaluations/evaluations.py +++ b/moabb/evaluations/evaluations.py @@ -714,7 +714,10 @@ def evaluate( self, dataset, pipelines, param_grid, process_pipeline, postprocess_pipeline=None ): if not self.is_valid(dataset): - raise AssertionError("Dataset is not appropriate for evaluation") + reason = self._get_incompatibility_reason(dataset) + raise AssertionError( + f"Dataset '{dataset.code}' is not appropriate for {self.__class__.__name__}: {reason}" + ) # this is a bit awkward, but we need to check if at least one pipe # have to be run before loading the data. If at least one pipeline # need to be run, we have to load all the data. From 3d6ef38422e582542e71fc28109a72adc96cb935 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 13:38:53 +0000 Subject: [PATCH 6/9] Fix pre-commit: add missing newline at end of file Co-authored-by: bruAristimunha <42702466+bruAristimunha@users.noreply.github.com> --- moabb/evaluations/evaluations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moabb/evaluations/evaluations.py b/moabb/evaluations/evaluations.py index 714287aa0..2cc731710 100644 --- a/moabb/evaluations/evaluations.py +++ b/moabb/evaluations/evaluations.py @@ -881,4 +881,3 @@ def _get_incompatibility_reason(self, dataset): f"but {self.__class__.__name__} requires at least 2 subjects" ) return "requirements not met" - From ebd44de2f700ba8173a666ef227771a5b23e1ca8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 14:12:33 +0000 Subject: [PATCH 7/9] Update what's new file with improved error messages enhancement Co-authored-by: bruAristimunha <42702466+bruAristimunha@users.noreply.github.com> --- docs/source/whats_new.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/whats_new.rst b/docs/source/whats_new.rst index 91ef815fe..55929e6a0 100644 --- a/docs/source/whats_new.rst +++ b/docs/source/whats_new.rst @@ -21,6 +21,7 @@ Enhancements ~~~~~~~~~~~~ - Introduce a new logo for the MOABB library (:gh:`858` by `Pierre Guetschel`_ and community) - Better verbosity control for initialization of the library (:gh:`850` by `Bruno Aristimunha`_) +- Improved error messages for dataset compatibility checks in evaluations - now provides specific reasons when datasets are incompatible (e.g., "dataset has only 1 session(s), but CrossSessionEvaluation requires at least 2 sessions") by `Bruno Aristimunha`_ - Ability to join rows from the tables of MOABB predictive performance scores and detailed CodeCarbon compute profiling metrics by the column `codecarbon_task_name` in MOABB results and the column `task_name` in CodeCarbon results (:gh:`866` by `Ethan Davis`_). - Adding two c-VEP datasets: :class:`moabb.datasets.MartinezCagigal2023Checker` and :class:`moabb.datasets.MartinezCagigal2023Pary` by `Victor Martinez-Cagigal`_ From 21ebf331a8d42573ffa66058db2a1b5bcb66aa1d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 14:16:51 +0000 Subject: [PATCH 8/9] Fix test_verbose.py to check for new specific error messages Co-authored-by: bruAristimunha <42702466+bruAristimunha@users.noreply.github.com> --- moabb/tests/test_verbose.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/moabb/tests/test_verbose.py b/moabb/tests/test_verbose.py index aae29cbaf..2d7151f01 100644 --- a/moabb/tests/test_verbose.py +++ b/moabb/tests/test_verbose.py @@ -19,8 +19,9 @@ def test_verbose_warning(caplog): with caplog.at_level(logging.WARNING): CrossSessionEvaluation(paradigm=paradigm, datasets=[dataset]) - # Check if warning was logged - assert "not compatible with evaluation" in caplog.text + # Check if warning was logged with specific incompatibility reason + assert "not compatible with CrossSessionEvaluation" in caplog.text + assert "requires at least 2 sessions" in caplog.text def test_verbose_error_suppression(caplog): @@ -35,7 +36,7 @@ def test_verbose_error_suppression(caplog): CrossSessionEvaluation(paradigm=paradigm, datasets=[dataset], verbose="ERROR") # Check if warning was suppressed - assert "not compatible with evaluation" not in caplog.text + assert "not compatible with CrossSessionEvaluation" not in caplog.text def test_verbose_false_warning(caplog): @@ -50,5 +51,6 @@ def test_verbose_false_warning(caplog): ): # Set to INFO to see if behavior is consistent CrossSessionEvaluation(paradigm=paradigm, datasets=[dataset], verbose=False) - # Check if warning was logged (since verbose=False -> WARNING) - assert "not compatible with evaluation" in caplog.text + # Check if warning was logged (since verbose=False -> WARNING) with new specific message + assert "not compatible with CrossSessionEvaluation" in caplog.text + assert "requires at least 2 sessions" in caplog.text From aae5911966ea72ea62339ef4c4be3eaaa3b70c3b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 20:24:37 +0000 Subject: [PATCH 9/9] Trigger CI: minor docstring formatting Co-authored-by: bruAristimunha <42702466+bruAristimunha@users.noreply.github.com> --- moabb/evaluations/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moabb/evaluations/base.py b/moabb/evaluations/base.py index 242a14854..68646bcec 100644 --- a/moabb/evaluations/base.py +++ b/moabb/evaluations/base.py @@ -342,6 +342,7 @@ def _get_incompatibility_reason(self, dataset): ------- str A human-readable reason for incompatibility. + """ return "requirements not met"