From e326fbca7443dfb1a873ace572090fbd72206936 Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Fri, 8 May 2026 16:12:43 +0000 Subject: [PATCH 1/8] WIP --- README.md | 53 ++++++++++++++++++++++++++ test_notebooks_sequential.py | 73 +++++++++++++++++++++++------------- 2 files changed, 100 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 4a7a8cb..5930fc2 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,56 @@ Static hosting: - Source: https://github.com/TomographicImaging/scripts - Rendered: https://tomographicimaging.github.io/scripts + +## Environment files for installing software + +### CIL-Demos +`cil_demos.yml` - for installing the latest release of CIL-Demos on a system with a GPU. +`cil_demos_cpu.yml` - for installing the latest release of CIL-Demos on a system without a GPU. + +### CIL +`cil.yml` - for installing the latest release of CIL. +`cil_test.yml` - for installing the latest + +### CILViewer +`cilviewer_ui` - for installing the latest release of CILViewer, with +## Script for Testing CIL demos + +`test_notebooks_sequential.py` is a script for testing the CIL-Demos + +To use it: +1. Clone [CIL-Demos](https://github.com/TomographicImaging/CIL-Demos), if you haven't already, and move to the branch of CIL-Demos you would like to test. +2. Clone [scripts](https://github.com/TomographicImaging/scripts) +3. Download all datasets used by the [CIL-Demos](https://github.com/TomographicImaging/CIL-Demos) notebooks you would like to test. +4. Create an environment using: + ```sh + conda env create -f https://tomographicimaging.github.io/scripts/env/cil_test_demos.yml + ``` +5. Clone [CIL](https://github.com/TomographicImaging/CIL), if you haven't already, and move to the branch of CIL you would like to test the demos with. +6. Activate the environment you created: + ```sh + conda activate cil_test_demos + ``` +7. Navigate into the CIL directory and install CIL into this environment: + ```sh + pip install -e . + ``` +8. The script uses `nbmake` and needs to point to the kernel to run the notebooks with. We need to make sure our conda environment is registered with the correct jupyter kernel name. Run: + ```sh + jupyter kernelspec list + ``` +If there is a kernel with name `cil_test_demos` then move to step 9. If not, run: + ```sh + python -m ipykernel install --user --name cil_test_demos --display-name "cil_test_demos" + ``` +9. At the top of the `test_notebooks_sequential.py` file, update these variables: + `CIL_DEMOS_DIR` - should point to your clone of CIL-Demos + `DATA_PATH` and `DATA_PATH_ALT` should point to directories where the CIL-Demos data is saved. +10. Run `test_notebooks_sequential.py`. This is likely to take quite a long time. It will create the following inside the `scripts` directory: + `test_notebooks_*.log` log file (with name including date and time of run) + This shows which notebooks have run and whether they have passed or failed. + `tmp_notebooks` directory + This contains the run and rendered `.ipynb` notebook files. + `html_outputs` directory + This contains run and rendered notebooks as `html` files. + diff --git a/test_notebooks_sequential.py b/test_notebooks_sequential.py index 5a58497..df8633f 100644 --- a/test_notebooks_sequential.py +++ b/test_notebooks_sequential.py @@ -10,7 +10,7 @@ DATA_PATH_ALT = '/mnt/share/materials/CIL/' SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -CIL_DEMOS_DIR = os.path.expanduser(os.path.join('~', 'CIL-Demos')) +CIL_DEMOS_DIR = os.path.expanduser("/home/lhe97136/CIL-Demos") PDF_OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'html_outputs') TMP_OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'tmp_notebooks') @@ -25,15 +25,42 @@ LOG_NAME = "test_notebooks_" + datetime.now().strftime("%Y%m%d_%H%M%S") + ".log" -def preprocess_notebook(original_path): - rel_path = os.path.relpath(original_path, CIL_DEMOS_DIR) - rel_path = rel_path.replace("..", "_") - tmp_path = os.path.join(TMP_OUTPUT_DIR, rel_path) - tmp_dir = os.path.dirname(tmp_path) - os.makedirs(tmp_dir, exist_ok=True) - tmp_path = tmp_path.replace('.ipynb', '_tmp.ipynb') +def copy_all_files(): + """ + Copy all *.ipynb and *.py files from `folders` into TMP_OUTPUT_DIR, + preserving relative directory structure. + Renames any *.ipynb files to *_tmp.ipynb + + Returns a list of tuples: (original_path, copied_path) for all notebooks. + """ + notebook_paths = [] + for folder in folders: + for root, _, files in os.walk(folder): + for file in files: + if file.endswith('.ipynb') or file.endswith('.py'): + original_path = os.path.join(root, file) + + rel_path = os.path.relpath(original_path, CIL_DEMOS_DIR) + rel_path = rel_path.replace("..", "_") + tmp_path = os.path.join(TMP_OUTPUT_DIR, rel_path) + tmp_dir = os.path.dirname(tmp_path) + os.makedirs(tmp_dir, exist_ok=True) + if file.endswith('.ipynb') and not file.endswith('_tmp.ipynb'): + tmp_path = tmp_path.replace('.ipynb', '_tmp.ipynb') + if os.path.exists(tmp_path): + with open(LOG_NAME, "a") as log_file: + log_file.write(f"Skipping (tmp exists): {original_path}\n") + + if not os.path.exists(tmp_path): + shutil.copy(original_path, tmp_path) + + if file.endswith('.ipynb'): + notebook_paths.append((original_path, tmp_path)) + return notebook_paths + - shutil.copy(original_path, tmp_path) + +def preprocess_notebook(original_path, tmp_path): with open(tmp_path, 'r') as f: notebook = nbformat.read(f, as_version=4) @@ -183,25 +210,19 @@ def cleanup_notebook_files(tmp_path): def main(): results = {} all_warnings = {} - for folder in folders: - for root, _, files in os.walk(folder): - for file in files: - if file.endswith('.ipynb') and not file.endswith('_tmp.ipynb'): - original_path = os.path.join(root, file) - rel_path = os.path.relpath(original_path, CIL_DEMOS_DIR) - rel_path = rel_path.replace("..", "_") - tmp_path = os.path.join(TMP_OUTPUT_DIR, rel_path).replace('.ipynb', '_tmp.ipynb') + with open(LOG_NAME, "a") as log: + log.write("=== Copying Files ===\n") - if os.path.exists(tmp_path): - with open(LOG_NAME, "a") as log_file: - log_file.write(f"Skipping (tmp exists): {original_path}\n") - continue + notebooks = copy_all_files() - with open(LOG_NAME, "a") as log_file: - log_file.write(f"\n=== Processing notebook: {original_path} ===\n") - tmp_path = preprocess_notebook(original_path) - passed = run_notebook_test(tmp_path, original_path, all_warnings) - results[original_path] = 'passed' if passed else 'failed' + for original_path, copied_path in notebooks: + + + with open(LOG_NAME, "a") as log_file: + log_file.write(f"\n=== Processing notebook: {original_path} ===\n") + tmp_path = preprocess_notebook(original_path, copied_path) + passed = run_notebook_test(tmp_path, original_path, all_warnings) + results[original_path] = 'passed' if passed else 'failed' with open(LOG_NAME, "a") as log_file: log_file.write("\n======== Notebook Test Summary =========\n") From 373a112c107588e045d06e4365c835a375e277f1 Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Fri, 8 May 2026 16:15:43 +0000 Subject: [PATCH 2/8] spacing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5930fc2..f43980b 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ To use it: ```sh jupyter kernelspec list ``` -If there is a kernel with name `cil_test_demos` then move to step 9. If not, run: + If there is a kernel with name `cil_test_demos` then move to step 9. If not, run: ```sh python -m ipykernel install --user --name cil_test_demos --display-name "cil_test_demos" ``` From 4436ea6954c1c8eb71c09d2355d1a31e1fb943a1 Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Fri, 8 May 2026 16:16:26 +0000 Subject: [PATCH 3/8] spacing --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index f43980b..67f88f3 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,13 @@ To use it: `CIL_DEMOS_DIR` - should point to your clone of CIL-Demos `DATA_PATH` and `DATA_PATH_ALT` should point to directories where the CIL-Demos data is saved. 10. Run `test_notebooks_sequential.py`. This is likely to take quite a long time. It will create the following inside the `scripts` directory: + `test_notebooks_*.log` log file (with name including date and time of run) This shows which notebooks have run and whether they have passed or failed. + `tmp_notebooks` directory This contains the run and rendered `.ipynb` notebook files. + `html_outputs` directory This contains run and rendered notebooks as `html` files. From e179d60208509e161517b13a1388af2f9dbfd493 Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Fri, 15 May 2026 17:15:11 +0000 Subject: [PATCH 4/8] docs --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 67f88f3..6bbca85 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,12 @@ Static hosting: ### CIL `cil.yml` - for installing the latest release of CIL. -`cil_test.yml` - for installing the latest +`cil_test.yml` - for installing the latest release of CIL with the requirements for running the tests. ### CILViewer -`cilviewer_ui` - for installing the latest release of CILViewer, with +`cilviewer_ui` - for installing the latest release of CILViewer, with packages needed to run the GUI. + + ## Script for Testing CIL demos `test_notebooks_sequential.py` is a script for testing the CIL-Demos From 87e9bf16664e4e994d6e805c560476b747594f4c Mon Sep 17 00:00:00 2001 From: Laura Murgatroyd <60604372+lauramurgatroyd@users.noreply.github.com> Date: Fri, 15 May 2026 18:25:05 +0100 Subject: [PATCH 5/8] Update test_notebooks_sequential.py --- test_notebooks_sequential.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_notebooks_sequential.py b/test_notebooks_sequential.py index df8633f..1ebbecb 100644 --- a/test_notebooks_sequential.py +++ b/test_notebooks_sequential.py @@ -10,7 +10,7 @@ DATA_PATH_ALT = '/mnt/share/materials/CIL/' SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -CIL_DEMOS_DIR = os.path.expanduser("/home/lhe97136/CIL-Demos") +CIL_DEMOS_DIR = os.path.expanduser(os.path.join('~', 'CIL-Demos')) PDF_OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'html_outputs') TMP_OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'tmp_notebooks') From 05267302ad7f66119104869801040ee057dc9865 Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Wed, 3 Jun 2026 13:58:38 +0000 Subject: [PATCH 6/8] Improve readme --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6bbca85..1208c4b 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,12 @@ Static hosting: ### CIL-Demos `cil_demos.yml` - for installing the latest release of CIL-Demos on a system with a GPU. + `cil_demos_cpu.yml` - for installing the latest release of CIL-Demos on a system without a GPU. ### CIL `cil.yml` - for installing the latest release of CIL. + `cil_test.yml` - for installing the latest release of CIL with the requirements for running the tests. ### CILViewer @@ -51,14 +53,14 @@ To use it: 9. At the top of the `test_notebooks_sequential.py` file, update these variables: `CIL_DEMOS_DIR` - should point to your clone of CIL-Demos `DATA_PATH` and `DATA_PATH_ALT` should point to directories where the CIL-Demos data is saved. -10. Run `test_notebooks_sequential.py`. This is likely to take quite a long time. It will create the following inside the `scripts` directory: +10. Run `test_notebooks_sequential.py`. This is likely to take quite a long time. It will create the following inside the current directory: `test_notebooks_*.log` log file (with name including date and time of run) + This shows which notebooks have run and whether they have passed or failed. `tmp_notebooks` directory + This contains the run and rendered `.ipynb` notebook files. - - `html_outputs` directory - This contains run and rendered notebooks as `html` files. + From 7b9393abb46279650012320867e8b35974f7a60b Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Wed, 3 Jun 2026 13:59:52 +0000 Subject: [PATCH 7/8] Improve readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1208c4b..38c8248 100644 --- a/README.md +++ b/README.md @@ -55,12 +55,12 @@ To use it: `DATA_PATH` and `DATA_PATH_ALT` should point to directories where the CIL-Demos data is saved. 10. Run `test_notebooks_sequential.py`. This is likely to take quite a long time. It will create the following inside the current directory: - `test_notebooks_*.log` log file (with name including date and time of run) + - `test_notebooks_*.log` log file (with name including date and time of run) This shows which notebooks have run and whether they have passed or failed. - `tmp_notebooks` directory - + - `tmp_notebooks` directory + This contains the run and rendered `.ipynb` notebook files. From a6bc07746a49a6884492b10bc0cf4109560e324c Mon Sep 17 00:00:00 2001 From: lauramurgatroyd Date: Wed, 3 Jun 2026 14:00:41 +0000 Subject: [PATCH 8/8] Improve readme --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 38c8248..833eaca 100644 --- a/README.md +++ b/README.md @@ -55,12 +55,8 @@ To use it: `DATA_PATH` and `DATA_PATH_ALT` should point to directories where the CIL-Demos data is saved. 10. Run `test_notebooks_sequential.py`. This is likely to take quite a long time. It will create the following inside the current directory: - - `test_notebooks_*.log` log file (with name including date and time of run) + `test_notebooks_*.log` log file (with name including date and time of run): This shows which notebooks have run and whether they have passed or failed. - This shows which notebooks have run and whether they have passed or failed. - - - `tmp_notebooks` directory - - This contains the run and rendered `.ipynb` notebook files. + `tmp_notebooks` directory: This contains the run and rendered `.ipynb` notebook files.