From cf3217c42cc5ddb8d47de91bc95d40b9d6d3217d Mon Sep 17 00:00:00 2001 From: Erica Neininger Date: Mon, 16 Mar 2026 16:48:18 +0000 Subject: [PATCH 1/4] Extract name of repo from source --- github_scripts/get_git_sources.py | 25 ++++++++-------------- github_scripts/merge_sources.py | 2 +- github_scripts/rose_stem_extract_source.py | 2 +- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/github_scripts/get_git_sources.py b/github_scripts/get_git_sources.py index 08efaa0e..257844be 100644 --- a/github_scripts/get_git_sources.py +++ b/github_scripts/get_git_sources.py @@ -88,18 +88,16 @@ def datetime_str() -> str: def clone_and_merge( - dependency: str, opts: Union[list, dict], loc: Path, use_mirrors: bool, mirror_loc: Path + opts: Union[list, dict], loc: Path, use_mirrors: bool, mirror_loc: Path ) -> None: """ Wrapper script for calling get_source and merge_source for a single dependency - dependency: name of the dependency opts: dict or list of dicts for a dependency in the dependencies file loc: path to location to clone to use_mirrors: bool, use local git mirrors if true mirror_loc: path to local git mirrors """ - if not isinstance(opts, list): opts = [opts] @@ -113,7 +111,6 @@ def clone_and_merge( values["source"], values["ref"], loc, - dependency, use_mirrors, mirror_loc, ) @@ -123,7 +120,6 @@ def clone_and_merge( values["source"], values["ref"], loc, - dependency, use_mirrors, mirror_loc, ) @@ -133,7 +129,6 @@ def get_source( source: str, ref: str, dest: Path, - repo: str, use_mirrors: bool = False, mirror_loc: Path = Path(""), ) -> None: @@ -144,18 +139,16 @@ def get_source( if ".git" in source: if use_mirrors: logger.info( - f"[{datetime_str()}] Cloning {repo} from {mirror_loc} at ref {ref}" + f"[{datetime_str()}] Cloning {dest.name} from {mirror_loc} at ref {ref}" ) - mirror_repo = repo - if "jules-internal" in source: - mirror_repo = "jules-internal" + mirror_repo = re.split("[:/]", source)[-1] mirror_loc = Path(mirror_loc) / "MetOffice" / mirror_repo clone_repo_mirror(source, ref, mirror_loc, dest) else: - logger.info(f"[{datetime_str()}] Cloning {repo} from {source} at ref {ref}") + logger.info(f"[{datetime_str()}] Cloning {dest.name} from {source} at ref {ref}") clone_repo(source, ref, dest) else: - logger.info(f"[{datetime_str()}] Syncing {repo} at ref {ref}") + logger.info(f"[{datetime_str()}] Syncing {dest.name} at ref {ref}") sync_repo(source, ref, dest) @@ -163,7 +156,6 @@ def merge_source( source: str, ref: str, dest: Path, - repo: str, use_mirrors: bool = False, mirror_loc: Path = Path(""), ) -> None: @@ -174,11 +166,12 @@ def merge_source( logger.info( f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Merging " - f"{source} at ref {ref} into {repo}" + f"{source} at ref {ref} into {dest.name}" ) if use_mirrors: - remote_path = Path(mirror_loc) / "MetOffice" / repo + mirror_repo = re.split("[:/]", source)[-1] + remote_path = Path(mirror_loc) / "MetOffice" / mirror_repo else: remote_path = source run_command(f"git -C {dest} remote add local {remote_path}") @@ -194,7 +187,7 @@ def merge_source( if result.returncode: unmerged_files = get_unmerged(dest) if unmerged_files: - handle_merge_conflicts(source, ref, dest, repo) + handle_merge_conflicts(source, ref, dest, dest.name) else: raise subprocess.CalledProcessError( result.returncode, command, result.stdout, result.stderr diff --git a/github_scripts/merge_sources.py b/github_scripts/merge_sources.py index 87d74644..acb63c22 100755 --- a/github_scripts/merge_sources.py +++ b/github_scripts/merge_sources.py @@ -82,7 +82,7 @@ def main(): for dependency, sources in dependencies.items(): dest = args.path / dependency - clone_and_merge(dependency, sources, dest, args.mirrors, args.mirror_loc) + clone_and_merge(sources, dest, args.mirrors, args.mirror_loc) if __name__ == "__main__": diff --git a/github_scripts/rose_stem_extract_source.py b/github_scripts/rose_stem_extract_source.py index d7bbf553..26033b93 100755 --- a/github_scripts/rose_stem_extract_source.py +++ b/github_scripts/rose_stem_extract_source.py @@ -45,7 +45,7 @@ def main() -> None: for dependency, sources in dependencies.items(): loc = clone_loc / dependency - clone_and_merge(dependency, sources, loc, use_mirrors, mirror_loc) + clone_and_merge(sources, loc, use_mirrors, mirror_loc) if __name__ == "__main__": From 01d9155df75807b70eaf9e1aa975f441e73914cd Mon Sep 17 00:00:00 2001 From: Erica Neininger Date: Tue, 17 Mar 2026 10:16:03 +0000 Subject: [PATCH 2/4] Account for working copies when extracting from mirrors --- github_scripts/get_git_sources.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/github_scripts/get_git_sources.py b/github_scripts/get_git_sources.py index 257844be..0e08b221 100644 --- a/github_scripts/get_git_sources.py +++ b/github_scripts/get_git_sources.py @@ -169,6 +169,10 @@ def merge_source( f"{source} at ref {ref} into {dest.name}" ) + if not any([g for g in ["https:", "git@", "localmirrors:"] if source.startswith(g)]): + # Source is local working copy - cannot determine mirror_repo + use_mirrors = False + if use_mirrors: mirror_repo = re.split("[:/]", source)[-1] remote_path = Path(mirror_loc) / "MetOffice" / mirror_repo From c0823cbfc05ddb3e1783ce8ccd6981694369b869 Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:36:02 +0000 Subject: [PATCH 3/4] fix merge conflict --- github_scripts/get_git_sources.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/github_scripts/get_git_sources.py b/github_scripts/get_git_sources.py index 925cde76..3c9a3b6f 100644 --- a/github_scripts/get_git_sources.py +++ b/github_scripts/get_git_sources.py @@ -148,7 +148,9 @@ def get_source( mirror_loc = Path(mirror_loc) / "MetOffice" / mirror_repo clone_repo_mirror(source, ref, mirror_loc, dest) else: - logger.info(f"[{datetime_str()}] Cloning {dest.name} from {source} at ref {ref}") + logger.info( + f"[{datetime_str()}] Cloning {dest.name} from {source} at ref {ref}" + ) clone_repo(source, ref, dest) else: logger.info(f"[{datetime_str()}] Syncing {dest.name} at ref {ref}") @@ -172,14 +174,9 @@ def merge_source( f"{source} at ref {ref} into {dest.name}" ) - if not any([g for g in ["https:", "git@", "localmirrors:"] if source.startswith(g)]): - # Source is local working copy - cannot determine mirror_repo - use_mirrors = False - - if use_mirrors: - mirror_repo = re.split("[:/]", source)[-1] - remote_path = Path(mirror_loc) / "MetOffice" / mirror_repo - else: + if not any( + [g for g in ["https:", "git@", "localmirrors:"] if source.startswith(g)] + ): if not ref: raise Exception( f"Cannot merge local source '{source}' with empty ref.\n" @@ -188,6 +185,14 @@ def merge_source( ) remote_path = source fetch = ref + elif use_mirrors: + mirror_repo = re.split("[:/]", source)[-1] + remote_path = Path(mirror_loc) / "MetOffice" / mirror_repo + fetch = determine_mirror_fetch(source, ref) + else: + # Not using a clone or the mirrors + remote_path = source + fetch = ref run_command(f"git -C {dest} remote add local {remote_path}") From c7fd7ca2dfd3929c5d4f23f5db38e0b59f24e575 Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Wed, 18 Mar 2026 14:51:01 +0000 Subject: [PATCH 4/4] unit tests working --- github_scripts/get_git_sources.py | 2 +- github_scripts/tests/test_get_git_sources.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/github_scripts/get_git_sources.py b/github_scripts/get_git_sources.py index 3c9a3b6f..f9fe168e 100644 --- a/github_scripts/get_git_sources.py +++ b/github_scripts/get_git_sources.py @@ -175,7 +175,7 @@ def merge_source( ) if not any( - [g for g in ["https:", "git@", "localmirrors:"] if source.startswith(g)] + [g for g in ["https:", "git@", "localmirrors:"] if str(source).startswith(g)] ): if not ref: raise Exception( diff --git a/github_scripts/tests/test_get_git_sources.py b/github_scripts/tests/test_get_git_sources.py index ea6685b1..081281fd 100644 --- a/github_scripts/tests/test_get_git_sources.py +++ b/github_scripts/tests/test_get_git_sources.py @@ -126,7 +126,6 @@ def test_merge_sources(setup_sources): "https://github.com/MetOffice/SimSys_Scripts.git", "main", target_clone, - "SimSys_Scripts", ) is None )