diff --git a/github_scripts/get_git_sources.py b/github_scripts/get_git_sources.py index 80374c13..f9fe168e 100644 --- a/github_scripts/get_git_sources.py +++ b/github_scripts/get_git_sources.py @@ -91,22 +91,16 @@ def datetime_str() -> str: def clone_and_merge( - dependency: str, - opts: Union[list, dict], - loc: Path, - use_mirrors: bool, - mirror_loc: Path, + opts: Union[list, dict], loc: Path, use_mirrors: bool, mirror_loc: Path ) -> None: """ Wrapper script for calling get_source and merge_source for a single dependency - dependency: name of the dependency opts: dict or list of dicts for a dependency in the dependencies file loc: path to location to clone to use_mirrors: bool, use local git mirrors if true mirror_loc: path to local git mirrors """ - if not isinstance(opts, list): opts = [opts] @@ -120,7 +114,6 @@ def clone_and_merge( values["source"], values["ref"], loc, - dependency, use_mirrors, mirror_loc, ) @@ -130,7 +123,6 @@ def clone_and_merge( values["source"], values["ref"], loc, - dependency, use_mirrors, mirror_loc, ) @@ -140,7 +132,6 @@ def get_source( source: str, ref: str, dest: Path, - repo: str, use_mirrors: bool = False, mirror_loc: Path = Path(""), ) -> None: @@ -151,18 +142,18 @@ def get_source( if ".git" in source: if use_mirrors: logger.info( - f"[{datetime_str()}] Cloning {repo} from {mirror_loc} at ref {ref}" + f"[{datetime_str()}] Cloning {dest.name} from {mirror_loc} at ref {ref}" ) - mirror_repo = repo - if "jules-internal" in source: - mirror_repo = "jules-internal" + mirror_repo = re.split("[:/]", source)[-1] mirror_loc = Path(mirror_loc) / "MetOffice" / mirror_repo clone_repo_mirror(source, ref, mirror_loc, dest) else: - logger.info(f"[{datetime_str()}] Cloning {repo} from {source} at ref {ref}") + logger.info( + f"[{datetime_str()}] Cloning {dest.name} from {source} at ref {ref}" + ) clone_repo(source, ref, dest) else: - logger.info(f"[{datetime_str()}] Syncing {repo} at ref {ref}") + logger.info(f"[{datetime_str()}] Syncing {dest.name} at ref {ref}") sync_repo(source, ref, dest) @@ -170,7 +161,6 @@ def merge_source( source: Union[Path, str], ref: str, dest: Path, - repo: str, use_mirrors: bool = False, mirror_loc: Path = Path(""), ) -> None: @@ -181,17 +171,12 @@ def merge_source( logger.info( f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Merging " - f"{source} at ref {ref} into {repo}" + f"{source} at ref {ref} into {dest.name}" ) - if ".git" in str(source): - if use_mirrors: - remote_path = Path(mirror_loc) / "MetOffice" / repo - fetch = determine_mirror_fetch(source, ref) - else: - remote_path = source - fetch = ref - else: + if not any( + [g for g in ["https:", "git@", "localmirrors:"] if str(source).startswith(g)] + ): if not ref: raise Exception( f"Cannot merge local source '{source}' with empty ref.\n" @@ -200,6 +185,14 @@ def merge_source( ) remote_path = source fetch = ref + elif use_mirrors: + mirror_repo = re.split("[:/]", source)[-1] + remote_path = Path(mirror_loc) / "MetOffice" / mirror_repo + fetch = determine_mirror_fetch(source, ref) + else: + # Not using a clone or the mirrors + remote_path = source + fetch = ref run_command(f"git -C {dest} remote add local {remote_path}") @@ -210,7 +203,7 @@ def merge_source( if result.returncode: unmerged_files = get_unmerged(dest) if unmerged_files: - handle_merge_conflicts(source, ref, dest, repo) + handle_merge_conflicts(source, ref, dest, dest.name) else: raise subprocess.CalledProcessError( result.returncode, command, result.stdout, result.stderr diff --git a/github_scripts/merge_sources.py b/github_scripts/merge_sources.py index 87d74644..acb63c22 100755 --- a/github_scripts/merge_sources.py +++ b/github_scripts/merge_sources.py @@ -82,7 +82,7 @@ def main(): for dependency, sources in dependencies.items(): dest = args.path / dependency - clone_and_merge(dependency, sources, dest, args.mirrors, args.mirror_loc) + clone_and_merge(sources, dest, args.mirrors, args.mirror_loc) if __name__ == "__main__": diff --git a/github_scripts/rose_stem_extract_source.py b/github_scripts/rose_stem_extract_source.py index d7bbf553..26033b93 100755 --- a/github_scripts/rose_stem_extract_source.py +++ b/github_scripts/rose_stem_extract_source.py @@ -45,7 +45,7 @@ def main() -> None: for dependency, sources in dependencies.items(): loc = clone_loc / dependency - clone_and_merge(dependency, sources, loc, use_mirrors, mirror_loc) + clone_and_merge(sources, loc, use_mirrors, mirror_loc) if __name__ == "__main__": diff --git a/github_scripts/tests/test_get_git_sources.py b/github_scripts/tests/test_get_git_sources.py index ea6685b1..081281fd 100644 --- a/github_scripts/tests/test_get_git_sources.py +++ b/github_scripts/tests/test_get_git_sources.py @@ -126,7 +126,6 @@ def test_merge_sources(setup_sources): "https://github.com/MetOffice/SimSys_Scripts.git", "main", target_clone, - "SimSys_Scripts", ) is None )