diff --git a/vinca/generate_azure.py b/vinca/generate_azure.py index 27853ae..602d004 100644 --- a/vinca/generate_azure.py +++ b/vinca/generate_azure.py @@ -26,9 +26,7 @@ def read_azure_script(fn): return (resources.files("vinca") / "azure_templates" / fn).read_text() -azure_linux_script = lu(read_azure_script("linux.sh")) -azure_osx_script = lu(read_azure_script("osx_64.sh")) -azure_osx_arm64_script = lu(read_azure_script("osx_arm64.sh")) +azure_unix_script = lu(read_azure_script("unix.sh")) azure_win_preconfig_script = lu(read_azure_script("win_preconfig.bat")) azure_win_script = lu(read_azure_script("win_build.bat")) @@ -66,6 +64,26 @@ def parse_command_line(argv): help="search for additional_recipes folder?", ) + parser.add_argument( + "-b", + "--batch_size", + dest="batch_size", + default=5, + type=int, + help="How many packages to build at most per stage", + ) + + parser.add_argument( + "--publish-mode", + dest="publish_mode", + choices=["immediate", "platform-finalize"], + default="immediate", + help=( + "When set to platform-finalize, batch jobs only build and publish pipeline " + "artifacts. A final stage uploads the full platform payload after all batches pass." + ), + ) + arguments = parser.parse_args(argv[1:]) config.parsed_args = arguments return arguments @@ -207,13 +225,158 @@ def add_additional_recipes(args): return additional_recipes +def get_batch_artifact_name(target, batch_key): + return f"{normalize_name(target)}-{batch_key}" + + +def get_build_output_path(target): + if target == "win-64": + return r"%CONDA_BLD_PATH%\\win-64" + return f"$HOME/conda-bld/{target}" + + +def get_publish_stage_name(target): + return f"publish_{normalize_name(target)}" + + +def get_unix_collect_script(target, batch_key): + artifact_name = get_batch_artifact_name(target, batch_key) + return lu( + f"""mkdir -p \"$(Build.ArtifactStagingDirectory)/{artifact_name}\" +if [ -d \"{get_build_output_path(target)}\" ]; then + cp -a \"{get_build_output_path(target)}/.\" \"$(Build.ArtifactStagingDirectory)/{artifact_name}/\" +fi""" + ) + + +def get_windows_collect_script(batch_key): + artifact_name = get_batch_artifact_name("win-64", batch_key) + return lu( + f"""if exist "%CONDA_BLD_PATH%\\win-64" ( + mkdir "$(Build.ArtifactStagingDirectory)\\{artifact_name}" 2>NUL + xcopy "%CONDA_BLD_PATH%\\win-64\\*" "$(Build.ArtifactStagingDirectory)\\{artifact_name}\\" /E /I /Y >NUL +)""" + ) + + +def get_unix_publish_script(target): + return lu( + f"""python3 -m pip install --user --disable-pip-version-check anaconda-client +export PATH="$HOME/.local/bin:$PATH" +shopt -s globstar nullglob +files=("$(Pipeline.Workspace)"/artifacts/**/*.conda "$(Pipeline.Workspace)"/artifacts/**/*.tar.bz2) +if (( ${{#files[@]}} == 0 )); then + echo "No built packages found for {target}" + exit 1 +fi +anaconda -t "$ANACONDA_API_TOKEN" upload "${{files[@]}}" --force""" + ) + + +def get_windows_publish_script(): + return lu( + r"""$files = Get-ChildItem -Path "$(Pipeline.Workspace) artifacts" -Recurse -Include *.conda,*.tar.bz2 -File +if ($files.Count -eq 0) { + throw "No built packages found for win-64" +} +foreach ($file in $files) { + anaconda -t $env:ANACONDA_API_TOKEN upload "$($file.FullName)" --force +}""".replace("\u000c", "\\") + ) + + +def append_azure_publish_stage( + azure_template, target, stage_names, pool, windows=False +): + publish_stage = { + "stage": get_publish_stage_name(target), + "dependsOn": stage_names, + "jobs": [ + { + "job": f"publish_{normalize_name(target)}", + "steps": [ + { + "task": "DownloadPipelineArtifact@2", + "displayName": f"Download built artifacts for {target}", + "inputs": { + "buildType": "current", + "targetPath": "$(Pipeline.Workspace)/artifacts", + }, + }, + ], + } + ], + } + + if windows: + publish_stage["jobs"][0]["variables"] = {"CONDA_BLD_PATH": "C:\\\\bld\\\\"} + publish_stage["jobs"][0]["steps"].extend( + [ + { + "task": "PythonScript@0", + "displayName": "Download Miniforge", + "inputs": { + "scriptSource": "inline", + "script": lu( + """import urllib.request +url = 'https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Windows-x86_64.exe' +path = r\"$(Build.ArtifactStagingDirectory)/Miniforge.exe\" +urllib.request.urlretrieve(url, path)""" + ), + }, + }, + { + "script": lu( + """start /wait \"\" %BUILD_ARTIFACTSTAGINGDIRECTORY%\\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=C:\\Miniforge""" + ), + "displayName": "Install Miniforge", + }, + { + "powershell": 'Write-Host "##vso[task.prependpath]C:\\Miniforge\\Scripts"', + "displayName": "Add conda to PATH", + }, + { + "script": lu( + """call activate base +mamba.exe install -c conda-forge --yes --quiet anaconda-client""" + ), + "displayName": "Install anaconda-client", + }, + { + "powershell": get_windows_publish_script(), + "env": { + "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", + }, + "displayName": "Publish built packages", + }, + ] + ) + else: + publish_stage["jobs"][0]["steps"].append( + { + "script": get_unix_publish_script(target), + "env": { + "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", + }, + "displayName": "Publish built packages", + } + ) + + if pool is not None: + publish_stage["pool"] = pool + + azure_template.setdefault("stages", []).append(publish_stage) + + def build_linux_pipeline( stages, trigger_branch, - script=azure_linux_script, + script=azure_unix_script, azure_template=None, docker_image=None, outfile="linux.yml", + target="linux-64", + publish_mode="immediate", ): # Build Linux pipeline if azure_template is None: @@ -223,27 +386,52 @@ def build_linux_pipeline( docker_image = "condaforge/linux-anvil-cos7-x86_64" azure_stages = [] - stage_names = [] for i, s in enumerate(stages): stage_name = f"stage_{i}" stage = {"stage": stage_name, "jobs": []} - stage_names.append(stage_name) for batch in s: - stage["jobs"].append( + batch_key = f"stage_{i}_job_{len(stage['jobs'])}" + steps = [ { - "job": f"stage_{i}_job_{len(stage['jobs'])}", - "steps": [ + "script": script, + "env": { + "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", + "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", + "DOCKER_IMAGE": docker_image, + "BUILD_TARGET": target, + "VINCA_SKIP_UPLOAD": "1" + if publish_mode == "platform-finalize" + else "0", + }, + "displayName": f"Build {' '.join([pkg for pkg in batch])}", + } + ] + if publish_mode == "platform-finalize": + artifact_name = get_batch_artifact_name(target, batch_key) + steps.extend( + [ + { + "script": get_unix_collect_script(target, batch_key), + "displayName": "Collect built artifacts", + "condition": "always()", + }, { - "script": script, - "env": { - "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", - "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", - "DOCKER_IMAGE": docker_image, + "task": "PublishPipelineArtifact@1", + "displayName": f"Publish workflow artifact {artifact_name}", + "condition": "always()", + "inputs": { + "targetPath": f"$(Build.ArtifactStagingDirectory)/{artifact_name}", + "artifact": artifact_name, }, - "displayName": f"Build {' '.join([pkg for pkg in batch])}", - } - ], + }, + ] + ) + + stage["jobs"].append( + { + "job": batch_key, + "steps": steps, } ) @@ -256,6 +444,14 @@ def build_linux_pipeline( if azure_stages: azure_template["stages"] = azure_stages + if publish_mode == "platform-finalize" and azure_stages: + append_azure_publish_stage( + azure_template, + target, + [stage["stage"] for stage in azure_stages], + azure_template.get("pool"), + ) + if not len(azure_stages): return @@ -268,33 +464,60 @@ def build_osx_pipeline( trigger_branch, vm_imagename="macOS-10.15", outfile="osx.yml", - script=azure_osx_script, + script=azure_unix_script, + target="osx-64", + publish_mode="immediate", ): # Build OSX pipeline azure_template = {"pool": {"vmImage": vm_imagename}} azure_stages = [] - stage_names = [] for i, s in enumerate(stages): stage_name = f"stage_{i}" stage = {"stage": stage_name, "jobs": []} - stage_names.append(stage_name) for batch in s: - stage["jobs"].append( + batch_key = f"stage_{i}_job_{len(stage['jobs'])}" + steps = [ { - "job": f"stage_{i}_job_{len(stage['jobs'])}", - "steps": [ + "script": script, + "env": { + "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", + "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", + "BUILD_TARGET": target, + "VINCA_SKIP_UPLOAD": "1" + if publish_mode == "platform-finalize" + else "0", + }, + "displayName": f"Build {' '.join([pkg for pkg in batch])}", + } + ] + if publish_mode == "platform-finalize": + artifact_name = get_batch_artifact_name(target, batch_key) + steps.extend( + [ + { + "script": get_unix_collect_script(target, batch_key), + "displayName": "Collect built artifacts", + "condition": "always()", + }, { - "script": script, - "env": { - "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", - "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", + "task": "PublishPipelineArtifact@1", + "displayName": f"Publish workflow artifact {artifact_name}", + "condition": "always()", + "inputs": { + "targetPath": f"$(Build.ArtifactStagingDirectory)/{artifact_name}", + "artifact": artifact_name, }, - "displayName": f"Build {' '.join([pkg for pkg in batch])}", - } - ], + }, + ] + ) + + stage["jobs"].append( + { + "job": batch_key, + "steps": steps, } ) @@ -307,6 +530,14 @@ def build_osx_pipeline( if azure_stages: azure_template["stages"] = azure_stages + if publish_mode == "platform-finalize" and azure_stages: + append_azure_publish_stage( + azure_template, + target, + [stage["stage"] for stage in azure_stages], + azure_template.get("pool"), + ) + if not len(azure_stages): return @@ -314,7 +545,12 @@ def build_osx_pipeline( fo.write(yaml.dump(azure_template, sort_keys=False)) -def build_win_pipeline(stages, trigger_branch, outfile="win.yml"): +def build_win_pipeline( + stages, + trigger_branch, + outfile="win.yml", + publish_mode="immediate", +): azure_template = {"pool": {"vmImage": "windows-2019"}} azure_stages = [] @@ -325,62 +561,86 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml"): with open(".scripts/build_win.bat", "r") as fi: script = lu(fi.read()) - stage_names = [] for i, s in enumerate(stages): stage_name = f"stage_{i}" stage = {"stage": stage_name, "jobs": []} - stage_names.append(stage_name) for batch in s: - stage["jobs"].append( + batch_key = f"stage_{i}_job_{len(stage['jobs'])}" + steps = [ { - "job": f"stage_{i}_job_{len(stage['jobs'])}", - "variables": {"CONDA_BLD_PATH": "C:\\\\bld\\\\"}, - "steps": [ - { - "task": "PythonScript@0", - "displayName": "Download Miniforge", - "inputs": { - "scriptSource": "inline", - "script": lu( - """import urllib.request + "task": "PythonScript@0", + "displayName": "Download Miniforge", + "inputs": { + "scriptSource": "inline", + "script": lu( + """import urllib.request url = 'https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Windows-x86_64.exe' path = r"$(Build.ArtifactStagingDirectory)/Miniforge.exe" urllib.request.urlretrieve(url, path)""" - ), - }, - }, - { - "script": lu( - """start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=C:\\Miniforge""" - ), - "displayName": "Install Miniforge", - }, - { - "powershell": 'Write-Host "##vso[task.prependpath]C:\\Miniforge\\Scripts"', - "displayName": "Add conda to PATH", - }, - { - "script": lu( - """call activate base + ), + }, + }, + { + "script": lu( + """start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=C:\\Miniforge""" + ), + "displayName": "Install Miniforge", + }, + { + "powershell": 'Write-Host "##vso[task.prependpath]C:\\Miniforge\\Scripts"', + "displayName": "Add conda to PATH", + }, + { + "script": lu( + """call activate base mamba.exe install -c conda-forge --yes --quiet conda-build pip ruamel.yaml anaconda-client""" - ), - "displayName": "Install conda-build, boa and activate environment", - }, + ), + "displayName": "Install conda-build, boa and activate environment", + }, + { + "script": azure_win_preconfig_script, + "displayName": "conda-forge build setup", + }, + { + "script": script, + "env": { + "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", + "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", + "PYTHONUNBUFFERED": 1, + "VINCA_SKIP_UPLOAD": "1" + if publish_mode == "platform-finalize" + else "0", + }, + "displayName": f"Build {' '.join([pkg for pkg in batch])}", + }, + ] + if publish_mode == "platform-finalize": + artifact_name = get_batch_artifact_name("win-64", batch_key) + steps.extend( + [ { - "script": azure_win_preconfig_script, - "displayName": "conda-forge build setup", + "script": get_windows_collect_script(batch_key), + "displayName": "Collect built artifacts", + "condition": "always()", }, { - "script": script, - "env": { - "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", - "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", - "PYTHONUNBUFFERED": 1, + "task": "PublishPipelineArtifact@1", + "displayName": f"Publish workflow artifact {artifact_name}", + "condition": "always()", + "inputs": { + "targetPath": f"$(Build.ArtifactStagingDirectory)/{artifact_name}", + "artifact": artifact_name, }, - "displayName": f"Build {' '.join([pkg for pkg in batch])}", }, - ], + ] + ) + + stage["jobs"].append( + { + "job": batch_key, + "variables": {"CONDA_BLD_PATH": "C:\\\\bld\\\\"}, + "steps": steps, } ) @@ -393,6 +653,15 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml"): if azure_stages: azure_template["stages"] = azure_stages + if publish_mode == "platform-finalize" and azure_stages: + append_azure_publish_stage( + azure_template, + "win-64", + [stage["stage"] for stage in azure_stages], + azure_template.get("pool"), + windows=True, + ) + if not len(azure_stages): return @@ -511,7 +780,7 @@ def main(): if len(filtered): filtered_stages.append(filtered) - stages = batch_stages(filtered_stages) + stages = batch_stages(filtered_stages, args.batch_size) print(stages) with open("buildorder.txt", "w") as fo: @@ -524,13 +793,21 @@ def main(): fo.write("\n".join(order)) if args.platform == "linux-64": - build_linux_pipeline(stages, args.trigger_branch, outfile="linux.yml") + build_linux_pipeline( + stages, + args.trigger_branch, + outfile="linux.yml", + target="linux-64", + publish_mode=args.publish_mode, + ) if args.platform == "osx-64": build_osx_pipeline( stages, args.trigger_branch, - script=azure_osx_script, + script=azure_unix_script, + target="osx-64", + publish_mode=args.publish_mode, ) if args.platform == "osx-arm64": @@ -539,7 +816,9 @@ def main(): args.trigger_branch, vm_imagename="macOS-11", outfile="osx_arm64.yml", - script=azure_osx_arm64_script, + script=azure_unix_script, + target="osx-arm64", + publish_mode=args.publish_mode, ) if args.platform == "linux-aarch64": @@ -560,8 +839,15 @@ def main(): azure_template=aarch64_azure_template, docker_image="condaforge/linux-anvil-aarch64", outfile="linux_aarch64.yml", + target="linux-aarch64", + publish_mode=args.publish_mode, ) # windows if args.platform == "win-64": - build_win_pipeline(stages, args.trigger_branch, outfile="win.yml") + build_win_pipeline( + stages, + args.trigger_branch, + outfile="win.yml", + publish_mode=args.publish_mode, + ) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 2a8481d..ff9825f 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -75,6 +75,17 @@ def parse_command_line(argv): help="How many packages to build at most per stage", ) + parser.add_argument( + "--publish-mode", + dest="publish_mode", + choices=["immediate", "platform-finalize"], + default="immediate", + help=( + "When set to platform-finalize, batch jobs only build and upload workflow " + "artifacts. A final job uploads the full platform payload after all batches pass." + ), + ) + arguments = parser.parse_args(argv[1:]) config.parsed_args = arguments return arguments @@ -241,6 +252,113 @@ def get_stage_name(batch): return " ".join(stage_name) +def get_batch_artifact_name(target, batch_key): + return f"{normalize_name(target)}-{batch_key}" + + +def get_build_artifact_path(target): + if target == "win-64": + return r"C:\\bld\\win-64" + return f"~/conda-bld/{target}" + + +def get_publish_job_key(target): + return f"publish_{normalize_name(target)}" + + +def get_github_publish_command(target): + if target == "win-64": + return lu( + f"""$files = Get-ChildItem -Path \"output/{target}\" -Recurse -Include *.conda,*.tar.bz2 -File +if ($files.Count -eq 0) {{ + throw \"No built packages found for {target}\" +}} +foreach ($file in $files) {{ + pixi run upload \"$($file.FullName)\" --force +}}""" + ) + + return lu( + f"""shopt -s globstar nullglob +files=(output/{target}/**/*.conda output/{target}/**/*.tar.bz2) +if (( ${{#files[@]}} == 0 )); then + echo \"No built packages found for {target}\" + exit 1 +fi +pixi run upload "${{files[@]}}" --force""" + ) + + +def get_artifact_upload_step(target, batch_key): + return { + "name": f"Upload built artifacts for {batch_key}", + "if": "always()", + "uses": "actions/upload-artifact@v6", + "with": { + "name": get_batch_artifact_name(target, batch_key), + "path": get_build_artifact_path(target), + "if-no-files-found": "ignore", + "retention-days": 7, + }, + } + + +def add_publish_job(azure_template, target, runs_on, needs): + publish_job_key = get_publish_job_key(target) + steps = [ + { + "name": "Checkout code", + "uses": "actions/checkout@v6", + }, + { + "name": "Setup pixi", + "uses": "prefix-dev/setup-pixi@v0.9.4", + "with": { + "frozen": True, + }, + }, + ] + + if target == "win-64": + steps.append( + { + "uses": "egor-tensin/cleanup-path@v5", + "with": { + "dirs": "C:\\Program Files\\Git\\usr\\bin;C:\\Program Files\\Git\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Git\\mingw64\\bin" + }, + } + ) + + steps.extend( + [ + { + "name": f"Download built artifacts for {target}", + "uses": "actions/download-artifact@v5", + "with": { + "pattern": f"{normalize_name(target)}-*", + "path": f"output/{target}", + "merge-multiple": True, + }, + }, + { + "name": f"Publish built packages for {target}", + "run": get_github_publish_command(target), + "shell": "powershell" if target == "win-64" else "bash", + "env": { + "ANACONDA_API_TOKEN": "${{ secrets.ANACONDA_API_TOKEN }}", + }, + }, + ] + ) + + azure_template["jobs"][publish_job_key] = { + "name": f"Publish {target}", + "runs-on": runs_on, + "needs": needs, + "steps": steps, + } + + def build_unix_pipeline( stages, trigger_branch, @@ -250,6 +368,7 @@ def build_unix_pipeline( outfile="linux.yml", pipeline_name="build_unix", target="", + publish_mode="immediate", ): blurb = {"jobs": {}, "name": pipeline_name} @@ -266,26 +385,33 @@ def build_unix_pipeline( batch_keys.append(batch_key) pretty_stage_name = get_stage_name(batch) + steps = [ + { + "name": "Checkout code", + "uses": "actions/checkout@v6", + }, + { + "name": f"Build {' '.join([pkg for pkg in batch])}", + "env": { + "ANACONDA_API_TOKEN": "${{ secrets.ANACONDA_API_TOKEN }}", + "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", + "BUILD_TARGET": target, + "VINCA_SKIP_UPLOAD": "1" + if publish_mode == "platform-finalize" + else "0", + }, + "run": script, + }, + ] + if publish_mode == "platform-finalize": + steps.append(get_artifact_upload_step(target, batch_key)) + azure_template["jobs"][batch_key] = { "name": pretty_stage_name, "runs-on": runs_on, "strategy": {"fail-fast": False}, "needs": prev_batch_keys, - "steps": [ - { - "name": "Checkout code", - "uses": "actions/checkout@v6", - }, - { - "name": f"Build {' '.join([pkg for pkg in batch])}", - "env": { - "ANACONDA_API_TOKEN": "${{ secrets.ANACONDA_API_TOKEN }}", - "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", - "BUILD_TARGET": target, # use for cross-compilation - }, - "run": script, - }, - ], + "steps": steps, } prev_batch_keys = batch_keys @@ -293,6 +419,9 @@ def build_unix_pipeline( if len(azure_template.get("jobs", [])) == 0: return + if publish_mode == "platform-finalize" and prev_batch_keys: + add_publish_job(azure_template, target, runs_on, prev_batch_keys) + azure_template["on"] = {"push": {"branches": [trigger_branch]}} dump_for_gha(azure_template, outfile) @@ -306,6 +435,7 @@ def build_linux_pipeline( runs_on="ubuntu-latest", outfile="linux.yml", pipeline_name="build_linux", + publish_mode="immediate", ): build_unix_pipeline( stages, @@ -316,6 +446,7 @@ def build_linux_pipeline( outfile=outfile, pipeline_name=pipeline_name, target="linux-64", + publish_mode=publish_mode, ) @@ -328,6 +459,7 @@ def build_osx_pipeline( script=azure_unix_script, target="osx-64", pipeline_name="build_osx64", + publish_mode="immediate", ): build_unix_pipeline( stages, @@ -338,10 +470,17 @@ def build_osx_pipeline( outfile=outfile, target=target, pipeline_name=pipeline_name, + publish_mode=publish_mode, ) -def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template=None): +def build_win_pipeline( + stages, + trigger_branch, + outfile="win.yml", + azure_template=None, + publish_mode="immediate", +): vm_imagename = "windows-2022" # Build Win pipeline blurb = {"jobs": {}, "name": "build_win"} @@ -365,6 +504,44 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template batch_keys.append(batch_key) pretty_stage_name = get_stage_name(batch) + steps = [ + {"name": "Checkout code", "uses": "actions/checkout@v6"}, + { + "name": "Setup pixi", + "uses": "prefix-dev/setup-pixi@v0.9.4", + "with": { + "pixi-version": "v0.63.2", + "cache": "true", + }, + }, + { + "uses": "egor-tensin/cleanup-path@v5", + "with": { + "dirs": "C:\\Program Files\\Git\\usr\\bin;C:\\Program Files\\Git\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Git\\mingw64\\bin" + }, + }, + { + "shell": "cmd", + "run": azure_win_preconfig_script, + "name": "conda-forge build setup", + }, + { + "shell": "cmd", + "run": script, + "env": { + "ANACONDA_API_TOKEN": "${{ secrets.ANACONDA_API_TOKEN }}", + "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", + "PYTHONUNBUFFERED": 1, + "VINCA_SKIP_UPLOAD": "1" + if publish_mode == "platform-finalize" + else "0", + }, + "name": f"Build {' '.join([pkg for pkg in batch])}", + }, + ] + if publish_mode == "platform-finalize": + steps.append(get_artifact_upload_step("win-64", batch_key)) + azure_template["jobs"][batch_key] = { "name": pretty_stage_name, "runs-on": vm_imagename, @@ -374,38 +551,7 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template "CONDA_BLD_PATH": "C:\\\\bld\\\\", "VINCA_CUSTOM_CMAKE_BUILD_DIR": "C:\\\\x\\\\", }, - "steps": [ - {"name": "Checkout code", "uses": "actions/checkout@v6"}, - { - "name": "Setup pixi", - "uses": "prefix-dev/setup-pixi@v0.9.4", - "with": { - "pixi-version": "v0.63.2", - "cache": "true", - }, - }, - { - "uses": "egor-tensin/cleanup-path@v5", - "with": { - "dirs": "C:\\Program Files\\Git\\usr\\bin;C:\\Program Files\\Git\\bin;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Git\\mingw64\\bin" - }, - }, - { - "shell": "cmd", - "run": azure_win_preconfig_script, - "name": "conda-forge build setup", - }, - { - "shell": "cmd", - "run": script, - "env": { - "ANACONDA_API_TOKEN": "${{ secrets.ANACONDA_API_TOKEN }}", - "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", - "PYTHONUNBUFFERED": 1, - }, - "name": f"Build {' '.join([pkg for pkg in batch])}", - }, - ], + "steps": steps, } prev_batch_keys = batch_keys @@ -413,6 +559,9 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template if len(azure_template.get("jobs", [])) == 0: return + if publish_mode == "platform-finalize" and prev_batch_keys: + add_publish_job(azure_template, "win-64", vm_imagename, prev_batch_keys) + azure_template["on"] = {"push": {"branches": [trigger_branch]}} dump_for_gha(azure_template, outfile) @@ -557,12 +706,14 @@ def main(): args.trigger_branch, outfile="linux.yml", pipeline_name="build_linux64", + publish_mode=args.publish_mode, ) if args.platform == "osx-64": build_osx_pipeline( stages, args.trigger_branch, + publish_mode=args.publish_mode, ) if args.platform == "osx-arm64": @@ -574,6 +725,7 @@ def main(): script=azure_unix_script, target=platform, pipeline_name="build_osx_arm64", + publish_mode=args.publish_mode, ) if args.platform == "linux-aarch64": @@ -585,11 +737,17 @@ def main(): outfile="linux_aarch64.yml", target=platform, pipeline_name="build_linux_aarch64", + publish_mode=args.publish_mode, ) # windows if args.platform == "win-64": - build_win_pipeline(stages, args.trigger_branch, outfile="win.yml") + build_win_pipeline( + stages, + args.trigger_branch, + outfile="win.yml", + publish_mode=args.publish_mode, + ) if args.platform == "emscripten-wasm32": build_unix_pipeline( @@ -598,4 +756,5 @@ def main(): outfile="emscripten_wasm32.yml", pipeline_name="build_emscripten_wasm32", target="emscripten-wasm32", + publish_mode=args.publish_mode, ) diff --git a/vinca/test_generate_publish_workflows.py b/vinca/test_generate_publish_workflows.py new file mode 100644 index 0000000..2162cdc --- /dev/null +++ b/vinca/test_generate_publish_workflows.py @@ -0,0 +1,133 @@ +from pathlib import Path + +import yaml + +from . import generate_azure, generate_gha + + +def _load_yaml(path: Path): + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def test_generate_gha_platform_finalize_linux(tmp_path): + outfile = tmp_path / "linux.yml" + + generate_gha.build_linux_pipeline( + [[["ros-demo-a"]], [["ros-demo-b"]]], + "buildbranch_linux", + outfile=str(outfile), + pipeline_name="build_linux64", + publish_mode="platform-finalize", + ) + + data = _load_yaml(outfile) + jobs = data["jobs"] + + build_job = jobs["stage_0_job_0"] + assert build_job["steps"][1]["env"]["VINCA_SKIP_UPLOAD"] == "1" + assert build_job["steps"][2]["uses"] == "actions/upload-artifact@v6" + assert build_job["steps"][2]["with"]["path"] == "~/conda-bld/linux-64" + + publish_job = jobs["publish_linux_64"] + assert publish_job["needs"] == ["stage_1_job_1"] + assert publish_job["steps"][2]["uses"] == "actions/download-artifact@v5" + assert publish_job["steps"][3]["name"] == "Publish built packages for linux-64" + + +def test_generate_gha_unix_artifact_upload_uses_tilde_home_path(): + assert generate_gha.get_build_artifact_path("linux-64") == "~/conda-bld/linux-64" + assert generate_gha.get_build_artifact_path("osx-64") == "~/conda-bld/osx-64" + + +def test_generate_gha_platform_finalize_windows(tmp_path): + outfile = tmp_path / "win.yml" + + generate_gha.build_win_pipeline( + [[["ros-demo-a"]]], + "buildbranch_win", + outfile=str(outfile), + publish_mode="platform-finalize", + ) + + data = _load_yaml(outfile) + jobs = data["jobs"] + + build_job = jobs["stage_0_job_0"] + assert build_job["steps"][4]["env"]["VINCA_SKIP_UPLOAD"] == "1" + assert build_job["steps"][5]["uses"] == "actions/upload-artifact@v6" + assert "publish_win_64" in jobs + + +def test_generate_azure_platform_finalize_linux(tmp_path): + outfile = tmp_path / "linux.yml" + + generate_azure.build_linux_pipeline( + [[["ros-demo-a"]]], + "buildbranch_linux", + outfile=str(outfile), + target="linux-64", + publish_mode="platform-finalize", + ) + + data = _load_yaml(outfile) + stages = data["stages"] + + build_stage = stages[0] + build_job = build_stage["jobs"][0] + assert build_job["steps"][0]["env"]["VINCA_SKIP_UPLOAD"] == "1" + assert build_job["steps"][2]["task"] == "PublishPipelineArtifact@1" + + publish_stage = stages[1] + assert publish_stage["stage"] == "publish_linux_64" + assert publish_stage["dependsOn"] == ["stage_0"] + + +def test_generate_azure_platform_finalize_windows(tmp_path): + outfile = tmp_path / "win.yml" + + generate_azure.build_win_pipeline( + [[["ros-demo-a"]]], + "buildbranch_win", + outfile=str(outfile), + publish_mode="platform-finalize", + ) + + data = _load_yaml(outfile) + stages = data["stages"] + + build_job = stages[0]["jobs"][0] + assert build_job["steps"][5]["env"]["VINCA_SKIP_UPLOAD"] == "1" + assert build_job["steps"][7]["task"] == "PublishPipelineArtifact@1" + + publish_stage = stages[1] + assert publish_stage["stage"] == "publish_win_64" + assert publish_stage["jobs"][0]["steps"][0]["task"] == "DownloadPipelineArtifact@2" + + +def test_generate_azure_platform_finalize_depends_on_emitted_stages_only(tmp_path): + outfile = tmp_path / "linux_skipped_stage.yml" + + generate_azure.build_linux_pipeline( + [[], [["ros-demo-a"]]], + "buildbranch_linux", + outfile=str(outfile), + target="linux-64", + publish_mode="platform-finalize", + ) + + data = _load_yaml(outfile) + publish_stage = data["stages"][-1] + assert publish_stage["stage"] == "publish_linux_64" + assert publish_stage["dependsOn"] == ["stage_1"] + + +def test_generate_azure_unix_publish_script_uses_anaconda_client(): + script = generate_azure.get_unix_publish_script("linux-64") + + assert ( + "python3 -m pip install --user --disable-pip-version-check anaconda-client" + in script + ) + assert 'anaconda -t "$ANACONDA_API_TOKEN" upload' in script + assert "pixi run upload" not in script + assert "pixi.sh/install.sh" not in script