diff --git a/docs/reference/config-reference.rst b/docs/reference/config-reference.rst index 445a71eb..80429790 100644 --- a/docs/reference/config-reference.rst +++ b/docs/reference/config-reference.rst @@ -1,6 +1,8 @@ Configuration Reference ======================= +.. currentmodule:: fromager.packagesettings + Per-package Settings -------------------- @@ -22,6 +24,41 @@ For example `flash_attn.yaml`. .. autopydantic_model:: fromager.packagesettings.SbomSettings +Source Resolver +^^^^^^^^^^^^^^^ + +.. autopydantic_model:: fromager.packagesettings.PyPISDistResolver + +.. autopydantic_model:: fromager.packagesettings.PyPIPrebuiltResolver + +.. autopydantic_model:: fromager.packagesettings.PyPIDownloadResolver + +.. autopydantic_model:: fromager.packagesettings.PyPIGitResolver + +.. autopydantic_model:: fromager.packagesettings.VersionMapResolver + +.. autopydantic_model:: fromager.packagesettings.GitHubTagDownloadResolver + :inherited-members: AbstractGitSourceResolver + +.. autopydantic_model:: fromager.packagesettings.GitHubTagCloneResolver + :inherited-members: AbstractGitSourceResolver + +.. autopydantic_model:: fromager.packagesettings.GitLabTagDownloadResolver + :inherited-members: AbstractGitSourceResolver + +.. autopydantic_model:: fromager.packagesettings.GitLabTagCloneResolver + :inherited-members: AbstractGitSourceResolver + +.. autopydantic_model:: fromager.packagesettings.NotAvailableResolver + +.. autopydantic_model:: fromager.packagesettings.HookResolver + +.. autoclass:: fromager.packagesettings.BuildSDist + + .. autoattribute:: pep517 + .. autoattribute:: tarball + + Global Settings --------------- diff --git a/src/fromager/packagesettings/__init__.py b/src/fromager/packagesettings/__init__.py index 3233b0a9..b2e4d937 100644 --- a/src/fromager/packagesettings/__init__.py +++ b/src/fromager/packagesettings/__init__.py @@ -12,6 +12,21 @@ VariantInfo, ) from ._pbi import PackageBuildInfo +from ._resolver import ( + DEFAULT_TAG_MATCHER, + BuildSDist, + GitHubTagCloneResolver, + GitHubTagDownloadResolver, + GitLabTagCloneResolver, + GitLabTagDownloadResolver, + HookResolver, + NotAvailableResolver, + PyPIDownloadResolver, + PyPIGitResolver, + PyPIPrebuiltResolver, + PyPISDistResolver, + VersionMapResolver, +) from ._settings import Settings, SettingsFile from ._templates import substitute_template from ._typedefs import ( @@ -31,21 +46,33 @@ ) __all__ = ( + "DEFAULT_TAG_MATCHER", "MODEL_CONFIG", "Annotations", "BuildDirectory", "BuildOptions", + "BuildSDist", "DownloadSource", "EnvKey", "EnvVars", + "GitHubTagCloneResolver", + "GitHubTagDownloadResolver", + "GitLabTagCloneResolver", + "GitLabTagDownloadResolver", "GitOptions", "GlobalChangelog", + "HookResolver", + "NotAvailableResolver", "Package", "PackageBuildInfo", "PackageSettings", "PackageVersion", "PatchMap", "ProjectOverride", + "PyPIDownloadResolver", + "PyPIGitResolver", + "PyPIPrebuiltResolver", + "PyPISDistResolver", "RawAnnotations", "ResolverDist", "SbomSettings", @@ -55,6 +82,7 @@ "Variant", "VariantChangelog", "VariantInfo", + "VersionMapResolver", "default_update_extra_environ", "get_extra_environ", "substitute_template", diff --git a/src/fromager/packagesettings/_models.py b/src/fromager/packagesettings/_models.py index 47d96f6d..b86fc272 100644 --- a/src/fromager/packagesettings/_models.py +++ b/src/fromager/packagesettings/_models.py @@ -15,6 +15,7 @@ from pydantic import Field from pydantic_core import core_schema +from ._resolver import SourceResolver from ._typedefs import ( MODEL_CONFIG, BuildDirectory, @@ -228,7 +229,7 @@ class VariantInfo(pydantic.BaseModel): VAR1: "value 1" VAR2: "2.0 wheel_server_url: https://pypi.org/simple/ - pre_build: False + pre_built: False """ model_config = MODEL_CONFIG @@ -249,6 +250,12 @@ class VariantInfo(pydantic.BaseModel): pre_built: bool = False """Use pre-built wheel from index server?""" + source: typing.Annotated[ + SourceResolver | None, + pydantic.Field(default=None, discriminator="provider"), + ] + """Source resolver and downloader""" + class GitOptions(pydantic.BaseModel): """Git repository cloning options @@ -371,6 +378,12 @@ class PackageSettings(pydantic.BaseModel): project_override: ProjectOverride = Field(default_factory=ProjectOverride) """Patch project settings""" + source: typing.Annotated[ + SourceResolver | None, + pydantic.Field(default=None, discriminator="provider"), + ] + """Source resolver and downloader""" + variants: Mapping[Variant, VariantInfo] = Field(default_factory=dict) """Variant configuration""" diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index 5c8458ea..00e4a280 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -20,6 +20,7 @@ ProjectOverride, VariantInfo, ) +from ._resolver import SourceResolver from ._templates import _resolve_template, substitute_template from ._typedefs import Annotations, PackageVersion, PatchMap, Template, Variant @@ -175,6 +176,14 @@ def pre_built(self) -> bool: return vi.pre_built return False + @property + def source_resolver(self) -> SourceResolver | None: + """Get source resolver settings (variant or global)""" + vi = self._ps.variants.get(self.variant) + if vi is not None and vi.source is not None: + return vi.source + return self._ps.source + @property def wheel_server_url(self) -> str | None: """Alternative package index for pre-build wheel""" diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py new file mode 100644 index 00000000..7ca6d8fc --- /dev/null +++ b/src/fromager/packagesettings/_resolver.py @@ -0,0 +1,513 @@ +from __future__ import annotations + +import enum +import logging +import re +import typing + +import pydantic + +from .. import resolver +from ._typedefs import MODEL_CONFIG, PackageVersion + +if typing.TYPE_CHECKING: + from .. import context, requirements_file + +logger = logging.getLogger(__name__) + +VERSION_QUOTED = "%7Bversion%7D" + + +class BuildSDist(enum.StrEnum): + pep517 = "pep517" + tarball = "tarball" + + +class AbstractResolver(pydantic.BaseModel): + model_config = MODEL_CONFIG + + provider: str + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.BaseProvider: + raise NotImplementedError + + +class PyPISDistResolver(AbstractResolver): + """Resolve version with PyPI, download sdist from PyPI + + The ``pypi-sdist`` provider uses :pep:`503` *Simple Repository API* or + :pep:`691` *JSON-based Simple API* to resolve packages on PyPI or a + PyPI-compatible index. + + The provider downloads source distributions (tarballs) from the index. + It ignores releases that have only wheels and no sdist. + + Example:: + + provider: pypi-sdist + index_url: https://pypi.test/simple + """ + + provider: typing.Literal["pypi-sdist"] + + index_url: pydantic.HttpUrl = pydantic.Field( + default=pydantic.HttpUrl("https://pypi.org/simple/"), + description="Python Package Index URL", + ) + + # It is not safe to use PEP 517 to re-generate a source distribution. + # Some PEP 517 backends require VCS to generate correct sdist. + build_sdist: typing.ClassVar[BuildSDist | None] = BuildSDist.tarball + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.PyPIProvider: + return resolver.PyPIProvider( + include_sdists=True, + include_wheels=False, + sdist_server_url=str(self.index_url), + constraints=ctx.constraints, + req_type=req_type, + ignore_platform=False, + ) + + +class PyPIPrebuiltResolver(AbstractResolver): + """Resolve version with PyPI, download pre-built wheel from PyPI + + The ``pypi-prebuilt`` provider uses :pep:`503` *Simple Repository API* or + :pep:`691` *JSON-based Simple API* to resolve packages on PyPI or a + PyPI-compatible index. + + The provider downloads pre-built wheels from the index. It ignores + versions that have no compatible wheels (sdist-only or incompatible + OS, CPU arch, or glibc version). + + Example:: + + provider: pypi-prebuilt + index_url: https://pypi.test/simple + """ + + provider: typing.Literal["pypi-prebuilt"] + + index_url: pydantic.HttpUrl = pydantic.Field( + default=pydantic.HttpUrl("https://pypi.org/simple/"), + description="Python Package Index URL", + ) + + build_sdist: typing.ClassVar[BuildSDist | None] = None + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.PyPIProvider: + return resolver.PyPIProvider( + include_sdists=False, + include_wheels=True, + sdist_server_url=str(self.index_url), + constraints=ctx.constraints, + req_type=req_type, + ignore_platform=False, + ) + + +class PyPIDownloadResolver(AbstractResolver): + """Resolve version with PyPI, download sdist from arbitrary URL + + The ``pypi-download`` provider uses :pep:`503` *Simple Repository API* or + :pep:`691` *JSON-based Simple API* to resolve packages on PyPI or a + PyPI-compatible index. + + The provider takes all releases into account (sdist-only, wheel-only, + even incompatible wheels). + + It downloads tarball from an alternative download location. The download + URL must contain a ``{version}`` template, e.g. + ``https://download.example/mypackage-{version}.tar.gz``. + + Example:: + + provider: pypi-download + index_url: https://pypi.test/simple + download_url: https://download.test/test_pypidownload-{version}.tar.gz + """ + + provider: typing.Literal["pypi-download"] + + index_url: pydantic.HttpUrl = pydantic.Field( + default=pydantic.HttpUrl("https://pypi.org/simple/"), + description="Python Package Index URL", + ) + + download_url: pydantic.HttpUrl + """Remote download URL + + URL must contain '{version}' template string. + """ + + build_sdist: typing.ClassVar[BuildSDist | None] = BuildSDist.tarball + + @pydantic.field_validator("download_url", mode="after") + @classmethod + def validate_download_url(cls, value: pydantic.HttpUrl) -> pydantic.HttpUrl: + if not value.path: + raise ValueError(f"url {value} has an empty path") + if VERSION_QUOTED not in value.path: + raise ValueError(f"missing '{{version}}' in url {value}") + return value + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.PyPIProvider: + return resolver.PyPIProvider( + include_sdists=True, + include_wheels=True, + sdist_server_url=str(self.index_url), + constraints=ctx.constraints, + req_type=req_type, + ignore_platform=True, + override_download_url=str(self.download_url).replace( + VERSION_QUOTED, "{version}" + ), + ) + + +class PyPIGitResolver(AbstractResolver): + """Resolve version with PyPI, build sdist from git clone + + The ``pypi-git`` provider uses :pep:`503` *Simple Repository API* or + :pep:`691` *JSON-based Simple API* to resolve packages on PyPI or a + PyPI-compatible index. + + The provider takes all releases into account (sdist-only, wheel-only, + even incompatible wheels). + + It clones and retrieves a git repo + recursive submodules at a specific + tag. The tag must contain ``{version}`` template. + + Example:: + + provider: pypi-git + index_url: https://pypi.test/simple + clone_url: https://code.test/project/repo.git + tag: 'v{version}' + build_sdist: pep517 + """ + + provider: typing.Literal["pypi-git"] + + index_url: pydantic.HttpUrl = pydantic.Field( + default=pydantic.HttpUrl("https://pypi.org/simple/"), + description="Python Package Index URL", + ) + + clone_url: pydantic.AnyUrl + """git clone URL + + https://git.test/repo.git + """ + + tag: str + + build_sdist: BuildSDist = BuildSDist.pep517 + """Source distribution build method""" + + @pydantic.field_validator("clone_url", mode="after") + @classmethod + def validate_clone_url(cls, value: pydantic.AnyUrl) -> pydantic.AnyUrl: + if value.scheme not in {"https", "ssh"}: + raise ValueError(f"invalid scheme in url {value}") + if not value.path: + raise ValueError(f"url {value} has an empty path") + return value + + @pydantic.field_validator("tag", mode="after") + @classmethod + def validate_tag(cls, value: str) -> str: + if "{version}" not in value: + raise ValueError(f"missing '{{version}}' in tag {value}") + return value + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.PyPIProvider: + download_url = f"git+{self.clone_url}@refs/tags/{self.tag}" + return resolver.PyPIProvider( + include_sdists=True, + include_wheels=True, + sdist_server_url=str(self.index_url), + constraints=ctx.constraints, + req_type=req_type, + ignore_platform=True, + override_download_url=download_url, + ) + + +class VersionMapResolver(AbstractResolver): + """Maps known versions to git commits. + + The ``versionmap-git`` provider maps known version numbers to known git + commits. It clones a git repo at the configured tag. + + Example:: + + provider: versionmap-git + clone_url: https://git.example/viking/viking.git + versionmap: + '1.0': abad1dea + '1.1': refs/tags/1.1 + """ + + provider: typing.Literal["versionmap-git"] + + clone_url: pydantic.AnyUrl + """git clone URL + + https://git.test/repo.git + """ + + versionmap: dict[PackageVersion, str] + + build_sdist: typing.ClassVar[BuildSDist | None] = BuildSDist.tarball + + @pydantic.field_validator("clone_url", mode="after") + @classmethod + def validate_clone_url(cls, value: pydantic.AnyUrl) -> pydantic.AnyUrl: + if value.scheme not in {"https", "ssh"}: + raise ValueError(f"invalid scheme in url {value}") + if not value.path: + raise ValueError(f"url {value} has an empty path") + return value + + +# matches versions like "v1.0" and "1.0" +DEFAULT_TAG_MATCHER = re.compile(r"^(v?\d.*)$") + + +class AbstractGitSourceResolver(AbstractResolver): + """Common abstract class for GitHub and GitLab resolver""" + + project_url: pydantic.HttpUrl + """Full project URL""" + + matcher_factory: pydantic.ImportString = DEFAULT_TAG_MATCHER + """Matcher import string (``package.module:name``) + + Matcher can be a :class:`re.Pattern` object or a factory function + that accepts *ctx* arg and returns a :class:`~fromager.resolver.MatchFunction`. + """ + + build_sdist: BuildSDist = BuildSDist.pep517 + """Source distribution build method""" + + @pydantic.field_validator("project_url", mode="after") + @classmethod + def validate_url(cls, value: pydantic.HttpUrl) -> pydantic.HttpUrl: + """Validate that URL is https URL with host and path""" + if value.scheme != "https" or not value.host or not value.path: + raise ValueError(f"invalid url {value}: expected https, hostname, and path") + if value.path.endswith(".git"): + raise ValueError(f"invalid url {value}: path ends with '.git'") + return value + + @pydantic.field_validator("matcher_factory", mode="after") + @classmethod + def validate_matcher( + cls, value: re.Pattern | typing.Callable + ) -> re.Pattern | typing.Callable: + """Validate that tag pattern has exactly one match group""" + if isinstance(value, re.Pattern): + if value.groups != 1: + raise ValueError( + "Expected a re pattern with exactly one match group, " + f"got {value.groups} groups for {value.pattern}." + ) + elif not callable(value): + raise TypeError(f"{value} is not callable") + return value + + def _get_matcher( + self, ctx: context.WorkContext + ) -> re.Pattern | resolver.MatchFunction: + if isinstance(self.matcher_factory, re.Pattern): + return self.matcher_factory + elif callable(self.matcher_factory): + return self.matcher_factory(ctx=ctx) # type: ignore + else: + raise TypeError(self.matcher_factory) + + def _github_provider( + self, + *, + ctx: context.WorkContext, + req_type: requirements_file.RequirementType, + override_download_url: str | None = None, + ) -> resolver.GitHubTagProvider: + if self.project_url.host != "github.com": + raise ValueError(f"Expected 'github.com' in {self.project_url}") + if not self.project_url.path or self.project_url.path.count("/") != 2: + raise ValueError( + f"Invalid path in {self.project_url}, expected two elements" + ) + organization, repo = self.project_url.path.lstrip("/").split("/") + return resolver.GitHubTagProvider( + organization=organization, + repo=repo, + constraints=ctx.constraints, + matcher=self._get_matcher(ctx), + req_type=req_type, + override_download_url=override_download_url, + ) + + def _gitlab_provider( + self, + *, + ctx: context.WorkContext, + req_type: requirements_file.RequirementType, + override_download_url: str | None = None, + ) -> resolver.GitLabTagProvider: + assert self.project_url.path # for type checker + return resolver.GitLabTagProvider( + project_path=self.project_url.path.lstrip("/"), + server_url=f"https://{self.project_url.host}", + constraints=ctx.constraints, + matcher=self._get_matcher(ctx), + req_type=req_type, + override_download_url=override_download_url, + ) + + +class GitHubTagDownloadResolver(AbstractGitSourceResolver): + """Resolve version from GitHub tags, build sdist from a git tarball download + + The ``github`` provider uses GitHub's REST API to resolve versions from tags. + + Example:: + + provider: github-tag-download + url: https://github.com/python-wheel-build/fromager + matcher_factory: fromager.packagesettings:DEFAULT_TAG_MATCHER + build_sdist: pep517 + """ + + provider: typing.Literal["github-tag-download"] + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.GitHubTagProvider: + return self._github_provider( + ctx=ctx, req_type=req_type, override_download_url="FIXME" + ) + + +class GitHubTagCloneResolver(AbstractGitSourceResolver): + """Resolve version from GitHub tags, build sdist from a git clone + + The ``github`` provider uses GitHub's REST API to resolve versions from tags. + + Example:: + + provider: github-tag-git + url: https://github.com/python-wheel-build/fromager + matcher_factory: fromager.packagesettings:DEFAULT_TAG_MATCHER + build_sdist: pep517 + """ + + provider: typing.Literal["github-tag-git"] + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.GitHubTagProvider: + return self._github_provider( + ctx=ctx, req_type=req_type, override_download_url="FIXME" + ) + + +class GitLabTagDownloadResolver(AbstractGitSourceResolver): + """Resolve version from GitLab tags, build sdist from a git tarball download + + The ``gitlab`` provider uses GitLab's REST API to resolve versions from tags. + + Example:: + + provider: gitlab-tag-download + url: https://gitlab.test/python-wheel-build/fromager + matcher_factory: fromager.packagesettings:DEFAULT_TAG_MATCHER + build_sdist: pep517 + """ + + provider: typing.Literal["gitlab-tag-download"] + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.GitLabTagProvider: + return self._gitlab_provider( + ctx=ctx, req_type=req_type, override_download_url="FIXME" + ) + + +class GitLabTagCloneResolver(AbstractGitSourceResolver): + """Resolve version from GitLab tags, build sdist from a git clone + + The ``gitlab`` provider uses GitLab's REST API to resolve versions from tags. + + Example:: + + provider: gitlab-tag-git + url: https://gitlab.test/python-wheel-build/fromager + matcher_factory: fromager.packagesettings:DEFAULT_TAG_MATCHER + build_sdist: pep517 + """ + + provider: typing.Literal["gitlab-tag-git"] + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.GitLabTagProvider: + return self._gitlab_provider( + ctx=ctx, req_type=req_type, override_download_url="FIXME" + ) + + +class NotAvailableResolver(pydantic.BaseModel): + """Prevent resolve and download""" + + model_config = MODEL_CONFIG + + provider: typing.Literal["not-available"] + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.BaseProvider: + raise ValueError("package is not available") + + +class HookResolver(pydantic.BaseModel): + """Call resolver_provider and download_source hook""" + + model_config = MODEL_CONFIG + + provider: typing.Literal["hook"] + + def resolver_provider( + self, ctx: context.WorkContext, req_type: requirements_file.RequirementType + ) -> resolver.BaseProvider: + # TODO + raise NotImplementedError + + +SourceResolver = ( + PyPISDistResolver + | PyPIPrebuiltResolver + | PyPIDownloadResolver + | PyPIGitResolver + | VersionMapResolver + | GitHubTagCloneResolver + | GitHubTagDownloadResolver + | GitLabTagCloneResolver + | GitLabTagDownloadResolver + | NotAvailableResolver + | HookResolver +) diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 402eebb0..baa81d56 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -8,7 +8,7 @@ from packaging.utils import NormalizedName from packaging.version import Version -from fromager import build_environment, context +from fromager import build_environment, context, requirements_file, resolver from fromager.packagesettings import ( Annotations, BuildDirectory, @@ -29,6 +29,14 @@ TEST_OTHER_PKG = "test-other-pkg" TEST_RELATED_PKG = "test-pkg-library" TEST_PREBUILT_PKG = "test-prebuilt-pkg" +TEST_GITHUB_DOWNLOAD = "test-github-download" +TEST_GITHUB_GIT = "test-github-git" +TEST_GITLAB_DOWNLOAD = "test-gitlab-download" +TEST_GITLAB_GIT = "test-gitlab-git" +TEST_PYPIDOWNLOAD = "test-pypidownload" +TEST_PYPIPREBUILT = "test-pypiprebuilt" +TEST_PYPIGIT = "test-pypigit" +TEST_PYPISDIST = "test-pypisdist" FULL_EXPECTED: dict[str, typing.Any] = { "annotations": { @@ -85,6 +93,7 @@ "ignore_platform": True, "use_pypi_org_metadata": True, }, + "source": None, "variants": { "cpu": { "annotations": { @@ -93,6 +102,7 @@ "env": {"EGG": "spam ${EGG}", "EGG_AGAIN": "$EGG"}, "wheel_server_url": "https://wheel.test/simple", "pre_built": False, + "source": None, }, "rocm": { "annotations": { @@ -101,12 +111,14 @@ "env": {"SPAM": ""}, "wheel_server_url": None, "pre_built": True, + "source": None, }, "cuda": { "annotations": None, "env": {}, "wheel_server_url": None, "pre_built": False, + "source": None, }, }, } @@ -146,6 +158,7 @@ "ignore_platform": False, "use_pypi_org_metadata": None, }, + "source": None, "variants": {}, } @@ -186,12 +199,14 @@ "ignore_platform": False, "use_pypi_org_metadata": None, }, + "source": None, "variants": { "cpu": { "annotations": None, "env": {}, "pre_built": True, "wheel_server_url": None, + "source": None, }, }, } @@ -504,6 +519,14 @@ def test_settings_overrides(testdata_context: context.WorkContext) -> None: TEST_OTHER_PKG, TEST_RELATED_PKG, TEST_PREBUILT_PKG, + TEST_GITHUB_DOWNLOAD, + TEST_GITHUB_GIT, + TEST_GITLAB_DOWNLOAD, + TEST_GITLAB_GIT, + TEST_PYPIDOWNLOAD, + TEST_PYPIGIT, + TEST_PYPIPREBUILT, + TEST_PYPISDIST, } @@ -549,11 +572,19 @@ def test_global_changelog(testdata_context: context.WorkContext) -> None: def test_settings_list(testdata_context: context.WorkContext) -> None: assert testdata_context.settings.list_overrides() == { + TEST_PKG, TEST_EMPTY_PKG, TEST_OTHER_PKG, - TEST_PKG, TEST_RELATED_PKG, TEST_PREBUILT_PKG, + TEST_GITHUB_DOWNLOAD, + TEST_GITHUB_GIT, + TEST_GITLAB_DOWNLOAD, + TEST_GITLAB_GIT, + TEST_PYPIDOWNLOAD, + TEST_PYPIGIT, + TEST_PYPIPREBUILT, + TEST_PYPISDIST, } assert testdata_context.settings.list_pre_built() == {TEST_PREBUILT_PKG} assert testdata_context.settings.variant_changelog() == [] @@ -899,3 +930,99 @@ def test_version_none_no_reference( result = pbi.get_extra_environ(template_env={}, version=None) assert result["FOO"] == "bar" assert "__version__" not in result + + +@pytest.mark.parametrize( + "name,expected", + [ + ( + TEST_GITHUB_DOWNLOAD, + { + "provider": "github-tag-download", + # "matcher_factory": "", + "project_url": "https://github.com/python-wheel-build/fromager", + }, + ), + ( + TEST_GITHUB_GIT, + { + "provider": "github-tag-git", + # "matcher_factory": "", + "project_url": "https://github.com/python-wheel-build/fromager", + }, + ), + ( + TEST_GITLAB_DOWNLOAD, + { + "provider": "gitlab-tag-download", + # "matcher_factory": "", + "project_url": "https://gitlab.test/python-wheel-build/fromager", + }, + ), + ( + TEST_GITLAB_GIT, + { + "provider": "gitlab-tag-git", + # "matcher_factory": "", + "project_url": "https://gitlab.test/python-wheel-build/fromager", + }, + ), + ( + TEST_PYPIDOWNLOAD, + { + "provider": "pypi-download", + "index_url": "https://pypi.test/simple", + "download_url": "https://download.test/test_pypidownload-%7Bversion%7D.tar.gz", + }, + ), + ( + TEST_PYPIGIT, + { + "provider": "pypi-git", + "index_url": "https://pypi.test/simple", + "clone_url": "https://github.com/python-wheel-build/fromager.git", + "tag": "v{version}", + }, + ), + ( + TEST_PYPIPREBUILT, + { + "provider": "pypi-prebuilt", + "index_url": "https://pypi.test/simple", + }, + ), + ( + TEST_PYPISDIST, + { + "provider": "pypi-sdist", + "index_url": "https://pypi.test/simple", + }, + ), + ], +) +def test_source_resolvers( + name: str, expected: dict, testdata_context: context.WorkContext +) -> None: + pbi = testdata_context.settings.package_build_info(name) + assert pbi.source_resolver + assert pbi.source_resolver.provider == expected["provider"] + assert pbi.serialize(mode="json")["source"] == expected + + resolver_provider = pbi.source_resolver.resolver_provider( + ctx=testdata_context, + req_type=requirements_file.RequirementType.TOP_LEVEL, + ) + assert isinstance(resolver_provider, resolver.BaseProvider) + + +def test_source_resolver_variant(testdata_context: context.WorkContext) -> None: + pbi = testdata_context.settings.package_build_info(TEST_PYPISDIST) + assert pbi.source_resolver + assert pbi.source_resolver.provider == "pypi-sdist" + assert str(pbi.source_resolver.index_url) == "https://pypi.test/simple" + + testdata_context.settings.variant = Variant("rocm") + pbi = testdata_context.settings.package_build_info(TEST_PYPISDIST) + assert pbi.source_resolver + assert pbi.source_resolver.provider == "pypi-sdist" + assert str(pbi.source_resolver.index_url) == "https://rocm.test/simple" diff --git a/tests/testdata/context/overrides/settings/test_github-download.yaml b/tests/testdata/context/overrides/settings/test_github-download.yaml new file mode 100644 index 00000000..cc1f13a1 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_github-download.yaml @@ -0,0 +1,5 @@ +source: + provider: github-tag-download + project_url: https://github.com/python-wheel-build/fromager + matcher_factory: fromager.packagesettings:DEFAULT_TAG_MATCHER + build_sdist: pep517 diff --git a/tests/testdata/context/overrides/settings/test_github-git.yaml b/tests/testdata/context/overrides/settings/test_github-git.yaml new file mode 100644 index 00000000..e27c2eaa --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_github-git.yaml @@ -0,0 +1,5 @@ +source: + provider: github-tag-git + project_url: https://github.com/python-wheel-build/fromager + matcher_factory: fromager.packagesettings:DEFAULT_TAG_MATCHER + build_sdist: pep517 diff --git a/tests/testdata/context/overrides/settings/test_gitlab-download.yaml b/tests/testdata/context/overrides/settings/test_gitlab-download.yaml new file mode 100644 index 00000000..1996d31e --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_gitlab-download.yaml @@ -0,0 +1,5 @@ +source: + provider: gitlab-tag-download + project_url: https://gitlab.test/python-wheel-build/fromager + matcher_factory: fromager.packagesettings:DEFAULT_TAG_MATCHER + build_sdist: pep517 diff --git a/tests/testdata/context/overrides/settings/test_gitlab-git.yaml b/tests/testdata/context/overrides/settings/test_gitlab-git.yaml new file mode 100644 index 00000000..df874368 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_gitlab-git.yaml @@ -0,0 +1,5 @@ +source: + provider: gitlab-tag-git + project_url: https://gitlab.test/python-wheel-build/fromager + matcher_factory: fromager.packagesettings:DEFAULT_TAG_MATCHER + build_sdist: pep517 diff --git a/tests/testdata/context/overrides/settings/test_pypidownload.yaml b/tests/testdata/context/overrides/settings/test_pypidownload.yaml new file mode 100644 index 00000000..1045e714 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_pypidownload.yaml @@ -0,0 +1,4 @@ +source: + provider: pypi-download + index_url: https://pypi.test/simple + download_url: https://download.test/test_pypidownload-{version}.tar.gz diff --git a/tests/testdata/context/overrides/settings/test_pypigit.yaml b/tests/testdata/context/overrides/settings/test_pypigit.yaml new file mode 100644 index 00000000..521b3489 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_pypigit.yaml @@ -0,0 +1,6 @@ +source: + provider: pypi-git + index_url: https://pypi.test/simple + clone_url: https://github.com/python-wheel-build/fromager.git + tag: 'v{version}' + build_sdist: pep517 diff --git a/tests/testdata/context/overrides/settings/test_pypiprebuilt.yaml b/tests/testdata/context/overrides/settings/test_pypiprebuilt.yaml new file mode 100644 index 00000000..8fe01538 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_pypiprebuilt.yaml @@ -0,0 +1,3 @@ +source: + provider: pypi-prebuilt + index_url: https://pypi.test/simple diff --git a/tests/testdata/context/overrides/settings/test_pypisdist.yaml b/tests/testdata/context/overrides/settings/test_pypisdist.yaml new file mode 100644 index 00000000..9b5ab385 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_pypisdist.yaml @@ -0,0 +1,8 @@ +source: + provider: pypi-sdist + index_url: https://pypi.test/simple +variants: + rocm: + source: + provider: pypi-sdist + index_url: https://rocm.test/simple