From 58840f8e0a4678f1b098be2e5382a359a76ccce2 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 13 Apr 2026 09:58:22 -0400 Subject: [PATCH] feat(packagesettings): add dependency-chained build tags Implements support for explicit package dependencies in settings to enable recursive build tag calculation. When a dependency's changelog changes, all packages that depend on it automatically get incremented build tags. Key features: - New 'dependencies' field in PackageSettings (list of package names) - Recursive build tag calculation: own_changelog + sum(dep_build_tags) - Transitive dependency resolution through entire dependency chain - Circular dependency detection with clear error messages - Support for "fake packages" (platform dependencies like CUDA, ROCm) - Fully backward compatible (empty dependencies = original behavior) Implementation: - PackageSettings.dependencies field with comprehensive docstring - PackageBuildInfo._calculate_build_tag() for recursive logic - 9 new unit tests covering all edge cases - 6 test data files for dependency chains and circular dependencies - Comprehensive how-to guide with examples and best practices All 458 project tests pass, type checking clean, documentation builds successfully. Closes: #478 Co-Authored-By: Claude Sonnet 4.5 Signed-off-by: Doug Hellmann --- docs/how-tos/dependency-build-tags.rst | 250 ++++++++++++++++++ docs/how-tos/index.rst | 1 + src/fromager/packagesettings/_models.py | 15 ++ src/fromager/packagesettings/_pbi.py | 59 ++++- tests/test_packagesettings.py | 118 +++++++++ .../overrides/settings/test_circular_a.yaml | 5 + .../overrides/settings/test_circular_b.yaml | 5 + .../overrides/settings/test_dep_chain_a.yaml | 5 + .../overrides/settings/test_dep_chain_b.yaml | 5 + .../overrides/settings/test_dep_chain_c.yaml | 3 + .../overrides/settings/test_fake_cuda.yaml | 5 + 11 files changed, 466 insertions(+), 5 deletions(-) create mode 100644 docs/how-tos/dependency-build-tags.rst create mode 100644 tests/testdata/context/overrides/settings/test_circular_a.yaml create mode 100644 tests/testdata/context/overrides/settings/test_circular_b.yaml create mode 100644 tests/testdata/context/overrides/settings/test_dep_chain_a.yaml create mode 100644 tests/testdata/context/overrides/settings/test_dep_chain_b.yaml create mode 100644 tests/testdata/context/overrides/settings/test_dep_chain_c.yaml create mode 100644 tests/testdata/context/overrides/settings/test_fake_cuda.yaml diff --git a/docs/how-tos/dependency-build-tags.rst b/docs/how-tos/dependency-build-tags.rst new file mode 100644 index 00000000..ca5dfa32 --- /dev/null +++ b/docs/how-tos/dependency-build-tags.rst @@ -0,0 +1,250 @@ +Dependency-Chained Build Tags +============================== + +When building packages that depend on system libraries or other packages, you may +want the build tag to automatically increment when any dependency's build +configuration changes. This ensures downstream packages are rebuilt when their +dependencies are updated, even if the downstream package itself hasn't changed. + +Use Cases +--------- + +Dependency-chained build tags are particularly useful for: + +* **Platform Dependencies**: Packages that depend on CUDA, ROCm, or other + accelerator stacks where the platform version changes independently of your + package code. + +* **Midstream Builds**: When you maintain patches or configuration for upstream + packages and need downstream dependents to rebuild when you change those + patches. + +* **Transitive Rebuilds**: Ensuring an entire dependency chain rebuilds when a + base library changes (e.g., when updating OpenSSL, all packages that depend + on it should get new build tags). + +How It Works +------------ + +Build tags are calculated as: + +.. code-block:: text + + build_tag = own_changelog_count + sum(dependency_build_tags) + +Dependencies are resolved **recursively** and **transitively**. If package A +depends on B, and B depends on C, then A's build tag includes changes from both +B and C. + +Basic Example +------------- + +Consider a simple dependency chain where ``torch`` depends on ``cuda-toolkit``: + +**overrides/settings/cuda-toolkit.yaml**: + +.. code-block:: yaml + + changelog: + "12.8": + - "Initial CUDA 12.8 support" + "12.9": + - "Updated to CUDA 12.9" + +**overrides/settings/torch.yaml**: + +.. code-block:: yaml + + dependencies: + - cuda-toolkit + changelog: + "2.0.0": + - "Initial build" + +When you build ``torch==2.0.0`` with CUDA 12.9: + +* ``cuda-toolkit`` has 1 changelog entry for version 12.9 → build tag = ``1`` +* ``torch`` has 1 own changelog entry + 1 from cuda-toolkit → build tag = ``2`` + +If you later add another CUDA changelog entry: + +.. code-block:: yaml + + changelog: + "12.9": + - "Updated to CUDA 12.9" + - "Fixed memory issue" # New entry + +Now: + +* ``cuda-toolkit`` version 12.9 → build tag = ``2`` +* ``torch`` version 2.0.0 → build tag = ``3`` (automatically incremented!) + +The wheel filename becomes: ``torch-2.0.0-3-cp311-cp311-linux_x86_64.whl`` + +Transitive Dependencies +----------------------- + +Dependencies are resolved transitively through the entire chain. + +**overrides/settings/triton.yaml**: + +.. code-block:: yaml + + dependencies: + - cuda-toolkit + changelog: + "2.3.0": + - "Triton for CUDA 12.x" + +**overrides/settings/torch.yaml**: + +.. code-block:: yaml + + dependencies: + - triton + changelog: + "2.0.0": + - "Initial build" + +Build tags for ``torch==2.0.0``: + +* ``cuda-toolkit`` (version 12.9): 2 changelog entries → build tag = ``2`` +* ``triton`` (version 2.3.0): 1 own + 2 from cuda-toolkit → build tag = ``3`` +* ``torch`` (version 2.0.0): 1 own + 3 from triton → build tag = ``4`` + +Notice that ``torch`` **automatically includes** ``cuda-toolkit``'s changes even +though it only directly lists ``triton`` as a dependency. + +Multiple Dependencies +--------------------- + +A package can depend on multiple packages. All dependency build tags are summed. + +**overrides/settings/vllm.yaml**: + +.. code-block:: yaml + + dependencies: + - torch + - triton + - cuda-toolkit + changelog: + "0.3.0": + - "Initial vLLM build" + +If each dependency has a build tag of 2, then: + +.. code-block:: text + + vllm build_tag = 1 (own) + 2 (torch) + 2 (triton) + 2 (cuda-toolkit) = 7 + +"Fake Packages" for Platform Dependencies +------------------------------------------ + +You can create settings files for platform dependencies (like CUDA, ROCm) that +don't actually exist as Python packages. These act as markers to track platform +version changes. + +**overrides/settings/cuda-toolkit.yaml**: + +.. code-block:: yaml + + # No source code - just a version marker + changelog: + "12.8": + - "CUDA 12.8.0 release" + "12.9": + - "CUDA 12.9.0 release" + +**overrides/settings/rocm-runtime.yaml**: + +.. code-block:: yaml + + changelog: + "6.0": + - "ROCm 6.0 release" + "6.1": + - "ROCm 6.1 release" + +Now packages can depend on these platform markers: + +.. code-block:: yaml + + dependencies: + - cuda-toolkit # or rocm-runtime for ROCm builds + +Circular Dependency Detection +------------------------------ + +Fromager automatically detects and prevents circular dependencies: + +**overrides/settings/package-a.yaml**: + +.. code-block:: yaml + + dependencies: + - package-b + +**overrides/settings/package-b.yaml**: + +.. code-block:: yaml + + dependencies: + - package-a + +This will raise an error: + +.. code-block:: text + + ValueError: Circular dependency detected: package-a appears in + dependency chain: package-a -> package-b -> package-a + +Scope and Limitations +---------------------- + +**Version-Independent** + +Dependencies apply to **all versions** of a package. You cannot specify +different dependencies for different versions, or use version constraints: + +.. code-block:: yaml + + dependencies: + - torch # ✓ Correct + - torch>=2.0 # ✗ Not supported + - torch; sys_platform=='linux' # ✗ Not supported + +**Per-Package, Not Per-Variant** + +Dependencies are package-level, not variant-level. If you need different +dependencies for CUDA vs ROCm variants, use separate packages or platform markers. + +**Build Tags Only** + +The ``dependencies`` field only affects build tag calculation. It does **not**: + +* Add runtime dependencies to wheel metadata +* Affect dependency resolution during bootstrap +* Change the build environment or compilation flags + +Best Practices +-------------- + +1. **Use Semantic Changelog Entries**: Write clear changelog entries that + explain what changed and why a rebuild is needed. + +2. **Minimize Dependencies**: Only list direct dependencies that actually affect + the build. Transitive dependencies are included automatically. + +3. **Platform Markers**: Use fake packages for system dependencies (CUDA, ROCm, + OpenSSL) to track platform version changes separately from Python packages. + +4. **Testing**: When adding a dependency, verify the build tag increments as + expected by checking the wheel filename. + +See Also +-------- + +* :doc:`/reference/config-reference` - Full configuration reference +* :doc:`/customization` - Comprehensive customization guide diff --git a/docs/how-tos/index.rst b/docs/how-tos/index.rst index fd0465b0..3e641ac0 100644 --- a/docs/how-tos/index.rst +++ b/docs/how-tos/index.rst @@ -46,6 +46,7 @@ Customize builds with overrides, variants, and version handling. :maxdepth: 1 pyproject-overrides + dependency-build-tags multiple-versions pre-release-versions diff --git a/src/fromager/packagesettings/_models.py b/src/fromager/packagesettings/_models.py index 47d96f6d..780acd41 100644 --- a/src/fromager/packagesettings/_models.py +++ b/src/fromager/packagesettings/_models.py @@ -332,6 +332,21 @@ class PackageSettings(pydantic.BaseModel): changelog: VariantChangelog = Field(default_factory=dict) """Changelog entries""" + dependencies: list[Package] = Field(default_factory=list) + """Package dependencies for build tag calculation + + When any dependency's build tag changes, this package's build tag + increases accordingly. Applies to all versions of the package. + Supports transitive dependencies and "fake packages" (platform + dependencies like CUDA, ROCm that have settings but no source). + + Example:: + + dependencies: + - torch + - cuda-toolkit + """ + config_settings: dict[str, str | list[str]] = Field(default_factory=dict) """PEP 517 arbitrary configuration for wheel builds diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index 5c8458ea..c04fcd17 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -50,6 +50,7 @@ class PackageBuildInfo: """ def __init__(self, settings: Settings, ps: PackageSettings) -> None: + self._settings = settings self._variant = typing.cast(Variant, settings.variant) self._patches_dir = settings.patches_dir self._variant_changelog = settings.variant_changelog() @@ -284,7 +285,18 @@ def get_changelog(self, version: Version) -> list[str]: return variant_changelog + package_changelog def build_tag(self, version: Version) -> BuildTag: - """Build tag for version's changelog and this variant + """Build tag for version's changelog and dependencies + + The build tag is calculated as: + own_changelog_count + sum(dependency_build_tags) + + Dependencies are resolved recursively and transitively. + + Args: + version: Package version to calculate build tag for + + Raises: + ValueError: If circular dependency detected .. versionchanged 0.54.0:: @@ -292,16 +304,53 @@ def build_tag(self, version: Version) -> BuildTag: the build tag from changelog, e.g. version `1.0.3+local.suffix` uses `1.0.3`. """ + return self._calculate_build_tag(version, visited=set()) + + def _calculate_build_tag( + self, version: Version, visited: set[NormalizedName] + ) -> BuildTag: + """Recursively calculate build tag including dependencies + + Args: + version: Package version to calculate build tag for + visited: Set of already-visited packages for cycle detection + + Raises: + ValueError: If circular dependency detected + """ if self.pre_built: # pre-built wheels have no built tag return () + + # Check for circular dependency + if self.package in visited: + raise ValueError( + f"Circular dependency detected: {self.package} appears in " + f"dependency chain: {' -> '.join(sorted(visited))} -> {self.package}" + ) + + # Add current package to visited set (immutable update) + visited = visited | {self.package} + + # Calculate own changelog count pv = typing.cast(PackageVersion, version) - release = len(self.get_changelog(pv)) - if release == 0: + own_changelog_count = len(self.get_changelog(pv)) + + # Calculate dependency contribution + dependency_contribution = 0 + for dep_pkg in self._ps.dependencies: + dep_pbi = self._settings.package_build_info(dep_pkg) + dep_tag = dep_pbi._calculate_build_tag(version, visited=visited) + if dep_tag: # Only count if dependency has a build tag + dependency_contribution += dep_tag[0] + + total = own_changelog_count + dependency_contribution + + if total == 0: return () - # suffix = "." + self.variant.replace("-", "_") + suffix = "" - return release, suffix + return total, suffix def get_extra_environ( self, diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 402eebb0..90d57fb4 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -46,6 +46,7 @@ Version("1.0.1"): ["fixed bug"], Version("1.0.2"): ["more bugs", "rebuild"], }, + "dependencies": [], "config_settings": { "setup-args": [ "-Dsystem-freetype=true", @@ -122,6 +123,7 @@ "exclusive_build": False, }, "changelog": {}, + "dependencies": [], "config_settings": {}, "env": {}, "download_source": { @@ -162,6 +164,7 @@ "changelog": { Version("1.0.1"): ["onboard"], }, + "dependencies": [], "config_settings": {}, "env": {}, "download_source": { @@ -504,6 +507,12 @@ def test_settings_overrides(testdata_context: context.WorkContext) -> None: TEST_OTHER_PKG, TEST_RELATED_PKG, TEST_PREBUILT_PKG, + "test-dep-chain-a", + "test-dep-chain-b", + "test-dep-chain-c", + "test-circular-a", + "test-circular-b", + "test-fake-cuda", } @@ -554,6 +563,12 @@ def test_settings_list(testdata_context: context.WorkContext) -> None: TEST_PKG, TEST_RELATED_PKG, TEST_PREBUILT_PKG, + "test-dep-chain-a", + "test-dep-chain-b", + "test-dep-chain-c", + "test-circular-a", + "test-circular-b", + "test-fake-cuda", } assert testdata_context.settings.list_pre_built() == {TEST_PREBUILT_PKG} assert testdata_context.settings.variant_changelog() == [] @@ -899,3 +914,106 @@ def test_version_none_no_reference( result = pbi.get_extra_environ(template_env={}, version=None) assert result["FOO"] == "bar" assert "__version__" not in result + + +def test_build_tag_no_dependencies(testdata_context: context.WorkContext) -> None: + """Test build tag without dependencies (backward compatibility).""" + pbi = testdata_context.settings.package_build_info("test-empty-pkg") + # Empty package has no changelog entries + assert pbi.build_tag(Version("1.0.0")) == () + + +def test_build_tag_simple_dependency(testdata_context: context.WorkContext) -> None: + """Test build tag with a simple dependency chain: C (no deps).""" + pbi = testdata_context.settings.package_build_info("test-dep-chain-c") + # C has 1 changelog entry, no dependencies + assert pbi.build_tag(Version("1.0.0")) == (1, "") + + +def test_build_tag_transitive_dependencies( + testdata_context: context.WorkContext, +) -> None: + """Test build tag includes all transitive dependencies: A -> B -> C.""" + # C: 1 changelog entry, no dependencies = 1 + pbi_c = testdata_context.settings.package_build_info("test-dep-chain-c") + assert pbi_c.build_tag(Version("1.0.0")) == (1, "") + + # B: 1 changelog entry + 1 from C = 2 + pbi_b = testdata_context.settings.package_build_info("test-dep-chain-b") + assert pbi_b.build_tag(Version("1.0.0")) == (2, "") + + # A: 1 changelog entry + 2 from B (which includes C) = 3 + pbi_a = testdata_context.settings.package_build_info("test-dep-chain-a") + assert pbi_a.build_tag(Version("1.0.0")) == (3, "") + + +def test_build_tag_circular_dependency(testdata_context: context.WorkContext) -> None: + """Test circular dependency detection: A -> B -> A.""" + pbi = testdata_context.settings.package_build_info("test-circular-a") + with pytest.raises(ValueError, match="Circular dependency detected"): + pbi.build_tag(Version("1.0.0")) + + +def test_build_tag_fake_package(testdata_context: context.WorkContext) -> None: + """Test fake package (platform dependency with settings but no source).""" + pbi = testdata_context.settings.package_build_info("test-fake-cuda") + # Fake package has changelog for version 12.9 (1 entry) + assert pbi.build_tag(Version("12.9")) == (1, "") + # Fake package has changelog for version 12.8 (1 entry) + assert pbi.build_tag(Version("12.8")) == (1, "") + # No changelog for other versions + assert pbi.build_tag(Version("1.0.0")) == () + + +def test_build_tag_with_pre_built_dependency( + testdata_context: context.WorkContext, +) -> None: + """Test that pre-built dependencies contribute 0 to build tag.""" + # test-prebuilt-pkg has pre_built: True for cpu variant + # Even though it has a changelog, pre-built packages return () + pbi = testdata_context.settings.package_build_info("test-prebuilt-pkg") + assert pbi.pre_built is True + assert pbi.build_tag(Version("1.0.1")) == () + + +def test_dependencies_field_default() -> None: + """Test that dependencies field defaults to empty list.""" + ps = PackageSettings.from_default("test-pkg") + assert ps.dependencies == [] + + +def test_dependencies_field_from_yaml(testdata_context: context.WorkContext) -> None: + """Test that dependencies are parsed from YAML.""" + pbi = testdata_context.settings.package_build_info("test-dep-chain-a") + ps = pbi._ps + assert ps.dependencies == ["test-dep-chain-b"] + + +def test_build_tag_non_existent_dependency( + testdata_context: context.WorkContext, +) -> None: + """Test that non-existent dependencies create default settings with empty changelog.""" + # Create a package with a dependency that doesn't exist + ps = PackageSettings.from_string( + "test-nonexist-dep", + """ +dependencies: + - nonexistent-package +changelog: + "1.0.0": + - "My change" +""", + ) + + # Manually create a Settings object to test + settings = Settings( + settings=SettingsFile(), + package_settings=[ps], + variant="cpu", + patches_dir=testdata_context.settings.patches_dir, + max_jobs=1, + ) + + pbi = settings.package_build_info("test-nonexist-dep") + # Should be 1 (own changelog) + 0 (non-existent dependency has no changelog) = 1 + assert pbi.build_tag(Version("1.0.0")) == (1, "") diff --git a/tests/testdata/context/overrides/settings/test_circular_a.yaml b/tests/testdata/context/overrides/settings/test_circular_a.yaml new file mode 100644 index 00000000..d287eb75 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_circular_a.yaml @@ -0,0 +1,5 @@ +dependencies: + - test-circular-b +changelog: + "1.0.0": + - "Package A" diff --git a/tests/testdata/context/overrides/settings/test_circular_b.yaml b/tests/testdata/context/overrides/settings/test_circular_b.yaml new file mode 100644 index 00000000..3f0819d5 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_circular_b.yaml @@ -0,0 +1,5 @@ +dependencies: + - test-circular-a +changelog: + "1.0.0": + - "Package B" diff --git a/tests/testdata/context/overrides/settings/test_dep_chain_a.yaml b/tests/testdata/context/overrides/settings/test_dep_chain_a.yaml new file mode 100644 index 00000000..4d3171c7 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_dep_chain_a.yaml @@ -0,0 +1,5 @@ +dependencies: + - test-dep-chain-b +changelog: + "1.0.0": + - "Initial release" diff --git a/tests/testdata/context/overrides/settings/test_dep_chain_b.yaml b/tests/testdata/context/overrides/settings/test_dep_chain_b.yaml new file mode 100644 index 00000000..67bbbdbe --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_dep_chain_b.yaml @@ -0,0 +1,5 @@ +dependencies: + - test-dep-chain-c +changelog: + "1.0.0": + - "Base package update" diff --git a/tests/testdata/context/overrides/settings/test_dep_chain_c.yaml b/tests/testdata/context/overrides/settings/test_dep_chain_c.yaml new file mode 100644 index 00000000..cb3cff52 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_dep_chain_c.yaml @@ -0,0 +1,3 @@ +changelog: + "1.0.0": + - "Foundation change" diff --git a/tests/testdata/context/overrides/settings/test_fake_cuda.yaml b/tests/testdata/context/overrides/settings/test_fake_cuda.yaml new file mode 100644 index 00000000..81f3eb61 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_fake_cuda.yaml @@ -0,0 +1,5 @@ +changelog: + "12.9": + - "Updated to CUDA 12.9" + "12.8": + - "Updated to CUDA 12.8"