feat(resolver): wire source_resolver into resolver and download pipelines

Activate the `source` field on `PackageSettings` and `VariantInfo` so
YAML-configured providers are used by the resolver and download
pipelines.  This is the wiring layer for the model classes introduced
in PR #1052.

- Uncomment `source: SourceResolver | None` in `PackageSettings` and
  `VariantInfo`
- Add `PackageBuildInfo.source_resolver` property (variant overrides
  package level)
- `default_resolver_provider()` checks `pbi.source_resolver` before
  returning the default `PyPIProvider`
- `default_download_source()` detects `git+https://` / `git+ssh://`
  URLs and routes to `download_git_source()`
- Add `GitOptions.remove_dot_git` field (default `True`) and wire it
  into `download_git_source()`
- Update existing test snapshots and add 20 new tests

Closes: #1048
Co-Authored-By: Claude <claude@anthropic.com>
Signed-off-by: Shanmukh Pawan <smoparth@redhat.com>

Author: smoparth

src/fromager/packagesettings/_models.py

@@ -15,7 +15,7 @@
 from pydantic import AnyUrl, Field
 from pydantic_core import core_schema
 
-# from ._resolver import SourceResolver
+from ._resolver import SourceResolver
 from ._typedefs import (
     MODEL_CONFIG,
     BuildDirectory,
@@ -324,9 +324,10 @@ class VariantInfo(pydantic.BaseModel):
     pre_built: bool = False
     """Use pre-built wheel from index server?"""
 
-    # TODO
-    # source: SourceResolver | None
-    # """Source resolver and downloader"""
+    source: typing.Annotated[
+        SourceResolver | None,
+        pydantic.Field(default=None, discriminator="provider"),
+    ] = None
 
 
 class GitOptions(pydantic.BaseModel):
@@ -336,6 +337,7 @@ class GitOptions(pydantic.BaseModel):
 
         submodules: False
         submodule_paths: []
+        remove_dot_git: True
     """
 
     model_config = MODEL_CONFIG
@@ -358,6 +360,18 @@ class GitOptions(pydantic.BaseModel):
     - ["vendor/lib1", "vendor/lib2"]
     """
 
+    remove_dot_git: bool = False
+    """Remove ``.git`` directory after cloning?
+
+    When True, the ``.git`` directory is removed from the cloned source
+    tree so it does not end up in the built sdist.  Defaults to False
+    to preserve backward compatibility with existing ``req.url`` git
+    clones that rely on ``.git`` for version detection (e.g. via
+    setuptools-scm).
+
+    .. versionadded:: 0.85
+    """
+
 
 _DictStrAny = dict[str, typing.Any]
 
@@ -452,9 +466,10 @@ class PackageSettings(pydantic.BaseModel):
     project_override: ProjectOverride = Field(default_factory=ProjectOverride)
     """Patch project settings"""
 
-    # TODO
-    # source: SourceResolver | None
-    # """Source resolver and downloader"""
+    source: typing.Annotated[
+        SourceResolver | None,
+        pydantic.Field(default=None, discriminator="provider"),
+    ] = None
 
     variants: Mapping[Variant, VariantInfo] = Field(default_factory=dict)
     """Variant configuration"""

src/fromager/packagesettings/_pbi.py

@@ -26,6 +26,7 @@
 
 if typing.TYPE_CHECKING:
     from .. import build_environment
+    from ._resolver import SourceResolver
     from ._settings import Settings
 
 logger = logging.getLogger(__name__)
@@ -176,6 +177,18 @@ def pre_built(self) -> bool:
             return vi.pre_built
         return False
 
+    @property
+    def source_resolver(self) -> SourceResolver | None:
+        """Effective source resolver for this package and variant.
+
+        Returns the variant-level ``source`` override if set, otherwise
+        the package-level ``source``, or ``None`` when neither is configured.
+        """
+        vi = self._ps.variants.get(self._variant)
+        if vi is not None and vi.source is not None:
+            return vi.source
+        return self._ps.source
+
     @property
     def wheel_server_url(self) -> str | None:
         """Alternative package index for pre-build wheel"""

src/fromager/resolver.py

@@ -119,14 +119,23 @@ def default_resolver_provider(
     include_wheels: bool,
     req_type: RequirementType | None = None,
     ignore_platform: bool = False,
-) -> (
-    PyPIProvider
-    | GenericProvider
-    | GitHubTagProvider
-    | GitLabTagProvider
-    | VersionMapProvider
-):
-    """Lookup resolver provider to resolve package versions"""
+) -> BaseProvider:
+    """Lookup resolver provider to resolve package versions.
+
+    When the package has a ``source`` configuration, the provider is
+    created from the declarative resolver model.  Otherwise the default
+    ``PyPIProvider`` is returned.
+    """
+    pbi = ctx.package_build_info(req)
+    source = pbi.source_resolver
+    if source is not None:
+        logger.info(
+            "%s: using source resolver provider %r",
+            req.name,
+            source.provider,
+        )
+        return source.resolver_provider(ctx, typing.cast(RequirementType, req_type))
+
     return PyPIProvider(
         include_sdists=include_sdists,
         include_wheels=include_wheels,

src/fromager/sources.py

@@ -199,6 +199,30 @@ def resolve_source(
         raise
 
 
+def _is_git_url(url: str) -> bool:
+    """Return True if *url* is a VCS-style ``git+https://`` or ``git+ssh://`` URL."""
+    return url.startswith("git+https://") or url.startswith("git+ssh://")
+
+
+def _parse_git_url(url: str) -> tuple[str, str | None]:
+    """Split a VCS URL into clone URL and optional ref.
+
+    ``git+https://host/repo@ref`` -> ``(https://host/repo, ref)``
+    ``git+https://host/repo``     -> ``(https://host/repo, None)``
+    """
+    clone_url = url
+    if clone_url.startswith("git+"):
+        clone_url = clone_url[len("git+") :]
+
+    parsed = urlparse(clone_url)
+    ref: str | None = None
+    if "@" in parsed.path:
+        new_path, _, ref = parsed.path.rpartition("@")
+        clone_url = parsed._replace(path=new_path).geturl()
+
+    return clone_url, ref
+
+
 def default_download_source(
     ctx: context.WorkContext,
     req: Requirement,
@@ -208,10 +232,24 @@ def default_download_source(
 ) -> pathlib.Path:
     "Download the requirement and return the name of the output path."
     pbi = ctx.package_build_info(req)
-    destination_filename = pbi.download_source_destination_filename(version=version)
     url = pbi.download_source_url(version=version, default=download_url)
     if url is None:
         raise ValueError(f"Could not determine download URL for {req}")
+
+    if _is_git_url(url):
+        clone_url, ref = _parse_git_url(url)
+        download_path = ctx.work_dir / f"{req.name}-{version}" / f"{req.name}-{version}"
+        download_path.mkdir(parents=True, exist_ok=True)
+        download_git_source(
+            ctx=ctx,
+            req=req,
+            url_to_clone=clone_url,
+            destination_dir=download_path,
+            ref=ref,
+        )
+        return download_path
+
+    destination_filename = pbi.download_source_destination_filename(version=version)
     if destination_filename is None:
         url_filename = resolver.extract_filename_from_url(url)
         if url_filename.endswith(".zip"):
@@ -239,21 +277,22 @@ def download_git_source(
     destination_dir: pathlib.Path,
     ref: str | None = None,
 ) -> None:
+    """Clone a git repository into *destination_dir*.
+
+    Applies ``git_options`` from the package settings (submodules,
+    ``remove_dot_git``).
+    """
     if url_to_clone.startswith("git+"):
         url_to_clone = url_to_clone[len("git+") :]
 
     logger.info(f"cloning source from {url_to_clone}@{ref} to {destination_dir}")
-    # Get git options from package settings
     pbi = ctx.package_build_info(req)
     git_opts = pbi.git_options
 
-    # Configure submodules based on package settings
     submodules: bool | list[str] = False
     if git_opts.submodule_paths:
-        # If specific paths are configured, use those
         submodules = git_opts.submodule_paths
     elif git_opts.submodules:
-        # If general submodule support is enabled, clone all submodules
         submodules = True
 
     gitutils.git_clone(
@@ -265,6 +304,12 @@ def download_git_source(
         ref=ref,
     )
 
+    if git_opts.remove_dot_git:
+        dot_git = destination_dir / ".git"
+        if dot_git.exists():
+            logger.info("removing %s", dot_git)
+            shutil.rmtree(dot_git)
+
 
 # Helper method to check whether .zip /.tar / .tgz is able to extract and check its content.
 # It will throw exception if any other file is encountered. Eg: index.html

tests/test_packagesettings.py

@@ -71,6 +71,7 @@
     "git_options": {
         "submodules": False,
         "submodule_paths": [],
+        "remove_dot_git": False,
     },
     "name": "test-pkg",
     "has_config": True,
@@ -88,6 +89,7 @@
         "use_pypi_org_metadata": True,
         "min_release_age": None,
     },
+    "source": None,
     "variants": {
         "cpu": {
             "annotations": {
@@ -96,6 +98,7 @@
             "env": {"EGG": "spam ${EGG}", "EGG_AGAIN": "$EGG"},
             "wheel_server_url": "https://wheel.test/simple",
             "pre_built": False,
+            "source": None,
         },
         "rocm": {
             "annotations": {
@@ -104,12 +107,14 @@
             "env": {"SPAM": ""},
             "wheel_server_url": None,
             "pre_built": True,
+            "source": None,
         },
         "cuda": {
             "annotations": None,
             "env": {},
             "wheel_server_url": None,
             "pre_built": False,
+            "source": None,
         },
     },
 }
@@ -134,6 +139,7 @@
     "git_options": {
         "submodules": False,
         "submodule_paths": [],
+        "remove_dot_git": False,
     },
     "has_config": True,
     "purl": None,
@@ -150,6 +156,7 @@
         "use_pypi_org_metadata": None,
         "min_release_age": None,
     },
+    "source": None,
     "variants": {},
 }
 
@@ -175,6 +182,7 @@
     "git_options": {
         "submodules": False,
         "submodule_paths": [],
+        "remove_dot_git": False,
     },
     "has_config": True,
     "purl": None,
@@ -191,12 +199,14 @@
         "use_pypi_org_metadata": None,
         "min_release_age": None,
     },
+    "source": None,
     "variants": {
         "cpu": {
             "annotations": None,
             "env": {},
             "pre_built": True,
             "wheel_server_url": None,
+            "source": None,
         },
     },
 }