diff --git a/providers/git/docs/connections/git.rst b/providers/git/docs/connections/git.rst index 8bde13c1719e1..9799bbdd0f169 100644 --- a/providers/git/docs/connections/git.rst +++ b/providers/git/docs/connections/git.rst @@ -116,3 +116,39 @@ Extra (optional) "strict_host_key_checking": "yes", "known_hosts_file": "/path/to/known_hosts" } + + **GitHub App authentication:** + + In order to use GitHub App authentication the ``github`` extra needs to be installed: + + .. code-block:: bash + + pip install 'apache-airflow-providers-git[github]' + + * ``github_app_id``: The App ID of your GitHub App. Note that the GitHub App Client ID can also be used. + * ``github_installation_id``: The installation ID of your GitHub app. + * ``key_file``: Path to a PEM-encoded private key file for your GitHub App. + * ``private_key``: An inline PEM-encoded private key string. When provided, the hook writes it + to a temporary file and uses it for the GitHub App connection. + Mutually exclusive with ``key_file``. + + + Example with key file: + + .. code-block:: json + + { + "github_app_id": "1234567", + "github_installation_id": "67890", + "key_file": "/path/to/private-key.pem" + } + + Example with inline private key: + + .. code-block:: json + + { + "github_app_id": "1234567", + "github_installation_id": "67890", + "private_key": "" + } diff --git a/providers/git/docs/index.rst b/providers/git/docs/index.rst index 164d26ed52373..de0337148b95d 100644 --- a/providers/git/docs/index.rst +++ b/providers/git/docs/index.rst @@ -100,6 +100,23 @@ PIP package Version required ``GitPython`` ``>=3.1.44`` ========================================== ================== +Optional dependencies +--------------------- + +These extras install optional third-party libraries that enable additional features of the provider. +Install them when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-git[github] + + +========== =================== +Extra Dependencies +========== =================== +``github`` ``PyGithub>=2.1.1`` +========== =================== + Downloading official packages ----------------------------- diff --git a/providers/git/pyproject.toml b/providers/git/pyproject.toml index 3975fffe56b0c..4e47ce78faf7a 100644 --- a/providers/git/pyproject.toml +++ b/providers/git/pyproject.toml @@ -64,6 +64,13 @@ dependencies = [ "GitPython>=3.1.44", ] +# The optional dependencies should be modified in place in the generated file +# Any change in the dependencies is preserved when the file is regenerated +[project.optional-dependencies] +github = [ + "PyGithub>=2.1.1", +] + [dependency-groups] dev = [ "apache-airflow", @@ -71,6 +78,7 @@ dev = [ "apache-airflow-devel-common", "apache-airflow-providers-common-compat", # Additional devel dependencies (do not remove this line and add extra development dependencies) + "PyGithub>=2.1.1" ] # To build docs: diff --git a/providers/git/src/airflow/providers/git/hooks/git.py b/providers/git/src/airflow/providers/git/hooks/git.py index 0eafece941a8f..0f59fde3f2b84 100644 --- a/providers/git/src/airflow/providers/git/hooks/git.py +++ b/providers/git/src/airflow/providers/git/hooks/git.py @@ -26,11 +26,13 @@ import stat import tempfile import warnings +from collections.abc import Generator +from datetime import datetime, timedelta, timezone from typing import Any from urllib.parse import quote as urlquote from airflow.exceptions import AirflowProviderDeprecationWarning -from airflow.providers.common.compat.sdk import BaseHook +from airflow.providers.common.compat.sdk import AirflowOptionalProviderFeatureException, BaseHook log = logging.getLogger(__name__) @@ -53,6 +55,10 @@ class GitHook(BaseHook): * ``ssh_config_file`` — path to a custom SSH config file. * ``host_proxy_cmd`` — SSH ProxyCommand string (e.g. for bastion/jump hosts). * ``ssh_port`` — non-default SSH port. + * ``github_app_id`` — GitHub App ID used for GitHub App authentication. Requires the GitHub App + private key to be provided as a PEM-encoded key via either ``private_key`` (inline) or + ``key_file`` (path to key file). + * ``github_installation_id`` — GitHub App installation ID used for GitHub App authentication. """ conn_name_attr = "git_conn_id" @@ -80,6 +86,8 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: "ssh_config_file": "", "host_proxy_cmd": "", "ssh_port": "", + "github_app_id": "", + "github_installation_id": "", } ) }, @@ -110,6 +118,11 @@ def __init__( self.host_proxy_cmd = extra.get("host_proxy_cmd") self.ssh_port: int | None = int(extra["ssh_port"]) if extra.get("ssh_port") else None + # GitHub App Auth Options + self.github_app_id = extra.get("github_app_id") + self.github_installation_id = extra.get("github_installation_id") + self.github_app_token_exp: datetime | None = None + self.env: dict[str, str] = {} if self.key_file and self.private_key: @@ -127,6 +140,21 @@ def __init__( AirflowProviderDeprecationWarning, stacklevel=2, ) + github_app_fields = (self.github_app_id, self.github_installation_id) + if any(github_app_fields) and not all(github_app_fields): + raise ValueError( + "Both 'github_app_id' and 'github_installation_id' must be provided to use GitHub App Authentication" + ) + if all(github_app_fields): + if self.auth_token: + raise ValueError("Password field must be empty to use GitHub App Auth") + if not (self.repo_url or "").startswith(("https://", "http://")): + raise ValueError( + f"GitHub App authentication requires an HTTPS repository URL, but got: {self.repo_url!r}" + ) + if self.key_file and not self.private_key: + with open(self.key_file, encoding="utf-8") as key_file: + self.private_key = key_file.read() self._process_git_auth_url() _VALID_STRICT_HOST_KEY_CHECKING = frozenset({"yes", "no", "accept-new", "off", "ask"}) @@ -183,7 +211,89 @@ def _build_ssh_command(self, key_path: str | None = None) -> str: return " ".join(parts) - def _process_git_auth_url(self): + def _get_github_app_token(self): + try: + from github import Auth, GithubIntegration + except ImportError as exc: + raise AirflowOptionalProviderFeatureException( + "The PyGithub library is required for GitHub App authentication. Please install it with 'pip install apache-airflow-providers-git[github]'" + ) from exc + + auth = Auth.AppAuth(self.github_app_id, self.private_key) + integration = GithubIntegration(auth=auth) + access_token = integration.get_access_token(installation_id=self.github_installation_id) + github_app_token_exp = access_token.expires_at + log.info( + "Successfully obtained GitHub App installation access token (expires at: %s)", + github_app_token_exp, + ) + + return "x-access-token", access_token.token, github_app_token_exp + + def _ensure_github_app_token(self) -> None: + TOKEN_REFRESH_BUFFER = timedelta(minutes=5) + if ( + self.github_app_token_exp is None + or self.github_app_token_exp < datetime.now(timezone.utc) + TOKEN_REFRESH_BUFFER + ): + log.info( + "GitHub App token is missing or near expiry (expires at: %s). Refreshing token.", + self.github_app_token_exp, + ) + self.user_name, self.auth_token, self.github_app_token_exp = self._get_github_app_token() + + @contextlib.contextmanager + def _github_app_askpass_env(self) -> Generator[None]: + if not self.auth_token: + yield + return + + token = shlex.quote(self.auth_token) + with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete=True) as askpass_script: + askpass_script.write( + "#!/bin/sh\n" + 'case "$1" in\n' + " *Username*) echo x-access-token;;\n" + f" *Password*) echo {token};;\n" + f" *) echo {token};;\n" + "esac\n" + ) + askpass_script.flush() + os.chmod(askpass_script.name, stat.S_IRWXU) + + old_askpass = os.environ.get("GIT_ASKPASS") + old_lc_all = os.environ.get("LC_ALL") + old_terminal_prompt = os.environ.get("GIT_TERMINAL_PROMPT") + try: + os.environ["GIT_ASKPASS"] = askpass_script.name + os.environ["GIT_TERMINAL_PROMPT"] = "0" + self.env["GIT_ASKPASS"] = askpass_script.name + self.env["LC_ALL"] = "C" + self.env["GIT_TERMINAL_PROMPT"] = "0" + yield + finally: + if old_askpass is None: + self.env.pop("GIT_ASKPASS", None) + os.environ.pop("GIT_ASKPASS", None) + else: + self.env["GIT_ASKPASS"] = old_askpass + os.environ["GIT_ASKPASS"] = old_askpass + + if old_lc_all is None: + self.env.pop("LC_ALL", None) + os.environ.pop("LC_ALL", None) + else: + self.env["LC_ALL"] = old_lc_all + os.environ["LC_ALL"] = old_lc_all + + if old_terminal_prompt is None: + self.env.pop("GIT_TERMINAL_PROMPT", None) + os.environ.pop("GIT_TERMINAL_PROMPT", None) + else: + self.env["GIT_TERMINAL_PROMPT"] = old_terminal_prompt + os.environ["GIT_TERMINAL_PROMPT"] = old_terminal_prompt + + def _process_git_auth_url(self) -> None: if not isinstance(self.repo_url, str): return if self.auth_token and self.repo_url.startswith("https://"): @@ -240,6 +350,12 @@ def _passphrase_askpass_env(self): @contextlib.contextmanager def configure_hook_env(self): + if self.github_app_id is not None and self.github_installation_id is not None: + self._ensure_github_app_token() + with self._github_app_askpass_env(): + yield + return + if self.private_key: with tempfile.NamedTemporaryFile(mode="w", delete=True) as tmp_keyfile: tmp_keyfile.write(self.private_key) diff --git a/providers/git/tests/unit/git/hooks/test_git.py b/providers/git/tests/unit/git/hooks/test_git.py index 484791fe19c34..30b8717dd708c 100644 --- a/providers/git/tests/unit/git/hooks/test_git.py +++ b/providers/git/tests/unit/git/hooks/test_git.py @@ -51,6 +51,12 @@ def bundle_temp_dir(tmp_path): CONN_ONLY_INLINE_KEY = "my_git_conn_only_inline_key" CONN_BOTH_PATH_INLINE = "my_git_conn_both_path_inline" CONN_NO_REPO_URL = "my_git_conn_no_repo_url" +CONN_APP_INLINE_KEY = "git_app_inline_key" +CONN_APP_ONLY_APP_ID = "git_app_only_app_id" +CONN_APP_ONLY_INSTALLATION_ID = "git_app_only_installation_id" +CONN_APP_NO_KEY = "git_app_no_key" +CONN_APP_INVALID_APP_ID = "git_app_invalid_app_id" +CONN_APP_INVALID_INSTALLATION_ID = "git_app_invalid_installation_id" @pytest.fixture @@ -122,6 +128,85 @@ def setup_connections(self, create_connection_without_db): }, ) ) + create_connection_without_db( + Connection( + conn_id=CONN_BOTH_PATH_INLINE, + host="path/to/repo", + conn_type="git", + extra={ + "key_file": "path/to/key", + "private_key": "inline_key", + }, + ) + ) + create_connection_without_db( + Connection( + conn_id="my_git_conn_strict", + host=AIRFLOW_GIT, + conn_type="git", + extra='{"key_file": "/files/pkey.pem", "strict_host_key_checking": "yes"}', + ) + ) + create_connection_without_db( + Connection( + conn_id=CONN_APP_INLINE_KEY, + host=AIRFLOW_HTTPS_URL, + conn_type="git", + extra={ + "github_app_id": "12345", + "github_installation_id": "67890", + "private_key": "inline_pem_key", + }, + ) + ) + create_connection_without_db( + Connection( + conn_id=CONN_APP_ONLY_APP_ID, + host=AIRFLOW_HTTPS_URL, + conn_type="git", + extra={"github_app_id": "12345"}, + ) + ) + create_connection_without_db( + Connection( + conn_id=CONN_APP_ONLY_INSTALLATION_ID, + host=AIRFLOW_HTTPS_URL, + conn_type="git", + extra={"github_installation_id": "67890"}, + ) + ) + create_connection_without_db( + Connection( + conn_id=CONN_APP_NO_KEY, + host=AIRFLOW_HTTPS_URL, + conn_type="git", + extra={"github_app_id": "12345", "github_installation_id": "67890"}, + ) + ) + create_connection_without_db( + Connection( + conn_id=CONN_APP_INVALID_APP_ID, + host=AIRFLOW_HTTPS_URL, + conn_type="git", + extra={ + "github_app_id": "not_an_int", + "github_installation_id": "67890", + "private_key": "inline_pem_key", + }, + ) + ) + create_connection_without_db( + Connection( + conn_id=CONN_APP_INVALID_INSTALLATION_ID, + host=AIRFLOW_HTTPS_URL, + conn_type="git", + extra={ + "github_app_id": "12345", + "github_installation_id": "not_an_int", + "private_key": "inline_pem_key", + }, + ) + ) @pytest.mark.parametrize( ("conn_id", "hook_kwargs", "expected_repo_url", "warns_on_default"), @@ -413,3 +498,206 @@ def test_passphrase_askpass_cleaned_up(self, create_connection_without_db): assert os.path.exists(askpass_path) # Both the askpass script and the temp key file should be cleaned up assert not os.path.exists(askpass_path) + + # --- GitHub App auth tests --- + + def test_only_app_id_without_installation_id_raises(self): + with pytest.raises( + ValueError, match="Both 'github_app_id' and 'github_installation_id' must be provided" + ): + GitHook(git_conn_id=CONN_APP_ONLY_APP_ID) + + def test_only_installation_id_without_app_id_raises(self): + with pytest.raises( + ValueError, + match="Both 'github_app_id' and 'github_installation_id' must be provided", + ): + GitHook(git_conn_id=CONN_APP_ONLY_INSTALLATION_ID) + + def test_app_id_and_installation_id_without_key_does_not_raise_on_init(self): + hook = GitHook(git_conn_id=CONN_APP_NO_KEY) + + assert hook.github_app_id == "12345" + assert hook.github_installation_id == "67890" + assert hook.private_key is None + + def test_app_auth_with_key_file_reads_file(self, create_connection_without_db, tmp_path, monkeypatch): + key_file = tmp_path / "app_key.pem" + key_file.write_text("file_pem_key_content") + create_connection_without_db( + Connection( + conn_id="git_app_key_file", + host=AIRFLOW_HTTPS_URL, + conn_type="git", + extra={ + "github_app_id": "12345", + "github_installation_id": "67890", + "key_file": str(key_file), + }, + ) + ) + from datetime import datetime, timedelta, timezone + + mock_expiry = datetime.now(timezone.utc) + timedelta(hours=1) + monkeypatch.setattr( + "airflow.providers.git.hooks.git.GitHook._get_github_app_token", + lambda self: ("x-access-token", "ghs_test_token", mock_expiry), + ) + with pytest.warns(AirflowProviderDeprecationWarning, match="accept-new"): + hook = GitHook(git_conn_id="git_app_key_file") + + assert hook.private_key == "file_pem_key_content" + + def test_app_auth_with_missing_key_file_raises(self, create_connection_without_db): + create_connection_without_db( + Connection( + conn_id="git_app_missing_key_file", + host=AIRFLOW_HTTPS_URL, + conn_type="git", + extra={ + "github_app_id": "12345", + "github_installation_id": "67890", + "key_file": "/nonexistent/path/key.pem", + }, + ) + ) + with pytest.warns(AirflowProviderDeprecationWarning, match="accept-new"): + with pytest.raises(FileNotFoundError): + GitHook(git_conn_id="git_app_missing_key_file") + + def test_app_auth_defers_token_fetch(self, monkeypatch): + """GitHub App token is not fetched in __init__, only on configure_hook_env.""" + from datetime import datetime, timedelta, timezone + + mock_called = [] + + def mock_get_token(self): + mock_called.append(True) + return ("x-access-token", "ghs_test_token", datetime.now(timezone.utc) + timedelta(hours=1)) + + monkeypatch.setattr( + "airflow.providers.git.hooks.git.GitHook._get_github_app_token", + mock_get_token, + ) + # __init__ should NOT call _get_github_app_token + with pytest.warns(AirflowProviderDeprecationWarning, match="accept-new"): + hook = GitHook(git_conn_id=CONN_APP_INLINE_KEY) + assert len(mock_called) == 0 + assert hook.auth_token is None + assert hook.github_app_id == "12345" + assert hook.github_installation_id == "67890" + + # First call to configure_hook_env should trigger the token fetch + with hook.configure_hook_env(): + assert len(mock_called) == 1 + assert hook.auth_token == "ghs_test_token" + assert hook.user_name == "x-access-token" + + def test_app_auth_success_stores_app_id_and_installation_id(self): + """App ID and installation ID are stored at __init__ time.""" + with pytest.warns(AirflowProviderDeprecationWarning, match="accept-new"): + hook = GitHook(git_conn_id=CONN_APP_INLINE_KEY) + assert hook.github_app_id == "12345" + assert hook.github_installation_id == "67890" + + @pytest.mark.parametrize( + ("app_id", "installation_id"), + [ + ("12345", "67890"), + (12345, 67890), + ], + ) + def test_app_id_and_installation_id_are_stored_as_provided( + self, app_id, installation_id, create_connection_without_db, monkeypatch + ): + from datetime import datetime, timedelta, timezone + + create_connection_without_db( + Connection( + conn_id="git_app_int_check", + host=AIRFLOW_HTTPS_URL, + conn_type="git", + extra={ + "github_app_id": app_id, + "github_installation_id": installation_id, + "private_key": "inline_pem_key", + }, + ) + ) + monkeypatch.setattr( + "airflow.providers.git.hooks.git.GitHook._get_github_app_token", + lambda self: ("x-access-token", "token", datetime.now(timezone.utc) + timedelta(hours=1)), + ) + with pytest.warns(AirflowProviderDeprecationWarning, match="accept-new"): + hook = GitHook(git_conn_id="git_app_int_check") + assert hook.github_app_id == app_id + assert hook.github_installation_id == installation_id + + def test_github_app_token_refresh_near_expiry(self, monkeypatch): + """Token is refreshed when near expiry during configure_hook_env.""" + from datetime import datetime, timedelta, timezone + + mock_get_token_call_count = [0] + + def mock_get_token(self): + mock_get_token_call_count[0] += 1 + # First call returns token expiring in 3 minutes + if mock_get_token_call_count[0] == 1: + return ( + "x-access-token", + f"token_{mock_get_token_call_count[0]}", + datetime.now(timezone.utc) + timedelta(minutes=3), + ) + # Second call (refresh) returns token expiring in 1 hour + return ( + "x-access-token", + f"token_{mock_get_token_call_count[0]}", + datetime.now(timezone.utc) + timedelta(hours=1), + ) + + monkeypatch.setattr( + "airflow.providers.git.hooks.git.GitHook._get_github_app_token", + mock_get_token, + ) + with pytest.warns(AirflowProviderDeprecationWarning, match="accept-new"): + hook = GitHook(git_conn_id=CONN_APP_INLINE_KEY) + assert mock_get_token_call_count[0] == 0 # No call in __init__ + + # First configure_hook_env triggers first token fetch + with hook.configure_hook_env(): + assert mock_get_token_call_count[0] == 1 + assert hook.auth_token == "token_1" + + # Second configure_hook_env triggers refresh (token near expiry) + with hook.configure_hook_env(): + assert mock_get_token_call_count[0] == 2 + assert hook.auth_token == "token_2" + + def test_github_app_integration_call_shape(self, monkeypatch): + """Verify GithubIntegration is called with correct arguments.""" + from datetime import datetime, timedelta, timezone + from unittest import mock + + mock_integration = mock.MagicMock() + mock_access_token = mock.MagicMock() + mock_access_token.token = "ghs_test_token" + mock_access_token.expires_at = datetime.now(timezone.utc) + timedelta(hours=1) + mock_integration.get_access_token.return_value = mock_access_token + + import sys + from types import SimpleNamespace + + fake_github = SimpleNamespace( + Auth=SimpleNamespace(AppAuth=lambda app_id, key: "auth"), + GithubIntegration=lambda auth: mock_integration, + ) + monkeypatch.setitem(sys.modules, "github", fake_github) + + with pytest.warns(AirflowProviderDeprecationWarning, match="accept-new"): + hook = GitHook(git_conn_id=CONN_APP_INLINE_KEY) + with hook.configure_hook_env(): + # Verify get_access_token was called with installation_id kwarg + assert mock_integration.get_access_token.call_count == 1 + _, kwargs = mock_integration.get_access_token.call_args + assert "installation_id" in kwargs + assert str(kwargs["installation_id"]) == "67890" diff --git a/uv.lock b/uv.lock index 53c2e5eca77a4..21a57bbd5b720 100644 --- a/uv.lock +++ b/uv.lock @@ -5425,12 +5425,18 @@ dependencies = [ { name = "gitpython" }, ] +[package.optional-dependencies] +github = [ + { name = "pygithub" }, +] + [package.dev-dependencies] dev = [ { name = "apache-airflow" }, { name = "apache-airflow-devel-common" }, { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-task-sdk" }, + { name = "pygithub" }, ] docs = [ { name = "apache-airflow-devel-common", extra = ["docs"] }, @@ -5441,7 +5447,9 @@ requires-dist = [ { name = "apache-airflow", editable = "." }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "gitpython", specifier = ">=3.1.44" }, + { name = "pygithub", marker = "extra == 'github'", specifier = ">=2.1.1" }, ] +provides-extras = ["github"] [package.metadata.requires-dev] dev = [ @@ -5449,6 +5457,7 @@ dev = [ { name = "apache-airflow-devel-common", editable = "devel-common" }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-task-sdk", editable = "task-sdk" }, + { name = "pygithub", specifier = ">=2.1.1" }, ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }]