fix: ensure that headers keys are unique in link_content (#10111)

* fix: ensure that headers keys are unique in link_content

* Add release note

* Fix mypy issue

* Rename variable

* Add unit tests + improve release note

* Revert mypy issue

Author: ZeJ0hn

haystack/components/fetchers/link_content.py

@@ -22,7 +22,6 @@
 
 logger = logging.getLogger(__name__)
 
-
 DEFAULT_USER_AGENT = f"haystack/LinkContentFetcher/{__version__}"
 
 REQUEST_HEADERS = {
@@ -33,6 +32,25 @@
 }
 
 
+def _merge_headers(*args: dict[str, str]) -> dict[str, str]:
+    """
+    Merge a list of dict using case-insensitively
+
+    :param args: a list of dict to merge
+    :returns: The merged dict
+    """
+    merged = {}
+    keymap = {}
+
+    for d in args:
+        for k, v in d.items():
+            kl = k.lower()
+            keymap[kl] = k
+            merged[kl] = v
+
+    return {keymap[kl]: v for kl, v in merged.items()}
+
+
 def _text_content_handler(response: httpx.Response) -> ByteStream:
     """
     Handles text content.
@@ -169,17 +187,24 @@ def __init__(  # pylint: disable=too-many-positional-arguments
             after=self._switch_user_agent,
         )
         def get_response(url):
-            # Build headers with precedence:
-            # client defaults -> component defaults -> user-provided -> rotating UA
-            base = dict(self._client.headers)
-            headers = {**base, **REQUEST_HEADERS, **self.request_headers}
-            headers["User-Agent"] = self.user_agents[self.current_user_agent_idx]  # rotation wins
-            response = self._client.get(url, headers=headers)
+            response = self._client.get(url, headers=self._get_headers())
             response.raise_for_status()
             return response
 
         self._get_response: Callable = get_response
 
+    def _get_headers(self):
+        """
+        Build headers with precedence
+
+        client defaults -> component defaults -> user-provided -> rotating UA
+        """
+        base = dict(self._client.headers)
+        headers = _merge_headers(
+            base, REQUEST_HEADERS, self.request_headers, {"User-Agent": self.user_agents[self.current_user_agent_idx]}
+        )
+        return headers
+
     def __del__(self):
         """
         Clean up resources when the component is deleted.
@@ -378,10 +403,7 @@ async def _get_response_async(self, url: str, client: httpx.AsyncClient) -> http
 
         while attempt <= self.retry_attempts:
             try:
-                base = dict(client.headers)
-                headers = {**base, **REQUEST_HEADERS, **self.request_headers}
-                headers["User-Agent"] = self.user_agents[self.current_user_agent_idx]
-                response = await client.get(url, headers=headers)
+                response = await client.get(url, headers=self._get_headers())
                 response.raise_for_status()
                 return response
             except (httpx.HTTPStatusError, httpx.RequestError) as e:

releasenotes/notes/fix-bad-request-on-link-content-f021c7012d63a60e.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    Ensure request header keys are unique in link_content to prevent 400 Bad Request errors.
+
+    Some image providers return a 400 Bad Request when using ImageContent.from_url() because the User-Agent
+    header appears multiple times with different casing (e.g., user-agent, User-Agent).
+    This update normalizes header keys in a case-insensitive way, removes duplicates, and
+    preserves only the last occurrence.

test/components/fetchers/test_link_content_fetcher.py

@@ -404,3 +404,38 @@ async def test_request_headers_merging_and_ua_override(self):
             assert sent_headers["X-Async"] == "true"
             assert sent_headers["Accept-Language"] == "de-DE"
             assert sent_headers["User-Agent"] == "ua-async-1"  # rotating UA wins
+
+    @pytest.mark.asyncio
+    async def test_duplicated_request_headers_merging(self):
+        # Patch the AsyncClient class to control the instance created by LinkContentFetcher
+        with patch("haystack.components.fetchers.link_content.httpx.AsyncClient") as AsyncClientMock:
+            aclient = AsyncClientMock.return_value
+            aclient.headers = {}  # base headers used in the merge
+
+            mock_response = Mock(status_code=200, text="OK", headers={"Content-Type": "text/plain"})
+            aclient.get = AsyncMock(return_value=mock_response)
+
+            fetcher = LinkContentFetcher(
+                request_headers={
+                    "x-test-header": "header-1",
+                    "X-Test-Header": "agent-2",
+                    "X-TEST-HEADER": "agent-3",
+                    "X-TeSt-HeAdEr": "good-one",
+                }
+            )
+
+            _ = (await fetcher.run_async(urls=["https://example.com"]))["streams"]
+
+            assert aclient.get.await_count == 1
+            sent_headers = aclient.get.call_args.kwargs["headers"]
+            existing_keys = {}
+            for key, value in sent_headers.items():
+                lower_key = key.lower()
+                if lower_key in existing_keys:
+                    assert False
+                elif lower_key == "x-test-header":
+                    assert value == "good-one"
+                existing_keys[lower_key] = key
+
+            assert "x-test-header" in existing_keys
+            assert existing_keys["x-test-header"] == "X-TeSt-HeAdEr"