Merge branch 'main' into go-v1.26-and-go-fix

Author: gammazero

.github/workflows/gateway-conformance.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.14
         with:
           output: fixtures
           merged: true
@@ -47,7 +47,7 @@ jobs:
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests without IPNS and DNSLink
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.13
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.14
         with:
           gateway-url: http://127.0.0.1:8040
           subdomain-url: http://example.net:8040
@@ -84,7 +84,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.14
         with:
           output: fixtures
           merged: true
@@ -114,7 +114,7 @@ jobs:
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests without IPNS and DNSLink
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.13
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.14
         with:
           gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway
           subdomain-url: http://example.net:8040
@@ -152,7 +152,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.14
         with:
           output: fixtures
           merged: true
@@ -182,7 +182,7 @@ jobs:
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests without IPNS and DNSLink
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.13
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.14
         with:
           gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway
           subdomain-url: http://example.net:8040

CHANGELOG.md

@@ -16,14 +16,20 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+- ✨ `gateway`: responses now include the `Ipfs-Uri` header with a canonical `ipfs://` or `ipns://` URI for the requested content path, and expose it via the default `Access-Control-Expose-Headers`. The header carries the content root in canonical form (base32 CIDv1 for `/ipfs/`, base36 CIDv1 for cryptographic `/ipns/` names, lowercase FQDN for DNSLink) with percent-encoded path segments, so clients get a value that is safe in HTTP field context regardless of bytes in the underlying path. [IPIP-548](https://github.com/ipfs/specs/pull/548) [#1209](https://github.com/ipfs/boxo/pull/1209)
+
 ### Changed
 
  - updated Go in `go.mod` to 1.26.0
+- 🛠 `gateway`: the deprecated `X-Ipfs-Path` response header is no longer sent by default; its value cannot represent all UnixFS file names and it is superseded by `Ipfs-Uri`. **Action required:** consumers that read `X-Ipfs-Path` should migrate to `Ipfs-Uri`; to restore the legacy header meanwhile, set `Config.DeprecatedXIpfsPath` and call `Headers.WithDeprecatedXIpfsPath` before `Headers.ApplyCors` so it is listed in `Access-Control-Expose-Headers` again. Even with the flag set, the header is omitted for content paths with bytes that cannot appear in an HTTP field value (Section 5.5 of RFC 9110), such as raw non-ASCII UnixFS file names: gateway-conformance fails a gateway that sends such values, and only `Ipfs-Uri` carries those paths. [IPIP-548](https://github.com/ipfs/specs/pull/548) [#1209](https://github.com/ipfs/boxo/pull/1209)
+- `gateway`: conformance CI runs [gateway-conformance v0.14.0](https://github.com/ipfs/gateway-conformance/releases/tag/v0.14.0), the first release with the IPIP-548 `Ipfs-Uri` tests. [#1209](https://github.com/ipfs/boxo/pull/1209)
 
 ### Removed
 
 ### Fixed
 
+- `gateway`: `X-Ipfs-Path` values no longer carry bytes that are invalid in an HTTP field value (Section 5.5 of RFC 9110). The header used to echo raw UnixFS file names, so non-ASCII paths arrived garbled or broke strict clients; when the header is enabled, it is now omitted for such paths, which only the percent-encoded `Ipfs-Uri` can carry. [#1209](https://github.com/ipfs/boxo/pull/1209)
+
 ### Security
 
 ## [v0.42.2]

docs/CODEOWNERS

@@ -4,6 +4,3 @@
 
 # Deafult
 * @ipfs/kubo-maintainers
-
-# HTTP Gateway
-gateway/  @lidel

gateway/gateway.go

@@ -79,6 +79,21 @@ type Config struct {
 	// is being proxied by other service, which wants to use the error message.
 	DisableHTMLErrors bool
 
+	// DeprecatedXIpfsPath configures the gateway to send the legacy
+	// X-Ipfs-Path response header, deprecated by [IPIP-0548]. Disabled by
+	// default: the legacy value cannot represent all UnixFS file names (raw
+	// non-ASCII bytes are mangled per RFC 9110 field-value rules) and it is
+	// superseded by the Ipfs-Uri header, which is sent whenever the content
+	// root can be normalized. Enable only for backward compatibility with
+	// legacy consumers that still expect X-Ipfs-Path. When enabled, pair it
+	// with [Headers.WithDeprecatedXIpfsPath] so the header is also listed in
+	// Access-Control-Expose-Headers. Even when enabled, the header is
+	// omitted for content paths that contain bytes that cannot appear in an
+	// HTTP field value (Section 5.5 of RFC 9110), per [IPIP-0548].
+	//
+	// [IPIP-0548]: https://github.com/ipfs/specs/pull/548
+	DeprecatedXIpfsPath bool
+
 	// PublicGateways configures the behavior of known public gateways. Each key is
 	// a fully qualified domain name (FQDN). To be used with WithHostname.
 	PublicGateways map[string]*PublicGateway

gateway/gateway_test.go

@@ -380,6 +380,101 @@ func TestHeaders(t *testing.T) {
 		test(dagCborResponseFormat, dagCborPath, dagCborRoots)
 	})
 
+	t.Run("Ipfs-Uri contains expected values", func(t *testing.T) {
+		test := func(responseFormat string, path string, uri string) {
+			t.Run(responseFormat+" "+path, func(t *testing.T) {
+				url := ts.URL + path
+				req := mustNewRequest(t, http.MethodGet, url, nil)
+				req.Header.Add("Accept", responseFormat)
+				res := mustDoWithoutRedirect(t, req)
+				_, err := io.Copy(io.Discard, res.Body)
+				require.NoError(t, err)
+				defer res.Body.Close()
+				require.Equal(t, http.StatusOK, res.StatusCode)
+				require.Equal(t, uri, res.Header.Get("Ipfs-Uri"))
+			})
+		}
+
+		test("", filePath, "ipfs://"+rootCID+"/subdir/fnord")
+		test("text/html", dirPath, "ipfs://"+rootCID+"/subdir/")
+		test("text/html", "/ipfs/"+rootCID+"/", "ipfs://"+rootCID+"/")
+		test("text/html", hamtFilePath, "ipfs://"+rootCID+"/hamt/685.txt")
+		test(dagJsonResponseFormat, dagCborPath, "ipfs://"+rootCID+"/subdir/dag-cbor-document")
+	})
+
+	t.Run("Ipfs-Uri is sent with 412 from Cache-Control: only-if-cached", func(t *testing.T) {
+		// Valid CID that is missing from the fixture, so only-if-cached
+		// returns 412 before the response format is known.
+		missingCID := "bafkreicm2cerwpdtah2rd7rxg5jcaqsj52blfuaprkurromr5y6p3a5zlu"
+		req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+missingCID, nil)
+		req.Header.Add("Cache-Control", "only-if-cached")
+		res := mustDoWithoutRedirect(t, req)
+		_, err := io.Copy(io.Discard, res.Body)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		require.Equal(t, http.StatusPreconditionFailed, res.StatusCode)
+		require.Equal(t, "ipfs://"+missingCID, res.Header.Get("Ipfs-Uri"))
+	})
+
+	t.Run("Ipfs-Uri is sent with 400 from malformed Accept header", func(t *testing.T) {
+		req := mustNewRequest(t, http.MethodGet, ts.URL+filePath, nil)
+		req.Header.Add("Accept", "application/vnd.ipld.car; version=1; version=2")
+		res := mustDoWithoutRedirect(t, req)
+		_, err := io.Copy(io.Discard, res.Body)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		require.Equal(t, http.StatusBadRequest, res.StatusCode)
+		require.Equal(t, "ipfs://"+rootCID+"/subdir/fnord", res.Header.Get("Ipfs-Uri"))
+	})
+
+	t.Run("X-Ipfs-Path is not sent by default", func(t *testing.T) {
+		req := mustNewRequest(t, http.MethodGet, ts.URL+filePath, nil)
+		res := mustDoWithoutRedirect(t, req)
+		_, err := io.Copy(io.Discard, res.Body)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		require.Equal(t, http.StatusOK, res.StatusCode)
+		require.Empty(t, res.Header.Get("X-Ipfs-Path"))
+		require.NotContains(t, res.Header.Values("Access-Control-Expose-Headers"), "X-Ipfs-Path")
+		require.Contains(t, res.Header.Values("Access-Control-Expose-Headers"), "Ipfs-Uri")
+	})
+
+	t.Run("X-Ipfs-Path is sent when DeprecatedXIpfsPath is enabled", func(t *testing.T) {
+		ts := newTestServerWithConfig(t, backend, Config{
+			DeserializedResponses: true,
+			DeprecatedXIpfsPath:   true,
+		})
+		req := mustNewRequest(t, http.MethodGet, ts.URL+filePath, nil)
+		res := mustDoWithoutRedirect(t, req)
+		_, err := io.Copy(io.Discard, res.Body)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		require.Equal(t, http.StatusOK, res.StatusCode)
+		require.Equal(t, filePath, res.Header.Get("X-Ipfs-Path"))
+		require.Equal(t, "ipfs://"+rootCID+"/subdir/fnord", res.Header.Get("Ipfs-Uri"))
+		require.Contains(t, res.Header.Values("Access-Control-Expose-Headers"), "X-Ipfs-Path")
+	})
+
+	t.Run("X-Ipfs-Path is omitted for field-value-unsafe paths even when DeprecatedXIpfsPath is enabled", func(t *testing.T) {
+		// The content path /ipfs/{cid}/łódź.txt contains raw non-ASCII
+		// bytes, which cannot appear in an HTTP field value (Section 5.5
+		// of RFC 9110), so the legacy header must be omitted despite the
+		// flag; Ipfs-Uri carries the percent-encoded path instead.
+		backend, root := newMockBackend(t, "dir-with-tricky-filenames.car")
+		ts := newTestServerWithConfig(t, backend, Config{
+			DeserializedResponses: true,
+			DeprecatedXIpfsPath:   true,
+		})
+		req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+root.String()+"/%C5%82%C3%B3d%C5%BA.txt", nil)
+		res := mustDoWithoutRedirect(t, req)
+		_, err := io.Copy(io.Discard, res.Body)
+		require.NoError(t, err)
+		defer res.Body.Close()
+		require.Equal(t, http.StatusOK, res.StatusCode)
+		require.Empty(t, res.Header.Get("X-Ipfs-Path"))
+		require.Equal(t, "ipfs://"+root.String()+"/%C5%82%C3%B3d%C5%BA.txt", res.Header.Get("Ipfs-Uri"))
+	})
+
 	t.Run("If-None-Match with wrong value forces path resolution, but X-Ipfs-Roots is correct (regression)", func(t *testing.T) {
 		test := func(responseFormat string, path string, roots string) {
 			t.Run(responseFormat, func(t *testing.T) {

gateway/handler.go

@@ -28,6 +28,7 @@ import (
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
+	"golang.org/x/net/idna"
 )
 
 var log = logging.Logger("boxo/gateway")
@@ -36,6 +37,11 @@ const (
 	ipfsPathPrefix        = "/ipfs/"
 	ipnsPathPrefix        = ipns.NamespacePrefix
 	immutableCacheControl = "public, max-age=29030400, immutable"
+
+	// maxIpfsUriLength caps the value of the Ipfs-Uri response header.
+	// Longer values are not sent (IPIP-0548 allows omission) to stay under
+	// common per-field limits in reverse proxies.
+	maxIpfsUriLength = 8192
 )
 
 var (
@@ -261,6 +267,16 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}()
 
+	// Set as soon as the content path is known, so redirect and error
+	// responses below carry it too (IPIP-0548). The content path does not
+	// change after this point.
+	if i.config.DeprecatedXIpfsPath && isFieldValueSafe(contentPath.String()) {
+		w.Header().Set("X-Ipfs-Path", contentPath.String())
+	}
+	if uri, ok := ipfsUriHeaderValue(contentPath); ok && len(uri) <= maxIpfsUriLength {
+		w.Header().Set("Ipfs-Uri", uri)
+	}
+
 	if i.handleOnlyIfCached(w, r, contentPath) {
 		return
 	}
@@ -274,8 +290,6 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 	trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat))
 	i.requestTypeMetric.WithLabelValues(contentPath.Namespace(), responseFormat).Inc()
 
-	w.Header().Set("X-Ipfs-Path", contentPath.String())
-
 	// Fail fast if unsupported request type was sent to a Trustless Gateway.
 	if !i.isDeserializedResponsePossible(r) && !i.isTrustlessRequest(contentPath, responseFormat) {
 		err := errors.New("only trustless requests are accepted on this gateway: https://specs.ipfs.tech/http-gateways/trustless-gateway/")
@@ -545,6 +559,123 @@ func setIpfsRootsHeader(w http.ResponseWriter, rq *requestData, md *ContentPathM
 	w.Header().Set("X-Ipfs-Roots", rootCidList)
 }
 
+// isFieldValueSafe reports whether s can be carried byte-for-byte in an HTTP
+// field value: only HTAB (0x09), SP (0x20), and visible ASCII (0x21-0x7E)
+// are allowed (Section 5.5 of RFC 9110). CR, LF, and NUL are rejected or
+// replaced by HTTP software, other control characters are invalid, and
+// non-ASCII bytes arrive garbled, so the legacy X-Ipfs-Path header MUST be
+// omitted for such content paths (IPIP-0548); Ipfs-Uri percent-encodes them
+// instead.
+func isFieldValueSafe(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if c := s[i]; c != '\t' && (c < 0x20 || c > 0x7E) {
+			return false
+		}
+	}
+	return true
+}
+
+// ipfsUriHeaderValue builds the value of the Ipfs-Uri response header for the
+// given content path (IPIP-0548): an ipfs:// or ipns:// URI with the content
+// root in its canonical text form, and a URI path that mirrors the content
+// path remainder (everything after "/{namespace}/{root}") with every segment
+// percent-encoded, so a trailing slash is kept. It returns false when the
+// namespace has no URI scheme or the content root cannot be normalized, in
+// which case the header is not sent.
+func ipfsUriHeaderValue(contentPath path.Path) (string, bool) {
+	segments := contentPath.Segments()
+	if len(segments) < 2 {
+		return "", false
+	}
+
+	var authority string
+	switch contentPath.Namespace() {
+	case path.IPFSNamespace:
+		// Canonical form: CIDv1 in lowercase base32.
+		c, err := cid.Decode(segments[1])
+		if err != nil {
+			return "", false
+		}
+		authority, err = cid.NewCidV1(c.Type(), c.Hash()).StringOfBase(multibase.Base32)
+		if err != nil {
+			return "", false
+		}
+	case path.IPNSNamespace:
+		if name, err := ipns.NameFromString(segments[1]); err == nil {
+			// Canonical form for cryptographic IPNS names: CIDv1 in
+			// lowercase base36, preserving the multicodec. ipns.Name
+			// covers libp2p-key, the only codec in use today; roots with
+			// other codecs fall through to the DNSLink branch, fail its
+			// checks, and the header is omitted, as IPIP-0548 allows.
+			authority = name.String()
+		} else {
+			// DNSLink: canonical form is the lowercase FQDN as A-labels,
+			// without the optional trailing dot. Omit the header when the
+			// root does not convert to A-labels or the result is not a
+			// multi-label DNS name.
+			host, err := idna.Lookup.ToASCII(strings.TrimSuffix(segments[1], "."))
+			if err != nil || !strings.Contains(host, ".") ||
+				strings.Contains(host, "..") || strings.HasPrefix(host, ".") || strings.HasSuffix(host, ".") {
+				return "", false
+			}
+			authority = host
+		}
+	default:
+		return "", false
+	}
+
+	var b strings.Builder
+	b.WriteString(contentPath.Namespace())
+	b.WriteString("://")
+	b.WriteString(authority)
+	// The URI path mirrors the content path remainder: split on "/", each
+	// segment percent-encoded, rejoined with "/". The content path is already
+	// normalized (path.NewPath collapses duplicate slashes and dot segments
+	// but keeps a trailing slash), so the remainder here is "", "/", or
+	// "/segment..." with an optional trailing slash, all of which the header
+	// value reproduces.
+	remainder := strings.TrimPrefix(contentPath.String(), "/"+segments[0]+"/"+segments[1])
+	if remainder != "" {
+		for _, segment := range strings.Split(remainder[1:], "/") {
+			b.WriteByte('/')
+			b.WriteString(encodeIpfsUriSegment(segment))
+		}
+	}
+	return b.String(), true
+}
+
+// encodeIpfsUriSegment percent-encodes a single content path segment for use
+// in an Ipfs-Uri value: every byte outside the RFC 3986 unreserved set becomes
+// %XX with uppercase hex, so the output is ASCII-only and byte-identical
+// across implementations. This is stricter than url.PathEscape, which leaves
+// sub-delims like "&" and "=" unencoded. A segment that is exactly "." or
+// ".." is emitted fully percent-encoded (IPIP-0548), so it cannot be taken
+// for a relative dot segment; such segments never survive the gateway's
+// request path normalization, but the encoding is total for any input.
+func encodeIpfsUriSegment(segment string) string {
+	switch segment {
+	case ".":
+		return "%2E"
+	case "..":
+		return "%2E%2E"
+	}
+	const upperhex = "0123456789ABCDEF"
+	var b strings.Builder
+	b.Grow(len(segment))
+	for i := 0; i < len(segment); i++ {
+		c := segment[i]
+		if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' ||
+			c == '-' || c == '.' || c == '_' || c == '~' {
+			b.WriteByte(c)
+		} else {
+			b.WriteByte('%')
+			b.WriteByte(upperhex[c>>4])
+			b.WriteByte(upperhex[c&0xF])
+		}
+	}
+	return b.String()
+}
+
 // lastModifiedMatch returns true if we can respond with HTTP 304 Not Modified
 // It compares If-Modified-Since with logical modification time read from DAG
 // (e.g. UnixFS 1.5 modtime, if present)