test: add downstream client conformance coverage

Signed-off-by: lucarlig <luca.carlig@ibm.com>

Author: lucarlig

.github/workflows/mcp_conformance.yml

@@ -28,6 +28,12 @@ jobs:
       - name: Test conformance reporter
         run: tests/conformance/report-baseline-diff-test.sh
 
+      - name: Test client conformance adapter
+        run: |
+          python3 -m py_compile tests/conformance/write_client_config.py
+          tests/conformance/client-under-test-test.sh
+          tests/conformance/bless-client-baseline-test.sh
+
       - name: Download the conformance binary
         uses: actions/download-artifact@v8.0.1
         with:
@@ -96,9 +102,12 @@ jobs:
         id: runner
         run: tests/conformance/run-conformance.sh
 
+      - name: Run scoped MCP 2026-07-28 client conformance
+        run: tests/conformance/run-client-conformance.sh
+
       - name: Report conformance baseline diff
         if: always()
-        run: tests/conformance/report-baseline-diff.sh
+        run: tests/conformance/report-baseline-diff.sh conformance-results/server
 
       - name: Print live stack logs
         if: always()

Makefile

@@ -29,7 +29,7 @@ conformance: ## Build the data plane and run official MCP 2026-07-28 conformance
 	docker build -t "$(CF_DATAPLANE_IMAGE)" -f docker/conformance.Dockerfile .
 	CF_DATAPLANE_IMAGE="$(CF_DATAPLANE_IMAGE)" tests/conformance/run-local.sh
 
-conformance-bless: ## Run conformance and update the expected-failure baseline
+conformance-bless: ## Run conformance and update the server and client expected-failure baselines
 	MCP_CONFORMANCE_BLESS=true $(MAKE) conformance
 
 docs-serve: ## Serve the wiki book locally at http://127.0.0.1:3000

_context/wiki/testing.md

@@ -37,25 +37,39 @@ These run in `cargo nextest run` with no Docker dependencies.
 ## MCP Conformance CI
 
 `.github/workflows/mcp_conformance.yml` runs the pinned official conformance
-suite `0.2.0-alpha.11` with `--requirements 2026-07-28`. Its small live path is
-official runner → nginx → checked-out dataplane → fixture proxy → official
-fixture, with the published `latest` control plane registering and publishing
+suite `0.2.0-alpha.11` for MCP `2026-07-28` in both directions. The server leg
+is official client → nginx → checked-out dataplane → fixture proxy → official
+server, with the published `latest` control plane registering and publishing
 the fixture through Redis. The backend-only proxy rewrites `Host` to
 `localhost:3000`, which the official fixture's DNS-rebinding protection
 requires, while leaving dataplane header protections unchanged. The control
 plane uses ephemeral SQLite, so PostgreSQL is unnecessary. The harness lives
 in `tests/conformance/`.
 
+The scoped client leg then treats the dataplane as an MCP client: the official
+runner starts a scenario backend, the adapter publishes an isolated route to
+Redis, and a downstream `tools/call` makes the dataplane connect to that
+backend. It covers tool calls, per-request client metadata and protocol-version
+retry, standard MCP headers, and custom parameter headers. The control plane is
+stopped first so its periodic publisher cannot replace the scenario route or
+probe the observation backend. OAuth client scenarios remain a control-plane
+responsibility. Server and client results are written below `server/` and
+`client/`, with separate `expected-failures.yml` and
+`client-expected-failures.yml` baselines.
+
+`make conformance` runs both legs locally, while `make conformance-bless` runs
+both and refreshes both expected-failure baselines from that run.
+
 Because this conformance CLI cannot set a bearer header, nginx adds an
 ephemeral control-plane token when one is absent; there is no auth proxy or
 repository-owned JavaScript. A route probe prevents control-plane fallback.
-Counts and the official fixture log appear directly in the Actions log, and
-`expected-failures.yml` guards the current baseline. The job does not retain a
-separate conformance artifact. `upstream-fixture-failures.yml` records the
-pinned fixture's seven scored failures and one warning; its other 47 failures
-are extension or pending scenarios and are already unscored. CI prints the
-exact actual-versus-baseline diff, adds annotations for unexpected and stale
-entries, and writes the same comparison to the job summary.
+Counts and the official fixture log appear directly in the Actions log. The
+job does not retain a separate conformance artifact.
+`upstream-fixture-failures.yml` records the pinned fixture's seven scored
+failures and one warning; its other 47 failures are extension or pending
+scenarios and are already unscored. CI prints the exact server
+actual-versus-baseline diff, adds annotations for unexpected and stale entries,
+and writes the same comparison to the job summary.
 
 ## Full-Stack Integration Harness
 

tests/conformance/bless-client-baseline-test.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-client-baseline-test.XXXXXX")"
+results_dir="${state_dir}/results"
+baseline_file="${state_dir}/client-expected-failures.yml"
+
+cleanup() {
+  rm -rf -- "${state_dir}"
+}
+trap cleanup EXIT INT TERM
+
+mkdir -p \
+  "${results_dir}/tools_call-2026-08-20T00-00-00-000Z" \
+  "${results_dir}/request-metadata-2026-08-20T00-00-00-000Z" \
+  "${results_dir}/http-standard-headers-2026-08-20T00-00-00-000Z" \
+  "${results_dir}/http-custom-headers-2026-08-20T00-00-00-000Z"
+
+cat > "${baseline_file}" <<'EOF'
+client: []
+EOF
+for scenario in tools_call request-metadata http-standard-headers; do
+  cat > "${results_dir}/${scenario}-2026-08-20T00-00-00-000Z/checks.json" <<'EOF'
+[{"id":"passes","status":"SUCCESS"}]
+EOF
+done
+cat > "${results_dir}/http-custom-headers-2026-08-20T00-00-00-000Z/checks.json" <<'EOF'
+[
+  {"id":"mirrors","status":"SUCCESS"},
+  {"id":"mirrors","status":"FAILURE"},
+  {"id":"encodes","status":"WARNING"},
+  {"id":"omits-null","status":"SUCCESS"}
+]
+EOF
+
+"${script_dir}/bless-client-baseline.sh" "${results_dir}" "${baseline_file}"
+
+cat > "${state_dir}/expected.yml" <<'EOF'
+# Generated by `make conformance-bless` from scoped client findings.
+# OAuth client scenarios are control-plane responsibilities and are not run here.
+client:
+  - http-custom-headers:encodes
+  - http-custom-headers:mirrors
+EOF
+diff -u "${state_dir}/expected.yml" "${baseline_file}"
+
+second_output="$("${script_dir}/bless-client-baseline.sh" "${results_dir}" "${baseline_file}")"
+grep --fixed-strings --quiet -- 'was already current' <<< "${second_output}"
+
+echo 'client conformance baseline tests passed'

tests/conformance/bless-client-baseline.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ "$#" -gt 2 ]; then
+  echo "Usage: bless-client-baseline.sh [results-dir [baseline-file]]" >&2
+  exit 2
+fi
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+repo_root="$(cd -- "${script_dir}/../.." && pwd)"
+results_dir="${1:-${repo_root}/conformance-results/client}"
+baseline_file="${2:-${script_dir}/client-expected-failures.yml}"
+scenarios=(tools_call request-metadata http-standard-headers http-custom-headers)
+
+for command in awk cmp cp jq mktemp sed sort; do
+  if ! command -v "${command}" > /dev/null 2>&1; then
+    echo "Required command not found: ${command}" >&2
+    exit 2
+  fi
+done
+if [ ! -d "${results_dir}" ]; then
+  echo "No client conformance results directory: ${results_dir}" >&2
+  exit 2
+fi
+if [ ! -f "${baseline_file}" ]; then
+  echo "Client conformance baseline not found: ${baseline_file}" >&2
+  exit 2
+fi
+
+state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-client-baseline.XXXXXX")"
+findings="${state_dir}/findings.txt"
+baseline_entries="${state_dir}/baseline-entries.txt"
+candidate="${state_dir}/client-expected-failures.yml"
+
+cleanup() {
+  rm -f -- "${findings}" "${baseline_entries}" "${candidate}"
+  rmdir -- "${state_dir}"
+}
+trap cleanup EXIT INT TERM
+
+: > "${findings}"
+for scenario in "${scenarios[@]}"; do
+  matches=("${results_dir}/${scenario}-"*/checks.json)
+  if [ ! -e "${matches[0]}" ]; then
+    echo "No client conformance results found for scenario: ${scenario}" >&2
+    exit 2
+  fi
+  checks_file="${matches[$((${#matches[@]} - 1))]}"
+  jq --raw-output --arg scenario "${scenario}" '
+    def severity($status):
+      if $status == "FAILURE" then 3
+      elif $status == "WARNING" then 2
+      elif $status == "SUCCESS" then 1
+      else 0
+      end;
+
+    reduce (.[] | select(.status != "INFO")) as $check
+      ({};
+       ($check.id) as $id |
+       if .[$id] == null or severity($check.status) >= severity(.[$id].status)
+       then .[$id] = $check
+       else .
+       end) |
+    to_entries[] |
+    select(.value.status == "FAILURE" or .value.status == "WARNING") |
+    ($scenario + ":" + .key)
+  ' "${checks_file}" >> "${findings}"
+done
+LC_ALL=C sort -u -o "${findings}" "${findings}"
+
+awk '
+  /^client:$/ { in_client = 1; next }
+  in_client && /^[^[:space:]]/ { exit }
+  in_client && /^[[:space:]]*-[[:space:]]+/ {
+    line = $0
+    sub(/^[[:space:]]*-[[:space:]]+/, "", line)
+    sub(/[[:space:]]+#.*$/, "", line)
+    print line
+  }
+' "${baseline_file}" | LC_ALL=C sort -u > "${baseline_entries}"
+
+{
+  # shellcheck disable=SC2016 # Backticks are literal Markdown.
+  echo '# Generated by `make conformance-bless` from scoped client findings.'
+  echo '# OAuth client scenarios are control-plane responsibilities and are not run here.'
+  if [ -s "${findings}" ]; then
+    echo 'client:'
+    sed 's/^/  - /' "${findings}"
+  else
+    echo 'client: []'
+  fi
+} > "${candidate}"
+
+if cmp --silent "${findings}" "${baseline_entries}"; then
+  echo "Client conformance baseline was already current: ${baseline_file}"
+else
+  cp "${candidate}" "${baseline_file}"
+  echo "Client conformance baseline updated: ${baseline_file}"
+fi

tests/conformance/client-expected-failures.yml

@@ -0,0 +1,10 @@
+# Dataplane-owned upstream MCP client findings for the scoped client lane.
+# OAuth scenarios are control-plane responsibilities and are not run here.
+client:
+  # The upstream client does not yet mirror x-mcp-header tool arguments into
+  # Mcp-Param-* request headers. Keep the null/omission checks as required
+  # passes by baselining only the affected checks, not the whole scenario.
+  - http-custom-headers:sep-2243-client-supports-custom-headers
+  - http-custom-headers:sep-2243-client-mirrors-designated-params
+  - http-custom-headers:sep-2243-client-encode-values
+  - http-custom-headers:sep-2243-client-base64-unsafe

tests/conformance/client-under-test-test.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-client-adapter-test.XXXXXX")"
+fake_bin="${state_dir}/bin"
+docker_args="${state_dir}/docker-args"
+curl_bodies="${state_dir}/curl-bodies"
+
+cleanup() {
+  rm -rf -- "${state_dir}"
+}
+trap cleanup EXIT INT TERM
+
+mkdir -p "${fake_bin}"
+cat > "${fake_bin}/docker" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' "$*" > "${FAKE_DOCKER_ARGS}"
+EOF
+cat > "${fake_bin}/curl" <<'EOF'
+#!/usr/bin/env bash
+while [ "$#" -gt 0 ]; do
+  if [ "$1" = "--data" ]; then
+    shift
+    printf '%s\n' "$1" >> "${FAKE_CURL_BODIES}"
+  fi
+  shift
+done
+printf '%s\n' 'data: {"jsonrpc":"2.0","id":1,"result":{"content":[]}}'
+EOF
+chmod +x "${fake_bin}/docker" "${fake_bin}/curl"
+
+export PATH="${fake_bin}:${PATH}"
+export FAKE_DOCKER_ARGS="${docker_args}"
+export FAKE_CURL_BODIES="${curl_bodies}"
+export MCP_CONFORMANCE_PROTOCOL_VERSION=2026-07-28
+export MCP_CONFORMANCE_SUBJECT=test-subject
+export MCP_CONFORMANCE_CLIENT_SERVER_ID=test-client-server
+export MCP_CONFORMANCE_PORT=18080
+export MCP_CONFORMANCE_SCENARIO=http-custom-headers
+export MCP_CONFORMANCE_CONTEXT='{
+  "name": "http-custom-headers",
+  "toolCalls": [
+    {"name": "first", "arguments": {"region": "west"}},
+    {"name": "second", "arguments": {"verbose": null}}
+  ]
+}'
+
+"${script_dir}/client-under-test.sh" "http://localhost:43123/mcp"
+
+grep --fixed-strings --quiet -- 'http://host.docker.internal:43123/mcp' "${docker_args}"
+grep --fixed-strings --quiet -- '["first","second"]' "${docker_args}"
+test "$(wc -l < "${curl_bodies}" | tr -d '[:space:]')" -eq 2
+jq --exit-status --slurp '
+  length == 2 and
+  .[0].method == "tools/call" and
+  .[0].params.name == "first" and
+  .[0].params.arguments.region == "west" and
+  .[0].params._meta["io.modelcontextprotocol/protocolVersion"] == "2026-07-28" and
+  .[1].params.name == "second" and
+  .[1].params.arguments.verbose == null
+' "${curl_bodies}" > /dev/null
+
+echo 'client conformance adapter tests passed'