feat(llm-request-router): route LLM workers through backend router

Route worker gRPC registration by HTTP/2 authority and reverse QUIC by SNI through a highly available backend router. Wire it into the LLM addon, expose the required Gateway routes, harden PKI and render validation, and make local self-managed testing use the same path.

Author: mikeyrcamp

deploy/helm/gateway-routes/Makefile

@@ -14,12 +14,15 @@
 # limitations under the License.
 
 
-.PHONY: test test-render-routes test-vanity-gateway-route
+.PHONY: test test-render-routes test-vanity-gateway-route check-llm-worker-routes
 
-test: test-render-routes test-vanity-gateway-route
+test: test-render-routes test-vanity-gateway-route check-llm-worker-routes
 
 test-render-routes:
 	@sh ./scripts/test-render-routes.sh
 
 test-vanity-gateway-route:
 	@sh ./scripts/test-vanity-gateway-route.sh
+
+check-llm-worker-routes:
+	@sh ./scripts/check-llm-worker-routes.sh

deploy/helm/gateway-routes/README.md

@@ -4,17 +4,26 @@ This repository contains the Helm chart for deploying NVCF ingress routes via th
 
 ## Overview
 
-The chart deploys `HTTPRoute`, `TCPRoute`, and `ReferenceGrant` resources that attach to an existing Gateway provisioned separately by the cluster operator (e.g. Envoy Gateway, Istio, Traefik, Kong). It also includes optional `PodMonitor` resources for scraping Envoy Gateway proxy metrics with Prometheus.
-
-The chart deploys routing configuration only. It does not include any container images. Backend services referenced by the routes (`api`, `nvct-api`, `api-keys`, `invocation`, `llm-api-gateway`, `vanity-gateway`, `reval`, `sis`, `grpc`, `nats`) must already be deployed separately.
+The chart deploys `HTTPRoute`, `GRPCRoute`, `TCPRoute`, `UDPRoute`, and
+`ReferenceGrant` resources that attach to an existing Gateway provisioned
+separately by the cluster operator, such as Envoy Gateway, Istio, Traefik, or
+Kong. It also includes optional `PodMonitor` resources for scraping Envoy
+Gateway proxy metrics with Prometheus.
+
+The chart deploys routing configuration only. It does not include any
+container images. Backend services referenced by the routes (`api`,
+`nvct-api`, `api-keys`, `invocation`, `llm-api-gateway`,
+`llm-request-router-backend-router`, `vanity-gateway`, `reval`, `sis`, `grpc`,
+`nats`) must already be deployed separately.
 
 ## Prerequisites
 
 - Kubernetes cluster
 - Helm 3.x
 - `kubectl`
 - A Gateway API compatible controller installed in the cluster
-- An existing `Gateway` resource with an HTTP listener (and TCP listeners if the gRPC or NATS routes are enabled)
+- Existing `Gateway` resources with the listeners required by each enabled route
+- A Gateway controller with `UDPRoute` support when LLM worker routing is enabled
 - The backend services that the routes target, deployed in their respective namespaces
 
 ## Getting Started
@@ -57,6 +66,8 @@ Important settings to review before deployment:
 - `nvcfGatewayRoutes.gateways.shared.*` for the HTTP Gateway name, namespace, and listener
 - `nvcfGatewayRoutes.gateways.grpc.*` for the TCP Gateway name, namespace, and listener
 - `nvcfGatewayRoutes.gateways.nats.*` for the NATS TCP Gateway name, namespace, and listener
+- `nvcfGatewayRoutes.gateways.llmGrpc.*` for the LLM worker gRPC TCP listener
+- `nvcfGatewayRoutes.gateways.llmQuic.*` for the LLM reverse-tunnel UDP listener
 - `nvcfGatewayRoutes.routes.<route>.enabled` to toggle individual routes
 - `nvcfGatewayRoutes.routes.nvcfApi.grpc.enabled` and
   `nvcfGatewayRoutes.routes.nvctApi.grpc.enabled` to expose API gRPC routes
@@ -87,6 +98,7 @@ Enabled `HTTPRoute` entries must not share a resolved hostname because each `HTT
 | `grpc` | TCPRoute | Not rendered | `grpc.nvcf:10081` |
 | `grpcWorker` | TCPRoute (disabled by default) | Not rendered | `grpc.nvcf:10086` |
 | `nats` | TCPRoute (disabled by default) | Not rendered | `nats.nats-system:4222` |
+| `llmWorker` | TCPRoute and UDPRoute (disabled by default) | Not rendered | `llm-request-router-backend-router.<backend namespace>:50071/TCP,50072/UDP` |
 
 Cross-namespace routing is supported via `ReferenceGrant` resources rendered into each backend namespace.
 
@@ -97,3 +109,12 @@ Cross-namespace routing is supported via `ReferenceGrant` resources rendered int
 - The `grpc` TCPRoute does not enforce HTTP hostname matching at the Gateway layer. Configure DNS or TCP load balancer routing outside this chart.
 - The `grpcWorker` TCPRoute is beta support for split or multi-cluster gRPC worker callbacks. It carries HTTP/1 CONNECT callback traffic only. Enable it only when the control-plane grpc-proxy runs one replica with HPA disabled. Multi-replica grpc-proxy requires pod-specific callback routing and is not supported by this shared TCPRoute.
 - Enabling the `nats` route requires a reachable TCP listener for NATS on the referenced Gateway. The HTTP Gateway address does not imply NATS reachability unless that same Gateway also has the NATS TCP listener configured.
+- The `llmWorker` routes target Stargate's authority/SNI-aware backend router.
+  Set `nvcfGatewayRoutes.routes.llmWorker.backend.namespace` to the effective
+  namespace of the `llm-request-router` release. The gateway chart cannot
+  derive the namespace of a separate Helm release.
+  Use `nvcfGatewayRoutes.routes.llmWorker.backend.grpcPort` for registration
+  traffic and `nvcfGatewayRoutes.routes.llmWorker.backend.quicPort` for reverse
+  tunnels; this route does not use the generic `backend.port` setting.
+  Keep the TCP and UDP Gateways separate when the infrastructure requires
+  separate load balancers for each protocol.

deploy/helm/gateway-routes/chart/templates/_helpers.tpl

@@ -53,6 +53,10 @@ app.kubernetes.io/name: {{ include "nvcf-gateway.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{- define "nvcf-gateway.llmWorkerBackendNamespace" -}}
+{{- required "nvcfGatewayRoutes.routes.llmWorker.backend.namespace is required when llmWorker.enabled is true" .Values.nvcfGatewayRoutes.routes.llmWorker.backend.namespace -}}
+{{- end }}
+
 {{/*
 Validate that enabled HTTPRoutes do not compete for the same hostname and
 root PathPrefix match on the shared Gateway. All HTTPRoute templates in this

deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+{{- if and .Values.nvcfGatewayRoutes.enabled .Values.nvcfGatewayRoutes.routes.llmWorker.enabled }}
+{{- $backendNamespace := include "nvcf-gateway.llmWorkerBackendNamespace" . }}
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: ReferenceGrant
+metadata:
+  name: allow-llm-worker-routes
+  namespace: {{ $backendNamespace }}
+  labels:
+    {{- include "nvcf-gateway.labels" . | nindent 4 }}
+spec:
+  from:
+    - group: gateway.networking.k8s.io
+      kind: TCPRoute
+      namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }}
+    - group: gateway.networking.k8s.io
+      kind: UDPRoute
+      namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }}
+  to:
+    - group: ""
+      kind: Service
+      name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.name }}
+{{- end }}

deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+{{- if and .Values.nvcfGatewayRoutes.enabled .Values.nvcfGatewayRoutes.routes.llmWorker.enabled }}
+{{- $backendNamespace := include "nvcf-gateway.llmWorkerBackendNamespace" . }}
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: TCPRoute
+metadata:
+  name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.name }}-grpc
+  namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }}
+  labels:
+    {{- include "nvcf-gateway.labels" . | nindent 4 }}
+    app.kubernetes.io/component: llm-worker-grpc-route
+  {{- with .Values.nvcfGatewayRoutes.routes.llmWorker.routeAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  parentRefs:
+    - name: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.name }}
+      namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }}
+      sectionName: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.listenerName }}
+  rules:
+    - backendRefs:
+        - name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.name }}
+          namespace: {{ $backendNamespace }}
+          port: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.grpcPort }}
+{{- end }}

deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+{{- if and .Values.nvcfGatewayRoutes.enabled .Values.nvcfGatewayRoutes.routes.llmWorker.enabled }}
+{{- $backendNamespace := include "nvcf-gateway.llmWorkerBackendNamespace" . }}
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: UDPRoute
+metadata:
+  name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.name }}-quic
+  namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }}
+  labels:
+    {{- include "nvcf-gateway.labels" . | nindent 4 }}
+    app.kubernetes.io/component: llm-worker-quic-route
+  {{- with .Values.nvcfGatewayRoutes.routes.llmWorker.routeAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  parentRefs:
+    - name: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.name }}
+      namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }}
+      sectionName: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.listenerName }}
+  rules:
+    - backendRefs:
+        - name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.name }}
+          namespace: {{ $backendNamespace }}
+          port: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.quicPort }}
+{{- end }}

deploy/helm/gateway-routes/chart/values.yaml

@@ -51,6 +51,17 @@ nvcfGatewayRoutes:
       namespace: gateway
       # Listener (sectionName) on the Gateway to attach the route to
       listenerName: nats
+    # TCP Gateway for LLM worker registration and Stargate watches.
+    llmGrpc:
+      name: llm-grpc-gateway
+      namespace: gateway
+      listenerName: llm-grpc
+    # UDP Gateway for LLM reverse tunnels. This may be a separate load
+    # balancer from the TCP Gateway.
+    llmQuic:
+      name: llm-quic-gateway
+      namespace: gateway
+      listenerName: llm-quic
 
   # HTTPRoute configurations
   routes:
@@ -239,6 +250,21 @@ nvcfGatewayRoutes:
         namespace: nats-system
         port: 4222
       routeAnnotations: {}
+
+    # Backend-facing LLM registration/watch and reverse-tunnel routes.
+    # The backend router selects the correct Stargate pod by gRPC authority
+    # and QUIC SNI, so this route supports multi-replica request routers.
+    llmWorker:
+      enabled: false
+      name: llm-worker
+      backend:
+        name: llm-request-router-backend-router
+        # Required when llmWorker is enabled. Set this to the effective
+        # llm-request-router namespace.
+        namespace: ""
+        grpcPort: 50071
+        quicPort: 50072
+      routeAnnotations: {}
 
     # NVCF Worker Container needs to be able to fetch secrets for function pods
     # In split/multi cluster planes, ess resides in control-plane and requires

deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+chart_dir="${script_dir}/../chart"
+rendered="$(mktemp)"
+disabled="$(mktemp)"
+invalid_backend_namespace_error="$(mktemp)"
+trap 'rm -f "$rendered" "$disabled" "$invalid_backend_namespace_error"' EXIT
+
+helm template nvcf-gateway-routes "$chart_dir" \
+  --namespace gateway \
+  --set nvcfGatewayRoutes.routes.llmWorker.enabled=true \
+  --set nvcfGatewayRoutes.gateways.llmGrpc.name=llm-grpc-gateway \
+  --set nvcfGatewayRoutes.gateways.llmGrpc.namespace=gateway \
+  --set nvcfGatewayRoutes.gateways.llmQuic.name=llm-quic-gateway \
+  --set nvcfGatewayRoutes.gateways.llmQuic.namespace=gateway \
+  --set nvcfGatewayRoutes.routes.llmWorker.backend.namespace=router-system \
+  >"$rendered"
+
+assert_contains() {
+  local pattern="$1"
+  local message="$2"
+  if ! grep -Fq -- "$pattern" "$rendered"; then
+    echo "FAIL: ${message}" >&2
+    exit 1
+  fi
+}
+
+assert_contains "kind: TCPRoute" \
+  "LLM worker routing must expose gRPC registration over TCP"
+assert_contains "kind: UDPRoute" \
+  "LLM worker routing must expose reverse tunnels over UDP"
+assert_contains "name: llm-request-router-backend-router" \
+  "LLM worker routes must target the authority/SNI-aware backend router"
+assert_contains "name: allow-llm-worker-routes" \
+  "ReferenceGrant must permit cross-namespace LLM worker routes"
+assert_contains "sectionName: llm-grpc" \
+  "TCPRoute must attach to the configured LLM gRPC listener"
+assert_contains "sectionName: llm-quic" \
+  "UDPRoute must attach to the configured LLM QUIC listener"
+
+reference_grant_service_name="$(awk '
+  $0 == "kind: ReferenceGrant" { in_grant = 1; target_grant = 0; in_to = 0 }
+  in_grant && !target_grant && $1 == "name:" && $2 == "allow-llm-worker-routes" { target_grant = 1 }
+  target_grant && $0 == "  to:" { in_to = 1 }
+  target_grant && in_to && $1 == "name:" { print $2; exit }
+' "$rendered")"
+if [[ "$reference_grant_service_name" != "llm-request-router-backend-router" ]]; then
+  echo "FAIL: LLM worker ReferenceGrant must stay scoped to the configured backend Service" >&2
+  exit 1
+fi
+
+backend_namespace_references="$(grep -Fc -- "namespace: router-system" "$rendered" || true)"
+if [[ "$backend_namespace_references" != "3" ]]; then
+  echo "FAIL: LLM worker routes and ReferenceGrant must use the configured backend namespace" >&2
+  exit 1
+fi
+
+if helm template nvcf-gateway-routes "$chart_dir" \
+  --namespace gateway \
+  --set nvcfGatewayRoutes.routes.llmWorker.enabled=true \
+  --set-string nvcfGatewayRoutes.routes.llmWorker.backend.namespace= \
+  >/dev/null 2>"$invalid_backend_namespace_error"; then
+  echo "FAIL: enabled LLM worker routing must require an explicit backend namespace" >&2
+  exit 1
+fi
+if ! grep -Fq -- "nvcfGatewayRoutes.routes.llmWorker.backend.namespace is required when llmWorker.enabled is true" "$invalid_backend_namespace_error"; then
+  echo "FAIL: missing LLM worker backend namespace must return the expected validation error" >&2
+  exit 1
+fi
+
+helm template nvcf-gateway-routes "$chart_dir" \
+  --namespace gateway \
+  --set nvcfGatewayRoutes.routes.llmWorker.enabled=false \
+  >"$disabled"
+
+if grep -Eq '^  name: (llm-worker-(grpc|quic)|allow-llm-worker-routes)$' "$disabled"; then
+  echo "FAIL: disabled LLM worker routing must not render route resources" >&2
+  exit 1
+fi
+
+echo "PASS: LLM worker Gateway routes render correctly"

deploy/helm/gateway-routes/scripts/test-render-routes.sh

@@ -164,6 +164,9 @@ assert_resource_count "$default_render" GRPCRoute nvcf-api-grpc gateway 0
 assert_resource_count "$default_render" GRPCRoute nvct-api-grpc gateway 0
 assert_resource_count "$default_render" TCPRoute grpc-worker gateway 0
 assert_resource_count "$default_render" TCPRoute nats gateway 0
+assert_resource_count "$default_render" TCPRoute llm-worker-grpc gateway 0
+assert_resource_count "$default_render" UDPRoute llm-worker-quic gateway 0
+assert_resource_count "$default_render" ReferenceGrant allow-llm-worker-routes nvcf 0
 assert_resource_count "$default_render" ReferenceGrant allow-tcproute-to-nats nats-system 0
 
 helm template nvcf-gateway-routes "$repo_root/chart" \
@@ -172,6 +175,8 @@ helm template nvcf-gateway-routes "$repo_root/chart" \
   --set nvcfGatewayRoutes.routes.nvctApi.grpc.enabled=true \
   --set nvcfGatewayRoutes.routes.grpcWorker.enabled=true \
   --set nvcfGatewayRoutes.routes.nats.enabled=true \
+  --set nvcfGatewayRoutes.routes.llmWorker.enabled=true \
+  --set nvcfGatewayRoutes.routes.llmWorker.backend.namespace=nvcf \
   --set nvcfGatewayRoutes.gateways.nats.name=nats-gateway \
   --set nvcfGatewayRoutes.gateways.nats.namespace=gateway \
   --set nvcfGatewayRoutes.gateways.nats.listenerName=nats \
@@ -225,6 +230,32 @@ assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.rules[0].ba
 assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.hostnames' null
 assert_resource_field "$enabled_render" TCPRoute nats gateway '.metadata.annotations' null
 
+assert_resource_count "$enabled_render" TCPRoute llm-worker-grpc gateway 1
+assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.metadata.labels."app.kubernetes.io/component"' llm-worker-grpc-route
+assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.parentRefs[0].name' llm-grpc-gateway
+assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.parentRefs[0].namespace' gateway
+assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.parentRefs[0].sectionName' llm-grpc
+assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.rules[0].backendRefs[0].name' llm-request-router-backend-router
+assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.rules[0].backendRefs[0].namespace' nvcf
+assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.rules[0].backendRefs[0].port' 50071
+
+assert_resource_count "$enabled_render" UDPRoute llm-worker-quic gateway 1
+assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.metadata.labels."app.kubernetes.io/component"' llm-worker-quic-route
+assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.parentRefs[0].name' llm-quic-gateway
+assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.parentRefs[0].namespace' gateway
+assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.parentRefs[0].sectionName' llm-quic
+assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.rules[0].backendRefs[0].name' llm-request-router-backend-router
+assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.rules[0].backendRefs[0].namespace' nvcf
+assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.rules[0].backendRefs[0].port' 50072
+
+assert_resource_count "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf 1
+assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.from[0].kind' TCPRoute
+assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.from[0].namespace' gateway
+assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.from[1].kind' UDPRoute
+assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.from[1].namespace' gateway
+assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.to[0].kind' Service
+assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.to[0].name' llm-request-router-backend-router
+
 assert_resource_count "$enabled_render" ReferenceGrant allow-tcproute-to-nats nats-system 1
 assert_resource_field "$enabled_render" ReferenceGrant allow-tcproute-to-nats nats-system '.spec.from[0].kind' TCPRoute
 assert_resource_field "$enabled_render" ReferenceGrant allow-tcproute-to-nats nats-system '.spec.from[0].namespace' gateway

deploy/helm/llm-request-router/Makefile

@@ -27,7 +27,7 @@ OCI_REGISTRY_NAMESPACE ?= <your-org>
 CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml)
 CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml)
 
-.PHONY: install uninstall status lint template validate clean package push-oci test check-pki-render check-multi-replica-render
+.PHONY: install uninstall status lint template validate clean package push-oci test check-pki-render check-multi-replica-render check-backend-router-render
 
 install:
 ifndef values
@@ -65,14 +65,17 @@ template:
 validate: template
 	kubeconform -strict -summary -output pretty -kubernetes-version 1.31.5 bin/manifest.yaml
 
-test: check-pki-render check-multi-replica-render
+test: check-pki-render check-multi-replica-render check-backend-router-render
 
 check-pki-render:
 	sh ./scripts/check-pki-render.sh
 
 check-multi-replica-render:
 	sh ./scripts/check-multi-replica-render.sh
 
+check-backend-router-render:
+	sh ./scripts/check-backend-router-render.sh
+
 clean:
 	rm -rf ./packaged-charts bin