diff --git a/docs/voice-gateway-core-proposal.md b/docs/voice-gateway-core-proposal.md new file mode 100644 index 0000000..759e351 --- /dev/null +++ b/docs/voice-gateway-core-proposal.md @@ -0,0 +1,108 @@ +# Core proposal: ephemeral client credentials for realtime AI sessions + +**Status:** userland proof of concept shipped in openclaWP +(`includes/class-openclawp-voice-session.php` + `voice-gateway/`). +**Audience:** php-ai-client / wp-ai-client (WordPress AI Team). + +## Problem + +Realtime AI sessions — voice (Gemini Live, OpenAI Realtime), screen sharing, +live transcription — run over long-lived WebSocket/WebRTC connections that PHP +cannot hold. The session is therefore established by a *client*: a browser, a +native app, or a sidecar daemon next to WordPress. + +That client needs a provider credential. Today the only options are: + +1. **Configure the provider API key in the client too** — two sources of + truth, long-lived secret outside WordPress, no way to revoke per client. +2. **Hand the stored key out over REST** — the key wp-ai-client guards in its + credential store leaks to every client that may start a session. + +Both contradict the premise of the AI Client credential store: WordPress owns +the key. + +## What providers already offer + +Providers solve this with **short-lived, scoped credentials** minted +server-side from the long-lived key: + +- **Google Gemini Live**: `v1alpha/auth_tokens` — single-use tokens with + `expireTime`, `newSessionExpireTime` and `liveConnectConstraints` (pin the + model and Live config the token can be used with). Passed as + `?access_token=` on the Bidi WebSocket. +- **OpenAI Realtime**: `POST /v1/realtime/sessions` returns an ephemeral + `client_secret` for the browser to open the WebRTC/WS session. + +The shape is identical: *exchange the server-held key for a constrained, +expiring client credential.* + +## Proposal + +php-ai-client adds a provider capability for minting ephemeral client +credentials, and wp-ai-client exposes it to consumers: + +```php +interface ProviderWithEphemeralCredentialsInterface +{ + public static function ephemeralCredentials(): EphemeralCredentialHandlerInterface; +} + +interface EphemeralCredentialHandlerInterface +{ + /** + * Exchanges the provider's stored request authentication for a + * short-lived client credential, optionally constrained. + */ + public function createCredential(EphemeralCredentialConstraints $constraints): EphemeralCredential; +} + +final class EphemeralCredential +{ + public function getType(): string; // e.g. 'ephemeral_token' + public function getValue(): string; // never the long-lived key + public function getExpiresAt(): DateTimeImmutable; + public function getConnectionInfo(): array; // e.g. ws_url, auth param style +} +``` + +Consumer code (a voice feature, a realtime block, a sidecar handshake +endpoint) becomes provider-neutral: + +```php +$credential = AiClient::defaultRegistry() + ->createEphemeralCredential( 'google', $constraints ); +``` + +WordPress keeps sole custody of the long-lived key; clients receive only +expiring, scoped credentials; revocation and auditing happen where the key +lives. + +## The proof of concept in this repo + +`POST /openclawp/v1/voice/session` implements the consumer side today: + +1. Reads the Google key the canonical way — + `AiClient::defaultRegistry()->getProviderRequestAuthentication( 'google' )` + (the key never enters the REST response). +2. Mints a single-use Live token (`v1alpha/auth_tokens`, constrained to the + session's model, 30-minute expiry). +3. Returns `{credential, ws_url, model}` to the voice client. + +Returning the raw key is behind an off-by-default filter +(`openclawp_voice_session_allow_api_key`) precisely because the core-shaped +contract is "the stored key never leaves the server". + +If the proposal lands, step 2 collapses into the registry call above and the +endpoint loses all provider-specific code — which is the point: today the +Google `auth_tokens` HTTP call is written by hand in a plugin, where it can +drift; in core it would sit next to the provider that owns the protocol. + +## Why core (and not just plugins) + +- The credential store is already core's (`wp-ai-client`, WP 7.0). Ephemeral + exchange is a property *of the stored credential*, not of any product. +- Every realtime consumer (voice UIs, live blocks, agents talking on the + phone) needs the same exchange; without core support each plugin + re-implements per-provider token minting against undocumented-in-PHP APIs. +- It is small: one interface, one DTO, per-provider implementations of a + single HTTP call. No session management, no transport, no UI. diff --git a/includes/class-openclawp-bootstrap.php b/includes/class-openclawp-bootstrap.php index 21444b6..7cea735 100644 --- a/includes/class-openclawp-bootstrap.php +++ b/includes/class-openclawp-bootstrap.php @@ -71,6 +71,7 @@ public static function init(): void { OpenclaWP_Agency_Rest::register(); OpenclaWP_Decisions_Rest::register(); OpenclaWP_Agenttic_Bridge::register(); + OpenclaWP_Voice_Session::register(); OpenclaWP_Agent_Card::register(); OpenclaWP_A2a_Client_Bridge::register(); OpenclaWP_Canonical_Chat_Handler::register(); diff --git a/includes/class-openclawp-voice-session.php b/includes/class-openclawp-voice-session.php new file mode 100644 index 0000000..41870ba --- /dev/null +++ b/includes/class-openclawp-voice-session.php @@ -0,0 +1,287 @@ + WP_REST_Server::CREATABLE, + 'callback' => array( __CLASS__, 'handle' ), + 'permission_callback' => array( __CLASS__, 'check_permission' ), + 'args' => array( + 'agent' => array( + 'type' => 'string', + 'required' => false, + 'sanitize_callback' => 'sanitize_title', + ), + 'model' => array( + 'type' => 'string', + 'required' => false, + 'sanitize_callback' => 'sanitize_text_field', + ), + ), + ) + ); + } + + public static function check_permission( WP_REST_Request $request ) { + $default = current_user_can( 'manage_options' ); + + /** + * Filters whether the current user may mint a voice session + * credential. Mirrors `openclawp_agenttic_bridge_permission`: a voice + * client that may chat with the agent should normally also be allowed + * here. + * + * @since 0.6.0 + * + * @param bool $allowed Default: manage_options. + * @param WP_REST_Request $request Current request. + */ + $allowed = (bool) apply_filters( 'openclawp_voice_session_permission', $default, $request ); + + if ( ! $allowed ) { + return new WP_Error( + 'openclawp_forbidden', + __( 'You do not have permission to start a voice session.', 'openclawp' ), + array( 'status' => 403 ) + ); + } + + return true; + } + + public static function handle( WP_REST_Request $request ) { + $model = (string) $request->get_param( 'model' ); + if ( '' === $model ) { + $model = self::DEFAULT_MODEL; + } + + /** + * Filters the Gemini Live model a voice session is constrained to. + * + * @since 0.6.0 + * + * @param string $model Model slug (no `models/` prefix). + * @param WP_REST_Request $request Current request. + */ + $model = (string) apply_filters( 'openclawp_voice_session_model', $model, $request ); + + $api_key = self::get_provider_api_key( 'google' ); + if ( '' === $api_key ) { + return new WP_Error( + 'openclawp_voice_no_credentials', + __( 'No Google AI credentials are configured in the AI Client.', 'openclawp' ), + array( 'status' => 501 ) + ); + } + + $token = self::mint_google_token( $api_key, $model ); + if ( ! is_wp_error( $token ) ) { + return rest_ensure_response( + array_merge( + array( + 'provider' => 'google', + 'model' => $model, + 'credential' => $token, + 'ws_url' => 'wss://' . wp_parse_url( self::GOOGLE_API_BASE, PHP_URL_HOST ) + . '/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent', + ), + self::agent_info( (string) $request->get_param( 'agent' ) ) + ) + ); + } + + /** + * Filters whether the endpoint may fall back to returning the + * long-lived API key when ephemeral-token minting fails. Off by + * default on purpose: the core-shaped contract is that the stored + * key never leaves the server. + * + * @since 0.6.0 + * + * @param bool $allowed Default false. + * @param WP_REST_Request $request Current request. + */ + if ( apply_filters( 'openclawp_voice_session_allow_api_key', false, $request ) ) { + return rest_ensure_response( + array_merge( + array( + 'provider' => 'google', + 'model' => $model, + 'credential' => array( + 'type' => 'api_key', + 'value' => $api_key, + ), + 'ws_url' => 'wss://' . wp_parse_url( self::GOOGLE_API_BASE, PHP_URL_HOST ) + . '/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent', + ), + self::agent_info( (string) $request->get_param( 'agent' ) ) + ) + ); + } + + return $token; + } + + /** + * Reads a provider API key from the AI Client credential store — the + * core-owned path: WP stores it, `AiClient::defaultRegistry()` exposes it + * to PHP, and it is never serialized into a REST response. + */ + private static function get_provider_api_key( string $provider_id ): string { + if ( ! class_exists( '\\WordPress\\AiClient\\AiClient' ) ) { + return ''; + } + + try { + $registry = \WordPress\AiClient\AiClient::defaultRegistry(); + $auth = $registry->getProviderRequestAuthentication( $provider_id ); + if ( is_object( $auth ) && method_exists( $auth, 'getApiKey' ) ) { + return (string) $auth->getApiKey(); + } + } catch ( Throwable $e ) { + return ''; + } + + return ''; + } + + /** + * Mints a single-use ephemeral token for the Live API + * (`v1alpha/auth_tokens`). The token can open exactly one Live session, + * constrained to $model, and expires in 30 minutes. + * + * @return array{type: string, value: string, expires_at: string}|WP_Error + */ + private static function mint_google_token( string $api_key, string $model ) { + $expire_at = gmdate( 'Y-m-d\TH:i:s\Z', time() + 30 * MINUTE_IN_SECONDS ); + $new_session_by = gmdate( 'Y-m-d\TH:i:s\Z', time() + 2 * MINUTE_IN_SECONDS ); + + $body = array( + 'uses' => 1, + 'expireTime' => $expire_at, + 'newSessionExpireTime' => $new_session_by, + 'liveConnectConstraints' => array( + 'model' => 'models/' . $model, + ), + ); + + /** + * Filters the auth_tokens:create request body, e.g. to pin the full + * Live config into the token constraints. + * + * @since 0.6.0 + * + * @param array $body Request body. + * @param string $model Model slug. + */ + $body = (array) apply_filters( 'openclawp_voice_token_request', $body, $model ); + + $response = wp_remote_post( + self::GOOGLE_API_BASE . '/v1alpha/auth_tokens', + array( + 'timeout' => 15, + 'headers' => array( + 'Content-Type' => 'application/json', + 'x-goog-api-key' => $api_key, + ), + 'body' => wp_json_encode( $body ), + ) + ); + + if ( is_wp_error( $response ) ) { + return new WP_Error( + 'openclawp_voice_mint_failed', + $response->get_error_message(), + array( 'status' => 502 ) + ); + } + + $code = (int) wp_remote_retrieve_response_code( $response ); + $data = json_decode( wp_remote_retrieve_body( $response ), true ); + + if ( 200 !== $code || empty( $data['name'] ) ) { + return new WP_Error( + 'openclawp_voice_mint_failed', + sprintf( + /* translators: %d: HTTP status code from the Google API. */ + __( 'Ephemeral token minting failed (HTTP %d).', 'openclawp' ), + $code + ), + array( + 'status' => 502, + 'upstream' => is_array( $data ) ? ( $data['error']['message'] ?? null ) : null, + ) + ); + } + + return array( + 'type' => 'ephemeral_token', + 'value' => (string) $data['name'], + 'expires_at' => $expire_at, + ); + } + + /** + * Optional agent display info so the voice shell can introduce itself + * without extra round-trips. Never includes the agent's instructions. + * + * @return array{agent?: array{slug: string, label: string}} + */ + private static function agent_info( string $agent_slug ): array { + if ( '' === $agent_slug || ! function_exists( 'wp_get_agent' ) ) { + return array(); + } + + $agent = wp_get_agent( $agent_slug ); + if ( ! $agent ) { + return array(); + } + + $label = is_object( $agent ) && method_exists( $agent, 'get_label' ) + ? (string) $agent->get_label() + : $agent_slug; + + return array( + 'agent' => array( + 'slug' => $agent_slug, + 'label' => $label, + ), + ); + } +} diff --git a/voice-gateway/.env.example b/voice-gateway/.env.example new file mode 100644 index 0000000..b41eab6 --- /dev/null +++ b/voice-gateway/.env.example @@ -0,0 +1,26 @@ +# openclaWP Voice Gateway — copy to .env and fill in. + +# Gemini Live (AI Studio key). GOOGLE_API_KEY works as fallback. +GEMINI_API_KEY= + +# WordPress site that hosts the agent (no trailing slash). +OPENCLAWP_VOICE_WP_BASE=https://example.com + +# Agent slug as registered with wp_register_agent(). e.g. carpeta-bot +OPENCLAWP_VOICE_AGENT= +# Human label spoken by the voice shell. e.g. Menta +OPENCLAWP_VOICE_AGENT_LABEL= + +# JSON file with {"user": "...", "app_password": "..."} — chmod 600. +# OPENCLAWP_VOICE_AUTH_FILE=~/.openclawp-voice/auth.json + +# Optional extra persona lines appended to the voice system instruction. +# OPENCLAWP_VOICE_PERSONA=Sos la voz de Menta, asistente del campo. Rioplatense seco. +# OPENCLAWP_VOICE_PERSONA_FILE= + +# Tuning (defaults shown). +# OPENCLAWP_VOICE_MODEL=gemini-3.1-flash-live-preview +# OPENCLAWP_VOICE_NAME=Puck +# OPENCLAWP_VOICE_TZ=America/Montevideo +# OPENCLAWP_VOICE_HOST=127.0.0.1 +# OPENCLAWP_VOICE_PORT=8766 diff --git a/voice-gateway/.gitignore b/voice-gateway/.gitignore new file mode 100644 index 0000000..67b5e38 --- /dev/null +++ b/voice-gateway/.gitignore @@ -0,0 +1,3 @@ +.venv/ +.env +__pycache__/ diff --git a/voice-gateway/README.md b/voice-gateway/README.md new file mode 100644 index 0000000..798e742 --- /dev/null +++ b/voice-gateway/README.md @@ -0,0 +1,136 @@ +# openclaWP Voice Gateway + +Full-duplex voice for any agent registered with the Agents API on a WordPress +site running openclaWP. **Gemini Live is only the voice shell** (native +speech-to-text, text-to-speech, turn-taking, barge-in); the agent stays the +single brain. Every user request is forwarded through one tool — `ask_agent` — +to openclaWP's agenttic chat endpoint, so the agent's memory, tool gating +(destructive-action confirmations) and transcripts all stay in WordPress. + +```text +browser mic (PCM 16 kHz over WSS) ⇄ gateway.py ⇄ Gemini Live API + │ ask_agent(consulta) + ▼ + POST {site}/wp-json/openclawp/v1/agenttic/{agent} + (JSON-RPC `message/send`, app-password auth, sticky sessionId) +``` + +This is a sidecar daemon, not WordPress code — same pattern as wp-carpeta's +`bridge/`. It needs Python ≥ 3.10. + +## Setup + +```bash +cd voice-gateway +python3 -m venv .venv && .venv/bin/pip install -r requirements.txt +cp .env.example .env # fill in key, site, agent slug +``` + +Create the auth file (an Application Password for a user allowed to chat with +the agent): + +```bash +mkdir -p ~/.openclawp-voice +cat > ~/.openclawp-voice/auth.json <<'EOF' +{ "user": "wpuser", "app_password": "xxxx xxxx xxxx xxxx xxxx xxxx" } +EOF +chmod 600 ~/.openclawp-voice/auth.json +``` + +Run: + +```bash +.venv/bin/python gateway.py +# → http://127.0.0.1:8766 (page + /ws/voice + /healthz) +``` + +`GET /healthz` reports missing config without burning a Gemini session. + +## Browser requirements + +`getUserMedia` needs a **secure origin** — serve through a TLS reverse proxy +(localhost is also fine for testing). The page computes the WS URL relative to +its own location, so serving under a path prefix works. + +### Caddy example (path prefix `/voz/`) + +```caddyfile +handle_path /voz/* { + reverse_proxy 127.0.0.1:8766 +} +``` + +### launchd example (macOS host) + +`~/Library/LaunchAgents/com.openclawp.voice-gateway.plist`: + +```xml + + + + Labelcom.openclawp.voice-gateway + ProgramArguments + /PATH/TO/voice-gateway/.venv/bin/python + /PATH/TO/voice-gateway/gateway.py + + WorkingDirectory/PATH/TO/voice-gateway + RunAtLoad + KeepAlive + StandardOutPath/tmp/openclawp-voice.log + StandardErrorPath/tmp/openclawp-voice.log + +``` + +```bash +launchctl load ~/Library/LaunchAgents/com.openclawp.voice-gateway.plist +``` + +## Configuration + +### Credentials: WordPress mints them (preferred) + +On session start the gateway calls `POST /openclawp/v1/voice/session`, which +exchanges the Google key already stored in WordPress' AI Client credential +store for a **single-use ephemeral Live token** (the long-lived key never +leaves WP). See `docs/voice-gateway-core-proposal.md` — this flow is shaped as +a proof of concept of what wp-ai-client could support natively. + +`GEMINI_API_KEY` in the local `.env` is only a fallback for sites whose +openclaWP predates `/voice/session` or has no Google credentials stored. + +| Variable | Default | Meaning | +|---|---|---| +| `GEMINI_API_KEY` | — | Optional fallback AI Studio key (`GOOGLE_API_KEY` works too) | +| `OPENCLAWP_VOICE_WP_BASE` | — | Site URL, no trailing slash | +| `OPENCLAWP_VOICE_AGENT` | — | Agent slug (`wp_register_agent`) | +| `OPENCLAWP_VOICE_AGENT_LABEL` | slug | Name the voice shell uses | +| `OPENCLAWP_VOICE_AUTH_FILE` | `~/.openclawp-voice/auth.json` | `{user, app_password}` JSON | +| `OPENCLAWP_VOICE_PERSONA` / `_FILE` | — | Extra persona lines for the voice shell | +| `OPENCLAWP_VOICE_MODEL` | `gemini-3.1-flash-live-preview` | Gemini Live model | +| `OPENCLAWP_VOICE_NAME` | `Puck` | Gemini prebuilt voice | +| `OPENCLAWP_VOICE_TZ` | `America/Montevideo` | Temporal-context timezone | +| `OPENCLAWP_VOICE_HOST` / `_PORT` | `127.0.0.1` / `8766` | Bind address | + +## Design notes + +- **One brain.** The voice model is instructed to forward *everything* + domain-related through `ask_agent` and never answer from its own knowledge. + Agent turns take seconds; the tool runs async (Gemini Live keeps the audio + channel open) and the shell verbalizes the wait. +- **Session continuity.** The first agent reply returns an openclawp + `sessionId`; the gateway pins it for the rest of the voice session, so the + agent has conversational memory and WP keeps the transcript. +- **Gated tools.** If the agent answers with a pending-confirmation message, + the shell reads it out loud and relays the user's spoken decision back + through `ask_agent`. The site-side decision flow is unchanged. +- **Costs.** Session audio seconds (in/out) are logged on close. + +## Browser wire protocol + +Upstream: `{"audio": ""}` or `{"text": "..."}`. +Downstream: `{"audio": ""}`, +`{"type": "user_transcript"|"agent_transcript", "text", "final"}`, +`{"type": "agent_thought", "text"}` (what was forwarded to the agent), +`{"type": "interrupted"}` (barge-in: flush playback), `{"type": "ready"}`, +`{"text": ""}`. diff --git a/voice-gateway/gateway.py b/voice-gateway/gateway.py new file mode 100644 index 0000000..ccad101 --- /dev/null +++ b/voice-gateway/gateway.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +""" +openclaWP Voice Gateway — talk to any Agents API agent, full duplex. + +Architecture (one sentence): Gemini Live is the voice shell (native STT/TTS, +turn-taking, barge-in); the registered agent stays the only brain, reached +through a single tool (`ask_agent`) that forwards each request to openclaWP's +agenttic chat endpoint, so memory, tool gating and transcripts stay in WP. + + browser mic (PCM 16 kHz over WS) ⇄ this gateway ⇄ Gemini Live API + │ ask_agent(consulta) + ▼ + POST {WP_BASE}/wp-json/openclawp/v1/agenttic/{AGENT} + (JSON-RPC message/send, app-password auth, sticky sessionId) + +The browser protocol mirrors the proven laviere voice frontend: JSON frames +{audio: } / {text: } upstream; {audio: }, +{type: user_transcript|agent_transcript, text, final}, {type: interrupted} +downstream. + +Run: uvicorn is embedded — `python3 gateway.py` (see README.md). +""" + +import asyncio +import base64 +import json +import logging +import os +import time +import uuid +from datetime import datetime, timedelta +from pathlib import Path +from zoneinfo import ZoneInfo + +import httpx +import websockets +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles + +try: + from dotenv import load_dotenv + + load_dotenv(Path(__file__).parent / ".env") +except ImportError: + pass + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger("voice-gateway") + +# ── Config ──────────────────────────────────────────────────────────────────── + +GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY", "") +GEMINI_MODEL = os.environ.get("OPENCLAWP_VOICE_MODEL", "gemini-3.1-flash-live-preview") +GEMINI_VOICE = os.environ.get("OPENCLAWP_VOICE_NAME", "Puck") +GEMINI_WS_URL = ( + "wss://generativelanguage.googleapis.com/ws/" + "google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" +) + +WP_BASE = os.environ.get("OPENCLAWP_VOICE_WP_BASE", "").rstrip("/") +AGENT_SLUG = os.environ.get("OPENCLAWP_VOICE_AGENT", "") +AGENT_LABEL = os.environ.get("OPENCLAWP_VOICE_AGENT_LABEL", AGENT_SLUG) +AUTH_FILE = Path( + os.environ.get("OPENCLAWP_VOICE_AUTH_FILE", str(Path.home() / ".openclawp-voice" / "auth.json")) +) +TIME_ZONE = os.environ.get("OPENCLAWP_VOICE_TZ", "America/Montevideo") +HOST = os.environ.get("OPENCLAWP_VOICE_HOST", "127.0.0.1") +PORT = int(os.environ.get("OPENCLAWP_VOICE_PORT", "8766")) + +# Extra persona text appended to the system instruction (or a file with it). +PERSONA = os.environ.get("OPENCLAWP_VOICE_PERSONA", "") +PERSONA_FILE = os.environ.get("OPENCLAWP_VOICE_PERSONA_FILE", "") +if not PERSONA and PERSONA_FILE and Path(PERSONA_FILE).exists(): + PERSONA = Path(PERSONA_FILE).read_text(encoding="utf-8") + + +def validate_config() -> list: + missing = [] + if not WP_BASE: + missing.append("OPENCLAWP_VOICE_WP_BASE") + if not AGENT_SLUG: + missing.append("OPENCLAWP_VOICE_AGENT") + if not AUTH_FILE.exists(): + missing.append(f"auth file {AUTH_FILE} (JSON {{user, app_password}})") + # GEMINI_API_KEY is optional: the preferred path is an ephemeral token + # minted by WP (POST /openclawp/v1/voice/session) from the key the AI + # Client credential store already owns. + return missing + + +def read_auth() -> dict: + return json.loads(AUTH_FILE.read_text(encoding="utf-8")) + + +# ── System instruction ──────────────────────────────────────────────────────── + + +def build_system_instruction() -> str: + base = f"""Sos la voz de {AGENT_LABEL}, un agente que vive en un sitio WordPress. +Tu único trabajo es ser sus oídos y su boca; el razonamiento lo hace el agente. + +Reglas estrictas: +1. Para CUALQUIER consulta, dato, registro o acción que pida el usuario, llamá la + herramienta `ask_agent` pasando el pedido completo y fiel del usuario. No + respondas de memoria ni inventes datos: si la pregunta es sobre el dominio + del agente, va por `ask_agent` siempre. +2. La consulta del agente puede tardar unos segundos. Apenas llames la + herramienta avisale al usuario con una frase corta y natural ("dame un + segundo, lo busco", "ya te digo") y esperá la respuesta. +3. Cuando llegue la respuesta, transmitila fiel y completa en voz natural. No + agregues información que no esté en la respuesta. Si la respuesta pide una + confirmación, pedísela al usuario y mandá su decisión de vuelta con + `ask_agent` (ej.: "sí, confirmo"). +4. Solo charla trivial (saludos, "¿me escuchás?") se responde directo, breve. +5. Hablás español rioplatense, tono cercano y conciso. Sin emojis, sin listas + leídas como listas: convertí todo a frases habladas naturales.""" + if PERSONA: + base += "\n\nPersona adicional:\n" + PERSONA.strip() + return base + + +def temporal_context(now: datetime = None) -> str: + """Same role as the WhatsApp bridge's buildTemporalContext(): pin relative + dates per turn so the agent doesn't drift across a long voice session.""" + tz = ZoneInfo(TIME_ZONE) + now = now or datetime.now(tz) + today = now.strftime("%Y-%m-%d") + tomorrow = (now + timedelta(days=1)).strftime("%Y-%m-%d") + yesterday = (now - timedelta(days=1)).strftime("%Y-%m-%d") + return ( + "[Contexto temporal del turno]\n" + f"Hoy ({TIME_ZONE}) es {today}. Mañana es {tomorrow}. Ayer fue {yesterday}.\n" + "Usá estas fechas para interpretar fechas relativas. No copies estas líneas en respuestas.\n" + "\n[Mensaje del usuario (por voz)]\n" + ) + + +# ── ask_agent: forward a turn to the openclaWP agenttic endpoint ────────────── + + +async def ask_agent(consulta: str, session: dict) -> str: + """POST the user's request to the agent and return its text reply. + + `session` carries the sticky openclawp sessionId for this voice session so + the agent keeps conversational memory across turns. + """ + auth = read_auth() + req_id = f"voz-{int(time.time() * 1000)}-{uuid.uuid4().hex[:6]}" + params = { + "id": f"task-{req_id}", + "message": { + "role": "user", + "parts": [{"type": "text", "text": temporal_context() + consulta}], + "messageId": f"msg-{req_id}", + "kind": "message", + }, + } + if session.get("sessionId"): + params["sessionId"] = session["sessionId"] + + body = {"jsonrpc": "2.0", "id": req_id, "method": "message/send", "params": params} + url = f"{WP_BASE}/wp-json/openclawp/v1/agenttic/{AGENT_SLUG}" + + async with httpx.AsyncClient(timeout=120.0) as client: + res = await client.post(url, json=body, auth=(auth["user"], auth["app_password"])) + if res.status_code != 200: + logger.error("ask_agent HTTP %s: %s", res.status_code, res.text[:200]) + return "El agente no está disponible en este momento. Probá de nuevo en un rato." + data = res.json() + if data.get("error"): + logger.error("ask_agent rpc error: %s", json.dumps(data["error"])[:200]) + return "El agente devolvió un error. Probá reformular el pedido." + + result = data.get("result") or {} + if result.get("sessionId"): + session["sessionId"] = result["sessionId"] + parts = ((result.get("status") or {}).get("message") or {}).get("parts") or [] + text = "\n".join( + p["text"] for p in parts if p.get("type") == "text" and isinstance(p.get("text"), str) + ).strip() + return text or "El agente no devolvió respuesta." + + +# ── Live credential: WP-minted ephemeral token, env API key as fallback ────── + + +async def get_live_credential() -> tuple: + """Returns (ws_url_with_credential, source). + + Preferred: ask WordPress to mint a single-use ephemeral Live token from + the provider key stored in the AI Client (the key never reaches us). + Fallback: GEMINI_API_KEY from the local environment, for sites whose + openclaWP predates /voice/session or has no Google credentials stored. + """ + auth = read_auth() + url = f"{WP_BASE}/wp-json/openclawp/v1/voice/session" + try: + async with httpx.AsyncClient(timeout=20.0) as client: + res = await client.post( + url, + json={"agent": AGENT_SLUG, "model": GEMINI_MODEL}, + auth=(auth["user"], auth["app_password"]), + ) + if res.status_code == 200: + data = res.json() + cred = data.get("credential") or {} + ws_url = data.get("ws_url") or GEMINI_WS_URL + if cred.get("value"): + logger.info("Using WP-minted credential (%s).", cred.get("type")) + # Ephemeral tokens authenticate via ?access_token= (v1alpha + # only); plain API keys via ?key=. + param = "access_token" if cred.get("type") == "ephemeral_token" else "key" + return f"{ws_url}?{param}={cred['value']}", cred.get("type", "unknown") + else: + logger.warning("voice/session HTTP %s: %s", res.status_code, res.text[:200]) + except Exception as e: # noqa: BLE001 — fall back to env key + logger.warning("voice/session unreachable: %s", e) + + if GEMINI_API_KEY: + logger.info("Falling back to local GEMINI_API_KEY.") + return f"{GEMINI_WS_URL}?key={GEMINI_API_KEY}", "env_api_key" + return "", "" + + +# ── Gemini Live session setup ───────────────────────────────────────────────── + + +def build_setup_message() -> dict: + return { + "setup": { + "model": f"models/{GEMINI_MODEL}", + "generation_config": { + "response_modalities": ["AUDIO"], + "speech_config": { + "voice_config": {"prebuilt_voice_config": {"voice_name": GEMINI_VOICE}} + }, + }, + "input_audio_transcription": {}, + "output_audio_transcription": {}, + "system_instruction": {"parts": [{"text": build_system_instruction()}]}, + "tools": [ + { + "function_declarations": [ + { + "name": "ask_agent", + "description": ( + f"Envía el pedido del usuario al agente {AGENT_LABEL} y devuelve su " + "respuesta. Usala para TODA consulta de datos, registros o acciones." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "consulta": { + "type": "STRING", + "description": "El pedido del usuario, completo y fiel.", + } + }, + "required": ["consulta"], + }, + } + ] + } + ], + } + } + + +# ── WebSocket bridge ────────────────────────────────────────────────────────── + +app = FastAPI() + + +@app.get("/healthz") +async def healthz(): + missing = validate_config() + return {"ok": not missing, "missing": missing, "agent": AGENT_SLUG, "model": GEMINI_MODEL} + + +@app.websocket("/ws/voice") +async def voice_endpoint(websocket: WebSocket): + await websocket.accept() + missing = validate_config() + if missing: + await websocket.send_json({"text": f"Gateway sin configurar: faltan {', '.join(missing)}"}) + await websocket.close(code=1008) + return + + session = {"sessionId": None} + audio_in = 0 # bytes of PCM uploaded (16 kHz mono s16 → 32000 B/s) + audio_out = 0 # bytes of PCM played back (24 kHz mono s16 → 48000 B/s) + user_text = "" + model_text = "" + + url, cred_source = await get_live_credential() + if not url: + await websocket.send_json( + {"text": "Sin credenciales para Gemini Live: configurá la key de Google en el AI Client de WP o GEMINI_API_KEY local."} + ) + await websocket.close(code=1008) + return + logger.info( + "Voice session start: agent=%s model=%s cred=%s", AGENT_SLUG, GEMINI_MODEL, cred_source + ) + + try: + async with websockets.connect(url, max_size=16 * 1024 * 1024) as gemini_ws: + send_lock = asyncio.Lock() + + async def send_to_gemini(payload: dict): + async with send_lock: + await gemini_ws.send(json.dumps(payload)) + + await send_to_gemini(build_setup_message()) + + async def run_tool(call: dict): + name = call.get("name", "") + args = call.get("args") or {} + call_id = call.get("id") + if name == "ask_agent": + consulta = str(args.get("consulta", "")).strip() + logger.info("ask_agent: %s", consulta[:120]) + await websocket.send_json({"type": "agent_thought", "text": consulta}) + try: + output = await ask_agent(consulta, session) + except Exception as e: # noqa: BLE001 — must always answer Gemini + logger.exception("ask_agent failed") + output = f"Error consultando al agente: {e}" + else: + output = f"Herramienta desconocida: {name}" + func_resp = {"name": name, "response": {"output": output}} + if call_id: + func_resp["id"] = call_id + await send_to_gemini({"toolResponse": {"functionResponses": [func_resp]}}) + + async def forward_to_gemini(): + nonlocal audio_in + async for message in websocket.iter_text(): + data = json.loads(message) + if "audio" in data: + audio_in += len(data["audio"]) * 3 // 4 # b64 → bytes, close enough + await send_to_gemini( + { + "realtime_input": { + "audio": { + "mime_type": "audio/pcm;rate=16000", + "data": data["audio"], + } + } + } + ) + elif "text" in data: + await send_to_gemini( + { + "client_content": { + "turns": [ + {"role": "user", "parts": [{"text": data["text"]}]} + ], + "turn_complete": True, + } + } + ) + + async def forward_from_gemini(): + nonlocal audio_out, user_text, model_text + async for raw in gemini_ws: + if isinstance(raw, bytes): + raw = raw.decode("utf-8") + response = json.loads(raw) + + if "setupComplete" in response or "setup_complete" in response: + await websocket.send_json({"type": "ready"}) + continue + + content = response.get("serverContent") or response.get("server_content") + if content: + if content.get("interrupted"): + await websocket.send_json({"type": "interrupted"}) + model_text = "" + continue + + in_tr = content.get("inputTranscription") or content.get( + "input_transcription" + ) + if in_tr and in_tr.get("text"): + user_text += in_tr["text"] + await websocket.send_json( + {"type": "user_transcript", "text": user_text, "final": False} + ) + + out_tr = content.get("outputTranscription") or content.get( + "output_transcription" + ) + if out_tr and out_tr.get("text"): + model_text += out_tr["text"] + await websocket.send_json( + {"type": "agent_transcript", "text": model_text, "final": False} + ) + + turn = content.get("modelTurn") or content.get("model_turn") + if turn: + if user_text: + await websocket.send_json( + {"type": "user_transcript", "text": user_text, "final": True} + ) + user_text = "" + for part in turn.get("parts", []): + inline = part.get("inlineData") or part.get("inline_data") + if inline and inline.get("data"): + audio_out += len(inline["data"]) * 3 // 4 + await websocket.send_json({"audio": inline["data"]}) + + if content.get("turnComplete") or content.get("turn_complete"): + if model_text: + await websocket.send_json( + {"type": "agent_transcript", "text": model_text, "final": True} + ) + model_text = "" + + tool_call = response.get("toolCall") or response.get("tool_call") + if tool_call: + calls = tool_call.get("functionCalls") or tool_call.get("function_calls") or [] + for call in calls: + asyncio.create_task(run_tool(call)) + + done, pending = await asyncio.wait( + [ + asyncio.create_task(forward_to_gemini()), + asyncio.create_task(forward_from_gemini()), + ], + return_when=asyncio.FIRST_COMPLETED, + ) + for task in pending: + task.cancel() + for task in done: + if not task.cancelled() and task.exception(): + raise task.exception() + except WebSocketDisconnect: + pass + except Exception: + logger.exception("Voice session error") + try: + await websocket.send_json({"text": "Se cortó la sesión de voz. Recargá para reintentar."}) + except Exception: # noqa: BLE001 — socket may already be gone + pass + finally: + secs_in = audio_in / 32000.0 + secs_out = audio_out / 48000.0 + logger.info( + "Voice session end: %.1fs in, %.1fs out, openclawp session=%s", + secs_in, + secs_out, + session.get("sessionId"), + ) + + +# Static frontend — served last so /ws/voice and /healthz win. +STATIC_DIR = Path(__file__).parent / "static" + + +@app.get("/") +async def index(): + return FileResponse(STATIC_DIR / "index.html") + + +app.mount("/", StaticFiles(directory=str(STATIC_DIR)), name="static") + + +if __name__ == "__main__": + import uvicorn + + missing = validate_config() + if missing: + logger.warning("Config incompleta (el WS la rechaza hasta resolverla): %s", missing) + uvicorn.run(app, host=HOST, port=PORT) diff --git a/voice-gateway/requirements.txt b/voice-gateway/requirements.txt new file mode 100644 index 0000000..4a0811c --- /dev/null +++ b/voice-gateway/requirements.txt @@ -0,0 +1,5 @@ +fastapi>=0.110 +uvicorn>=0.29 +websockets>=12.0 +httpx>=0.27 +python-dotenv>=1.0 diff --git a/voice-gateway/static/index.html b/voice-gateway/static/index.html new file mode 100644 index 0000000..b01e392 --- /dev/null +++ b/voice-gateway/static/index.html @@ -0,0 +1,191 @@ + + + + + +Voz — openclaWP + + + +
+

Hablar con el agente

+ Desconectado +
+
+
+ + +
+ + +