Implement a dora-openai-websocket server

Cargo.toml

@@ -36,6 +36,7 @@ members = [
     "node-hub/dora-rerun",
     "node-hub/terminal-print",
     "node-hub/openai-proxy-server",
+    "node-hub/dora-openai-websocket",
     "node-hub/dora-kit-car",
     "node-hub/dora-object-to-pose",
     "node-hub/dora-mistral-rs",

binaries/cli/src/command/coordinator.rs

@@ -9,6 +9,7 @@ use dora_tracing::TracingBuilder;
 use eyre::Context;
 use std::net::{IpAddr, SocketAddr};
 use tokio::runtime::Builder;
+#[cfg(feature = "tracing")]
 use tracing::level_filters::LevelFilter;
 
 #[derive(Debug, clap::Args)]

binaries/cli/src/command/daemon.rs

@@ -14,6 +14,7 @@ use std::{
     path::PathBuf,
 };
 use tokio::runtime::Builder;
+#[cfg(feature = "tracing")]
 use tracing::level_filters::LevelFilter;
 
 #[derive(Debug, clap::Args)]

binaries/cli/src/command/mod.rs

@@ -16,22 +16,22 @@ mod up;
 
 pub use run::run_func;
 
-use build::Build;
-use check::Check;
-use coordinator::Coordinator;
-use daemon::Daemon;
-use destroy::Destroy;
-use eyre::Context;
-use graph::Graph;
-use list::ListArgs;
-use logs::LogsArgs;
-use new::NewArgs;
-use run::Run;
-use runtime::Runtime;
-use self_::SelfSubCommand;
-use start::Start;
-use stop::Stop;
-use up::Up;
+pub use build::Build;
+pub use check::Check;
+pub use coordinator::Coordinator;
+pub use daemon::Daemon;
+pub use destroy::Destroy;
+pub use eyre::Context;
+pub use graph::Graph;
+pub use list::ListArgs;
+pub use logs::LogsArgs;
+pub use new::NewArgs;
+pub use run::Run;
+pub use runtime::Runtime;
+pub use self_::SelfSubCommand;
+pub use start::Start;
+pub use stop::Stop;
+pub use up::Up;
 
 /// dora-rs cli client
 #[derive(Debug, clap::Subcommand)]

binaries/cli/src/command/run.rs

@@ -12,6 +12,7 @@ use crate::{
     session::DataflowSession,
 };
 use dora_daemon::{Daemon, LogDestination, flume};
+#[cfg(feature = "tracing")]
 use dora_tracing::TracingBuilder;
 use eyre::Context;
 use tokio::runtime::Builder;

binaries/cli/src/command/start/mod.rs

@@ -31,28 +31,28 @@ mod attach;
 pub struct Start {
     /// Path to the dataflow descriptor file
     #[clap(value_name = "PATH")]
-    dataflow: String,
+    pub dataflow: String,
     /// Assign a name to the dataflow
     #[clap(long)]
-    name: Option<String>,
+    pub name: Option<String>,
     /// Address of the dora coordinator
     #[clap(long, value_name = "IP", default_value_t = LOCALHOST)]
-    coordinator_addr: IpAddr,
+    pub coordinator_addr: IpAddr,
     /// Port number of the coordinator control server
     #[clap(long, value_name = "PORT", default_value_t = DORA_COORDINATOR_PORT_CONTROL_DEFAULT)]
-    coordinator_port: u16,
+    pub coordinator_port: u16,
     /// Attach to the dataflow and wait for its completion
     #[clap(long, action)]
-    attach: bool,
+    pub attach: bool,
     /// Run the dataflow in background
     #[clap(long, action)]
-    detach: bool,
+    pub detach: bool,
     /// Enable hot reloading (Python only)
     #[clap(long, action)]
-    hot_reload: bool,
+    pub hot_reload: bool,
     // Use UV to run nodes.
     #[clap(long, action)]
-    uv: bool,
+    pub uv: bool,
 }
 
 impl Executable for Start {

binaries/cli/src/lib.rs

@@ -5,7 +5,7 @@ use std::{
     path::PathBuf,
 };
 
-mod command;
+pub mod command;
 mod common;
 mod formatting;
 pub mod output;

examples/openai-realtime/README.md

@@ -0,0 +1,78 @@
+# Dora-OpenAI-Realtime (ROOT Repo)
+
+## Front End
+
+### Build Client
+
+```bash
+git clone [email protected]:moxin-org/moly.git
+cd moly
+cargo build --release
+```
+
+### Run Client
+
+```bash
+cd moly
+cargo run -r
+```
+
+## Server
+
+### Build server
+
+```bash
+uv venv --seed -p 3.11
+source .venv/bin/activate
+uv pip install dora-rs-cli dora-rs
+dora build whisper-template-metal.yml --uv ## very long process
+```
+
+### Run server
+
+```bash
+source .venv/bin/activate
+dora up
+cargo run --release -p dora-openai-websocket
+```
+
+## On finish
+
+```bash
+dora destroy
+```
+
+## GUI
+
+- Go to MolyServer Tab
+- Add a custom Provider
+- In API Host, use:
+
+  - Name: dora-websocket
+  - API Host: ws://127.0.0.1:8123
+  - Type: OpenAI Realtime
+
+- Then go to Chat Tab
+- New Chat
+- ( Make sure the servver is running with: `cargo run --release -p dora-openai-websocket`)
+- On bottom right, click on 🎧 icon.
+  > If nothing happen is that the server is not found.
+- Click on start
+- Wait for the first AI greeting
+- Start speaking!
+- You should get AI response!
+
+### WIP: Moyoyo
+
+## {Recommended} Install git-lfs
+
+```bash
+brew install git-lfs # MacOS
+```
+
+## Clone Moxin Voice Chat
+
+```bash
+git lfs install
+git clone https://github.com/moxin-org/moxin-voice-chat.git
+```

examples/openai-realtime/whisper-1-template-metal.yml

@@ -0,0 +1,71 @@
+nodes:
+  - id: NODE_ID
+    build: pip install -e ../../node-hub/dora-openai-websocket
+    path: dynamic
+    inputs:
+      audio: tts/audio
+      transcript: stt/text
+      text: llm/text
+      speech_started: dora-vad/timestamp_start
+      speech_stopped: dora-vad/timestamp_end
+    outputs:
+      - audio
+      - text
+      - function_call_output
+
+  - id: dora-vad
+    build: pip install -e ../../node-hub/dora-vad
+    path: dora-vad
+    inputs:
+      audio:
+        source: NODE_ID/audio
+        queue_size: 10
+    outputs:
+      - audio
+      - timestamp_start
+      - timestamp_end
+    env:
+      MIN_SPEECH_DURATION_MS: 500
+      MIN_SILENCE_DURATION_MS: 500
+      THRESHOLD: 0.6
+
+  - id: stt
+    build: pip install -e ../../node-hub/dora-distil-whisper
+    path: dora-distil-whisper
+    inputs:
+      audio:
+        source: dora-vad/audio
+        queue_size: 10
+    outputs:
+      - text
+      - word
+      - speech_started
+    env:
+      TARGET_LANGUAGE: english
+
+  - id: llm
+    build: pip install -e ../../node-hub/dora-qwen
+    path: dora-qwen
+    inputs:
+      text: stt/text
+      text_to_audio: NODE_ID/text
+      text_tool_response: NODE_ID/function_call_output
+    outputs:
+      - text
+    env:
+      MODEL_NAME_OR_PATH: LLM_ID
+      MODEL_FILE_PATTERN: "*[qQ]6_[kK].[gG][gG][uU][fF]"
+      MAX_TOKENS: 10_000
+      CONTEXT_SIZE: 10_000
+      TOOLS_JSON: |
+        TOOLS_ID
+      SYSTEM_PROMPT: |
+        SYSTEM_PROMPT_ID
+
+  - id: tts
+    build: pip install -e ../../node-hub/dora-kokoro-tts
+    path: dora-kokoro-tts
+    inputs:
+      text: llm/text
+    outputs:
+      - audio

examples/openai-realtime/whisper-template-metal.yml

@@ -0,0 +1,70 @@
+nodes:
+  - id: NODE_ID
+    build: pip install -e ../../node-hub/dora-openai-websocket
+    path: dynamic
+    inputs:
+      audio: tts/audio
+      transcript: stt/text
+      text: llm/text
+      speech_started: dora-vad/timestamp_start
+      speech_stopped: dora-vad/timestamp_end
+    outputs:
+      - audio
+      - response.create
+      - function_call_output
+
+  - id: dora-vad
+    build: pip install -e ../../node-hub/dora-vad
+    path: dora-vad
+    inputs:
+      audio:
+        source: NODE_ID/audio
+        queue_size: 3
+    outputs:
+      - audio
+      - timestamp_start
+      - timestamp_end
+    env:
+      MIN_SPEECH_DURATION_MS: 1000
+      MIN_SILENCE_DURATION_MS: 1000
+      THRESHOLD: 0.5
+
+  - id: stt
+    build: pip install -e ../../node-hub/dora-distil-whisper
+    path: dora-distil-whisper
+    inputs:
+      audio: dora-vad/audio
+      text_noise: llm/text
+    outputs:
+      - text
+      - word
+      - speech_started
+    env:
+      TARGET_LANGUAGE: english
+
+  - id: llm
+    build: pip install -e ../../node-hub/dora-qwen
+    path: dora-qwen
+    inputs:
+      text: stt/text
+      response.create: NODE_ID/response.create
+      text_tool_response: NODE_ID/function_call_output
+    outputs:
+      - text
+    env:
+      MODEL_NAME_OR_PATH: LLM_ID
+      MODEL_FILE_PATTERN: "*[qQ]6_[kK].[gG][gG][uU][fF]"
+      MAX_TOKENS: 30_000
+      CONTEXT_SIZE: 30_000
+      TOOLS_JSON: >
+        TOOLS_ID
+      SYSTEM_PROMPT: |
+        SYSTEM_PROMPT_ID
+
+  - id: tts
+    build: pip install -e ../../node-hub/dora-kokoro-tts
+    path: dora-kokoro-tts
+    inputs:
+      text: llm/text
+    outputs:
+      - audio