feat(platform): add Speech

Author: Guikingone

demo/tests/Blog/Command/StreamCommandTest.php

@@ -16,8 +16,13 @@
 use Symfony\AI\Agent\AgentInterface;
 use Symfony\AI\Platform\Message\MessageBag;
 use Symfony\AI\Platform\Metadata\Metadata;
+use Symfony\AI\Platform\Result\DeferredResult;
+use Symfony\AI\Platform\Result\InMemoryRawResult;
 use Symfony\AI\Platform\Result\RawResultInterface;
 use Symfony\AI\Platform\Result\ResultInterface;
+use Symfony\AI\Platform\Result\TextResult;
+use Symfony\AI\Platform\Speech\Speech;
+use Symfony\AI\Platform\Test\PlainConverter;
 use Symfony\Component\Console\Input\ArrayInput;
 use Symfony\Component\Console\Output\BufferedOutput;
 use Symfony\Component\Console\Style\SymfonyStyle;
@@ -52,6 +57,15 @@ public function getRawResult(): ?RawResultInterface
                 public function setRawResult(RawResultInterface $rawResult): void
                 {
                 }
+
+                public function addSpeech(Speech $speech): void
+                {
+                }
+
+                public function getSpeech(string $identifier): Speech
+                {
+                    return new Speech([], new DeferredResult(new PlainConverter(new TextResult('foo')), new InMemoryRawResult()), 'bar');
+                }
             });
 
         $input = new ArrayInput([]);

docs/components/platform.rst

@@ -501,6 +501,63 @@ This allows fast and isolated testing of AI-powered features without relying on
 
     This requires `cURL` and the `ext-curl` extension to be installed.
 
+Speech support
+~~~~~~~~~~~~~~
+
+Using speech to send messages / receive answers as audio is a common use case when integrating agents and/or chats.
+
+Speech support can be enable using ``Symfony\AI\Platform\Speech\SpeechProviderListener``::
+
+    use Symfony\AI\Agent\Agent;
+    use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechProvider;
+    use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
+    use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
+    use Symfony\AI\Platform\Message\Message;
+    use Symfony\AI\Platform\Message\MessageBag;
+    use Symfony\AI\Platform\Speech\SpeechConfiguration;
+    use Symfony\AI\Platform\Speech\SpeechProviderListener;
+    use Symfony\Component\EventDispatcher\EventDispatcher;
+
+    $eventDispatcher = new EventDispatcher();
+    $eventDispatcher->addSubscriber(new SpeechProviderListener([
+        new ElevenLabsSpeechProvider(PlatformFactory::create(
+            apiKey: $elevenLabsApiKey,
+            httpClient: http_client(),
+            speechConfiguration: new SpeechConfiguration(
+                ttsModel: 'eleven_multilingual_v2',
+                ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
+                sttModel: 'eleven_multilingual_v2'
+            )),
+        ),
+    ], []));
+
+    $platform = OpenAiPlatformFactory::create($openAiApiKey, httpClient: HttpClient::create(), eventDispatcher: $eventDispatcher);
+
+    $agent = new Agent($platform, 'gpt-4o');
+    $answer = $agent->call(new MessageBag(
+        Message::ofUser('Tina has one brother and one sister. How many sisters do Tina\'s siblings have?'),
+    ));
+
+    echo $answer->getSpeech('eleven_labs')->asBinary();
+
+When using the bundle, the configuration allows to configure models and voices::
+
+    ai:
+        platform:
+            eleven_labs:
+                api_key: '%env(ELEVEN_LABS_API_KEY)%'
+
+        speech:
+            eleven_labs:
+                tts_model: 'eleven_multilingual_v2'
+                tts_voice: '%env(ELEVEN_LABS_VOICE_IDENTIFIER)%'
+                tts_extra_options:
+                    foo: bar
+
+.. note::
+
+    Please be aware that enabling speech support requires to define corresponding platforms.
+
 Code Examples
 ~~~~~~~~~~~~~
 

examples/speech/README.md

@@ -0,0 +1,10 @@
+# Speech Examples
+
+Speech is mainly used to transform text to audio and vice versa, it can also be used to create an audio to audio pipeline.
+
+To run the examples, you can use additional tools like (mpg123)[https://www.mpg123.de/]:
+
+```bash
+php speech/agent-eleven-labs-speech-tts.php | mpg123 -
+php speech/agent-eleven-labs-speech-sts.php | mpg123 -
+```

examples/speech/agent-eleven-labs-speech-sts.php

@@ -0,0 +1,54 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Agent\Agent;
+use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechListener;
+use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechProvider;
+use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
+use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
+use Symfony\AI\Platform\Message\Content\Audio;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+use Symfony\AI\Platform\Speech\SpeechConfiguration;
+use Symfony\AI\Platform\Speech\SpeechProviderListener;
+use Symfony\Component\EventDispatcher\EventDispatcher;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+$eventDispatcher = new EventDispatcher();
+$eventDispatcher->addSubscriber(new SpeechProviderListener([
+    new ElevenLabsSpeechProvider(PlatformFactory::create(
+        apiKey: env('ELEVEN_LABS_API_KEY'),
+        httpClient: http_client(),
+        speechConfiguration: new SpeechConfiguration(
+            ttsModel: 'eleven_multilingual_v2',
+            ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
+            sttModel: 'eleven_multilingual_v2'
+        )),
+    ),
+], [
+    new ElevenLabsSpeechListener(PlatformFactory::create(
+        apiKey: env('ELEVEN_LABS_API_KEY'),
+        httpClient: http_client(),
+        speechConfiguration: new SpeechConfiguration(
+            sttModel: 'scribe_v1'
+        )),
+    ),
+]));
+
+$platform = OpenAiPlatformFactory::create(env('OPENAI_API_KEY'), httpClient: http_client(), eventDispatcher: $eventDispatcher);
+
+$agent = new Agent($platform, 'gpt-4o');
+$answer = $agent->call(new MessageBag(
+    Message::ofUser(Audio::fromFile(dirname(__DIR__, 2).'/fixtures/audio.mp3'))
+));
+
+echo $answer->getSpeech('eleven_labs')->asBinary();

examples/speech/agent-eleven-labs-speech-stt.php

@@ -0,0 +1,43 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Agent\Agent;
+use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechListener;
+use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
+use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
+use Symfony\AI\Platform\Message\Content\Audio;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+use Symfony\AI\Platform\Speech\SpeechConfiguration;
+use Symfony\AI\Platform\Speech\SpeechProviderListener;
+use Symfony\Component\EventDispatcher\EventDispatcher;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+$eventDispatcher = new EventDispatcher();
+$eventDispatcher->addSubscriber(new SpeechProviderListener([], [
+    new ElevenLabsSpeechListener(PlatformFactory::create(
+        apiKey: env('ELEVEN_LABS_API_KEY'),
+        httpClient: http_client(),
+        speechConfiguration: new SpeechConfiguration(
+            sttModel: 'scribe_v1'
+        )),
+    ),
+]));
+
+$platform = OpenAiPlatformFactory::create(env('OPENAI_API_KEY'), httpClient: http_client(), eventDispatcher: $eventDispatcher);
+
+$agent = new Agent($platform, 'gpt-4o');
+$answer = $agent->call(new MessageBag(
+    Message::ofUser(Audio::fromFile(dirname(__DIR__, 2).'/fixtures/audio.mp3'))
+));
+
+echo $answer->getContent();

examples/speech/agent-eleven-labs-speech-tts.php

@@ -0,0 +1,44 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Agent\Agent;
+use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechProvider;
+use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
+use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+use Symfony\AI\Platform\Speech\SpeechConfiguration;
+use Symfony\AI\Platform\Speech\SpeechProviderListener;
+use Symfony\Component\EventDispatcher\EventDispatcher;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+$eventDispatcher = new EventDispatcher();
+$eventDispatcher->addSubscriber(new SpeechProviderListener([
+    new ElevenLabsSpeechProvider(PlatformFactory::create(
+        apiKey: env('ELEVEN_LABS_API_KEY'),
+        httpClient: http_client(),
+        speechConfiguration: new SpeechConfiguration(
+            ttsModel: 'eleven_multilingual_v2',
+            ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
+            sttModel: 'eleven_multilingual_v2'
+        )),
+    ),
+], []));
+
+$platform = OpenAiPlatformFactory::create(env('OPENAI_API_KEY'), httpClient: http_client(), eventDispatcher: $eventDispatcher);
+
+$agent = new Agent($platform, 'gpt-4o');
+$answer = $agent->call(new MessageBag(
+    Message::ofUser('Tina has one brother and one sister. How many sisters do Tina\'s siblings have?'),
+));
+
+echo $answer->getSpeech('eleven_labs')->asBinary();

src/agent/src/Output.php

@@ -13,6 +13,7 @@
 
 use Symfony\AI\Platform\Message\MessageBag;
 use Symfony\AI\Platform\Result\ResultInterface;
+use Symfony\AI\Platform\Speech\Speech;
 
 /**
  * @author Christopher Hertel <[email protected]>
@@ -27,6 +28,7 @@ public function __construct(
         private ResultInterface $result,
         private readonly MessageBag $messageBag,
         private readonly array $options = [],
+        private ?Speech $speech = null,
     ) {
     }
 
@@ -57,4 +59,14 @@ public function getOptions(): array
     {
         return $this->options;
     }
+
+    public function setSpeech(?Speech $speech): void
+    {
+        $this->speech = $speech;
+    }
+
+    public function getSpeech(): ?Speech
+    {
+        return $this->speech;
+    }
 }

src/ai-bundle/CHANGELOG.md

@@ -34,3 +34,4 @@ CHANGELOG
    - Token usage metadata in agent results including prompt, completion, total, cached, and thinking tokens
    - Rate limit information tracking for supported platforms
  * Add support for configuring chats and message stores
+ * Add support for configuring speeches

src/ai-bundle/config/options.php

@@ -978,6 +978,23 @@
                     ->end()
                 ->end()
             ->end()
+            ->arrayNode('speech')
+                ->children()
+                    ->arrayNode('elevenlabs')
+                        ->children()
+                            ->stringNode('tts_model')->end()
+                            ->stringNode('tts_voice')->end()
+                            ->arrayNode('tts_extra_options')
+                                ->scalarPrototype()->end()
+                            ->end()
+                            ->stringNode('stt_model')->end()
+                            ->arrayNode('stt_extra_options')
+                                ->scalarPrototype()->end()
+                            ->end()
+                        ->end()
+                    ->end()
+                ->end()
+            ->end()
             ->arrayNode('vectorizer')
                 ->info('Vectorizers for converting strings to Vector objects and transforming TextDocument arrays to VectorDocument arrays')
                 ->useAttributeAsKey('name')

src/ai-bundle/config/services.php

@@ -63,6 +63,7 @@
 use Symfony\AI\Platform\Contract\JsonSchema\DescriptionParser;
 use Symfony\AI\Platform\Contract\JsonSchema\Factory as SchemaFactory;
 use Symfony\AI\Platform\Serializer\StructuredOutputSerializer;
+use Symfony\AI\Platform\Speech\SpeechProviderListener;
 use Symfony\AI\Platform\StructuredOutput\PlatformSubscriber;
 use Symfony\AI\Platform\StructuredOutput\ResponseFormatFactory;
 use Symfony\AI\Platform\StructuredOutput\ResponseFormatFactoryInterface;
@@ -241,5 +242,13 @@
                 tagged_locator('ai.message_store', 'name'),
             ])
             ->tag('console.command')
+
+        // listeners
+        ->set('ai.speech_provider.listener', SpeechProviderListener::class)
+            ->args([
+                tagged_iterator('ai.speech_provider', 'name'),
+                tagged_iterator('ai.speech_listener', 'name'),
+            ])
+            ->tag('kernel.event_subscriber')
     ;
 };