ScreenCaptureKit: AVAudioPCMBuffer Conversion Issue

[ssdurante]

Hi folks,

I'm trying to generate auto-subtitles in real-time using ScreenCaptureKit API and SwiftWhisper, but whisper always returns no transcript to me.

I suspect it is the conversion of AVAudioPCMBuffer to Float[], but I have tried to convert in several ways but none were effective.

Does anyone know what the problem might be?

Here is my code:

import AVFoundation
import Combine
import Foundation
import OSLog
import ScreenCaptureKit
import SwiftUI

class Capturer: ObservableObject, @unchecked Sendable {
    private let logger = Logger()
    private let captureEngine = CaptureEngine()
    private let whisper = Whisper()
    private var audioBuffer: [Float] = []
    
    @Published var isRunning = false
    @Published var currentCaption = ""
    
    private let samplesPerProcess = 16000
    
    private var streamConfiguration: SCStreamConfiguration {
        let config = SCStreamConfiguration()
        config.capturesAudio = true
        config.minimumFrameInterval = CMTime(value: 1, timescale: 60)
        config.queueDepth = 5
        return config
    }
    
    private func processAudioBuffer(_ newSamples: [Float]) async {
        audioBuffer.append(contentsOf: newSamples)
        
        while audioBuffer.count >= samplesPerProcess {
            let chunk = Array(audioBuffer.prefix(samplesPerProcess))
            let text = await whisper.extractTextFromAudio(chunk)
            print("Whisper output: \(text)")
            audioBuffer.removeFirst(samplesPerProcess)
        }
    }

    func start() async {
        guard !isRunning else { return }
        
        do {
            let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
            guard let display = content.displays.first else { return }
            
            let excludedApps = content.applications.filter {
                Bundle.main.bundleIdentifier == $0.bundleIdentifier
            }
            
            let filter = SCContentFilter(
                display: display,
                excludingApplications: excludedApps,
                exceptingWindows: []
            )
            
            DispatchQueue.main.async {
                self.isRunning = true
            }
            
            for try await frames in captureEngine.startCapture(configuration: streamConfiguration, filter: filter) {
                await processAudioBuffer(frames as! [Float])
            }
            
        } catch {
            logger.error("\(error.localizedDescription)")
            DispatchQueue.main.async {
                self.isRunning = false
            }
        }
    }

    func stop() async {
        await captureEngine.stopCapture()
        DispatchQueue.main.async {
            self.isRunning = false
            self.currentCaption = ""
        }
    }
}

class CaptureEngine: NSObject, @unchecked Sendable {
    private let logger = Logger()
    private(set) var stream: SCStream?
    private var streamOutput: CaptureEngineStreamOutput?
    private var continuation: AsyncThrowingStream<Any, Error>.Continuation?

    func startCapture(configuration: SCStreamConfiguration, filter: SCContentFilter) -> AsyncThrowingStream<Any, Error> {
        AsyncThrowingStream { continuation in
            let streamOutput = CaptureEngineStreamOutput(continuation)
            self.streamOutput = streamOutput
            self.continuation = continuation

            do {
                stream = SCStream(filter: filter, configuration: configuration, delegate: streamOutput)
                try stream?.addStreamOutput(streamOutput, type: .screen, sampleHandlerQueue: .global())
                try stream?.addStreamOutput(streamOutput, type: .audio, sampleHandlerQueue: .global())
                stream?.startCapture()
            } catch {
                logger.error("Capture start error: \(error)")
            }
        }
    }

    func stopCapture() async {
        do {
            try await stream?.stopCapture()
            continuation?.finish()
        } catch {
            continuation?.finish(throwing: error)
        }
    }
}

private class CaptureEngineStreamOutput: NSObject, SCStreamOutput, SCStreamDelegate {
    private let logger = Logger()
    private var continuation: AsyncThrowingStream<Any, Error>.Continuation?
    private lazy var formatConverter: AVAudioConverter? = nil
    
    init(_ continuation: AsyncThrowingStream<Any, Error>.Continuation?) {
        self.continuation = continuation
    }

    func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) {
        guard sampleBuffer.isValid, outputType == .audio,
              let buffer = sampleBuffer.asPCMBuffer else { return }

        if formatConverter == nil {
            let outputFormat = AVAudioFormat(
                commonFormat: .pcmFormatFloat32,
                sampleRate: 16000,
                channels: 1,
                interleaved: false
            )!
            formatConverter = AVAudioConverter(from: buffer.format, to: outputFormat)
        }
        
        guard let converter = formatConverter else { return }
        let duration = Double(buffer.frameCapacity) / buffer.format.sampleRate
        let outputFrameCapacity = AVAudioFrameCount(16000 * duration)
        
        guard let outputBuffer = AVAudioPCMBuffer(
            pcmFormat: converter.outputFormat,
            frameCapacity: outputFrameCapacity
        ) else { return }
        
        var error: NSError?
        let status = converter.convert(
            to: outputBuffer,
            error: &error,
            withInputFrom: { _, outStatus in
                outStatus.pointee = .haveData
                return buffer
            }
        )
        
        if status == .error {
            print("Conversion error: \(error?.localizedDescription ?? "unknown")")
            return
        }

        if let samples = try? decodePCMBuffer(outputBuffer) {
            continuation?.yield(samples)
        }
    }

    func stream(_ stream: SCStream, didStopWithError error: Error) {
        logger.error("Stream stopped: \(error)")
        continuation?.finish(throwing: error)
    }
    
    func decodePCMBuffer(_ buffer: AVAudioPCMBuffer) throws -> [Float] {
        guard let floatChannelData = buffer.floatChannelData else {
            throw NSError(domain: "Invalid PCM Buffer", code: 0)
        }
        
        let channelCount = Int(buffer.format.channelCount)
        let frameLength = Int(buffer.frameLength)
        
        var floats: [Float] = Array(repeating: 0, count: frameLength * channelCount)
        
        for channel in 0..<channelCount {
            let channelData = floatChannelData[channel]
            for frame in 0..<frameLength {
                floats[frame * channelCount + channel] = channelData[frame]
            }
        }
        
        return floats
    }
}

extension CMSampleBuffer {
    var asPCMBuffer: AVAudioPCMBuffer? {
        try? withAudioBufferList { audioBufferList, _ in
            guard let desc = formatDescription?.audioStreamBasicDescription,
                  let format = AVAudioFormat(standardFormatWithSampleRate: desc.mSampleRate,
                                           channels: desc.mChannelsPerFrame) else { return nil }
            return AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: audioBufferList.unsafePointer)
        }
    }
}

[ssdurante]

Any ideas? cc @jordibruin

[offbyseven]

I’m trying to understand how you narrowed down the issue to stem from the conversion of AVAudioPCMBuffer to Float[]. If you haven’t added logging to validate that assumption, that might help.

From what I’ve seen, this is likely a sample rate problem, which is relatively common when working with ScreenCaptureKit.

Whisper requires audio to be 16kHz mono frames, and will internally resample audio if it’s not in this format. But ScreenCaptureKit gives you audio with a format based on the sampleRate and channelCount set in SCStreamConfiguration, with a default sample rate of 48000 Hz in a stereo format. If you pass that to Whisper as if it were 16kHz, Whisper effectively hears everything 3× too slow and pitched down, and returns nothing. Try resampling to 16kHz mono before passing frames to Whisper and see if that solves your problem.

If you’re trying to get live captions/transcripts, you could consider a desktop recording sdk to get speaker-labeled transcripts and make the audio capture process more reliable.