AOQ 経由で qwen3.5-omni-plus-realtime を使用したプッシュツートーク音声会話の構築

func onPushToTalkPressed() {
    hasAudioInCurrentTurn = true
    engine.enableSendMediaStream(.audio, enable: true)
}

func onPushToTalkReleased(base64Jpeg: String? = nil) {
    engine.enableSendMediaStream(.audio, enable: false)
    guard hasAudioInCurrentTurn else { return }
    if imageMode == .singleImage, let base64Jpeg {
        let imageEvent: [String: Any] = [
            "type": "input_image_buffer.append",
            "image": base64Jpeg
        ]
        if let data = try? JSONSerialization.data(withJSONObject: imageEvent) {
            let dataMessage = AoqDataMsg()
            dataMessage.data = data
            engine.sendDataMsg(dataMessage)
        }
    }
    for event in [
        ["type": "input_audio_buffer.commit"],
        ["type": "response.create"]
    ] {
        guard let data = try? JSONSerialization.data(withJSONObject: event) else { continue }
        let dataMessage = AoqDataMsg()
        dataMessage.data = data
        engine.sendDataMsg(dataMessage)
    }
    hasAudioInCurrentTurn = false
}

import Foundation
import AoqClientSdk

final class ManualPushToTalkClient: NSObject, AoqEngineDelegate {
    enum ImageMode: Equatable {
        case none
        case continuousVideo
        case singleImage
    }

    private var engine: AoqClientEngine!
    private let imageMode: ImageMode
    private var hasAudioInCurrentTurn = false

    init(workDir: String, connectConfig: AoqConnectConfig, imageMode: ImageMode) {
        self.imageMode = imageMode
        super.init()
        let createConfig = AoqCreateConfig()
        createConfig.workDir = workDir
        self.engine = AoqClientEngine.createEngine(createConfig, delegate: self)

        // 例の値です。これらの設定を選択したモデルとアプリケーションのフォーマットに合わせてください。
        let audioEncoderConfig = AoqAudioCodecConfig()
        audioEncoderConfig.trackType = .audio
        audioEncoderConfig.codecType = .audioPCM
        audioEncoderConfig.sampleRate = 16_000
        audioEncoderConfig.channel = 1
        engine.setAudioEncoderConfig(audioEncoderConfig)

        let audioDecoderConfig = AoqAudioCodecConfig()
        audioDecoderConfig.trackType = .audio
        audioDecoderConfig.codecType = .audioPCM
        audioDecoderConfig.sampleRate = 24_000
        audioDecoderConfig.channel = 1
        engine.setAudioDecoderConfig(audioDecoderConfig)

        let publishAudioTrack = AoqTrackParam()
        publishAudioTrack.trackType = .audio
        let publishDataTrack = AoqTrackParam()
        publishDataTrack.trackType = .data
        let subscribeAudioTrack = AoqTrackParam()
        subscribeAudioTrack.trackType = .audio
        let subscribeDataTrack = AoqTrackParam()
        subscribeDataTrack.trackType = .data

        connectConfig.publishTracks = [publishAudioTrack, publishDataTrack]
        connectConfig.subscribeTracks = [subscribeAudioTrack, subscribeDataTrack]

        if imageMode == .continuousVideo {
            let videoEncoderConfig = AoqVideoCodecConfig()
            videoEncoderConfig.trackType = .video
            videoEncoderConfig.codecType = .videoJpeg
            videoEncoderConfig.width = 960
            videoEncoderConfig.height = 540
            videoEncoderConfig.fps = 2
            videoEncoderConfig.bitrate = 500_000
            engine.setVideoEncoderConfig(videoEncoderConfig)

            let publishVideoTrack = AoqTrackParam()
            publishVideoTrack.trackType = .video
            connectConfig.publishTracks = [publishAudioTrack, publishVideoTrack, publishDataTrack]
        }

        let captureConfig = AoqAudioCaptureConfig()
        captureConfig.channel = 1
        captureConfig.isExternal = false
        engine.startAudioCapture(captureConfig)

        let playbackConfig = AoqAudioPlaybackConfig()
        playbackConfig.channel = 1
        playbackConfig.isExternal = false
        playbackConfig.isDefaultSpeaker = true
        engine.startAudioPlayer(playbackConfig)

        if imageMode == .continuousVideo {
            let videoCaptureConfig = AoqVideoCaptureConfig()
            videoCaptureConfig.width = 1280
            videoCaptureConfig.height = 720
            videoCaptureConfig.fps = 15
            engine.startVideoCapture(videoCaptureConfig)
        }

        engine.enableSendMediaStream(.audio, enable: false)
        if imageMode == .continuousVideo {
            engine.enableSendMediaStream(.video, enable: false)
        }
        engine.connect(connectConfig)
    }

    func onPushToTalkPressed() {
        hasAudioInCurrentTurn = true
        engine.enableSendMediaStream(.audio, enable: true)
    }

    func onPushToTalkReleased(base64Jpeg: String? = nil) {
        engine.enableSendMediaStream(.audio, enable: false)
        guard hasAudioInCurrentTurn else { return }
        if imageMode == .singleImage, let base64Jpeg {
            let imageEvent: [String: Any] = [
                "type": "input_image_buffer.append",
                "image": base64Jpeg
            ]
            if let data = try? JSONSerialization.data(withJSONObject: imageEvent) {
                let dataMessage = AoqDataMsg()
                dataMessage.data = data
                engine.sendDataMsg(dataMessage)
            }
        }
        for event in [
            ["type": "input_audio_buffer.commit"],
            ["type": "response.create"]
        ] {
            guard let data = try? JSONSerialization.data(withJSONObject: event) else { continue }
            let dataMessage = AoqDataMsg()
            dataMessage.data = data
            engine.sendDataMsg(dataMessage)
        }
        hasAudioInCurrentTurn = false
    }

    private func sendSessionUpdate() {
        let event: [String: Any] = [
            "type": "session.update",
            "session": [
                "modalities": ["text", "audio"],
                "voice": "Ethan",
                "audio": [
                    "input": ["format": ["type": "pcm", "sample_rate": 16_000]],
                    "output": ["format": ["type": "pcm", "sample_rate": 24_000]]
                ],
                "turn_detection": NSNull()
            ]
        ]
        guard let data = try? JSONSerialization.data(withJSONObject: event) else { return }
        let dataMessage = AoqDataMsg()
        dataMessage.data = data
        engine.sendDataMsg(dataMessage)
    }

    func close() {
        engine.disconnect()
        AoqClientEngine.destroy()
    }

    func onConnectionStatusChange(_ status: AoqConnectionStatus) {
        if status == .connected { sendSessionUpdate() }
    }

    func onDataMsg(_ msg: AoqDataMsg) {
        guard let event = try? JSONSerialization.jsonObject(with: msg.data) as? [String: Any],
              let type = event["type"] as? String else { return }
        if type == "session.updated", imageMode == .continuousVideo {
            engine.enableSendMediaStream(.video, enable: true)
        }
    }

    func onError(_ code: Int, message: String) {}
    func onWarning(_ code: Int, message: String) {}
    func onStats(_ stats: AoqStats) {}
    func onAudioDeviceStateChanged(_ state: AoqAudioDeviceState) {}
    func onAudioDeviceRouteChanged(_ routeType: Int) {}
    func onAudioDeviceInterrupted(_ interrupt: Bool) {}
    func onAudioFileState(_ state: AoqAudioFileState) {}
    func onVideoDeviceStateChanged(_ state: AoqVideoDeviceState) {}
}