Qwen-Omni

# Installation instructions for pyaudio:
# APPLE Mac OS X
#   brew install portaudio
#   pip install pyaudio
# Debian/Ubuntu
#   sudo apt-get install python-pyaudio python3-pyaudio
#   or
#   pip install pyaudio
# CentOS
#   sudo yum install -y portaudio portaudio-devel && pip install pyaudio
# Microsoft Windows
#   python -m pip install pyaudio

import os
from openai import OpenAI
import base64
import numpy as np
import soundfile as sf

client = OpenAI(
    # 新加坡和北京地區的API Key不同。擷取API Key：https://www.alibabacloud.com/help/zh/model-studio/get-api-key
    api_key=os.getenv("DASHSCOPE_API_KEY"),
    # 以下為新加坡地區URL，調用時請將{WorkspaceId}替換為真實的業務空間ID，各地區的URL不同。
    base_url="https://{WorkspaceId}.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1",
)

completion = client.chat.completions.create(
    model="qwen3.5-omni-plus", # 模型為Qwen3-Omni-Flash時，請在非思考模式下運行
    messages=[{"role": "user", "content": "你是誰"}],
    # 設定輸出資料的模態，當前支援兩種：["text","audio"]、["text"]
    modalities=["text", "audio"],
    audio={"voice": "Tina", "format": "wav"},
    # stream 必須設定為 True，否則會報錯
    stream=True,
    stream_options={"include_usage": True},
)

# 方式1: 待產生結束後再進行解碼
audio_string = ""
for chunk in completion:
    if chunk.choices:
        if hasattr(chunk.choices[0].delta, "audio"):
            try:
                audio_string += chunk.choices[0].delta.audio["data"]
            except Exception as e:
                print(chunk.choices[0].delta.content)
    else:
        print(chunk.usage)

wav_bytes = base64.b64decode(audio_string)
audio_np = np.frombuffer(wav_bytes, dtype=np.int16)
sf.write("audio_assistant_py.wav", audio_np, samplerate=24000)

# 方式2: 邊產生邊解碼(使用方式2請將方式1的代碼進行注釋)
# # 初始化 PyAudio
# import pyaudio
# import time
# p = pyaudio.PyAudio()
# # 建立音頻流
# stream = p.open(format=pyaudio.paInt16,
#                 channels=1,
#                 rate=24000,
#                 output=True)

# for chunk in completion:
#     if chunk.choices:
#         if hasattr(chunk.choices[0].delta, "audio"):
#             try:
#                 audio_string = chunk.choices[0].delta.audio["data"]
#                 wav_bytes = base64.b64decode(audio_string)
#                 audio_np = np.frombuffer(wav_bytes, dtype=np.int16)
#                 # 直接播放音頻資料
#                 stream.write(audio_np.tobytes())
#             except Exception as e:
#                 print(chunk.choices[0].delta.content)

# time.sleep(0.8)
# # 清理資源
# stream.stop_stream()
# stream.close()
# p.terminate()

// 運行前的準備工作:
// Windows/Mac/Linux 通用:
// 1. 確保已安裝 Node.js (建議版本 >= 14)
// 2. 運行以下命令安裝必要的依賴:
//    npm install openai wav
//
// 如果要使用即時播放功能 (方式2), 還需要:
// Windows:
//    npm install speaker
// Mac:
//    brew install portaudio
//    npm install speaker
// Linux (Ubuntu/Debian):
//    sudo apt-get install libasound2-dev
//    npm install speaker

import OpenAI from "openai";

const openai = new OpenAI(
    {
        // 新加坡和北京地區的API Key不同。擷取API Key：https://www.alibabacloud.com/help/zh/model-studio/get-api-key
        apiKey: process.env.DASHSCOPE_API_KEY,
        // 以下為新加坡地區URL，調用時請將{WorkspaceId}替換為真實的業務空間ID，各地區的URL不同。
        baseURL: "https://{WorkspaceId}.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1"
    }
);
const completion = await openai.chat.completions.create({
    model: "qwen3.5-omni-plus", // 模型為Qwen3-Omni-Flash時，請在非思考模式下運行
    messages: [
        {
            "role": "user",
            "content": "你是誰？"
        }],
    stream: true,
    stream_options: {
        include_usage: true
    },
    modalities: ["text", "audio"],
    audio: { voice: "Tina", format: "wav" }
});

// 方式1: 待產生結束後再進行解碼
// 需要安裝: npm install wav
import { createWriteStream } from 'node:fs';  // node:fs 是 Node.js 內建模組，無需安裝
import { Writer } from 'wav';

async function convertAudio(audioString, audioPath) {
    try {
        // 解碼Base64字串為Buffer
        const wavBuffer = Buffer.from(audioString, 'base64');
        // 建立WAV檔案寫入流
        const writer = new Writer({
            sampleRate: 24000,  // 採樣率
            channels: 1,        // 單聲道
            bitDepth: 16        // 16位元深度
        });
        // 建立輸出檔案流並建立管道串連
        const outputStream = createWriteStream(audioPath);
        writer.pipe(outputStream);

        // 寫入PCM資料並結束寫入
        writer.write(wavBuffer);
        writer.end();

        // 使用Promise等待檔案寫入完成
        await new Promise((resolve, reject) => {
            outputStream.on('finish', resolve);
            outputStream.on('error', reject);
        });

        // 添加額外等待時間確保音頻完整
        await new Promise(resolve => setTimeout(resolve, 800));

        console.log(`音頻檔案已成功儲存為 ${audioPath}`);
    } catch (error) {
        console.error('處理過程中發生錯誤:', error);
    }
}

let audioString = "";
for await (const chunk of completion) {
    if (Array.isArray(chunk.choices) && chunk.choices.length > 0) {
        if (chunk.choices[0].delta.audio) {
            if (chunk.choices[0].delta.audio["data"]) {
                audioString += chunk.choices[0].delta.audio["data"];
            }
        }
    } else {
        console.log(chunk.usage);
    }
}
// 執行轉換
convertAudio(audioString, "audio_assistant_mjs.wav");

// 方式2: 邊產生邊即時播放
// 需要先按照上方系統對應的說明安裝必要組件
// import Speaker from 'speaker'; // 引入音頻播放庫

// // 建立擴音器執行個體（配置與 WAV 檔案參數一致）
// const speaker = new Speaker({
//     sampleRate: 24000,  // 採樣率
//     channels: 1,        // 聲道數
//     bitDepth: 16,       // 位深
//     signed: true        // 有符號 PCM
// });
// for await (const chunk of completion) {
//     if (Array.isArray(chunk.choices) && chunk.choices.length > 0) {
//         if (chunk.choices[0].delta.audio) {
//             if (chunk.choices[0].delta.audio["data"]) {
//                 const pcmBuffer = Buffer.from(chunk.choices[0].delta.audio.data, 'base64');
//                 // 直接寫入擴音器播放
//                 speaker.write(pcmBuffer);
//             }
//         }
//     } else {
//         console.log(chunk.usage);
//     }
// }
// speaker.on('finish', () => console.log('播放完成'));
// speaker.end(); // 根據實際 API 流結束情況調用

# Installation instructions for pyaudio:
# APPLE Mac OS X
#   brew install portaudio
#   pip install pyaudio
# Debian/Ubuntu
#   sudo apt-get install python-pyaudio python3-pyaudio
#   or
#   pip install pyaudio
# CentOS
#   sudo yum install -y portaudio portaudio-devel && pip install pyaudio
# Microsoft Windows
#   python -m pip install pyaudio

import os
from openai import OpenAI
import base64
import numpy as np
import soundfile as sf

import queue
import threading

client = OpenAI(
    # 新加坡和北京地區的API Key不同。擷取API Key：https://www.alibabacloud.com/help/zh/model-studio/get-api-key
    api_key=os.getenv("DASHSCOPE_API_KEY"),
    # 以下為新加坡地區URL，調用時請將{WorkspaceId}替換為真實的業務空間ID，各地區的URL不同。
    base_url="https://{WorkspaceId}.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1",
)

# 方式2: 邊產生邊解碼(使用方式2請將方式1的代碼進行注釋)
# # 初始化 PyAudio
import pyaudio
import time
# 建立一個隊列用於儲存音頻資料
audio_queue = queue.Queue()
# 設定是否已經開始播放
started_playing = False
# 設定緩衝時間（秒）
buffer_time = 5

# 音頻播放函數（將在單獨線程中運行）
def play_audio():
    global started_playing

    p = pyaudio.PyAudio()
    stream = p.open(format=pyaudio.paInt16,
                    channels=1,
                    rate=24000,
                    output=True)

    # 收集的音頻資料（用於緩衝）
    buffer_data = bytearray()

    try:
        while True:
            # 如果隊列為空白且已經開始播放，等待一小段時間
            if audio_queue.empty():
                if started_playing:
                    time.sleep(0.1)
                    # 如果隊列持續為空白，可能意味著音頻結束了
                    if audio_queue.empty():
                        # 播放剩餘的緩衝資料
                        if buffer_data:
                            stream.write(bytes(buffer_data))
                            buffer_data = bytearray()
                        continue
                else:
                    time.sleep(0.1)
                    continue

            # 從隊列擷取音頻資料
            audio_np = audio_queue.get()

            # 將資料添加到緩衝區
            buffer_data.extend(audio_np.tobytes())

            # 如果還沒開始播放且緩衝區大小足夠，開始播放
            samples_per_second = 24000 * 2  # 採樣率 * 每個樣本的位元組數（16位=2位元組）
            buffer_size_threshold = int(samples_per_second * buffer_time)

            if not started_playing and len(buffer_data) >= buffer_size_threshold:
                started_playing = True

            # 如果已經開始播放，按塊播放資料
            if started_playing:
                # 每次播放一小塊資料（例如0.1秒的資料）
                chunk_size = int(samples_per_second * 0.1)
                while len(buffer_data) >= chunk_size:
                    chunk = buffer_data[:chunk_size]
                    buffer_data = buffer_data[chunk_size:]
                    stream.write(bytes(chunk))

            # 標記任務完成
            audio_queue.task_done()
    finally:
        # 清理資源
        stream.stop_stream()
        stream.close()
        p.terminate()

# 啟動播放線程
audio_thread = threading.Thread(target=play_audio, daemon=True)
audio_thread.start()

completion = client.chat.completions.create(
    model="qwen3.5-omni-plus",
    messages=[{"role": "user", "content": "你是誰"}],
    # 設定輸出資料的模態，當前支援兩種：["text","audio"]、["text"]
    modalities=["text", "audio"],
    audio={"voice": "Tina", "format": "wav"},
    # stream 必須設定為 True，否則會報錯
    stream=True,
    stream_options={"include_usage": True},
)

# 接收音頻資料並放入隊列
for chunk in completion:
    if chunk.choices:
        if hasattr(chunk.choices[0].delta, "audio"):
            try:
                audio_string = chunk.choices[0].delta.audio["data"]
                wav_bytes = base64.b64decode(audio_string)
                audio_np = np.frombuffer(wav_bytes, dtype=np.int16)
                # 將音頻資料放入隊列，而不是直接播放
                audio_queue.put(audio_np)
            except Exception as e:
                print(chunk.choices[0].delta.audio["transcript"])

# 等待所有音頻資料播放完畢
audio_queue.join()
# 額外等待一段時間，確保最後的音頻都播放完畢
time.sleep(2)