AOQ と qwen-audio-3.1-asr-flash-streaming による音声入力の実装

// Create the engine
AoqClientEngine.AoqCreateConfig create = new AoqClientEngine.AoqCreateConfig();
create.workDir = getCacheDir().getAbsolutePath();
engine = AoqClientEngine.createEngine(this, create, new AoqClientListener() {
    @Override
    public void onConnectionStatusChange(
            @NonNull AoqClientEngine.AoqConnectionStatus state) {
        if (state == AoqClientEngine.AoqConnectionStatus.AoqConnectionStatusConnected) {
            // Connected: send run-task (see step 4)
        }
    }

    @Override
    public void onDataMsg(@NonNull AoqClientEngine.AoqDataMsg msg) {
        // Parse server events (see step 5)
    }

    @Override
    public void onError(int code, String message) {
        // Handle errors
    }
});

// Build the connection configuration after obtaining credentials from AppServer
AoqClientEngine.AoqConnectConfig config = new AoqClientEngine.AoqConnectConfig();
config.token = aoqTokenForClient;
config.sid = sid;
config.certFingerprint = clientRelayCertFingerprint;
config.workspaceIdHash = workspaceIdHash;
// Parse relayEndpoints from the response and add them to the configuration

// Publish Audio + Data tracks
AoqClientEngine.AoqTrackParam audioTrack = new AoqClientEngine.AoqTrackParam();
audioTrack.trackType = AoqClientEngine.AoqTrackType.AoqTrackTypeAudio;
config.publishTracks.add(audioTrack);

AoqClientEngine.AoqTrackParam dataTrack = new AoqClientEngine.AoqTrackParam();
dataTrack.trackType = AoqClientEngine.AoqTrackType.AoqTrackTypeData;
config.publishTracks.add(dataTrack);

// Subscribe to the Data track
AoqClientEngine.AoqTrackParam subData = new AoqClientEngine.AoqTrackParam();
subData.trackType = AoqClientEngine.AoqTrackType.AoqTrackTypeData;
config.subscribeTracks.add(subData);

// Establish the connection
engine.connect(config);

{
  "header": {
    "action": "run-task",
    "task_id": "9fb212ea-6f83-4aef-9b8e-1bdd71f1d4d3",
    "streaming": "duplex"
  },
  "payload": {
    "task_group": "audio",
    "task": "asr",
    "function": "recognition",
    "model": "qwen-audio-3.1-asr-flash-streaming",
    "parameters": {
      "format": "pcm",
      "sample_rate": 16000
    },
    "input": {}
  }
}

String taskId = UUID.randomUUID().toString();
JSONObject root = new JSONObject()
        .put("header", new JSONObject()
                .put("action", "run-task")
                .put("task_id", taskId)
                .put("streaming", "duplex"))
        .put("payload", new JSONObject()
                .put("task_group", "audio")
                .put("task", "asr")
                .put("function", "recognition")
                .put("model", "qwen-audio-3.1-asr-flash-streaming")
                .put("parameters", new JSONObject()
                        .put("format", "pcm")
                        .put("sample_rate", 16000))
                .put("input", new JSONObject()));

AoqClientEngine.AoqDataMsg msg = new AoqClientEngine.AoqDataMsg();
msg.data = root.toString().getBytes(StandardCharsets.UTF_8);
engine.sendDataMsg(msg);

byte[] pcm = pcmQueue.poll(640);  // Read one PCM frame

if (pcm != null) {
    JSONObject event = new JSONObject()
            .put("type", "input_audio_buffer.append")
            .put("event_id", "event_" + UUID.randomUUID())
            .put("audio", Base64.encodeToString(pcm, Base64.NO_WRAP));

    AoqClientEngine.AoqDataMsg msg = new AoqClientEngine.AoqDataMsg();
    msg.data = event.toString().getBytes(StandardCharsets.UTF_8);
    engine.sendDataMsg(msg);
}

JSONObject root = new JSONObject(new String(msg.data, StandardCharsets.UTF_8));
String event = root.optJSONObject("header").optString("event");

if ("task-started".equals(event)) {
    // Model ready: send buffered audio at double speed, then stream in real time
} else if ("result-generated".equals(event)) {
    // Parse recognition results (see step 6)
    JSONObject sentence = root.optJSONObject("payload")
            .optJSONObject("output").optJSONObject("sentence");
    if (sentence != null && !sentence.optBoolean("heartbeat")) {
        String text = sentence.optString("text", "");
        boolean isFinal = sentence.optBoolean("sentence_end");
        // Display recognition results
    }
} else if ("task-finished".equals(event)) {
    // Task finished
} else if ("task-failed".equals(event)) {
    // Task failed
}

private void handleRecognitionResult(JSONObject root) {
    JSONObject payload = root.optJSONObject("payload");
    JSONObject output = payload == null ? null : payload.optJSONObject("output");
    JSONObject sentence = output == null ? null : output.optJSONObject("sentence");
    if (sentence == null || sentence.optBoolean("heartbeat")) return;

    String text = sentence.optString("text", "");
    if (text.isEmpty()) return;
    boolean isFinal = sentence.optBoolean("sentence_end");

    // Display interim results in real time; append final results
    listener.onResult(text, isFinal);
}

{
    "header": {
        "action": "finish-task",
        "task_id": "UUID",
        "streaming": "duplex"
    },
    "payload": {
        "input": {}
    }
}

engine.stopAudioCapture();  // Stop capture

// Wait for the buffer to drain, then send finish-task
JSONObject root = new JSONObject()
        .put("header", new JSONObject()
                .put("action", "finish-task")
                .put("task_id", taskId)
                .put("streaming", "duplex"))
        .put("payload", new JSONObject()
                .put("input", new JSONObject()));

AoqClientEngine.AoqDataMsg msg = new AoqClientEngine.AoqDataMsg();
msg.data = root.toString().getBytes(StandardCharsets.UTF_8);
engine.sendDataMsg(msg);