Transcribe real-time speech with fun-asr-realtime over AOQ

import android.content.Context;

import com.alibaba.aoq.clientsdk.AoqClientEngine;
import com.alibaba.aoq.clientsdk.AoqClientListener;

import org.json.JSONArray;
import org.json.JSONObject;

import java.nio.charset.StandardCharsets;
import java.util.UUID;

public final class AsrClient {
    public interface ResultListener {
        void onResult(String text, boolean sentenceEnd);

        void onTaskFinished();

        void onError(String message);
    }

    private final AoqClientEngine engine;
    private final ResultListener resultListener;
    private String taskId;
    private boolean connected;
    private boolean taskStarted;

    public AsrClient(Context context, AoqClientEngine.AoqConnectConfig connectConfig,
                     ResultListener resultListener) {
        this.resultListener = resultListener;
        AoqClientListener listener = new AoqClientListener() {
            @Override
            public void onConnectionStatusChange(AoqClientEngine.AoqConnectionStatus status) {
                connected = status == AoqClientEngine.AoqConnectionStatus
                        .AoqConnectionStatusConnected;
                if (connected) {
                    beginRecognition();
                }
            }

            @Override
            public void onDataMsg(AoqClientEngine.AoqDataMsg msg) {
                handleServerEvent(msg);
            }
        };

        AoqClientEngine.AoqCreateConfig createConfig = new AoqClientEngine.AoqCreateConfig();
        createConfig.workDir = context.getFilesDir().getAbsolutePath();
        engine = AoqClientEngine.createEngine(context, createConfig, listener);

        configureAudioEncoder();
        configureTracks(connectConfig);
        startAudioCapture();
        engine.connect(connectConfig);
    }

    private void configureAudioEncoder() {
        AoqClientEngine.AoqAudioCodecConfig encoder = new AoqClientEngine.AoqAudioCodecConfig();
        encoder.trackType = AoqClientEngine.AoqTrackType.AoqTrackTypeAudio;
        encoder.codecType = AoqClientEngine.AoqEncoderType.AoqEncoderTypeAudioPCM;
        encoder.sampleRate = 16000;
        encoder.channel = 1;
        encoder.bitrate = 24000;
        engine.setAudioEncoderConfig(encoder);
    }

    private static void configureTracks(AoqClientEngine.AoqConnectConfig connectConfig) {
        addTrack(connectConfig, true, AoqClientEngine.AoqTrackType.AoqTrackTypeAudio);
        addTrack(connectConfig, true, AoqClientEngine.AoqTrackType.AoqTrackTypeData);
        addTrack(connectConfig, false, AoqClientEngine.AoqTrackType.AoqTrackTypeData);
    }

    private void startAudioCapture() {
        AoqClientEngine.AoqAudioCaptureConfig capture =
                new AoqClientEngine.AoqAudioCaptureConfig();
        capture.isExternal = false;
        capture.isVoipMode = true;
        capture.channel = 1;
        engine.startAudioCapture(capture);
        engine.enableSendMediaStream(AoqClientEngine.AoqTrackType.AoqTrackTypeAudio, false);
    }

    /** Starts another recognition task on the existing AOQ connection. */
    public void beginRecognition() {
        if (!connected || taskStarted || taskId != null) {
            return;
        }
        taskId = UUID.randomUUID().toString();
        JSONObject header = createHeader("run-task");
        JSONObject parameters = new JSONObject()
                .put("format", "pcm")
                .put("sample_rate", 16000);
        JSONObject payload = new JSONObject()
                .put("task_group", "audio")
                .put("task", "asr")
                .put("function", "recognition")
                .put("model", "fun-asr-realtime")
                .put("parameters", parameters)
                .put("input", new JSONObject());
        send(new JSONObject().put("header", header).put("payload", payload));
    }

    /** Stops the current task. Wait for task-finished before closing the connection. */
    public void finishRecognition() {
        if (taskId == null) {
            return;
        }
        engine.enableSendMediaStream(AoqClientEngine.AoqTrackType.AoqTrackTypeAudio, false);
        JSONObject payload = new JSONObject().put("input", new JSONObject());
        send(new JSONObject()
                .put("header", createHeader("finish-task"))
                .put("payload", payload));
    }

    public void close() {
        engine.enableSendMediaStream(AoqClientEngine.AoqTrackType.AoqTrackTypeAudio, false);
        engine.stopAudioCapture();
        engine.disconnect();
        AoqClientEngine.destroy();
    }

    private void handleServerEvent(AoqClientEngine.AoqDataMsg msg) {
        if (msg == null || msg.data == null) {
            return;
        }
        JSONObject event = new JSONObject(new String(msg.data, StandardCharsets.UTF_8));
        JSONObject header = event.optJSONObject("header");
        if (header == null) {
            return;
        }

        String eventName = header.optString("event", "");
        if ("task-started".equals(eventName)) {
            taskStarted = true;
            engine.enableSendMediaStream(
                    AoqClientEngine.AoqTrackType.AoqTrackTypeAudio, true);
        } else if ("result-generated".equals(eventName)) {
            handleRecognitionResult(event);
        } else if ("task-finished".equals(eventName)) {
            resetTaskState();
            resultListener.onTaskFinished();
        } else if ("task-failed".equals(eventName)) {
            String message = header.optString("error_message", "Recognition failed");
            resetTaskState();
            resultListener.onError(message);
        }
    }

    private void handleRecognitionResult(JSONObject event) {
        JSONObject payload = event.optJSONObject("payload");
        JSONObject output = payload == null ? null : payload.optJSONObject("output");
        JSONObject sentence = output == null ? null : output.optJSONObject("sentence");
        if (sentence == null || sentence.optBoolean("heartbeat", false)) {
            return;
        }
        String text = sentence.optString("text", "");
        if (!text.isEmpty()) {
            resultListener.onResult(text, sentence.optBoolean("sentence_end", false));
        }
    }

    private void resetTaskState() {
        engine.enableSendMediaStream(AoqClientEngine.AoqTrackType.AoqTrackTypeAudio, false);
        taskStarted = false;
        taskId = null;
    }

    private JSONObject createHeader(String action) {
        return new JSONObject()
                .put("action", action)
                .put("task_id", taskId)
                .put("streaming", "duplex");
    }

    private void send(JSONObject event) {
        AoqClientEngine.AoqDataMsg msg = new AoqClientEngine.AoqDataMsg();
        msg.data = event.toString().getBytes(StandardCharsets.UTF_8);
        engine.sendDataMsg(msg);
    }

    private static void addTrack(AoqClientEngine.AoqConnectConfig config, boolean publish,
                                 AoqClientEngine.AoqTrackType type) {
        AoqClientEngine.AoqTrackParam track = new AoqClientEngine.AoqTrackParam();
        track.trackType = type;
        if (publish) {
            config.publishTracks.add(track);
        } else {
            config.subscribeTracks.add(track);
        }
    }

    /** Maps the AppServer token response to an SDK connection configuration. */
    public static AoqClientEngine.AoqConnectConfig parseConnectConfig(String responseText) {
        JSONObject response = new JSONObject(responseText);
        AoqClientEngine.AoqConnectConfig config = new AoqClientEngine.AoqConnectConfig();
        config.token = response.optString("aoqTokenForClient", "");
        config.sid = response.optString("sid", "");
        config.certFingerprint = response.optString("clientRelayCertFingerprint", "");

        JSONArray endpoints = response.optJSONArray("clientRelayEndpoints");
        if (endpoints != null) {
            for (int i = 0; i < endpoints.length(); i++) {
                JSONObject item = endpoints.optJSONObject(i);
                if (item == null) {
                    continue;
                }
                AoqClientEngine.AoqRelayEndpoint endpoint =
                        new AoqClientEngine.AoqRelayEndpoint();
                endpoint.routeIndex = item.has("route_index")
                        ? item.optInt("route_index", i) : i;
                endpoint.endpoint = item.optString("endpoint", "");
                endpoint.port = item.optInt("port", 0);
                config.relayEndpoints.add(endpoint);
            }
        }

        JSONObject extraInfo = response.optJSONObject("extraInfo");
        config.workspaceIdHash = extraInfo == null
                ? "" : extraInfo.optString("workspaceIdHash", "");
        return config;
    }
}

private AsrClient client;

void startRecognition(Context context, String tokenResponseText) {
    AoqClientEngine.AoqConnectConfig config =
            AsrClient.parseConnectConfig(tokenResponseText);

    client = new AsrClient(context, config, new AsrClient.ResultListener() {
        @Override
        public void onResult(String text, boolean sentenceEnd) {
            // Update the UI with the partial or final sentence.
        }

        @Override
        public void onTaskFinished() {
            // Enable the start button or call beginRecognition() for another task.
        }

        @Override
        public void onError(String message) {
            // Display or log the error.
        }
    });
}

void onStopButtonClick() {
    // Finish the task and keep the AOQ connection open until task-finished arrives.
    client.finishRecognition();
}

void onPageDestroyed() {
    // Release local resources only when the page is closed.
    client.close();
}