Qwen-Audio-3.x-ASR-Flash-Streaming/Qwen-Audio-3.1-ASR-Flash-Message/Fun-ASR-Realtime Speech Recognition HarmonyOS SDK

{
  "dependencies": {
    "neonui": "file:libs/neonui.har"
  }
}

"requestPermissions": [
  {
    "name": "ohos.permission.INTERNET",
    "reason": "$string:reason_internet",
    "usedScene": { "abilities": ["EntryAbility"], "when": "always" }
  },
  {
    "name": "ohos.permission.MICROPHONE",
    "reason": "$string:reason_microphone",
    "usedScene": { "abilities": ["EntryAbility"], "when": "always" }
  }
]

{
  "url": "wss://dashscope.aliyuncs.com/api-ws/v1/inference",
  "device_id": "my_device_id",
  "service_mode": "1",
  "audio_update_manually": "false"
}

{
  "service_type": 4,
  "nls_config": {
    "model": "qwen-audio-3.0-asr-flash-streaming",
    "sr_format": "opus",
    "sample_rate": 16000
  }
}

import { AsrResult, Constants, INativeNuiCallback, KwsResult, NativeNui } from 'neonui';

constructor(mode_type: Constants.ModeType, flag?: string)

initialize(
  callback: INativeNuiCallback,
  parameters: string,
  level: number,
  save_log: boolean = false
): number

setParams(params: string): number

startDialog(vad_mode: Constants.VadMode, dialog_params: string): number

{
  "apikey": "st-****",
  "input_context": [
    { "role": "user", "content": [{ "type": "input_text", "text": "Example context" }] }
  ]
}

stopDialog(): number

cancelDialog(): number

dialogAction(action_params: string): number

{
  "type": "action",
  "command": "context",
  "context": [
    {
      "role": "user",
      "content": [
        { "text": "Example context", "type": "input_text" }
      ]
    }
  ]
}

updateAudio(data: ArrayBuffer, first_pack: boolean): number

pushReferenceData(data: ArrayBuffer, first_pack: boolean): number

release(): number

GetVersion(): string

refreshApikey(apikey: string, url: string = ''): string

onNuiEventCallback: (
  event: Constants.NuiEvent,
  resultCode: number,
  arg2: number,
  kwsResult: KwsResult,
  asrResult: AsrResult
) => void;

onNuiAudioStateChanged: (state: Constants.AudioState) => void;

onNuiNeedAudioData: (buffer: ArrayBuffer) => number;

onNuiAudioRMSChanged: (val: number) => number;

onNuiAssistEventCallback?: (
  event: Constants.NuiEvent,
  info: string,
  infoLen: number,
  data: ArrayBuffer
) => void;

onNuiLogTrackCallback: (level: Constants.LogLevel, log: string) => void;

import { AsrResult, Constants, INativeNuiCallback, KwsResult, NativeNui } from 'neonui';

const callback: INativeNuiCallback = {
  onNuiEventCallback: (event: Constants.NuiEvent, resultCode: number, arg2: number,
    kwsResult: KwsResult, asrResult: AsrResult): void => {
    if (event == Constants.NuiEvent.EVENT_ASR_PARTIAL_RESULT
      || event == Constants.NuiEvent.EVENT_SENTENCE_END) {
      // Parse payload.output.sentence.text from asrResult.allResponse.
    } else if (event == Constants.NuiEvent.EVENT_TRANSCRIBER_COMPLETE) {
      // Recognition is complete.
    } else if (event == Constants.NuiEvent.EVENT_ASR_ERROR) {
      // resultCode contains the error code.
    }
  },
  onNuiAudioStateChanged: (state: Constants.AudioState): void => {
    // Control AudioCapturer for STATE_OPEN, STATE_PAUSE, and STATE_CLOSE.
  },
  onNuiNeedAudioData: (buffer: ArrayBuffer): number => {
    // Copy recording data to buffer and return the number of bytes written.
    return 0;
  },
  onNuiAudioRMSChanged: (val: number): number => 0,
  onNuiLogTrackCallback: (level: Constants.LogLevel, log: string): void => {}
};

const nuiInstance = new NativeNui(Constants.ModeType.MODE_DIALOG);

const initParams: Record<string, Object> = {};
initParams['url'] = 'wss://dashscope.aliyuncs.com/api-ws/v1/inference';
initParams['device_id'] = 'my_device_id';
initParams['service_mode'] = Constants.ModeFullCloud;
initParams['audio_update_manually'] = 'false';

const initResult = nuiInstance.initialize(
  callback,
  JSON.stringify(initParams),
  Constants.LogLevel.LOG_LEVEL_DEBUG,
  false
);

if (initResult == Constants.NuiResultCode.SUCCESS) {
  const nlsConfig: Record<string, Object> = {
    'model': 'qwen-audio-3.0-asr-flash-streaming',
    'sr_format': 'opus',
    'sample_rate': 16000
  };
  const params: Record<string, Object> = {
    'service_type': Constants.kServiceTypeSpeechTranscriber,
    'nls_config': nlsConfig
  };
  nuiInstance.setParams(JSON.stringify(params));

  const dialogParams: Record<string, Object> = { 'apikey': 'st-****' };
  nuiInstance.startDialog(Constants.VadMode.TYPE_P2T, JSON.stringify(dialogParams));
}

// Stop recognition when the user finishes recording, and wait for EVENT_TRANSCRIBER_COMPLETE.
function stopRecognition(): void {
  nuiInstance.stopDialog();
}
// Call nuiInstance.release() from the EVENT_TRANSCRIBER_COMPLETE handler.

import { audio } from '@kit.AudioKit';

const options: audio.AudioCapturerOptions = {
  streamInfo: {
    samplingRate: audio.AudioSamplingRate.SAMPLE_RATE_16000,
    channels: audio.AudioChannel.CHANNEL_1,
    sampleFormat: audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE,
    encodingType: audio.AudioEncodingType.ENCODING_TYPE_RAW
  },
  capturerInfo: {
    source: audio.SourceType.SOURCE_TYPE_MIC,
    capturerFlags: 0
  }
};

const capturer = await audio.createAudioCapturer(options);
capturer.on('readData', (buffer: ArrayBuffer): void => {
  // Pull mode: enqueue data for onNuiNeedAudioData.
  // Push mode: call nuiInstance.updateAudio(buffer, firstPack).
});
await capturer.start();