<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>WebRTC access example</title>
</head>
<body>
<button id="start-session" type="button">Start translation</button>
<button id="finish-input" type="button" disabled>Finish input</button>
<button id="end-session" type="button" disabled>Disconnect after audio playback</button>
<button id="cancel-session" type="button" disabled>Cancel and disconnect</button>
<p id="session-status" role="status" aria-live="polite">Click "Start translation" to connect.</p>
<script>
const startSessionButton = document.getElementById("start-session");
const finishInputButton = document.getElementById("finish-input");
const endSessionButton = document.getElementById("end-session");
const cancelSessionButton = document.getElementById("cancel-session");
const sessionStatus = document.getElementById("session-status");
let translationFinished = false;
const pc = new RTCPeerConnection({ iceServers: [ ] });
const channels = new Set();
const senders = new Map();
const remoteAudio = document.createElement("audio");
remoteAudio.controls = true; // Users can play audio manually if autoplay is blocked.
remoteAudio.autoplay = true;
document.body.appendChild(remoteAudio);
let localStream = null;
let eventChannel = null;
let stopped = false;
let started = false;
let sessionCreated = false;
let updateSent = false;
let modelReady = false;
let mediaStarted = false;
let finishRequested = false;
const requestController = new AbortController();
const media = {
sendAudio: true,
receiveAudio: true,
};
function mediaDirection(send, receive) {
if (send && receive) return "sendrecv";
if (send) return "sendonly";
if (receive) return "recvonly";
return "inactive";
}
async function configureMedia() {
// Keep m=audio. The target service must support the selected directions.
const transceiver = pc.addTransceiver("audio", {
direction: mediaDirection(media.sendAudio, media.receiveAudio),
});
if (media.sendAudio) senders.set("audio", transceiver.sender);
if (!media.sendAudio) return;
const stream = await navigator.mediaDevices.getUserMedia({
audio: media.sendAudio,
video: false,
});
// The user may end the call while the permission prompt is open.
if (stopped) {
stream.getTracks().forEach(track => track.stop());
throw new Error("Session ended");
}
localStream = stream;
// sender.track is still empty, so no captured data is sent to the model.
}
// See Send and receive events over DataChannels for bindDataChannel.
pc.ondatachannel = ({ channel }) => bindDataChannel(channel);
const clientChannel = pc.createDataChannel("oai-events");
bindDataChannel(clientChannel);
pc.onconnectionstatechange = () => {
console.log("WebRTC state:", pc.connectionState);
if (pc.connectionState === "connected") {
tryInitializeModel();
} else if (["failed", "disconnected", "closed"].includes(pc.connectionState)) {
// This example ends the session; implement recovery as needed.
endSession();
}
};
function waitForIceComplete() {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => finish(new Error("ICE gathering timed out")), 15000);
function finish(error) {
clearTimeout(timer);
pc.removeEventListener("icegatheringstatechange", check);
requestController.signal.removeEventListener("abort", cancel);
error ? reject(error) : resolve();
}
function check() {
if (pc.iceGatheringState === "complete") finish();
}
function cancel() { finish(new Error("Session ended")); }
pc.addEventListener("icegatheringstatechange", check);
requestController.signal.addEventListener("abort", cancel, { once: true });
if (stopped) cancel();
else check();
});
}
function normalizeAnswerSdp(sdp) {
return String(sdp).trim().replace(/\r?\n/g, "\r\n") + "\r\n";
}
async function startSession() {
if (started || stopped) return;
started = true;
try {
await configureMedia();
if (stopped) return;
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
await waitForIceComplete();
// Implement this same-origin application endpoint; it is not a Model Studio API.
// AppServer forwards SDP with the API key and returns the raw Answer SDP.
const response = await fetch("/api/realtime/sdp", {
method: "POST",
headers: { "Content-Type": "application/sdp" },
body: pc.localDescription.sdp,
signal: requestController.signal,
});
if (!response.ok) {
throw new Error(`SDP exchange failed:${response.status} ${await response.text()}`);
}
const answerSdp = normalizeAnswerSdp(await response.text());
if (stopped) return;
await pc.setRemoteDescription({ type: "answer", sdp: answerSdp });
// Continue through connectionstatechange, DataChannel, and model events.
} catch (error) {
if (!stopped) console.error("Connection failed:", error);
endSession();
}
}
function tryInitializeModel() {
if (stopped || updateSent || !sessionCreated ||
pc.connectionState !== "connected" ||
eventChannel?.readyState !== "open") return;
sendModelEvent({
event_id: `event_${crypto.randomUUID()}`,
type: "session.update",
session: {
"output_modalities": media.receiveAudio ? ["text", "audio"] : ["text"],
"audio": {
"input": {
"turn_detection": {
"type": "speaker_detection",
"threshold": 0.5,
},
},
},
...(media.receiveAudio ? {
"voice": "Tina",
} : {}),
"translation": {
"language": "en",
},
},
});
updateSent = true;
}
async function handleModelEvent(event, channel) {
if (stopped) return;
if (event.type === "session.created") {
sessionCreated = true;
eventChannel = channel; // Send configuration through the channel that receives session events.
tryInitializeModel();
} else if (event.type === "session.updated" && updateSent) {
modelReady = true;
await startSendingMedia();
if (!finishRequested && !stopped) {
finishInputButton.disabled = false;
sessionStatus.textContent = "The model is ready. You can start speaking.";
}
} else if (event.type === "session.finished") {
translationFinished = true;
finishInputButton.disabled = true;
endSessionButton.disabled = false;
sessionStatus.textContent = "Translation is complete. Wait for remaining audio to finish before disconnecting.";
console.log("Translation processing has ended. Wait for remaining audio to finish before releasing resources.");
} else if (event.type === "error") {
console.error("Model error:", event.error);
endSession();
} else {
// Display text, transcripts, or response status as defined by the selected model. Receive audio over media tracks.
console.log("Model event:", event);
}
}
async function startSendingMedia() {
if (stopped || !modelReady || mediaStarted) return;
mediaStarted = true;
for (const track of localStream?.getTracks() ?? []) {
if (stopped) return;
const sender = senders.get(track.kind);
if (sender) await sender.replaceTrack(track);
}
}
// Register before startSession(). Tracks without streams are also received correctly.
const remoteAudioStream = new MediaStream();
pc.ontrack = ({ track }) => {
if (stopped) return;
if (track.kind === "audio" && media.receiveAudio) {
remoteAudioStream.addTrack(track);
remoteAudio.srcObject = remoteAudioStream;
remoteAudio.play().catch(() => {
console.info("Autoplay is blocked. Use the audio controls to play.");
});
}
};
async function finishTranslation() {
if (stopped || !modelReady || finishRequested) return;
finishRequested = true;
try {
localStream?.getAudioTracks().forEach(track => track.stop());
const sender = senders.get("audio");
if (sender) await sender.replaceTrack(null);
sendModelEvent({ type: "session.finish" });
} catch (error) {
finishRequested = false;
throw error;
}
}
function endSession() {
if (stopped) return;
stopped = true;
finishInputButton.disabled = true;
endSessionButton.disabled = true;
cancelSessionButton.disabled = true;
sessionStatus.textContent = "The connection is closed. Reload the page to translate again.";
modelReady = false;
sessionCreated = false;
updateSent = false;
mediaStarted = false;
requestController.abort(); // Cancel pending SDP requests and ICE gathering.
localStream?.getTracks().forEach(track => track.stop());
localStream = null;
channels.forEach(channel => channel.close());
channels.clear();
eventChannel = null;
pc.close();
senders.clear();
remoteAudioStream.getTracks().forEach(track => track.stop());
remoteAudio.pause();
remoteAudio.srcObject = null;
remoteAudio.remove();
}
function sendModelEvent(event, channel = eventChannel) {
if (stopped || pc.connectionState !== "connected" ||
channel?.readyState !== "open") {
throw new Error("Data channel is not ready");
}
channel.send(JSON.stringify(event));
}
function bindDataChannel(channel) {
channels.add(channel);
channel.onopen = () => {
if (stopped) return;
console.log("DataChannel opened:", channel.label);
tryInitializeModel();
};
channel.onmessage = ({ data }) => {
if (stopped) return;
let event;
try { event = JSON.parse(data); }
catch (error) {
console.warn("Cannot parse model event:", error);
return;
}
if (!event || typeof event !== "object") return;
handleModelEvent(event, channel).catch(error => {
if (!stopped) console.error("Model event handling failed:", error);
endSession();
});
};
channel.onerror = error => console.error("DataChannel error:", error);
channel.onclose = () => {
channels.delete(channel);
if (channel === eventChannel) endSession();
};
if (channel.readyState === "open") channel.onopen();
}
startSessionButton.addEventListener("click", () => {
startSessionButton.disabled = true;
cancelSessionButton.disabled = false;
sessionStatus.textContent = "Connecting...";
startSession();
});
finishInputButton.addEventListener("click", () => {
finishInputButton.disabled = true;
sessionStatus.textContent = "Finishing input...";
finishTranslation().then(() => {
if (!translationFinished && !stopped) {
sessionStatus.textContent = "Input has ended. Waiting for translation to complete.";
}
}).catch(error => {
if (!stopped) {
finishInputButton.disabled = false;
sessionStatus.textContent = "Unable to finish input. Try again.";
}
console.error("Failed to finish input:", error);
});
});
endSessionButton.addEventListener("click", endSession);
cancelSessionButton.addEventListener("click", endSession);
</script>
</body>
</html>
