de-id / live-streaming-demo Goto Github PK
View Code? Open in Web Editor NEWUse D-ID's live streaming API to stream a talking presenter
License: MIT License
Use D-ID's live streaming API to stream a talking presenter
License: MIT License
Demo requires placing of key in api.json file, which would be readable by all users in a production environment...how can D-ID web services be used securely via WebRTC without compromising security in this way?
How do I keep my connection always open?
Hey I want to know if streaming API supports for Driver Expression in config as it supports for standard talk API. If so, can you help me with documentation.
I tried to convert the convert the example to a react.js component and everything seems to be fine because I get no errors although the stream doesn't show the video or audio.
import React, { useState, useEffect, useRef, useCallback } from "react";
const DID_API = {
key: "your_api_key_here",
url: "https://api.d-id.com",
service: "clips",
};
function DiDAvatar() {
// States for all the labels and buttons
const [peerStatus, setPeerStatus] = useState("");
const [iceStatus, setIceStatus] = useState("");
const [iceGatheringStatus, setIceGatheringStatus] = useState("");
const [signalingStatus, setSignalingStatus] = useState("");
const [streamingStatus, setStreamingStatus] = useState("");
// Refs for the video element and the peer connection
const videoElement: any = useRef(null);
const [peerConnection, setpeerConnection] = useState<any | null>(null);
// Refs for IDs and statuses
let sessionClientAnswer;
const [streamId, setstreamId] = useState("");
const [newSessionId, setnewSessionId] = useState("");
let videoIsPlaying = false;
let lastBytesReceived;
const maxRetryCount = 3;
const maxDelaySec = 4;
const presenterInputByService = {
talks: {
source_url: "https://d-id-public-bucket.s3.amazonaws.com/or-roman.jpg",
},
clips: {
presenter_id: "rian-lZC6MmWfC1",
driver_id: "mXra4jY38i",
},
};
// Check API key
useEffect(() => {
return () => {
stopAllStreams();
closePC();
// Any other cleanup can go here
};
}, []);
useEffect(() => {
if (peerConnection) {
peerConnection.onicegatheringstatechange = onIceGatheringStateChange;
peerConnection.onicecandidate = onIceCandidate;
peerConnection.oniceconnectionstatechange = onIceConnectionStateChange;
peerConnection.onconnectionstatechange = onConnectionStateChange;
peerConnection.onsignalingstatechange = onSignalingStateChange;
peerConnection.ontrack = onTrack;
setpeerConnection(peerConnection);
console.log("peerConnection add Eventlisteners\n\n", peerConnection);
}
}, [peerConnection]);
const playIdleVideo = () => {
videoElement.current.srcObject =
"https://drive.google.com/file/d/1A8NqwOg36mw7XW-e7iY5gvQLLSKykzNq/view?usp=drive_link";
videoElement.current.loop = true;
};
const stopAllStreams = () => {
// Check if the peer connection exists and has streams
if (videoElement.current.srcObject) {
console.log("stopping video streams");
videoElement.current.srcObject
.getTracks()
.forEach((track) => track.stop());
videoElement.current.srcObject = null;
}
};
const closePC = () => {
if (peerConnection) {
// Close the peer connection
peerConnection.close();
setpeerConnection(null);
setPeerStatus("closed");
setIceStatus("");
setSignalingStatus("no signaling");
setStreamingStatus("Stream ended");
peerConnection.ontrack = null;
console.log("stopped peer connection");
}
};
// Handlers for peer connection state changes
const onIceGatheringStateChange = () => {
if (peerConnection) {
const status = peerConnection.iceGatheringState;
setIceGatheringStatus(status);
}
};
const onIceCandidate = (event) => {
if (event.candidate) {
const { candidate, sdpMid, sdpMLineIndex } = event.candidate;
fetch(`${DID_API.url}/${DID_API.service}/streams/${streamId}/ice`, {
method: "POST",
headers: {
Authorization: `Basic ${DID_API.key}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
candidate,
sdpMid,
sdpMLineIndex,
session_id: newSessionId,
}),
});
}
};
const onIceConnectionStateChange = () => {
const status = peerConnection.iceConnectionState;
setIceStatus(status);
if (status === "failed" || status === "closed") {
stopAllStreams();
closePC();
}
};
const onConnectionStateChange = () => {
const status = peerConnection.connectionState;
setPeerStatus(status);
};
const onSignalingStateChange = () => {
const status = peerConnection.signalingState;
setSignalingStatus(status);
};
const onVideoStatusChange = (videoIsPlaying, stream) => {
let status;
if (videoIsPlaying) {
status = "streaming";
const remoteStream = stream;
runVideoElement(remoteStream);
} else {
status = "empty";
playIdleVideo();
}
console.log("onVideoStatusChange", status);
};
const runVideoElement = (stream) => {
if (!stream) return;
videoElement.current.srcObject = stream;
videoElement.current.src =
"https://drive.google.com/file/d/1A8NqwOg36mw7XW-e7iY5gvQLLSKykzNq/view?usp=drive_link";
videoElement.current.loop = false;
// safari hotfix
if (videoElement.current.paused) {
videoElement.current
.play()
.then((_) => {})
.catch((e) => {});
}
};
const onTrack = async (event) => {
if (!event.track) return;
const stats = await peerConnection.getStats(event.track);
stats.forEach((report) => {
if (report.type === "inbound-rtp" && report.mediaType === "video") {
const videoStatusChanged =
videoIsPlaying !== report.bytesReceived > lastBytesReceived;
if (videoStatusChanged) {
videoIsPlaying = report.bytesReceived > lastBytesReceived;
onVideoStatusChange(videoIsPlaying, event.streams[0]);
}
lastBytesReceived = report.bytesReceived;
}
});
};
// Other utility functions like createPeerConnection, setVideoElement, etc.
const createPeerConnection = async (offer, iceServers) => {
const peerConnection = new RTCPeerConnection({ iceServers });
const remoteDescription = new RTCSessionDescription(offer);
await peerConnection.setRemoteDescription(remoteDescription);
console.log("set remote sdp OK");
const sessionClientAnswer = await peerConnection.createAnswer();
console.log("create local sdp OK");
await peerConnection.setLocalDescription(sessionClientAnswer);
console.log("set local sdp OK");
setpeerConnection(peerConnection);
return sessionClientAnswer;
};
const fetchWithRetries = async (url, options, retries = 1) => {
try {
return await fetch(url, options);
} catch (err) {
if (retries <= maxRetryCount) {
const delay =
Math.min(Math.pow(2, retries) / 4 + Math.random(), maxDelaySec) *
1000;
await new Promise((resolve) => setTimeout(resolve, delay));
console.log(
`Request failed, retrying ${retries}/${maxRetryCount}. Error ${err}`,
);
return fetchWithRetries(url, options, retries + 1);
} else {
throw new Error(`Max retries exceeded. error: ${err}`);
}
}
};
// Handler functions for buttons
const handleConnectClick = useCallback(async () => {
if (peerConnection && peerConnection.connectionState === "connected") {
return;
}
stopAllStreams();
closePC();
const sessionResponse = await fetchWithRetries(
`${DID_API.url}/${DID_API.service}/streams`,
{
method: "POST",
headers: {
Authorization: `Basic ${DID_API.key}`,
"Content-Type": "application/json",
},
body: JSON.stringify(presenterInputByService[DID_API.service]),
},
);
const {
id: newStreamId,
offer,
ice_servers: iceServers,
session_id: newSessionId,
} = await sessionResponse.json();
sessionStorage.setItem("streamId", newStreamId);
setstreamId(newStreamId);
setnewSessionId(newSessionId);
try {
sessionClientAnswer = await createPeerConnection(offer, iceServers);
} catch (e) {
console.log("error during streaming setup", e);
stopAllStreams();
closePC();
return;
}
const sdpResponse = await fetch(
`${DID_API.url}/${DID_API.service}/streams/${sessionStorage.getItem("streamId")}/sdp`,
{
method: "POST",
headers: {
Authorization: `Basic ${DID_API.key}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
answer: sessionClientAnswer,
session_id: newSessionId,
}),
},
);
}, [streamId]);
const handleStartClick = async () => {
// connectionState not supported in firefox
if (
peerConnection?.signalingState === "stable" ||
peerConnection?.iceConnectionState === "connected"
) {
const playResponse = await fetchWithRetries(
`${DID_API.url}/${DID_API.service}/streams/${streamId}`,
{
method: "POST",
headers: {
Authorization: `Basic ${DID_API.key}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
script: {
type: "text",
subtitles: "false",
provider: {
type: "microsoft",
voice_id: "en-US-JennyNeural",
// voice_config: {
// style: "string",
// rate: "0.5",
// pitch: "+2st",
// },
language: "English (United States)",
},
ssml: true,
input: "Hello World! And welcome to getitAI.",
},
...(DID_API.service === "clips" && {
background: {
color: "#FFFFFF",
},
}),
config: {
stitch: true,
},
session_id: newSessionId,
}),
},
);
console.log("playResponse", playResponse);
}
};
const handleDestroyClick = async () => {
await fetch(`${DID_API.url}/${DID_API.service}/streams/${streamId}`, {
method: "DELETE",
headers: {
Authorization: `Basic ${DID_API.key}`,
"Content-Type": "application/json",
},
body: JSON.stringify({ session_id: newSessionId }),
});
stopAllStreams();
closePC();
};
// Rendering
return (
<div>
<video width="400" height="400" ref={videoElement} autoPlay />
<div>
<li>iceGatheringStatus: {iceGatheringStatus} </li>
<li>iceStatus: {iceStatus} </li>
<li>PeerStatus: {peerStatus} </li>
<li>signalingStatus: {peerConnection?.signalingState} </li>
<li>streamingStatus: {streamingStatus} </li>
</div>
<button className="btn" onClick={handleConnectClick}>
Connect
</button>
<button className="btn" onClick={handleStartClick}>
Start
</button>
<button className="btn" onClick={handleDestroyClick}>
Destroy
</button>
</div>
);
}
export default DiDAvatar;
This does not seem to work on Safari. Any ideas on how to fix this?
I got the basics up and running, but it would be awesome to build this out a bit more to handle two-way conversations.
Your documentation (https://docs.d-id.com/reference/createtalkstream) says create talk endpoint will return an object with duration and session ID parameters. Still, when I tried it on my servers it did not return any of those parameters.
Response {type: 'cors', url: 'https://api.d-id.com/talks/streams/strm_jkooTEGYuED4HGPc3LvYg', redirected: false, status: 200, ok: true, …}body: ReadableStreambodyUsed: false headers: Headers {}ok: true redirected: false status: 200statusText: "OK" type: "cors"url: "https://api.d-id.com/talks/streams/strm_jkooTEGYuED4HGPc3LvYg"[[Prototype]]: Response
When turning the video, the returned video stream player's mouth is stuck and sometimes blank, which should be caused by unstable flow
const talkResponse = await fetchWithRetries(
`${DID_API.url}/talks/streams/${streamId?.current}`,
{
method: "POST",
headers: {
Authorization: `Basic ${DID_API.key}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
script: {
type: "text",
input: message,
provider: {
type: "elevenlabs",
voice_id: "U3YxbJLR6YHQuydI1Itp",
},
},
driver_url: "bank://lively/",
config: {
stitch: true,
},
session_id: sessionId?.current,
}),
}
);
response status code - 400
{"kind":"BadRequestError","description":"script sanitization failed"}
What is wrong?
Please, help with #6 It's not resolved yet
Could you please give us a python example of "live-streaming-demo"
We are having intermittent but fairly consistent CORS policy issues with the live-streaming-demo code. I am running your sample in Apache and using Google Chrome. I've tested both locally and on a hosted server with valid SSL.
Access to fetch at 'https://api.d-id.com/talks/streams/strm_wXeTNvuvh2TcZLuDE3fU6/ice' from origin 'http://127.0.0.1' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource. If an opaque response serves your needs, set the request's mode to 'no-cors' to fetch the resource with CORS disabled.
These issues occur both during the connection process (pressing Connect) and when using POST to Create a talk stream (pressing Talk). Even when a connection is established Pressing Talk seems to regularly fail the first time and then work on subsequent presses.
I've been working on a prototype for a client. Trying to work around these intermittent CORS policy issues is really eating into our credits. Any suggestions would be appreciated.
I try to use your demo with elevenlabs.
const talkResponse = await fetch(`${DID_API.url}/talks/streams/${streamId}`,
{
method: 'POST',
headers: { Authorization: `Basic ${DID_API.key}`, 'Content-Type': 'application/json' },
body: JSON.stringify({
"script": {
"type": "text",
"subtitles": "false",
"provider": {
"type": "elevenlabs",
"voice_id": "BMKpAZGNzkYvXm2hPSuP"
},
"input": "Hello, Niraj, my name is B K. Thank you for your help with this clone"
},
'driver_url': 'bank://lively/',
'config': {
'stitch': true,
},
'session_id': sessionId
})
});
but it generates an error
POST https://api.d-id.com/talks/streams/strm_PNmi1SRve84D4DLroBJ_2
payload
{
"script": {
"type": "text",
"subtitles": "false",
"provider": {
"type": "elevenlabs",
"voice_id": "BMKpAZGNzkYvXm2hPSuP"
},
"input": "Hello, Niraj, my name is B K. Thank you for your help with this clone!"
},
"driver_url": "bank://lively/",
"config": {
"stitch": true
},
"session_id": "AWSALB=Q5Gyd67+7D7aO/jGklXdslfpmIP5cWV+7hu6COtUKrcJ0lWMy+FYISrkSvK3EuFQV2fhLKAr1dD3wR2ekeW/ARpoLJ0bm/Y0BiZFxlwIlCFXzRWzcW/21KCSmh7P; Expires=Wed, 24 May 2023 13:57:14 GMT; Path=/; AWSALBCORS=Q5Gyd67+7D7aO/jGklXdslfpmIP5cWV+7hu6COtUKrcJ0lWMy+FYISrkSvK3EuFQV2fhLKAr1dD3wR2ekeW/ARpoLJ0bm/Y0BiZFxlwIlCFXzRWzcW/21KCSmh7P; Expires=Wed, 24 May 2023 13:57:14 GMT; Path=/; SameSite=None; Secure"
}
response status code - 400
{
"kind": "TextToSpeechProviderError",
"description": "SSML is only supported by Microsoft Azure",
"details": {
"provider": "elevenlabs"
}
}
What is wrong?
A declarative, efficient, and flexible JavaScript library for building user interfaces.
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google ❤️ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.