From b69b0dbcec642e13850253e8a1b29aa1951f44cd Mon Sep 17 00:00:00 2001 From: david Date: Mon, 31 Mar 2025 16:40:05 -0700 Subject: [PATCH] audio support --- src/server.ts | 120 ++++++++++++++++++++++++++++++---------------- static/index.html | 5 +- 2 files changed, 83 insertions(+), 42 deletions(-) diff --git a/src/server.ts b/src/server.ts index f3adad9..d473c9c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -41,35 +41,46 @@ class VideoStream extends Readable { // Function to start FFmpeg and capture raw video function startFFmpeg(): ChildProcessWithoutNullStreams { - const p = spawn('ffmpeg', [ + const p = spawn('ffmpeg', [ '-loglevel', 'debug', '-i', VIDEO_DEVICE, // Input device + + '-map', '0:v:0', '-vf', `scale=${WIDTH}:${HEIGHT}`, // Scale video resolution '-vcodec', 'rawvideo', // Output raw video codec '-pix_fmt', 'yuv420p', // Pixel format for WebRTC '-f', 'rawvideo', // Output format - 'pipe:1' // Pipe to stdout + 'pipe:3', // Pipe to stdout + + // Audio + '-map', '0:a:0', + '-acodec', 'pcm_s16le', + '-ac', '1', + '-ar', '48000', + '-f', 's16le', + 'pipe:4' + ], { -// stdio: ['ignore', 'pipe', 'pipe'], -// detached: true + stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe'] + // detached: true }); process.on('SIGINT', () => { console.log('🔻 Server shutting down... KILLING'); let b = p.kill('SIGINT'); // let b = process.kill(p.pid) process.exit(0); - }); - - process.on('SIGTERM', () => { + }); + + process.on('SIGTERM', () => { console.log('🔻 SIGTERM received'); p.kill('SIGTERM'); process.exit(0); - }); - process.on('exit', () => { + }); + process.on('exit', () => { p.kill('SIGHUP'); //this one let b = p.kill('SIGTERM'); - console.log("b ",b) - }); + console.log("b ", b) + }); return p; } @@ -84,32 +95,26 @@ const ffmpegProcess = startFFmpeg(); const videoSource = new nonstandard.RTCVideoSource(); +const audioSource = new nonstandard.RTCAudioSource(); // Function to create a WebRTC PeerConnection async function createPeerConnection(): Promise { - - const peerConnection = new RTCPeerConnection({iceServers: []} ); - - // Create a video source - - // const videoStream = new VideoStream('/dev/video0'); - - // track.addEventListener('') + const peerConnection = new RTCPeerConnection({ iceServers: [] }); + const videoStream = ffmpegProcess.stdio[3]; // pipe:3 + const audioStream = ffmpegProcess.stdio[4]; // pipe:4 // Start FFmpeg and pipe video frames to the source - - - ffmpegProcess.stdout.on('data', (chunk: Buffer) => { + videoStream.on('data', (chunk: Buffer) => { // Push video frames to the RTCVideoSource - + frameBuffer = Buffer.concat([frameBuffer, chunk]); while (frameBuffer.length >= FRAME_SIZE) { const frameData = frameBuffer.slice(0, FRAME_SIZE); frameBuffer = frameBuffer.slice(FRAME_SIZE); // Keep remaining data - + const frame: nonstandard.RTCVideoFrame = { width: WIDTH, height: HEIGHT, @@ -117,16 +122,52 @@ async function createPeerConnection(): Promise { } videoSource.onFrame(frame); - } + } }); - ffmpegProcess.stderr.on('data', (data: Buffer) => { + videoStream.on('data', (data: Buffer) => { // console.error('FFmpeg Error:', data.toString()); }); - ffmpegProcess.on('exit', (code) => { + videoStream.on('exit', (code) => { + console.log(`FFmpeg exited with code ${code}`); + }); + + // --- AUDIO handling --- + const AUDIO_FRAME_SIZE = 480 * 2; // 480 samples * 2 bytes (s16le) + let audioBuffer = Buffer.alloc(0); + + audioStream.on('data', (chunk: Buffer) => { + audioBuffer = Buffer.concat([audioBuffer, chunk]); + while (audioBuffer.length >= AUDIO_FRAME_SIZE) { + const frameData = audioBuffer.slice(0, AUDIO_FRAME_SIZE); + // const sampleBuffer = Buffer.from(frameData.buffer.; // makes an isolated buffer + audioBuffer = audioBuffer.slice(AUDIO_FRAME_SIZE); + + const samples = new Int16Array(480); + for (let i = 0; i < 480; i++) { + samples[i] = frameData.readInt16LE(i * 2); + } + + audioSource.onData({ + samples: samples, + sampleRate: 48000, + bitsPerSample: 16, + channelCount: 1, + numberOfFrames: 480 + }); + } + }); + + audioStream.on('data', (data: Buffer) => { + // console.error('FFmpeg Error:', data.toString()); + }); + + + + audioStream.on('exit', (code) => { console.log(`FFmpeg exited with code ${code}`); }); @@ -134,17 +175,16 @@ async function createPeerConnection(): Promise { // Add the track to the PeerConnection const track: MediaStreamTrack = videoSource.createTrack(); - console.log('vdei src ',videoSource.isScreencast) + const track1 = audioSource.createTrack(); + console.log('vdei src ', videoSource.isScreencast) const stream = new MediaStream() stream.addTrack(track) - console.log('enabled ',track.enabled, track.id, track.kind, track.label, track.readyState); + stream.addTrack(track1); + console.log('enabled ', track.enabled, track.id, track.kind, track.label, track.readyState); // track. - console.log('get',stream.getVideoTracks()[0].id) - peerConnection.addTrack(track, stream) - // peerConnection.addTransceiver(track, { direction: 'sendonly' }); // peerConnection.add - // peerConnection.addIceCandidate(); - // peerConnection - // console.log('Stream with track:', s.track.); + console.log('get', stream.getVideoTracks()[0].id) + peerConnection.addTrack(track, stream); + peerConnection.addTrack(track1, stream); return peerConnection; } @@ -159,10 +199,10 @@ wss.on('connection', async (ws: ws.WebSocket) => { console.log('Client connected'); ws.on('message', async (message: Buffer) => { - const { type, data} = JSON.parse(message.toString()); + const { type, data } = JSON.parse(message.toString()); console.log("message type", type) - if(type == 'offer') { + if (type == 'offer') { await peerConnection.setRemoteDescription(data); const answer = await peerConnection.createAnswer(); await peerConnection.setLocalDescription(answer); @@ -172,7 +212,7 @@ wss.on('connection', async (ws: ws.WebSocket) => { if (type === 'ice-candidate') { console.log('type ice') await peerConnection.addIceCandidate(data); - } + } }); @@ -182,13 +222,13 @@ wss.on('connection', async (ws: ws.WebSocket) => { console.error('ICE connection failed'); } }; - + // Send ICE candidates to the client peerConnection.onicecandidate = ({ candidate }) => { console.log("onicecandidate") if (candidate) { - ws.send(JSON.stringify({ type: 'ice-candidate', data: candidate })); + ws.send(JSON.stringify({ type: 'ice-candidate', data: candidate })); } }; diff --git a/static/index.html b/static/index.html index e39ec6a..f06515b 100644 --- a/static/index.html +++ b/static/index.html @@ -28,16 +28,17 @@ pc.ontrack = (event) => { console.log("Received track event", event.streams); video.srcObject = event.streams[0]; + // video.muted = false; }; pc.onicecandidate = ({ candidate }) => { - console.log("pc.onicecandidate") + // console.log("pc.onicecandidate") if (candidate) { ws.send(JSON.stringify({ type: 'ice-candidate', data: candidate })); // Use 'candidate' instead of 'ice-candidate' } }; pc.onicegatheringstatechange = () => { - console.log('ICE state:', pc.iceGatheringState); + // console.log('ICE state:', pc.iceGatheringState); }; ws.onopen = async () => {