How can we mix canvas stream with audio stream usi

Question

This question already has answers here:

MediaStream Capture Canvas and Audio Simultaneously (2 answers)

Closed 3 years ago.

I have a canvas stream using canvas.captureStream(). I have another video stream from webrtc video call. Now i want to mix canvas stream with audio tracks of the video stream.How can i do that?

Answer 1

Use the MediaStream constructor available in Firefox and Chrome 56, to combine tracks into a new stream:

let stream = new MediaStream([videoTrack, audioTrack]);

The following works for me in Firefox (Use https fiddle in Chrome, though it errors on recording):

navigator.mediaDevices.getUserMedia({audio: true})
  .then(stream => record(new MediaStream([stream.getTracks()[0],
                                          whiteNoise().getTracks()[0]]), 5000)
    .then(recording => {
      stop(stream);
      video.src = link.href = URL.createObjectURL(new Blob(recording));
      link.download = "recording.webm";
      link.innerHTML = "Download recording";
      log("Playing "+ recording[0].type +" recording:");
    })
    .catch(log))
  .catch(log);

var whiteNoise = () => {
  let ctx = canvas.getContext('2d');
  ctx.fillRect(0, 0, canvas.width, canvas.height);
  let p = ctx.getImageData(0, 0, canvas.width, canvas.height);
  requestAnimationFrame(function draw(){
    for (var i = 0; i < p.data.length; i++) {
      p.data[i++] = p.data[i++] = p.data[i++] = Math.random() * 255;
    }
    ctx.putImageData(p, 0, 0);
    requestAnimationFrame(draw);
  });
  return canvas.captureStream(60);
}

var record = (stream, ms) => {
  var rec = new MediaRecorder(stream), data = [];
  rec.ondataavailable = e => data.push(e.data);
  rec.start();
  log(rec.state + " for "+ (ms / 1000) +" seconds...");
  var stopped = new Promise((y, n) =>
      (rec.onstop = y, rec.onerror = e => n(e.error || e.name)));
  return Promise.all([stopped, wait(ms).then(_ => rec.stop())]).then(_ => data);
};

var stop = stream => stream.getTracks().forEach(track => track.stop());
var wait = ms => new Promise(resolve => setTimeout(resolve, ms));
var log = msg => div.innerHTML += "<br>" + msg;

<div id="div"></div><br>
<canvas id="canvas" width="160" height="120" hidden></canvas>
<video id="video" width="160" height="120" autoplay></video>
<a id="link"></a>