Close stream when finished

Question

I am streaming HTML 5 video with Express using fs.createReadStream(path).pipe(res); which eventually causes the error Error: EMFILE: too many open files if I refresh the page too many times (in the thousands). I assume I need to perform some kind of cleanup and close the file to prevent this but am unsure what needs to be done.

I am creating a read stream every time the user seeks to a part of the video which the client has not loaded which makes this problem happen in a matter of minutes. Task manager shows the same file open hundreds of times so it appears to never get closed.

Relevant part of the stream function:

fs.stat(path, function(err, stats) {
  var size = stats.size;
  var range = req.headers.range;

  if (range) {
    var parts = range.replace(/bytes=/, '').split('-');
    var start = parseInt(parts[0]);
    var end = parts[1] ? parseInt(parts[1]) : size - 1;
    var chunksize = (end - start) + 1;

    res.writeHead(206, {
      'Content-Range': 'bytes ' + start + '-' + end + '/' + size,
      'Accept-Ranges': 'bytes',
      'Content-Length': chunksize,
      'Content-Type': 'video/mp4'
    });

    fs.createReadStream(path, {start: start, end: end}).pipe(res);
  } else {
    res.writeHead(200, {
      'Content-Length': size,
      'Accept-Ranges': 'bytes',
      'Content-Type': 'video/mp4'
    });

    fs.createReadStream(path).pipe(res);
  }
});

Answer 1

On Linux/Unix EMFILE issues are resolved by ulimit. I'm not sure on windows. Google Windows ulimit EMFILE gets to this Stack Overflow

https://stackoverflow.com/a/729204/683017

Its saying 2048 file descriptors per process is limit on Windows. This would fit with what your saying

"in the thousands"

If this 2048 is a firm limit, and I'm not sure it is, (you'll have to google around to find out more about your specific OS).

Maybe think about rebalancing the load out to more processes, or more servers if possible.

Answer 2

It turns out the stream only gets destroyed and the file closed when the end is reached - which rarely happens as you would have to load the entire file without seeking or closing the browser.

To fix this you have to listen for the close event of the response and destroy the stream:

res.on('close', function() {
  stream.destroy();
});

Sometimes the close handler is never called, even when it is placed on the first line in the request function. A fairly hacky way I found to fix this is to check if the socket is destroyed before creating the stream:

if (req.socket.destroyed) {
  return;
}

The fixed stream function:

fs.stat(path, function(err, stats) {
  if (req.socket.destroyed) {
    return;
  }

  var stream;
  var size = stats.size;
  var range = req.headers.range;

  if (range) {
    var parts = range.replace(/bytes=/, '').split('-');
    var start = parseInt(parts[0]);
    var end = parts[1] ? parseInt(parts[1]) : size - 1;
    var chunksize = (end - start) + 1;

    res.writeHead(206, {
      'Content-Range': 'bytes ' + start + '-' + end + '/' + size,
      'Accept-Ranges': 'bytes',
      'Content-Length': chunksize,
      'Content-Type': 'video/mp4'
    });

    stream = fs.createReadStream(path, {start: start, end: end});
  } else {
    res.writeHead(200, {
      'Content-Length': size,
      'Accept-Ranges': 'bytes',
      'Content-Type': 'video/mp4'
    });

    stream = fs.createReadStream(path);
  }

  stream.pipe(res);

  res.on('close', function() {
    stream.destroy();
  });
});