Streaming an octet stream from request to S3 with

Question

I'm trying to stream an octet-stream straight to S3 using knox on node.js. The octet-stream is an XHR file upload from the browser. I assumed that I could just stream the request into putStream and everything would just work, but alas no.

Here's my code:

var client = knox.createClient({ 
           // AWS credentials here
         });
if (req.headers['content-type'].match(/application\/octet-stream/i)) {

  var filename = '/'+req.headers['x-file-name'];

  client.putStream(req, filename, function(err, res){
    // TODO: Catch errors
    body = '{"success":"true"}'
    res.writeHead(200, 
      { 'Content-Type':'text/html'
      , 'Content-Length':body.length
      })
    res.end(body)
  });

}

And the error I receive:

TypeError: Bad argument
    at Object.stat (fs.js:354:11)
    at Client.putStream (./lib/knox/client.js:181:6)

Answer 1

I believe client.putStream accepts 4 params, like this:

client.putStream(stream, filepath, {
  'Content-Length': file.length,
  'Content-Type': 'application/octet-stream',
  'x-amz-acl': 'private'
}, function(err, res) {
  ...
});

Answer 2

I'm doing something like this:

app.post('/uploadAmazon', function(req, res) {  
var params = req.query;

var request = client.request("PUT", '/' + req.header('x-file-name') + '?partNumber=' + params.partNumber 
        + '&uploadId=' + params.uploadId, {
    'Content-Length' : req.header('Content-Length')
} );

req.on('data', function(data){
    request.write(data);
});


request.on('response', function(response) {
    console.log('Partial ' + params.partNumber + ' statusCode: ' + response.statusCode);
    if (response.statusCode== 200) {
        uploadMap[params.id].currentSize++;
        uploadMap[params.id].completeXmlArray[+(params.partNumber) - 1] = '<Part><PartNumber>' + params.partNumber + '</PartNumber><ETag>' + response.headers.etag + '</ETag></Part>' ; 

        if (uploadMap[params.id].currentSize == uploadMap[params.id].totalSize) {
            uploadMap[params.id].uploadId = params.uploadId;
            completeSend(uploadMap[params.id]);
        }
    }
}).end();

res.end();

});

Assuming that I receive the file name, part number and uploadId from the post.

Answer 3

If you are using a version of node.js much older than 0.4.5 then upgrade.

Look in the util module and use util.pump to copy the file from the input stream to the output stream. If the file has to be downloaded first, just use a ReadStream from the file as the input stream.

Also, do have a look at the Javascript code for util.pump because I suspect that you haven't quite grasped how async I/O works in node.js.