Server crashing when downloading big files from ur

问题:

I'm trying to download a file (+200mb) from an url (that requires to be logged in) using the request module from nodejs but when it finishes the download the server starts to slow down until it crashes or gets really slow.

here's my current code (it downloads the whole file but my server crashes eventually):

//Required modules
var http = require('http'),
url = require("url"),
fs = require('fs'),
request = require('request'),
path = require("path"),
events = require("events"),
j = request.jar(),
request = request.defaults({ jar : j });

// make te request login in with cookies
console.log("downloading file :)");
request({
    url:"http://example.com/",
    method:"POST",
    form:{u: "username",p: "password"}
    },
    function(error,response,body){
        setTimeout(function(){
            request
              .get('http://example.com/test.ashx?file=15')
              .on('error', function(err) {
                console.log(err);
              })
              .pipe(fs.createWriteStream("/var/www/filesDir/CustomName.zip"));
                console.log(body);
        },1000)
    }
);

I've tried applying another solution from this answer but for some reason the file is not being downloaded properly, it only shows "Download progress: 0 bytes" all the time maybe it's something related with the login access.

here I put the other code I'm trying to implement from the last sentence:

var http = require('http');
var fs = require('fs');
var url = require("url");
var request = require('request');
var path = require("path");
var events = require("events");
var j = request.jar();
var request = request.defaults({ jar : j });


request({
    url:"http://example.com/",
    method:"POST",
    form:{u:"username",p:"password"}
    },  function(error,response,body){
                var downloadfile = "http://example.com/test.ashx?file=15";
                var host = url.parse(downloadfile).hostname;
                var filename = "1977.zip";
                var req = http.request({port: 80, host: host, method: 'GET'});
                console.log("Downloading file: " + filename);
                console.log("Before download request");
                req.end();

                dlprogress = 0;

                setInterval(function () {
                   console.log("Download progress: " + dlprogress + " bytes");
                }, 1000);

                req.addListener('response', function (response) {
                    var downloadfile = fs.createWriteStream(filename, {'flags': 'a'});
                    console.log("File size " + filename + ": " + response.headers['content-length'] + " bytes.");
                    response.addListener('data', function (chunk) {
                        dlprogress += chunk.length;
                        downloadfile.write(chunk, encoding='binary');
                    });
                    response.addListener("end", function() {
                        downloadfile.end();
                        console.log("Finished downloading " + filename);
                    });

                });
        }
);

It doesn't matter which way you decide to help me with.

回答1:

I ended up doing it like this, I've tested the code multiple times and the server didn't crash anymore:

var request = require('request');
var filed = require('filed');
var j = request.jar();
var request = request.defaults({ jar : j });

// make the request and login
request({
    url: "http://example.com/login",
    method:"POST",
    // 'u' and 'p' are the field names on the form
    form:{u:"username",p:"password"}
    }, function(error,response,body){
        setTimeout(function(){
            var downloadURL = 'http://example.com/download/file.zip';
            var downloadPath = "/path/to/download/localNameForFile.zip";

            var downloadFile = filed(downloadPath);
            var r = request(downloadURL).pipe(downloadFile);

            r.on('data', function(data) {
                console.log('binary data received');
            });
            downloadFile.on('end', function () {
                console.log(downloadPath, 'file downloaded to path');
            });

            downloadFile.on('error', function (err) {
                console.log(err, 'error downloading file');
            });
        },3000)
    }
);