$http.post: Large files do not work

Question

I am trying to upload files through my web app using the following code.

View:

  <form name="uploadForm" class="form-horizontal col-sm-12">
    <div class="col-sm-4">
      <input type="file" ng-model="rsdCtrl.viewData.file" name="file"/>
    </div>
    <div class="col-sm-4">
      <button class="btn btn-success" type="submit" ng-click="uploadFile()">Upload</button>
    </div>
  </form>

Controller:

function uploadFile(){
  if (uploadForm.file.$valid && file) {
    return uploadService.upload(vd.file, "Convictions Calculator", "PCCS").then(function(response){
      /* Some stuff */
    }).catch(handleServiceError);
  }
}

uploadService:

(function (){
'use strict';
angular.module('cica.common').service('uploadService', ['$http', '$routeParams', uploadService]);

function uploadService($http, $routeParams) {

    this.upload = function (file, name, type) {
        const fd = new FormData();
        fd.append('document', file);
        fd.append('jobId', $routeParams.jobId);
        fd.append('documentRename', name);
        fd.append('documentType', type);

        return $http.post('/document/upload', fd, {
            transformRequest: angular.identity,
            headers: {'Content-Type': undefined}
        }).catch(function(err){
            handleHttpError('Unable to upload document.', err);
        });
    };
  }
})();

routes.js:

    'POST /document/upload': {controller: 'DocumentController', action: 'uploadDocument'},

DocumentController:

"use strict";
const fs = require('fs');

module.exports = {
  uploadDocument: function (req, res) {
    console.log(req.allParams());   //Inserted as part of debugging
    const params = req.allParams();
    req.file('document').upload({
        // don't allow the total upload size to exceed ~100MB
        maxBytes: 100000000
    }, function whenDone(err, uploadedFiles) {
        if (err) {
            return res.serverError(err);
        }
        // If no files were uploaded, respond with an error.
        else if (uploadedFiles.length === 0) {
            return res.serverError('No file was uploaded');
        } else {
            const filePath = uploadedFiles[0].fd;
            const filename = uploadedFiles[0].filename;
            return fs.readFile(filePath, function (err, data) {
                if (err) {
                    return res.serverError(err);
                } else {
                    const jobId = params.jobId;
                    const jobVars =
                        {
                            filePath: results.filePath,
                            fileName: params.documentRename,
                            fileType: params.documentType
                        };
                    return DocumentService.uploadConvictions(req.session.sessionId, jobId, jobVars).then(function (response) {
                        return res.send("Document uploaded.");
                    }).catch(function (err) {
                        return res.serverError(err);
                    });
                }
            });
        }
    });
},

If I upload a .jpeg (around 11kB) the upload works exactly as expected, however, if I try to upload a larger .jpeg (around 170kB) it falls over. There is no immediate error thrown/caught though, what happens is the formData object created in the upload service seems to lose its data. If I breakpoint on its value, it returns empty for the larger file, which eventually causes an error when the function tries to use these variables further on. Is there some kind of limit set to the size of a file you can upload via this method, or have I configured this incorrectly?

Answer 1

Avoid using the FormData API when Uploading Large Files¹

The FormData API encodes data in base64 which add 33% extra overhead.

Instead of sending FormData, send the file directly:

app.service('fileUpload', function ($http) {
    this.uploadFileToUrl = function (url, file) {
        ̶v̶a̶r̶ ̶f̶d̶ ̶=̶ ̶n̶e̶w̶ ̶F̶o̶r̶m̶D̶a̶t̶a̶(̶)̶;̶
        ̶f̶d̶.̶a̶p̶p̶e̶n̶d̶(̶'̶f̶i̶l̶e̶'̶,̶ ̶f̶i̶l̶e̶)̶;̶
        ̶r̶e̶t̶u̶r̶n̶ ̶$̶h̶t̶t̶p̶.̶p̶o̶s̶t̶(̶u̶r̶l̶,̶ ̶f̶d̶,̶ ̶{̶
        return $http.post(url, file, {
            transformRequest: angular.identity,
            headers: { 'Content-Type': undefined }
        });
    };
});

When the browser sends FormData, it uses 'Content-Type': multipart/formdata and encodes each part using base64.

When the browser sends a file (or blob), it sets the content type to the MIME-type of the file (or blob). It puts the binary data in the body of the request.

---

How to enable <input type="file"> to work with ng-model²

Out of the box, the ng-model directive does not work with input type="file". It needs a directive:

app.directive("selectNgFile", function() {
  return {
    require: "ngModel",
    link: function postLink(scope,elem,attrs,ngModel) {
      elem.on("change", function(e) {
        var files = elem[0].files[0];
        ngModel.$setViewValue(files);
      })
    }
  }
});

Usage:

<input type="file" select-ng-file ng-model="rsdCtrl.viewData.file" name="file"/>

Answer 2

I take the chance and assume you are using bodyParser as middleware. bodyParser has a default limit of 100kb. Look at node_modules/body-parser/lib/types/urlencoded.js :

var limit = typeof options.limit !== 'number'
    ? bytes(options.limit || '100kb')
    : options.limit

You can change the limit in your app.js by

var bodyParser = require('body-parser');
...
app.use(bodyParser.urlencoded( { limit: 1048576 } )); //1mb