Load S3 file in lambda before main procedure

问题:

I am writing a lambda function that needs to load a key that is being stored in S3. It wont change often so I'd rather not grab it every time the lambda function is called, so I would like to load it once when the container is spun up and then keep that value for the lifetime of the lambda container.

However, due to the asynchronous method getObject this is causing an issue as the file may not have been loaded when the main module.export code gets run (particularly if this is the first run for a while and container is being created).

I've implemented a workaround using setTimeout but I wanted to see what the recommended way aorund this was, and are there any issues with my approach as it just doesn't feel right!

Sample code:

var AWS = require('aws-sdk')
var s3 = new AWS.S3();

var fileLoaded = false;

var params = {
  Bucket: 'bucket-name',
  Key: 'file-name'
};

s3.getObject(params, function(err, data) {
  if (err) console.log(err, err.stack); // an error occurred
  else {
      console.log('File loaded from S3');
      fileLoaded = true;
  }
});

exports.handler = (event, context, callback) => {

    console.log('I am in the main procedure, but i might not have the file yet')
    waitForFileLoadBeforeDoingSomething(event, context, callback)

};

function waitForFileLoadBeforeDoingSomething(event, context, callback){
    if(!fileLoaded){
        console.log('No file available to me yet, we will sleep')
        setTimeout(function(){
            waitForFileLoadBeforeDoingSomething(event, context, callback)
        }, 300)
    } else {
        console.log('I have the file!')
        doStuff(event, context, callback)
    }
}

function doStuff(event, context, callback){
    console.log('Now I can do the rest of the code')
    //Do the actual code here

    callback(null, 'success')
}

回答1:

Following pspi's answer and some more research I think a more correct solution would be this:

var AWS = require('aws-sdk')
var s3 = new AWS.S3();

var params = {
    Bucket: 'bucket-name',
    Key: 'file-name'
};

var fileLoaded = false;

exports.handler = (event, context, callback) => {

    if(!fileLoaded){
        s3.getObject(params, function (err, data) {
            if (err) console.log(err, err.stack);
            else {
                fileLoaded = true;
                doSomething(event, context, callback)
            }
        });
    } else {
        doSomething(event, context, callback)
    }

};

function doSomething(event, context, callback){

    //Do the actual work here

    callback(null, "success") //Then end
}

This seems to give the desired outcome of only loading the file from S3 once but not allowing code execution without the load having finished. It would allow multiple requests to S3 if it was called cold multiple times quickly but that's unlikely and still better than every time the function is called.

回答2:

He-hey, asynchronous code and modules gets out of hand pretty quick, doesn't it.

You'd like to call your callback as soon as the asynchronous s3.getObject() call is ready. The correct place is then to put the work callback inside the s3.getObject() callback. Like this, removing also the other fluff

var AWS = require('aws-sdk')
var s3 = new AWS.S3();

var params = {
    Bucket: 'bucket-name',
    Key: 'file-name'
};

exports.handler = (event, context, callback) => {

    s3.getObject(params, function (err, data) {
        if (err) console.log(err, err.stack);
        else {
            callback(null, 'success')  // do after we have completed s3.getObject()
        }
    });
};

The code should be tuned a little more (cache the result) if the exports.handler can be called multiple times. And I don't seem to recognized where the actual file data is passed on, but you get the gist of it.

回答3:

Here you have a bit cleaner version with promises:

var AWS = require('aws-sdk')
var s3 = new AWS.S3();

var params = {
    Bucket: 'bucket-name',
    Key: 'file-name'
};

var fileData = null;

exports.handler = (event, context, callback) => {
    if(!fileData) 
        s3.getObject(params).promise().then(data) => {
             fileData = data;
             doSomething(event, context, callback);
         }).catch((err) => {
             callback.done(err);
         });
     else 
        doSomething(event, context, callback);
};

function doSomething(event, context, callback){
    // Do the actual work here
    // you can use fileData variable now to use your downloaded file!!
    callback(null, "success") // Then end
}

回答4:

I like to pre-load the resource and use a promise to gate the flow. A "cold" container has to go get the file, but a warm container already has it. Loading it in the module setup give it just a LITTLE BIT of a head start by the time lambda actually calls your endpoint.

var AWS = require('aws-sdk')
var s3 = new AWS.S3();

var fileDataPromise = getFileDataPromise();

exports.handler = (event, context, callback) => {
    fileDataPromise.then((fileData) => {
        doSomething(event, context, callback, fileData);
    })
};

function getFileDataPromise() {
    var params = {
        Bucket: 'bucket-name',
        Key: 'file-name'
    };

    return new Promise((resolve, reject) => {
        s3.getObject(params, (err, data) => {
            if (err) console.log(err, err.stack);
            else {
                resolve(data)
            }
        })
    })
}

function doSomething(event, context, callback, fileData) {
    // Do the actual work here
    callback(null, "success")
}