phantomjs not waiting for “full” page load

Question

I'm using PhantomJS v1.4.1 to load some web pages. I don't have access to their server-side, I just getting links pointing to them. I'm using obsolete version of Phantom because I need to support Adobe Flash on that web pages.

The problem is many web-sites are loading their minor content async and that's why Phantom's onLoadFinished callback (analogue for onLoad in HTML) fired too early when not everything still has loaded. Can anyone suggest how can I wait for full load of a webpage to make, for example, a screenshot with all dynamic content like ads?

Answer 1

This is an implementation of Supr's answer. Also it uses setTimeout instead of setInterval as Mateusz Charytoniuk suggested.

Phantomjs will exit in 1000ms when there isn't any request or response.

// load the module
var webpage = require('webpage');
// get timestamp
function getTimestamp(){
    // or use Date.now()
    return new Date().getTime();
}

var lastTimestamp = getTimestamp();

var page = webpage.create();
page.onResourceRequested = function(request) {
    // update the timestamp when there is a request
    lastTimestamp = getTimestamp();
};
page.onResourceReceived = function(response) {
    // update the timestamp when there is a response
    lastTimestamp = getTimestamp();
};

page.open(html, function(status) {
    if (status !== 'success') {
        // exit if it fails to load the page
        phantom.exit(1);
    }
    else{
        // do something here
    }
});

function checkReadyState() {
    setTimeout(function () {
        var curentTimestamp = getTimestamp();
        if(curentTimestamp-lastTimestamp>1000){
            // exit if there isn't request or response in 1000ms
            phantom.exit();
        }
        else{
            checkReadyState();
        }
    }, 100);
}

checkReadyState();

Answer 2

I use a personnal blend of the phantomjs waitfor.js example.

This is my main.js file:

'use strict';

var wasSuccessful = phantom.injectJs('./lib/waitFor.js');
var page = require('webpage').create();

page.open('http://foo.com', function(status) {
  if (status === 'success') {
    page.includeJs('https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js', function() {
      waitFor(function() {
        return page.evaluate(function() {
          if ('complete' === document.readyState) {
            return true;
          }

          return false;
        });
      }, function() {
        var fooText = page.evaluate(function() {
          return $('#foo').text();
        });

        phantom.exit();
      });
    });
  } else {
    console.log('error');
    phantom.exit(1);
  }
});

And the lib/waitFor.js file (which is just a copy and paste of the waifFor() function from the phantomjs waitfor.js example):

function waitFor(testFx, onReady, timeOutMillis) {
    var maxtimeOutMillis = timeOutMillis ? timeOutMillis : 3000, //< Default Max Timout is 3s
        start = new Date().getTime(),
        condition = false,
        interval = setInterval(function() {
            if ( (new Date().getTime() - start < maxtimeOutMillis) && !condition ) {
                // If not time-out yet and condition not yet fulfilled
                condition = (typeof(testFx) === "string" ? eval(testFx) : testFx()); //< defensive code
            } else {
                if(!condition) {
                    // If condition still not fulfilled (timeout but condition is 'false')
                    console.log("'waitFor()' timeout");
                    phantom.exit(1);
                } else {
                    // Condition fulfilled (timeout and/or condition is 'true')
                    // console.log("'waitFor()' finished in " + (new Date().getTime() - start) + "ms.");
                    typeof(onReady) === "string" ? eval(onReady) : onReady(); //< Do what it's supposed to do once the condi>
                    clearInterval(interval); //< Stop this interval
                }
            }
        }, 250); //< repeat check every 250ms
}

This method is not asynchronous but at least am I assured that all the resources were loaded before I try using them.

Answer 3

I found this solution useful in a NodeJS app. I use it just in desperate cases because it launches a timeout in order to wait for the full page load.

The second argument is the callback function which is going to be called once the response is ready.

phantom = require('phantom');

var fullLoad = function(anUrl, callbackDone) {
    phantom.create(function (ph) {
        ph.createPage(function (page) {
            page.open(anUrl, function (status) {
                if (status !== 'success') {
                    console.error("pahtom: error opening " + anUrl, status);
                    ph.exit();
                } else {
                    // timeOut
                    global.setTimeout(function () {
                        page.evaluate(function () {
                            return document.documentElement.innerHTML;
                        }, function (result) {
                            ph.exit(); // EXTREMLY IMPORTANT
                            callbackDone(result); // callback
                        });
                    }, 5000);
                }
            });
        });
    });
}

var callback = function(htmlBody) {
    // do smth with the htmlBody
}

fullLoad('your/url/', callback);

Answer 4

Maybe you can use the onResourceRequested and onResourceReceived callbacks to detect asynchronous loading. Here's an example of using those callbacks from their documentation:

var page = require('webpage').create();
page.onResourceRequested = function (request) {
    console.log('Request ' + JSON.stringify(request, undefined, 4));
};
page.onResourceReceived = function (response) {
    console.log('Receive ' + JSON.stringify(response, undefined, 4));
};
page.open(url);

Also, you can look at examples/netsniff.js for a working example.

Answer 5

This the code I use:

var system = require('system');
var page = require('webpage').create();

page.open('http://....', function(){
      console.log(page.content);
      var k = 0;

      var loop = setInterval(function(){
          var qrcode = page.evaluate(function(s) {
             return document.querySelector(s).src;
          }, '.qrcode img');

          k++;
          if (qrcode){
             console.log('dataURI:', qrcode);
             clearInterval(loop);
             phantom.exit();
          }

          if (k === 50) phantom.exit(); // 10 sec timeout
      }, 200);
  });

Basically given the fact you're supposed to know that the page is full downloaded when a given element appears on the DOM. So the script is going to wait until this happens.

Answer 6

Another approach is to just ask PhantomJS to wait for a bit after the page has loaded before doing the render, as per the regular rasterize.js example, but with a longer timeout to allow the JavaScript to finish loading additional resources:

page.open(address, function (status) {
    if (status !== 'success') {
        console.log('Unable to load the address!');
        phantom.exit();
    } else {
        window.setTimeout(function () {
            page.render(output);
            phantom.exit();
        }, 1000); // Change timeout as required to allow sufficient time 
    }
});