How to Use CasperJS in node.js?

Question

I would like to use CasperJS in node.js.

I have referred to the following URL's to use CasperJS in node.js:

• https://github.com/sgentle/phantomjs-node
• http://casperjs.org/index.html#faq-executable

With the help of the above URLs I have written the following code:

//DISPLAY=:0 node test2.js
var phantom = require('phantom');
console.log('Hello, world!');
phantom.create(function (ph) {
    ph.casperPath = '/opt/libs/casperjs'
    ph.injectJs('/opt/libs/casperjs/bin/bootstrap.js');
    var casper = require('casper').create();
    casper.start('http://google.fr/');

    casper.thenEvaluate(function (term) {
        document.querySelector('input[name="q"]').setAttribute('value', term);
        document.querySelector('form[name="f"]').submit();
    }, {
        term: 'CasperJS'
    });

    casper.then(function () {
        // Click on 1st result link
        this.click('h3.r a');
    });

    casper.then(function () {
        console.log('clicked ok, new location is ' + this.getCurrentUrl());
    });

    casper.run();
});

---

When I run this code, I got the following error:

ERROR MSG:

tz@tz-ubuntu:/opt/workspaces/TestPhantomjs$ DISPLAY=:0 node test2.js 
Hello, world!
Error: Cannot find module 'casper'
    at Function._resolveFilename (module.js:332:11)
    at Function._load (module.js:279:25)
    at Module.require (module.js:354:17)
    at require (module.js:370:17)
    at /opt/workspaces/TestPhantomjs/test2.js:6:14
    at Object.<anonymous> (/opt/workspaces/TestPhantomjs/node_modules/phantom/phantom.js:82:43)
    at EventEmitter.<anonymous> (/opt/workspaces/TestPhantomjs/node_modules/phantom/node_modules/dnode/index.js:215:30)
    at EventEmitter.emit (events.js:67:17)
    at handleMethods (/opt/workspaces/TestPhantomjs/node_modules/phantom/node_modules/dnode-protocol/index.js:138:14)
    at EventEmitter.handle (/opt/workspaces/TestPhantomjs/node_modules/phantom/node_modules/dnode-protocol/index.js:98:13)
phantom stdout: Unable to load casper environment: Error: Failed to resolve module fs, tried fs

Answer 1

https://groups.google.com/group/casperjs/browse_thread/thread/641e9e6dff50fb0a/e67aaef5ab4ec918?hl=zh-CN#e67aaef5ab4ec918

Nicolas Perriault
2012/2/27 天猪 蓝虫. :

I wan to use casperjs in nodejs. and refs to: https://github.com/sgentle/phantomjs-node and http://casperjs.org/index.html#faq-executable

You can't run CasperJS that way; QtWebKit and V8 don't share the same js environment (and event loop), so your node.js app won't be able to load and use a CasperJS module. You have to run your CasperJS script separately using a subprocess call, like this one on github. I don't plan to make CasperJS compatible with phantomjs-node because it uses alert()-based dirty hacks I'm not easy with.

Cheers, -- Nicolas Perriault

Answer 2

One solution (which worked for me) is to start and stop your server on a per-test basis. For example, I have a runtests.coffee which looks like:

http = require 'http'
glob = require 'glob'
spawn = require('child_process').spawn

db = require './db' # Contains all database stuff.
webapp = require './webapp' # Contains all of the Express stuff.

db.connect 'test' # Connects to the db server and creates an empty test db.
server = http.createServer webapp.makeApp()
server.listen 0, ->
    port = server.address().port
    process.env.URL = "http://localhost:#{ port }"
    glob 'tests/*', (err, filenames) ->
        child = spawn 'casperjs', ['test'].concat(filenames)
        child.stdout.on 'data', (msg) -> process.stdout.write msg
        child.stderr.on 'data', (msg) -> process.stderr.write msg
        child.on 'exit', (code) ->
            db.disconnect() # Drops the test db.
            server.close()
            process.exit code

And my CasperJS tests in tests/ look like:

URL = require('system').env.URL # Note, Casper code here, not Node.

casper.test.begin 'Test something', 1, (test) ->
    casper.start "#{ URL }/welcome"
    casper.then ->
        test.assertHttpStatus 200
        # ....
    casper.run ->
        test.done()

Answer 3

You can use SpookyJS to drive CasperJS from Node.

Answer 4

CasperJS includes a web server to talk to the outside world. Node (using request, superagent etc) can now talk to casper over HTTP.

In scraper.js:

#!/usr/bin/env casperjs

// I AM NOT NODEJS
// I AM CASPER JS
// I RUN IN QTWEBKIT, NOT V8

var casper = require('casper').create();
var server = require('webserver').create();
var ipAndPort = '127.0.0.1:8585';

server.listen(ipAndPort, function(request, response) {

    casper.start('https://connect.data.com/login');
    casper.userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36");
    casper.then(function(){
        // lots of code here, and a few more cassper.then()s
    });

    casper.run(function(){
        console.log('\n\nFinished')
        response.statusCode = 200;
        var body = JSON.stringify({
            phoneNumber: '1800-YOLO-SWAG'
        })

        response.write(body);
        response.close();
    });
});

You can now run scraper.js as a web server:

chmod +x scraper.js
./scraper.js

You should run it as a Linux service just like you would for a node app.

Answer 5

It basically means that your script can't find Casper; have you checked the path and made sure that

/opt/libs/casperjs 

and:

/opt/libs/casperjs/bin/bootstrap.js

Are accessible by a website user ? considering the location it's probably not likely. /opt is a unix path, but the website will be looking in {websiterootpath}/opt.

I'd create a subfolder 'casperjs' in the root folder of your website and copy the contents of

/opt/libs/casperjs 

To there. Then change your paths from

/opt/libs/casperjs

To

/casperjs

Answer 6

I tried to run casper by node cron job too, here's my solution

in casper.js echo your response:

casper.then(function() {
    var comments = this.evaluate(getComments);
    this.echo(JSON.stringify(comments));
})

use node-cmd in node file casper_wrapper.js:

var cmd = require('node-cmd');

module.exports = function(url) {
    return new Promise(function(resolve, reject) {
        cmd.get(
            'casperjs casper.js ' + url, // casper takes args to run the script
            function(err, data, stderr){
                if (err) {
                    reject(err);
                    return;
                }
                var obj = JSON.parse(data);
                resolve(obj);
            }
        );
    });
}