From cb0c85a433dfab2611976f26f9fcb114109db12b Mon Sep 17 00:00:00 2001 From: Nolan Lawson Date: Sun, 6 Mar 2016 23:19:20 -0500 Subject: [PATCH] allow testing in Selenium locally using --selenium --- README.md | 20 +++ bin/zuul | 10 +- lib/Selenium.js | 187 +++++++++++++++++++++++ lib/zuul.js | 9 ++ package.json | 5 +- test/integration/jamine-selenium.js | 36 +++++ test/integration/mocha-qunit-selenium.js | 39 +++++ test/integration/mochabdd-selenium.js | 36 +++++ test/integration/tape-selenium.js | 54 +++++++ 9 files changed, 391 insertions(+), 5 deletions(-) create mode 100644 lib/Selenium.js create mode 100644 test/integration/jamine-selenium.js create mode 100644 test/integration/mocha-qunit-selenium.js create mode 100644 test/integration/mochabdd-selenium.js create mode 100644 test/integration/tape-selenium.js diff --git a/README.md b/README.md index fd8175c..244a6a5 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,26 @@ When iterating on your tests during development, simply use zuul `--local` mode See the [quickstart](https://github.com/defunctzombie/zuul/wiki/quickstart) page on the wiki for more details. +### Automated browser tests + +You can test in PhantomJS using: + + zuul --phantom + +Note that PhantomJS must be installed separately, e.g. `npm install phantomjs`. + +You can also test using Selenium against any browser you have installed locally. For instance: + + zuul --selenium + +will test in the default browser (Firefox). To test in another browser, you can do: + + zuul --selenium --browser-name chrome + +Or: + + zuul --selenium --browser-name firefox --browser-version 41.0.1 + ### Cross browser testing via Saucelabs The reason we go through all this trouble in the first place is to seamlessly run our tests against all those browsers we don't have installed. Luckily, [saucelabs](https://saucelabs.com/) runs some browsers and we can easily task zuul to test on those. diff --git a/bin/zuul b/bin/zuul index ea6b478..425e661 100755 --- a/bin/zuul +++ b/bin/zuul @@ -25,6 +25,7 @@ program .option('--phantom-remote-debugger-port [port]', 'connect phantom to remote debugger') .option('--phantom-remote-debugger-autorun', 'run tests automatically when --phantom-remote-debugger-port is specified') .option('--electron', 'run tests in electron. electron must be installed separately.') +.option('--selenium', 'run tests in Selenium, locally. Default browser: Chrome. Use --browser-name and --browser-version for others') .option('--tunnel-host ', 'specify a localtunnel server to use for forwarding') .option('--sauce-connect [tunnel-identifier]', 'use saucelabs with sauce connect instead of localtunnel. Optionally specify the tunnel-identifier') .option('--server ', 'specify a server script to be run') @@ -48,6 +49,7 @@ var config = { phantomRemoteDebuggerPort: program.phantomRemoteDebuggerPort, phantomRemoteDebuggerAutorun: program.phantomRemoteDebuggerAutorun, electron: program.electron, + selenium: program.selenium, prj_dir: process.cwd(), tunnel_host: program.tunnelHost, sauce_connect: program.sauceConnect, @@ -127,12 +129,14 @@ if (config.files.length === 0) { return process.exit(1); } -if ((program.browserVersion || program.browserPlatform) && !program.browserName) { +if (((program.browserVersion || program.browserPlatform) && !program.browserName) && + !program.selenium) { console.error('the browser name needs to be specified (via --browser-name)'); return process.exit(1); } -if ((program.browserName || program.browserPlatform) && !program.browserVersion) { +if (((program.browserName || program.browserPlatform) && !program.browserVersion) && + !program.selenium) { console.error('the browser version needs to be specified (via --browser-version)'); return process.exit(1); } @@ -188,7 +192,7 @@ if (config.local) { return zuul.run(function(passed) { }); } -else if (config.phantom || config.electron) { +else if (config.phantom || config.electron || config.selenium) { return zuul.run(function(passed) { process.exit(passed ? 0 : 1); }); diff --git a/lib/Selenium.js b/lib/Selenium.js new file mode 100644 index 0000000..2cdbc47 --- /dev/null +++ b/lib/Selenium.js @@ -0,0 +1,187 @@ +var path = require('path'); +var EventEmitter = require('events').EventEmitter; +var debug = require('debug')('zuul:selenium'); +var wd = require('wd'); + +var SELENIUM_VERSION = '2.52.0'; + +var setup_test_instance = require('./setup'); +require('colors'); + +function Selenium(opt) { + if (!(this instanceof Selenium)) { + return new Selenium(opt); + } + + var self = this; + self._opt = opt; + self._browserName = (opt.browsers && opt.browsers[0] && opt.browsers[0].name) || null; + self._browserVersion = (opt.browsers && opt.browsers[0] && opt.browsers[0].version) || null; + self.status = { + passed: 0, + failed: 0 + }; +} + +Selenium.prototype.__proto__ = EventEmitter.prototype; + +Selenium.prototype.start = function() { + var self = this; + + var seleniumClient; + var selenium = require('selenium-standalone'); + + function finish() { + if (self._finished) { + return; + } + self._finished = true; + reporter.removeAllListeners(); + if (seleniumClient) { + seleniumClient.quit(); + } + } + + self.controller = setup_test_instance(self._opt, function(err, url) { + if (err) { + console.log('Error: %s'.red, err); + self.emit('done', { + passed: false + }); + finish(); + } + + debug('url %s', url); + + var reporter = new EventEmitter(); + + reporter.on('console', function(msg) { + console.log.apply(console, msg.args); + }); + + reporter.on('test', function(test) { + console.log('starting', test.name.white); + }); + + reporter.on('test_end', function(test) { + if (!test.passed) { + console.log('failed', test.name.red); + return self.status.failed++; + } + + console.log('passed:', test.name.green); + self.status.passed++; + }); + + reporter.on('assertion', function(assertion) { + console.log('Error: %s'.red, assertion.message); + assertion.frames.forEach(function(frame) { + console.log(' %s %s:%d'.grey, frame.func, frame.filename, frame.line); + }); + console.log(); + }); + + reporter.on('done', function() { + finish(); + }); + + self.emit('init', url); + self.emit('start', reporter); + + var opts = {version: SELENIUM_VERSION}; + selenium.install(opts, function(err) { + if (err) { + console.log('Error: %s'.red, new Error( + 'Failed to install selenium')); + self.emit('done', { + passed: false + }); + finish(); + return; + } + selenium.start(opts, function() { + seleniumClient = wd.promiseChainRemote(); + onSeleniumReady(); + }); + }); + + function onSeleniumClientReady() { + var lastMessage = Date.now(); + var script = 'window.zuul_msg_bus ? ' + + 'window.zuul_msg_bus.splice(0, window.zuul_msg_bus.length) : ' + + '[]'; + + var interval = setInterval(poll, 2000); + + function onDoneMessage() { + clearInterval(interval); + seleniumClient.quit(function () { + seleniumClient = null; + self.emit('done', { + passed: self.status.passed, + failed: self.status.failed + }); + finish(); + }); + } + + function onGetMessages(err, messages) { + if (err) { + self.emit('error', err); + } else if (messages.length) { + lastMessage = Date.now(); + messages.forEach(function (msg) { + debug('msg: %j', msg); + if (msg.type === 'done') { + onDoneMessage(); + } else { + reporter.emit(msg.type, msg); + } + }); + } else if ((Date.now() - lastMessage) > testTimeout) { + clearInterval(interval); + console.log('Error: %s'.red, new Error( + 'selenium timeout after ' + testTimeout + ' ms')); + self.emit('done', { + passed: false + }); + finish(); + } + } + + function poll() { + seleniumClient.eval(script, onGetMessages); + } + } + + function onSeleniumReady() { + // TODO: maybe these should be configurable + var testTimeout = 120000; + var tunnelId = process.env.TRAVIS_JOB_NUMBER || 'tunnel-' + Date.now(); + var opts = { + tunnelTimeout: testTimeout, + name: self._browserName + ' - ' + tunnelId, + 'max-duration': 60 * 45, + 'command-timeout': 599, + 'idle-timeout': 599, + 'tunnel-identifier': tunnelId + }; + if (self._browserName) { + opts.browserName = self._browserName; + } + if (self._browserVersion) { + opts.version = self._browserVersion; + } + + seleniumClient.init(opts).get(url, onSeleniumClientReady); + } + }); +}; + +Selenium.prototype.shutdown = function() { + if (self.controller) { + self.controller.shutdown(); + } +}; + +module.exports = Selenium; diff --git a/lib/zuul.js b/lib/zuul.js index 6b3977a..6620162 100644 --- a/lib/zuul.js +++ b/lib/zuul.js @@ -10,6 +10,7 @@ var setup_test_instance = require('./setup'); var SauceBrowser = require('./SauceBrowser'); var PhantomBrowser = require('./PhantomBrowser'); var Electron = require('./Electron'); +var Selenium = require('./Selenium'); module.exports = Zuul; @@ -125,6 +126,14 @@ Zuul.prototype.run = function(done) { }); return electron.start(); } + if (config.selenium) { + var selenium = Selenium(config); + self.emit('browser', selenium); + selenium.once('done', function(results) { + done(results.failed === 0 && results.passed > 0); + }); + return selenium.start(); + } var batch = new Batch(); batch.concurrency(self._concurrency); diff --git a/package.json b/package.json index eacd283..55da984 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "lodash": "3.10.1", "opener": "1.4.0", "osenv": "0.0.3", + "selenium-standalone": "^5.0.0", "shallow-copy": "0.0.1", "shell-quote": "1.4.1", "stack-mapper": "0.2.2", @@ -36,7 +37,7 @@ "tap-finished": "0.0.1", "tap-parser": "0.7.0", "watchify": "3.7.0", - "wd": "0.3.11", + "wd": "0.4.0", "xtend": "2.1.2", "yamljs": "0.1.4", "zuul-localtunnel": "1.1.0" @@ -70,4 +71,4 @@ "scripts": { "test": "DEBUG=zuul* mocha --ui qunit --timeout 0 --bail -- test/index.js" } -} \ No newline at end of file +} diff --git a/test/integration/jamine-selenium.js b/test/integration/jamine-selenium.js new file mode 100644 index 0000000..dc1fefb --- /dev/null +++ b/test/integration/jamine-selenium.js @@ -0,0 +1,36 @@ +var Zuul = require('../../'); + +var after = require('after'); +var assert = require('assert'); + +test('jasmine - phantom', function(done) { + done = after(3, done); + + var config = { + ui: 'jasmine', + prj_dir: __dirname + '/../fixtures/jasmine', + selenium: true, + concurrency: 1, + files: [__dirname + '/../fixtures/jasmine/test.js'] + }; + + var zuul = Zuul(config); + + // each browser we test will emit as a browser + zuul.on('browser', function(browser) { + browser.on('init', function() { + done(); + }); + + browser.on('done', function(results) { + assert.equal(results.passed, 1); + assert.equal(results.failed, 1); + done(); + }); + }); + + zuul.run(function(passed) { + assert.ok(!passed); + done(); + }); +}); diff --git a/test/integration/mocha-qunit-selenium.js b/test/integration/mocha-qunit-selenium.js new file mode 100644 index 0000000..1fe87de --- /dev/null +++ b/test/integration/mocha-qunit-selenium.js @@ -0,0 +1,39 @@ +var Zuul = require('../../'); + +var after = require('after'); +var assert = require('assert'); + +test('mocha-qunit - phantom', function(done) { + done = after(3, done); + + var config = { + ui: 'mocha-qunit', + prj_dir: __dirname + '/../fixtures/mocha-qunit', + selenium: true, + concurrency: 1, + files: [__dirname + '/../fixtures/mocha-qunit/test.js'] + }; + var zuul = Zuul(config); + + zuul.on('browser', function(browser) { + browser.once('start', function(reporter) { + reporter.once('done', function(results) { + assert.equal(results.passed, false); + assert.equal(results.stats.passed, 1); + assert.equal(results.stats.failed, 1); + done(); + }); + }); + + browser.on('done', function(results) { + assert.equal(results.passed, 1); + assert.equal(results.failed, 1); + done(); + }); + }); + + zuul.run(function(passed) { + assert.ok(!passed); + done(); + }); +}); diff --git a/test/integration/mochabdd-selenium.js b/test/integration/mochabdd-selenium.js new file mode 100644 index 0000000..6ea3c8b --- /dev/null +++ b/test/integration/mochabdd-selenium.js @@ -0,0 +1,36 @@ +var Zuul = require('../../'); + +var after = require('after'); +var assert = require('assert'); + +test('mocha-bdd - phantom', function(done) { + done = after(3, done); + + var config = { + ui: 'mocha-bdd', + prj_dir: __dirname + '/../fixtures/mocha-bdd', + selenium: true, + concurrency: 1, + files: [__dirname + '/../fixtures/mocha-bdd/test.js'] + }; + + var zuul = Zuul(config); + + // each browser we test will emit as a browser + zuul.on('browser', function(browser) { + browser.on('init', function() { + done(); + }); + + browser.on('done', function(results) { + assert.equal(results.passed, 1); + assert.equal(results.failed, 1); + done(); + }); + }); + + zuul.run(function(passed) { + assert.ok(!passed); + done(); + }); +}); diff --git a/test/integration/tape-selenium.js b/test/integration/tape-selenium.js new file mode 100644 index 0000000..d57ba30 --- /dev/null +++ b/test/integration/tape-selenium.js @@ -0,0 +1,54 @@ +var Zuul = require('../../'); + +var after = require('after'); +var assert = require('assert'); + +test('tape - phantom', function(done) { + done = after(3, done); + var consoleOutput = []; + + var config = { + ui: 'tape', + prj_dir: __dirname + '/../fixtures/tape', + selenium: true, + concurrency: 1, + files: [__dirname + '/../fixtures/tape/test.js'] + }; + + var zuul = Zuul(config); + + // each browser we test will emit as a browser + zuul.on('browser', function(browser) { + browser.on('start', function(reporter) { + reporter.on('console', function(msg) { + consoleOutput.push(msg.args); + }); + }); + + browser.on('init', function() { + done(); + }); + + browser.on('done', function(results) { + var endOfOutput = consoleOutput.slice(-5); + + // check that we did output untill the end of the test suite + // this is the number of asserts in tape + assert.deepEqual(endOfOutput[0], ['1..9']); + assert.deepEqual(endOfOutput[1], ['# tests 9']); + assert.deepEqual(endOfOutput[2], ['# pass 5']); + assert.deepEqual(endOfOutput[3], ['# fail 4']); + assert.deepEqual(endOfOutput[4], ['']); + + // this is the number of passed/failed test() in tape + assert.equal(results.passed, 3); + assert.equal(results.failed, 3); + done(); + }); + }); + + zuul.run(function(passed) { + assert.ok(!passed); + done(); + }); +});