From 297c56e169749d89fc273040766bece7227af340 Mon Sep 17 00:00:00 2001 From: Torrie Fischer Date: Thu, 12 Mar 2015 12:51:21 -0700 Subject: [PATCH 1/6] Start building a beanstalkd worker --- Dockerfile | 2 ++ conf/workers.js | 9 +++++++++ docker-compose.yml | 18 ++++++++++++++++++ lib/config.js | 12 ++++++++++++ workers/launch.js | 5 +++++ 5 files changed, 46 insertions(+) create mode 100644 Dockerfile create mode 100644 conf/workers.js create mode 100644 docker-compose.yml create mode 100644 workers/launch.js diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6790509 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,2 @@ +FROM node:0.10-onbuild +EXPOSE 8000 diff --git a/conf/workers.js b/conf/workers.js new file mode 100644 index 0000000..0af6cf1 --- /dev/null +++ b/conf/workers.js @@ -0,0 +1,9 @@ +var path = require('path'); +var config = require('../lib/config'); + +module.exports = { + workers: config.get('beanstalk:workers'), + server: config.get('beanstalk:server'), + tubes: ['instance-launch'], + handlers: [path.resolve(__dirname, '../workers/launch')] +}; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f18e96c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +web: + build: . + command: node server.js + volumes: + - .:/usr/src/app + links: + - worker + ports: + - "8000:8000" +worker: + build: . + command: npm run worker + volumes: + - .:/usr/src/app + links: + - beanstalkd +beanstalkd: + image: kdihalas/beanstalkd diff --git a/lib/config.js b/lib/config.js index 9734b64..46d31d3 100644 --- a/lib/config.js +++ b/lib/config.js @@ -6,10 +6,22 @@ dotenv.load(); nconf.use('memory'); nconf.env(); +nconf.defaults({ + BEANSTALK_SERVER: '127.0.0.1:11300' +}); + +if (nconf.get('BEANSTALKD_PORT')) { + nconf.set('BEANSTALK_SERVER', nconf.get('BEANSTALKD_PORT_11300_TCP_ADDR')+':'+nconf.get('BEANSTALKD_PORT_11300_TCP_PORT')); +}; + nconf.defaults({ aws: { accessKeyId: nconf.get('AWS_ACCESS_KEY_ID'), secretAccessKey: nconf.get('AWS_SECRET_ACCESS_KEY') + }, + beanstalk: { + server: nconf.get('BEANSTALK_SERVER'), + workers: 1 } }); diff --git a/workers/launch.js b/workers/launch.js new file mode 100644 index 0000000..652cb71 --- /dev/null +++ b/workers/launch.js @@ -0,0 +1,5 @@ +module.exports.handlers = { + launch: function(data, done) { + console.log("Request to launch an instance!"); + } +} From a52a786ebe8fd63c5bd2b7641d2ac8d023a327ee Mon Sep 17 00:00:00 2001 From: Torrie Fischer Date: Thu, 12 Mar 2015 13:53:23 -0700 Subject: [PATCH 2/6] [FOLD] add missing changes --- README.md | 17 ++++++++++++++++- package.json | 7 ++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c981d6f..7fea053 100644 --- a/README.md +++ b/README.md @@ -15,5 +15,20 @@ to be loaded by dotenv: ``` npm install -node server +docker-compose up ``` + +This spins up the following: + +* A kdihalas/beanstalkd container +* A worker container that executes `npm run worker` +* A web container with ```node server.js``` + +## Job Processing + +compute-service contains a worker that can read jobs from a beanstalkd queue and +spin up instances. To configure, set the variable BEANSTALK_SERVER. The default +value is ```127.0.0.1:11300```. To run: + +```shell +$ ./node_modules/beanstalk_worker/bin/ diff --git a/package.json b/package.json index 5baf5cb..3fdb75b 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,8 @@ { + "scripts": { + "start": "node server.js", + "worker": "./node_modules/beanstalk_worker/bin/worker.js `pwd`/conf/workers.js" + }, "name": "compute-service", "version": "0.1.0", "description": "Codius host compute service", @@ -12,6 +16,7 @@ "aws-sdk": "^2.1.16", "bluebird": "^2.9.12", "dotenv": "^0.5.1", - "nconf": "^0.7.1" + "nconf": "^0.7.1", + "beanstalk_worker": "^0.2.1" } } From 0b56ed16a75a72f8fa9c3e4212887ad53786ef22 Mon Sep 17 00:00:00 2001 From: Torrie Fischer Date: Wed, 18 Mar 2015 16:58:14 -0700 Subject: [PATCH 3/6] Ignore vim swaps --- .gitignore | 1 + conf/workers.js | 9 --------- package.json | 8 +++++++- 3 files changed, 8 insertions(+), 10 deletions(-) delete mode 100644 conf/workers.js diff --git a/.gitignore b/.gitignore index 3c3629e..1bd7226 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules +*.swp diff --git a/conf/workers.js b/conf/workers.js deleted file mode 100644 index 0af6cf1..0000000 --- a/conf/workers.js +++ /dev/null @@ -1,9 +0,0 @@ -var path = require('path'); -var config = require('../lib/config'); - -module.exports = { - workers: config.get('beanstalk:workers'), - server: config.get('beanstalk:server'), - tubes: ['instance-launch'], - handlers: [path.resolve(__dirname, '../workers/launch')] -}; diff --git a/package.json b/package.json index 3fdb75b..897aa7a 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "scripts": { + "test": "istanbul cover _mocha", "start": "node server.js", "worker": "./node_modules/beanstalk_worker/bin/worker.js `pwd`/conf/workers.js" }, @@ -14,9 +15,14 @@ "license": "ISC", "dependencies": { "aws-sdk": "^2.1.16", + "beanstalk_worker": "^0.2.1", "bluebird": "^2.9.12", "dotenv": "^0.5.1", "nconf": "^0.7.1", - "beanstalk_worker": "^0.2.1" + "nodestalker": "^0.1.17" + }, + "devDependencies": { + "chai": "^2.1.1", + "sinon": "^1.13.0" } } From 56029fa9dbf321ac9ac890c004df93a0f0fc34d8 Mon Sep 17 00:00:00 2001 From: Torrie Fischer Date: Wed, 18 Mar 2015 16:58:41 -0700 Subject: [PATCH 4/6] Start writing a job queue --- lib/job-queue.js | 72 +++++++++++++++++++++++++++++++++++++++++++ test/lib.job-queue.js | 34 ++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 lib/job-queue.js create mode 100644 test/lib.job-queue.js diff --git a/lib/job-queue.js b/lib/job-queue.js new file mode 100644 index 0000000..c1229b8 --- /dev/null +++ b/lib/job-queue.js @@ -0,0 +1,72 @@ +var bluebird = require('bluebird'); +var Promise = bluebird.Promise; +var config = require('./config'); +var beanstalk = require('nodestalker'); + +module.exports = function(tubeName) { + var self = this; + self.client = beanstalk.Client(config.get('beanstalk:server')); + self.workers = {}; + self.tubeName = tubeName; + bluebird.promisifyAll(self.client, {promisifier: promisify}); + self.tubeDelay = self.client.useAsync(self.tubeName).then(function() { + return self.client.watchAsync(self.tubeName); + }); +} + +module.exports.prototype = { + useTube: function() { + return this.tubeDelay; + }, + + addWorker: function(name, worker) { + var self = this; + if (typeof(worker.run) !== 'function') { + throw new Error("Object with a run() method must be passed in"); + } + self.workers[name] = worker; + }, + + enqueueJob: function(type, data) { + var self = this; + var job = { + type: type, + data: data + } + return self.client.putAsync(JSON.stringify(job)); + }, + + processNextJob: function() { + var self = this; + var p = []; + var job; + return self.client.reserveAsync().then(function(_job) { + job = _job; + var realJob = JSON.parse(job.data); + if (realJob.type in self.workers) { + var r = self.workers[realJob.type].run(realJob.data); + if (typeof(r) === 'undefined') { + r = bluebird.resolve(); + } + return r.then(function() { + return self.client.deleteJobAsync(job.id); + }); + } else { + //FIXME: Log unhandled job + return self.client.releaseAsync(job.id); + } + }); + } +} + +function promisify(method) { + return function() { + var self = this; + var args = arguments; + return new bluebird.Promise(function(resolve, reject) { + var r = method.apply(self, args); + r.onSuccess(function(v) {resolve(v)}); + r.onError(function(err) {reject(new Error(err))}); + }); + } +} diff --git a/test/lib.job-queue.js b/test/lib.job-queue.js new file mode 100644 index 0000000..1d501c5 --- /dev/null +++ b/test/lib.job-queue.js @@ -0,0 +1,34 @@ +var bluebird = require('bluebird'); + +bluebird.longStackTraces(); + +var chai = require('chai'); +var expect = chai.expect; +var JobQueue = require('../lib/job-queue'); +var sinon = require('sinon'); + +var TestWorker = function() { +} + +TestWorker.prototype = { + run: function(data) {} +} + +describe('JobQueue', function() { + var queue, worker; + + beforeEach(function() { + queue = new JobQueue('test_'+Math.round(Math.random()*100)); + worker = new TestWorker(); + queue.addWorker('test', worker); + }); + + it('should run a job after it is enqueued', function() { + sinon.spy(worker, 'run'); + return queue.enqueueJob('test', {}).then(function() { + return queue.processNextJob(); + }).then(function() { + expect(worker.run.called).to.equal(true); + }); + }); +}); From 706993ee3060e66a514e5f5b363268092c068de8 Mon Sep 17 00:00:00 2001 From: Torrie Fischer Date: Wed, 18 Mar 2015 17:00:14 -0700 Subject: [PATCH 5/6] Start writing an instance launch task --- lib/config.js | 3 +++ package.json | 3 ++- test/workers.launch.js | 26 ++++++++++++++++++++++++++ workers/launch.js | 12 +++++++++--- 4 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 test/workers.launch.js diff --git a/lib/config.js b/lib/config.js index 46d31d3..7d8abbc 100644 --- a/lib/config.js +++ b/lib/config.js @@ -22,6 +22,9 @@ nconf.defaults({ beanstalk: { server: nconf.get('BEANSTALK_SERVER'), workers: 1 + }, + codius: { + endpoint: nconf.get('CODIUS_URI') } }); diff --git a/package.json b/package.json index 897aa7a..12aeada 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "bluebird": "^2.9.12", "dotenv": "^0.5.1", "nconf": "^0.7.1", - "nodestalker": "^0.1.17" + "nodestalker": "^0.1.17", + "request-promise": "^0.4.0" }, "devDependencies": { "chai": "^2.1.1", diff --git a/test/workers.launch.js b/test/workers.launch.js new file mode 100644 index 0000000..649ad88 --- /dev/null +++ b/test/workers.launch.js @@ -0,0 +1,26 @@ +var Promise = require('bluebird').Promise; +var chai = require('chai'); +var expect = chai.expect; +var JobQueue = require('../lib/job-queue'); +var sinon = require('sinon'); + +var LaunchWorker = require('../workers/launch'); + +describe('launch handler', function() { + var queue, worker; + + beforeEach(function() { + queue = new JobQueue('test_'+Math.random()*100); + worker = new LaunchWorker(); + queue.addWorker('instance_launch', worker); + }); + + it('should handle an instance launch job', function() { + sinon.spy(worker, 'run'); + return queue.enqueueJob('instance_launch', {}).then(function() { + return queue.processNextJob(); + }).then(function() { + expect(worker.run.called).to.equal(true); + }); + }); +}); diff --git a/workers/launch.js b/workers/launch.js index 652cb71..a3f8b26 100644 --- a/workers/launch.js +++ b/workers/launch.js @@ -1,5 +1,11 @@ -module.exports.handlers = { - launch: function(data, done) { - console.log("Request to launch an instance!"); +var request = require('request-promise'); +var config = require('../lib/config'); + +module.exports = function() { +} + +module.exports.prototype = { + run: function(data) { + return request(config.get('codius:endpoint')); } } From 9db56cf3d07968b59e9254df7595688b1a07c268 Mon Sep 17 00:00:00 2001 From: Torrie Fischer Date: Wed, 18 Mar 2015 17:02:32 -0700 Subject: [PATCH 6/6] Ignore coverage and .env --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 1bd7226..c9d43bc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ node_modules *.swp +coverage +.env