From e0a0ac92d7d1427bb3b59220d342117af2888702 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Mon, 18 Jan 2016 15:27:06 -0500 Subject: [PATCH 01/15] Adding CORS Support to yaml --- config/default.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/default.yaml b/config/default.yaml index 3d250d8..cfdc6c2 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -2,6 +2,11 @@ server: port: 5999 appRoot: "/bucky" +handlers: + http_headers: + Access-Control-Allow-Origin: * + + statsd: host: 'localhost' port: 8125 From c8dde1bc6cc3ca2871dcc3daff929b0b03878d45 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Tue, 19 Jan 2016 22:03:38 -0500 Subject: [PATCH 02/15] Adding security changes --- config/default.yaml | 9 +++++++-- server.coffee | 18 ++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index cfdc6c2..10331cc 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -4,7 +4,7 @@ server: handlers: http_headers: - Access-Control-Allow-Origin: * + Access-Control-Allow-Origin: www.app.dev.pardot.com statsd: @@ -38,6 +38,11 @@ modules: collectors: # Uncomment the modules that you'd like to use # - ./modules/collectionLogger - # - ./modules/statsd + - ./modules/statsd # - ./modules/openTSDB # - ./modules/influxdb + + whitelistedkeys: + - abc + - def + - ghi diff --git a/server.coffee b/server.coffee index d1976ef..71a1e88 100755 --- a/server.coffee +++ b/server.coffee @@ -12,6 +12,8 @@ configWrapper = require './lib/configWrapper' load = require './lib/load' MODULES = config.modules +ALLOW_ORIGIN_HEADERS = config.http_headers + loadLogger = -> if MODULES.logger load(MODULES.logger, {config}) @@ -27,7 +29,7 @@ loadConfig = (logger) -> configWrapper(config) setCORSHeaders = (req, res, next) -> - res.setHeader 'Access-Control-Allow-Origin', '*' + res.setHeader 'Access-Control-Allow-Origin', ALLOW_ORIGIN_HEADERS res.setHeader 'Access-Control-Allow-Methods', 'POST' res.setHeader 'Access-Control-Max-Age', '604800' res.setHeader 'Access-Control-Allow-Credentials', 'true' @@ -58,11 +60,23 @@ loadApp = (logger, loadedConfig) -> app = express() APP_ROOT = process.env.APP_ROOT ? loadedConfig.get('server.appRoot').get() ? '' + ALLOW_ORIGIN_HEADERS = loadedConfig.get('handlers.http_headers.Access-Control-Allow-Origin').get() ? '*' moduleGroups = {} - loadModuleGroup = (group) -> + loadKeys = (group) -> moduleGroups[group] = {} + if MODULES[group] + _.map MODULES[group], (name) -> + logger.log "Key found", name + else + [] + + appPromises = loadKeys 'whitelistedkeys' + + moduleGroups = {} + loadModuleGroup = (group) -> + moduleGroups[group] = {} if MODULES[group] _.map MODULES[group], (name) -> logger.log "Loading #{ group } Module", name From 64c0afca388f34de508fd3a50e0c04f40560a7cf Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Wed, 20 Jan 2016 10:08:19 -0500 Subject: [PATCH 03/15] security changes --- config/default.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 10331cc..a47f8bb 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -43,6 +43,6 @@ modules: # - ./modules/influxdb whitelistedkeys: - - abc - - def - - ghi + - key1 + - key2 + - key3 From e1b90a26aba77ebacaa3adcf3e8a4497cede4114 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Wed, 20 Jan 2016 10:29:10 -0500 Subject: [PATCH 04/15] security changes for whitelisting the keys --- config/default.yaml | 5 ----- server.coffee | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index a47f8bb..edd382f 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -2,11 +2,6 @@ server: port: 5999 appRoot: "/bucky" -handlers: - http_headers: - Access-Control-Allow-Origin: www.app.dev.pardot.com - - statsd: host: 'localhost' port: 8125 diff --git a/server.coffee b/server.coffee index 71a1e88..2fe0f1a 100755 --- a/server.coffee +++ b/server.coffee @@ -12,7 +12,6 @@ configWrapper = require './lib/configWrapper' load = require './lib/load' MODULES = config.modules -ALLOW_ORIGIN_HEADERS = config.http_headers loadLogger = -> if MODULES.logger @@ -29,7 +28,7 @@ loadConfig = (logger) -> configWrapper(config) setCORSHeaders = (req, res, next) -> - res.setHeader 'Access-Control-Allow-Origin', ALLOW_ORIGIN_HEADERS + res.setHeader 'Access-Control-Allow-Origin', '*' res.setHeader 'Access-Control-Allow-Methods', 'POST' res.setHeader 'Access-Control-Max-Age', '604800' res.setHeader 'Access-Control-Allow-Credentials', 'true' @@ -60,7 +59,6 @@ loadApp = (logger, loadedConfig) -> app = express() APP_ROOT = process.env.APP_ROOT ? loadedConfig.get('server.appRoot').get() ? '' - ALLOW_ORIGIN_HEADERS = loadedConfig.get('handlers.http_headers.Access-Control-Allow-Origin').get() ? '*' moduleGroups = {} loadKeys = (group) -> From 8d05ea461b22b25686e43e584c06e387f8cb6601 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Wed, 20 Jan 2016 12:56:40 -0500 Subject: [PATCH 05/15] security changes for whitelistingkeys --- modules/collectors.coffee | 3 +++ server.coffee | 12 ------------ 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/modules/collectors.coffee b/modules/collectors.coffee index 5dae58e..575a593 100644 --- a/modules/collectors.coffee +++ b/modules/collectors.coffee @@ -3,16 +3,19 @@ _ = require 'underscore' load = require "../lib/load" modules = require("config").modules +whitelistedkeys = '' module.exports = ({app, logger, config}, next) -> collectorHandler = (collectors) -> return (req, res) -> + #logger.log req.body res.send(204, '') for coll in collectors coll(req.body, {req, res}) logger.log "Loading collectors: #{ modules.collectors.join(', ') }" + whitelistedkeys = "#{modules.whitelistedkeys}".split(',') collectors = {} collPromises = [] diff --git a/server.coffee b/server.coffee index 2fe0f1a..829314e 100755 --- a/server.coffee +++ b/server.coffee @@ -60,18 +60,6 @@ loadApp = (logger, loadedConfig) -> APP_ROOT = process.env.APP_ROOT ? loadedConfig.get('server.appRoot').get() ? '' - moduleGroups = {} - loadKeys = (group) -> - moduleGroups[group] = {} - if MODULES[group] - _.map MODULES[group], (name) -> - logger.log "Key found", name - - else - [] - - appPromises = loadKeys 'whitelistedkeys' - moduleGroups = {} loadModuleGroup = (group) -> moduleGroups[group] = {} From 796829f944805fee10ad65f45b19af76c59f4d0b Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Wed, 20 Jan 2016 14:14:04 -0500 Subject: [PATCH 06/15] security changes for whitelistingkeys --- config/default.yaml | 20 +++++++++++++++++--- modules/collectors.coffee | 12 ++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index edd382f..5b55ff0 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -38,6 +38,20 @@ modules: # - ./modules/influxdb whitelistedkeys: - - key1 - - key2 - - key3 + - file.page.navigationStart + - file.page.unloadEventStart + - file.page.unloadEventEnd + - file.page.fetchStart + - file.page.domainLookupStart + - file.page.domainLookupEnd + - file.page.connectStart + - file.page.connectEnd + - file.page.requestStart + - file.page.responseStart + - file.page.responseEnd + - file.page.domLoading + - file.page.domInteractive + - file.page.domContentLoadedEventStart + - file.page.toJSON + - time.to.click.dont.click.button + - dont.click.button.clicks \ No newline at end of file diff --git a/modules/collectors.coffee b/modules/collectors.coffee index 575a593..20fbb90 100644 --- a/modules/collectors.coffee +++ b/modules/collectors.coffee @@ -7,9 +7,14 @@ whitelistedkeys = '' module.exports = ({app, logger, config}, next) -> collectorHandler = (collectors) -> + arrOfVals = [] return (req, res) -> - #logger.log req.body - res.send(204, '') + for fields of req.body + arrOfVals.push( fields ) + if arrayEqual(arrOfVals, whitelistedkeys) + res.send(204, '') + else + res.send(406, '') for coll in collectors coll(req.body, {req, res}) @@ -41,3 +46,6 @@ module.exports = ({app, logger, config}, next) -> collector[path] = collectorHandler(hls) next collector + + arrayEqual = (a, b) -> + a.length is b.length and a.every (elem, i) -> elem is b[i] \ No newline at end of file From 616ff4636659db2b7f46242d4a5b1a2fe4a1b1a0 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Wed, 20 Jan 2016 15:07:21 -0500 Subject: [PATCH 07/15] security changes for whitelistingkeys --- config/default.yaml | 38 +++++++++++++++++++------------------- modules/collectors.coffee | 17 +++++++++++------ 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 5b55ff0..78c6cf3 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -21,6 +21,7 @@ influxdb: # Acceptable version are: '0.8' and '0.9' version: '0.9' +useWhitelistedKeys: false modules: # The modules just get require'd in, so they don't have to be in the Bucky project. @@ -33,25 +34,24 @@ modules: collectors: # Uncomment the modules that you'd like to use # - ./modules/collectionLogger - - ./modules/statsd + # - ./modules/statsd # - ./modules/openTSDB # - ./modules/influxdb - whitelistedkeys: - - file.page.navigationStart - - file.page.unloadEventStart - - file.page.unloadEventEnd - - file.page.fetchStart - - file.page.domainLookupStart - - file.page.domainLookupEnd - - file.page.connectStart - - file.page.connectEnd - - file.page.requestStart - - file.page.responseStart - - file.page.responseEnd - - file.page.domLoading - - file.page.domInteractive - - file.page.domContentLoadedEventStart - - file.page.toJSON - - time.to.click.dont.click.button - - dont.click.button.clicks \ No newline at end of file + whitelistedKeys: + # Uncomment the keys that you'd like to whitelist + # - file.page.navigationStart + # - file.page.unloadEventStart + # - file.page.unloadEventEnd + # - file.page.fetchStart + # - file.page.domainLookupStart + # - file.page.domainLookupEnd + # - file.page.connectStart + # - file.page.connectEnd + # - file.page.requestStart + # - file.page.responseStart + # - file.page.responseEnd + # - file.page.domLoading + # - file.page.domInteractive + # - file.page.domContentLoadedEventStart + # - file.page.toJSON diff --git a/modules/collectors.coffee b/modules/collectors.coffee index 20fbb90..4f96718 100644 --- a/modules/collectors.coffee +++ b/modules/collectors.coffee @@ -3,24 +3,29 @@ _ = require 'underscore' load = require "../lib/load" modules = require("config").modules +useWhitelistedKeys = require("config").useWhitelistedKeys whitelistedkeys = '' module.exports = ({app, logger, config}, next) -> collectorHandler = (collectors) -> arrOfVals = [] return (req, res) -> - for fields of req.body - arrOfVals.push( fields ) - if arrayEqual(arrOfVals, whitelistedkeys) - res.send(204, '') + if useWhitelistedKeys + for fields of req.body + if (arrOfVals.indexOf( fields ) == -1) + arrOfVals.push( fields ) + if arrayEqual(arrOfVals, whitelistedkeys) + res.send(204, '') + else + res.send(406, '') else - res.send(406, '') + res.send(204, '') for coll in collectors coll(req.body, {req, res}) logger.log "Loading collectors: #{ modules.collectors.join(', ') }" - whitelistedkeys = "#{modules.whitelistedkeys}".split(',') + whitelistedkeys = "#{modules.whitelistedKeys}".split(',') collectors = {} collPromises = [] From 8d5061f9beeab0e86278a3a852c86e30b620b021 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Wed, 20 Jan 2016 15:15:00 -0500 Subject: [PATCH 08/15] security changes for whitelistingkeys --- config/default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.yaml b/config/default.yaml index 78c6cf3..a62b8c5 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -39,7 +39,7 @@ modules: # - ./modules/influxdb whitelistedKeys: - # Uncomment the keys that you'd like to whitelist + # The example below are timing metrics that Bucky includes by default. You will need to whitelist these if you use the default configuration of Bucky and enable whitelisting by setting useWhitelistedKeys to true # - file.page.navigationStart # - file.page.unloadEventStart # - file.page.unloadEventEnd From 9b8b46f5296cc5b9f6687c5a71de8f0e17a263a9 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Fri, 29 Jan 2016 12:12:53 -0500 Subject: [PATCH 09/15] Code refactoring --- config/default.yaml | 2 ++ modules/collectors.coffee | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index a62b8c5..8fd0e73 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -55,3 +55,5 @@ modules: # - file.page.domInteractive # - file.page.domContentLoadedEventStart # - file.page.toJSON + # - time.to.click.dont.click.button + # - dont.click.button.clicks diff --git a/modules/collectors.coffee b/modules/collectors.coffee index 4f96718..ef03bda 100644 --- a/modules/collectors.coffee +++ b/modules/collectors.coffee @@ -10,14 +10,16 @@ module.exports = ({app, logger, config}, next) -> collectorHandler = (collectors) -> arrOfVals = [] return (req, res) -> + arrOfVals = [] if useWhitelistedKeys for fields of req.body if (arrOfVals.indexOf( fields ) == -1) arrOfVals.push( fields ) - if arrayEqual(arrOfVals, whitelistedkeys) + if _.isEqual(arrOfVals, whitelistedkeys) res.send(204, '') else res.send(406, '') + return else res.send(204, '') @@ -50,7 +52,4 @@ module.exports = ({app, logger, config}, next) -> for path, hls of handlers collector[path] = collectorHandler(hls) - next collector - - arrayEqual = (a, b) -> - a.length is b.length and a.every (elem, i) -> elem is b[i] \ No newline at end of file + next collector \ No newline at end of file From cd02f9e3875f89e91693ed104bc51e9d2f84d383 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Fri, 29 Jan 2016 13:21:29 -0500 Subject: [PATCH 10/15] security changes for whitelistingkeys --- modules/collectors.coffee | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/modules/collectors.coffee b/modules/collectors.coffee index ef03bda..f160c8d 100644 --- a/modules/collectors.coffee +++ b/modules/collectors.coffee @@ -12,16 +12,10 @@ module.exports = ({app, logger, config}, next) -> return (req, res) -> arrOfVals = [] if useWhitelistedKeys - for fields of req.body - if (arrOfVals.indexOf( fields ) == -1) - arrOfVals.push( fields ) - if _.isEqual(arrOfVals, whitelistedkeys) - res.send(204, '') - else + if not _.every(_.keys(req.body), (v) -> _.contains(whitelistedkeys, v)) res.send(406, '') return - else - res.send(204, '') + res.send(204, '') for coll in collectors coll(req.body, {req, res}) From 60cd81ddafe4f26a5d07438ccbfd5ff2c445b23c Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Fri, 29 Jan 2016 13:23:56 -0500 Subject: [PATCH 11/15] security changes for whitelistingkeys --- modules/collectors.coffee | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/collectors.coffee b/modules/collectors.coffee index f160c8d..67c28eb 100644 --- a/modules/collectors.coffee +++ b/modules/collectors.coffee @@ -8,9 +8,7 @@ whitelistedkeys = '' module.exports = ({app, logger, config}, next) -> collectorHandler = (collectors) -> - arrOfVals = [] return (req, res) -> - arrOfVals = [] if useWhitelistedKeys if not _.every(_.keys(req.body), (v) -> _.contains(whitelistedkeys, v)) res.send(406, '') From f7548f6a0bcc76542a50b231364342b47b0f95c7 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Fri, 29 Jan 2016 15:03:59 -0500 Subject: [PATCH 12/15] security changes for CORS --- config/default.yaml | 2 +- modules/collectors.coffee | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 8fd0e73..7d3d832 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -21,7 +21,7 @@ influxdb: # Acceptable version are: '0.8' and '0.9' version: '0.9' -useWhitelistedKeys: false +onlyAcceptWhitelistedKeys: true modules: # The modules just get require'd in, so they don't have to be in the Bucky project. diff --git a/modules/collectors.coffee b/modules/collectors.coffee index 67c28eb..8d326d1 100644 --- a/modules/collectors.coffee +++ b/modules/collectors.coffee @@ -3,14 +3,15 @@ _ = require 'underscore' load = require "../lib/load" modules = require("config").modules -useWhitelistedKeys = require("config").useWhitelistedKeys +onlyAcceptWhitelistedKeys = require("config").onlyAcceptWhitelistedKeys whitelistedkeys = '' module.exports = ({app, logger, config}, next) -> collectorHandler = (collectors) -> return (req, res) -> - if useWhitelistedKeys + if onlyAcceptWhitelistedKeys if not _.every(_.keys(req.body), (v) -> _.contains(whitelistedkeys, v)) + console.log ("The key set you are trying to send is not whitelisted") res.send(406, '') return res.send(204, '') From cb9f24b2baf12b8975d0f99f460d6f7e107a8110 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Fri, 29 Jan 2016 15:05:31 -0500 Subject: [PATCH 13/15] security changes for whitelistingkeys --- server.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.coffee b/server.coffee index 829314e..be6bca8 100755 --- a/server.coffee +++ b/server.coffee @@ -12,7 +12,6 @@ configWrapper = require './lib/configWrapper' load = require './lib/load' MODULES = config.modules - loadLogger = -> if MODULES.logger load(MODULES.logger, {config}) @@ -63,6 +62,7 @@ loadApp = (logger, loadedConfig) -> moduleGroups = {} loadModuleGroup = (group) -> moduleGroups[group] = {} + if MODULES[group] _.map MODULES[group], (name) -> logger.log "Loading #{ group } Module", name From 1f1ba3b9e55c6590a68c07371e31d5e9c3943db3 Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Fri, 29 Jan 2016 16:23:43 -0500 Subject: [PATCH 14/15] security changes for whitelistingkeys --- config/default.yaml | 2 +- modules/collectors.coffee | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 7d3d832..b7c9b52 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -21,7 +21,7 @@ influxdb: # Acceptable version are: '0.8' and '0.9' version: '0.9' -onlyAcceptWhitelistedKeys: true +onlyAcceptWhitelistedKeys: false modules: # The modules just get require'd in, so they don't have to be in the Bucky project. diff --git a/modules/collectors.coffee b/modules/collectors.coffee index 8d326d1..54ee956 100644 --- a/modules/collectors.coffee +++ b/modules/collectors.coffee @@ -4,13 +4,12 @@ _ = require 'underscore' load = require "../lib/load" modules = require("config").modules onlyAcceptWhitelistedKeys = require("config").onlyAcceptWhitelistedKeys -whitelistedkeys = '' module.exports = ({app, logger, config}, next) -> collectorHandler = (collectors) -> return (req, res) -> if onlyAcceptWhitelistedKeys - if not _.every(_.keys(req.body), (v) -> _.contains(whitelistedkeys, v)) + if not _.every(_.keys(req.body), (v) -> _.contains(modules.whitelistedKeys, v)) console.log ("The key set you are trying to send is not whitelisted") res.send(406, '') return @@ -20,7 +19,6 @@ module.exports = ({app, logger, config}, next) -> coll(req.body, {req, res}) logger.log "Loading collectors: #{ modules.collectors.join(', ') }" - whitelistedkeys = "#{modules.whitelistedKeys}".split(',') collectors = {} collPromises = [] From 34f71e0546018f88b3051611855f320e7b5cdcae Mon Sep 17 00:00:00 2001 From: gmawalankar Date: Mon, 22 Feb 2016 11:19:46 -0500 Subject: [PATCH 15/15] Uncommenting the default keys sent by bucky --- config/default.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index b7c9b52..5eaded2 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -40,20 +40,20 @@ modules: whitelistedKeys: # The example below are timing metrics that Bucky includes by default. You will need to whitelist these if you use the default configuration of Bucky and enable whitelisting by setting useWhitelistedKeys to true - # - file.page.navigationStart - # - file.page.unloadEventStart - # - file.page.unloadEventEnd - # - file.page.fetchStart - # - file.page.domainLookupStart - # - file.page.domainLookupEnd - # - file.page.connectStart - # - file.page.connectEnd - # - file.page.requestStart - # - file.page.responseStart - # - file.page.responseEnd - # - file.page.domLoading - # - file.page.domInteractive - # - file.page.domContentLoadedEventStart - # - file.page.toJSON + - file.page.navigationStart + - file.page.unloadEventStart + - file.page.unloadEventEnd + - file.page.fetchStart + - file.page.domainLookupStart + - file.page.domainLookupEnd + - file.page.connectStart + - file.page.connectEnd + - file.page.requestStart + - file.page.responseStart + - file.page.responseEnd + - file.page.domLoading + - file.page.domInteractive + - file.page.domContentLoadedEventStart + - file.page.toJSON # - time.to.click.dont.click.button # - dont.click.button.clicks