From 8ddfb19a3f18ab059c2e00864d14de68621c4b6e Mon Sep 17 00:00:00 2001 From: Oskar Andero Date: Wed, 31 Aug 2016 15:19:54 +0200 Subject: [PATCH 1/6] Add compact body argument This feature will pack the json by removing indent spaces and newlines. This allows to circumvent: "Member must have length less than or equal to 51200" --- cumulus/MegaStack.py | 15 +++++++++++++-- cumulus/__init__.py | 6 +++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/cumulus/MegaStack.py b/cumulus/MegaStack.py index 9ea7aca..a2c3019 100644 --- a/cumulus/MegaStack.py +++ b/cumulus/MegaStack.py @@ -186,7 +186,7 @@ def check(self, stack_name=None): bool(stack.exists_in_cf( self.cf_desc_stacks)))) - def create(self, stack_name=None): + def create(self, stack_name=None, compact_body=False): """ Create all stacks in the yaml file. Any that already exist are skipped (no attempt to update) @@ -214,10 +214,15 @@ def create(self, stack_name=None): stack.read_template() self.logger.info("Creating: %s, %s" % ( stack.cf_stack_name, stack.get_params_tuples())) + stack_body = stack.template_body + if compact_body: + self.logger.info("Stack size: %u" % len(stack_body)) + stack_body = self.pack_body(stack.template_body) + self.logger.info("Packed size: %u" % len(stack_body)) try: self.cfconn.create_stack( stack_name=stack.cf_stack_name, - template_body=stack.template_body, + template_body=stack_body, parameters=stack.get_params_tuples(), capabilities=['CAPABILITY_IAM'], notification_arns=stack.sns_topic_arn, @@ -513,6 +518,12 @@ def watch_events(self, stack_name, while_status): time.sleep(5) return status + def pack_body(self, body): + """ + Pack the body by removing human readability chars like spaces and newlines + """ + return simplejson.dumps(simplejson.loads(body), separators=(',', ':')) + def _describe_all_stacks(self): """ Get all pages of stacks from describe_stacks API call. diff --git a/cumulus/__init__.py b/cumulus/__init__.py index 16ab438..4395cad 100755 --- a/cumulus/__init__.py +++ b/cumulus/__init__.py @@ -35,6 +35,10 @@ def main(): dest="stackname", required=False, help="The stack name, used with the watch action," " ignored for other actions") + conf_parser.add_argument( + "-c", "--compact", + action="store_true", dest="compactbody", required=False, + help="Compress each template body by removing spaces.") args = conf_parser.parse_args() # Validate that action is something we know what to do with @@ -80,7 +84,7 @@ def main(): # Run the method of the mega stack object for the action provided if args.action == 'create': - the_mega_stack.create(args.stackname) + the_mega_stack.create(args.stackname, args.compactbody) if args.action == 'check': the_mega_stack.check(args.stackname) From 2517ec89e3a3fa14e23367a0a73e96242f1c3e2d Mon Sep 17 00:00:00 2001 From: Oskar Andero Date: Wed, 31 Aug 2016 15:20:13 +0200 Subject: [PATCH 2/6] Add stack prefix argument Add --prefix argument to be able to override what the created stacks should be prefixed with. --- cumulus/CFStack.py | 11 ++++++++++- cumulus/MegaStack.py | 6 ++++-- cumulus/__init__.py | 6 +++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/cumulus/CFStack.py b/cumulus/CFStack.py index d686888..46ccbfb 100644 --- a/cumulus/CFStack.py +++ b/cumulus/CFStack.py @@ -12,10 +12,14 @@ class CFStack(object): region, template and what other stacks it depends on. """ def __init__(self, mega_stack_name, name, params, template_name, region, - sns_topic_arn, tags=None, depends_on=None): + sns_topic_arn, tags=None, depends_on=None, prefix=None): self.logger = logging.getLogger(__name__) + self.prefix = prefix + if mega_stack_name == name: self.cf_stack_name = name + elif prefix: + self.cf_stack_name = "%s-%s" % (prefix, name) else: self.cf_stack_name = "%s-%s" % (mega_stack_name, name) self.mega_stack_name = mega_stack_name @@ -31,6 +35,8 @@ def __init__(self, mega_stack_name, name, params, template_name, region, for dep in depends_on: if dep == mega_stack_name: self.depends_on.append(dep) + elif prefix: + self.depends_on.append("%s-%s" % (prefix, dep)) else: self.depends_on.append("%s-%s" % (mega_stack_name, dep)) self.region = region @@ -126,6 +132,9 @@ def _parse_param(self, param_name, param_dict): 'variable' in param_dict): if param_dict['source'] == self.mega_stack_name: source_stack = param_dict['source'] + elif self.prefix: + source_stack = ("%s-%s" % + (self.prefix, param_dict['source'])) else: source_stack = ("%s-%s" % (self.mega_stack_name, param_dict['source'])) diff --git a/cumulus/MegaStack.py b/cumulus/MegaStack.py index a2c3019..7325390 100644 --- a/cumulus/MegaStack.py +++ b/cumulus/MegaStack.py @@ -18,8 +18,9 @@ class MegaStack(object): Main worker class for cumulus. Holds array of CFstack objects and does most of the calls to CloudFormation API """ - def __init__(self, yamlFile): + def __init__(self, yamlFile, prefix=None): self.logger = logging.getLogger(__name__) + self.prefix = prefix # load the yaml file and turn it into a dict thefile = open(yamlFile, 'r') @@ -122,7 +123,8 @@ def __init__(self, yamlFile): region=self.region, sns_topic_arn=local_sns_arn, depends_on=the_stack.get('depends'), - tags=merged_tags + tags=merged_tags, + prefix=self.prefix ) ) diff --git a/cumulus/__init__.py b/cumulus/__init__.py index 4395cad..96ea636 100755 --- a/cumulus/__init__.py +++ b/cumulus/__init__.py @@ -39,6 +39,10 @@ def main(): "-c", "--compact", action="store_true", dest="compactbody", required=False, help="Compress each template body by removing spaces.") + conf_parser.add_argument( + "-p", "--prefix", metavar="PREFIX", + dest="cf_prefix", required=False, + help="The prefix of the created stacks. Default is the name of the mega stack.") args = conf_parser.parse_args() # Validate that action is something we know what to do with @@ -71,7 +75,7 @@ def main(): logging.getLogger('boto').setLevel(boto_numeric_level) # Create the mega_stack object and sort out dependencies - the_mega_stack = MegaStack(args.yamlfile) + the_mega_stack = MegaStack(args.yamlfile, args.cf_prefix) the_mega_stack.sort_stacks_by_deps() # Print some info about what we found in the yaml and dependency order From 50646e7c138651fdccba956d17a06b3bb6c0f8c3 Mon Sep 17 00:00:00 2001 From: Oskar Andero Date: Wed, 31 Aug 2016 15:20:26 +0200 Subject: [PATCH 3/6] Add global variables Some values might be passed in to every stack. Keep them as global variables instead of having to update in many places. The global vars are added in the mega stack: megastack: # Global vars vars: globalVar1: Value It can then be referenced in the params section of a stack: stack: params: type: global variable: globalVar1 --- cumulus/CFStack.py | 13 ++++++++++++- cumulus/MegaStack.py | 5 ++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cumulus/CFStack.py b/cumulus/CFStack.py index 46ccbfb..cd77ee9 100644 --- a/cumulus/CFStack.py +++ b/cumulus/CFStack.py @@ -12,7 +12,7 @@ class CFStack(object): region, template and what other stacks it depends on. """ def __init__(self, mega_stack_name, name, params, template_name, region, - sns_topic_arn, tags=None, depends_on=None, prefix=None): + sns_topic_arn, tags=None, depends_on=None, prefix=None, global_dict=None): self.logger = logging.getLogger(__name__) self.prefix = prefix @@ -48,6 +48,11 @@ def __init__(self, mega_stack_name, name, params, template_name, region, else: self.tags = tags + if global_dict is None: + self.global_dict = {} + else: + self.global_dict = global_dict + try: open(template_name, 'r') except: @@ -125,6 +130,12 @@ def _parse_param(self, param_name, param_dict): # Static value set, so use it if 'value' in param_dict: return str(param_dict['value']) + # Handle global variable dereference + elif ('type' in param_dict and + param_dict['type'] == "global" and + 'variable' in param_dict and + param_dict['variable'] in self.global_dict): + return str(self.global_dict[param_dict['variable']]) # No static value set, but if we have a source, # type and variable can try getting from CF elif ('source' in param_dict and diff --git a/cumulus/MegaStack.py b/cumulus/MegaStack.py index 7325390..cbef578 100644 --- a/cumulus/MegaStack.py +++ b/cumulus/MegaStack.py @@ -73,6 +73,8 @@ def __init__(self, yamlFile, prefix=None): # Array for holding CFStack objects once we create them self.stack_objs = [] + self.global_dict = self.stackDict[self.name].get('vars', {}) + # Get the names of the sub stacks from the yaml file and sort in array self.cf_stacks = self.stackDict[self.name]['stacks'].keys() @@ -124,7 +126,8 @@ def __init__(self, yamlFile, prefix=None): sns_topic_arn=local_sns_arn, depends_on=the_stack.get('depends'), tags=merged_tags, - prefix=self.prefix + prefix=self.prefix, + global_dict=self.global_dict ) ) From a71e1f9355ba57f7bbcfdcb0a6a302afbfa28b43 Mon Sep 17 00:00:00 2001 From: Oskar Andero Date: Wed, 31 Aug 2016 15:20:36 +0200 Subject: [PATCH 4/6] examples: add global var --- examples/legostack/legostack.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/legostack/legostack.yaml b/examples/legostack/legostack.yaml index c6423c6..43ca655 100644 --- a/examples/legostack/legostack.yaml +++ b/examples/legostack/legostack.yaml @@ -7,6 +7,9 @@ legostack: # Working on N.California. You can change it just remember to also update availablilty zones below and AMI ids region: us-west-1 highlight-output: true + # Add global variables + vars: + globalInstanceType: 't2.small' # Tags defined here are spread across every resource created by sub-stacks tags: globaltag: tagvalue @@ -125,7 +128,8 @@ legostack: - dmz-sub params: ParamInstanceType: - value: 't2.small' + type: global + variable: globalInstanceType ParamUseElasticIP: value: 'true' ParamInstanceAMI: @@ -198,7 +202,8 @@ legostack: - nat params: ParamInstanceType: - value: 't2.small' + type: global + variable: globalInstanceType ParamInstanceAMI: # Ubuntu 12.04 Precise HVM EBS value: 'ami-0dad4949' From 9517fca9f13eaacd5470de9846c138594782d998 Mon Sep 17 00:00:00 2001 From: Oskar Andero Date: Wed, 31 Aug 2016 15:20:46 +0200 Subject: [PATCH 5/6] Add argument for overriding global variables To easily override one of your globals you can now pass one or many --override-global arguments. This can be useful to override for instance a cognito pool or an email adress depending on which account you are deploying the stack. --- cumulus/MegaStack.py | 6 +++++- cumulus/__init__.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cumulus/MegaStack.py b/cumulus/MegaStack.py index cbef578..dfed947 100644 --- a/cumulus/MegaStack.py +++ b/cumulus/MegaStack.py @@ -18,7 +18,7 @@ class MegaStack(object): Main worker class for cumulus. Holds array of CFstack objects and does most of the calls to CloudFormation API """ - def __init__(self, yamlFile, prefix=None): + def __init__(self, yamlFile, prefix=None, override_dict=None): self.logger = logging.getLogger(__name__) self.prefix = prefix @@ -74,6 +74,10 @@ def __init__(self, yamlFile, prefix=None): self.stack_objs = [] self.global_dict = self.stackDict[self.name].get('vars', {}) + if override_dict: + for override in override_dict: + if override in self.global_dict: + self.global_dict[override] = override_dict[override] # Get the names of the sub stacks from the yaml file and sort in array self.cf_stacks = self.stackDict[self.name]['stacks'].keys() diff --git a/cumulus/__init__.py b/cumulus/__init__.py index 96ea636..fd45c4b 100755 --- a/cumulus/__init__.py +++ b/cumulus/__init__.py @@ -43,6 +43,10 @@ def main(): "-p", "--prefix", metavar="PREFIX", dest="cf_prefix", required=False, help="The prefix of the created stacks. Default is the name of the mega stack.") + conf_parser.add_argument( + "-o", "--override-global", + action='append', dest="override_global", required=False, + help="Override a global variable. Example: --override-global \"globalVariable=newValue\"") args = conf_parser.parse_args() # Validate that action is something we know what to do with @@ -74,8 +78,17 @@ def main(): exit(1) logging.getLogger('boto').setLevel(boto_numeric_level) + if args.override_global: + try: + override_dict = dict(k for k in (x.split('=') for x in args.override_global)) + except ValueError as exception: + print "Illegal global override parameter in: %s" % args.override_global + exit(1) + else: + override_dict = None + # Create the mega_stack object and sort out dependencies - the_mega_stack = MegaStack(args.yamlfile, args.cf_prefix) + the_mega_stack = MegaStack(args.yamlfile, args.cf_prefix, override_dict) the_mega_stack.sort_stacks_by_deps() # Print some info about what we found in the yaml and dependency order From 88aadd116880f8604e921e5c91d1cbe6a707d859 Mon Sep 17 00:00:00 2001 From: Oskar Andero Date: Wed, 31 Aug 2016 15:20:57 +0200 Subject: [PATCH 6/6] Add --[no-]highlight output argument Depending on what environment you are running on you might want colors or not. This allows you to configure it when calling. --- cumulus/MegaStack.py | 6 +++++- cumulus/__init__.py | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cumulus/MegaStack.py b/cumulus/MegaStack.py index dfed947..d55c9b1 100644 --- a/cumulus/MegaStack.py +++ b/cumulus/MegaStack.py @@ -18,7 +18,7 @@ class MegaStack(object): Main worker class for cumulus. Holds array of CFstack objects and does most of the calls to CloudFormation API """ - def __init__(self, yamlFile, prefix=None, override_dict=None): + def __init__(self, yamlFile, prefix=None, override_dict=None, highlight_arg=None): self.logger = logging.getLogger(__name__) self.prefix = prefix @@ -39,6 +39,10 @@ def __init__(self, yamlFile, prefix=None, override_dict=None): # that must be the mega stack name self.name = self.stackDict.keys()[0] + # Override highlight-output with argument if present. + if highlight_arg is not None: + self.stackDict[self.name]['highlight-output'] = highlight_arg + # Find and set the mega stacks region. Exit if we can't find it if 'region' in self.stackDict[self.name]: self.region = self.stackDict[self.name]['region'] diff --git a/cumulus/__init__.py b/cumulus/__init__.py index fd45c4b..19379b8 100755 --- a/cumulus/__init__.py +++ b/cumulus/__init__.py @@ -47,6 +47,14 @@ def main(): "-o", "--override-global", action='append', dest="override_global", required=False, help="Override a global variable. Example: --override-global \"globalVariable=newValue\"") + conf_parser.add_argument( + "--highlight", + action="store_true", dest="highlight", required=False, + help="Highlight output. This takes precedence over highlight-output in the yaml file.") + conf_parser.add_argument( + "--no-highlight", + action="store_false", dest="highlight", required=False, + help="Don't highlight output. This takes precedence over highlight-output in the yaml file.") args = conf_parser.parse_args() # Validate that action is something we know what to do with @@ -88,7 +96,7 @@ def main(): override_dict = None # Create the mega_stack object and sort out dependencies - the_mega_stack = MegaStack(args.yamlfile, args.cf_prefix, override_dict) + the_mega_stack = MegaStack(args.yamlfile, args.cf_prefix, override_dict, args.highlight) the_mega_stack.sort_stacks_by_deps() # Print some info about what we found in the yaml and dependency order