diff --git a/cumulus/CFStack.py b/cumulus/CFStack.py index d686888..cd77ee9 100644 --- a/cumulus/CFStack.py +++ b/cumulus/CFStack.py @@ -12,10 +12,14 @@ class CFStack(object): region, template and what other stacks it depends on. """ def __init__(self, mega_stack_name, name, params, template_name, region, - sns_topic_arn, tags=None, depends_on=None): + sns_topic_arn, tags=None, depends_on=None, prefix=None, global_dict=None): self.logger = logging.getLogger(__name__) + self.prefix = prefix + if mega_stack_name == name: self.cf_stack_name = name + elif prefix: + self.cf_stack_name = "%s-%s" % (prefix, name) else: self.cf_stack_name = "%s-%s" % (mega_stack_name, name) self.mega_stack_name = mega_stack_name @@ -31,6 +35,8 @@ def __init__(self, mega_stack_name, name, params, template_name, region, for dep in depends_on: if dep == mega_stack_name: self.depends_on.append(dep) + elif prefix: + self.depends_on.append("%s-%s" % (prefix, dep)) else: self.depends_on.append("%s-%s" % (mega_stack_name, dep)) self.region = region @@ -42,6 +48,11 @@ def __init__(self, mega_stack_name, name, params, template_name, region, else: self.tags = tags + if global_dict is None: + self.global_dict = {} + else: + self.global_dict = global_dict + try: open(template_name, 'r') except: @@ -119,6 +130,12 @@ def _parse_param(self, param_name, param_dict): # Static value set, so use it if 'value' in param_dict: return str(param_dict['value']) + # Handle global variable dereference + elif ('type' in param_dict and + param_dict['type'] == "global" and + 'variable' in param_dict and + param_dict['variable'] in self.global_dict): + return str(self.global_dict[param_dict['variable']]) # No static value set, but if we have a source, # type and variable can try getting from CF elif ('source' in param_dict and @@ -126,6 +143,9 @@ def _parse_param(self, param_name, param_dict): 'variable' in param_dict): if param_dict['source'] == self.mega_stack_name: source_stack = param_dict['source'] + elif self.prefix: + source_stack = ("%s-%s" % + (self.prefix, param_dict['source'])) else: source_stack = ("%s-%s" % (self.mega_stack_name, param_dict['source'])) diff --git a/cumulus/MegaStack.py b/cumulus/MegaStack.py index 9ea7aca..d55c9b1 100644 --- a/cumulus/MegaStack.py +++ b/cumulus/MegaStack.py @@ -18,8 +18,9 @@ class MegaStack(object): Main worker class for cumulus. Holds array of CFstack objects and does most of the calls to CloudFormation API """ - def __init__(self, yamlFile): + def __init__(self, yamlFile, prefix=None, override_dict=None, highlight_arg=None): self.logger = logging.getLogger(__name__) + self.prefix = prefix # load the yaml file and turn it into a dict thefile = open(yamlFile, 'r') @@ -38,6 +39,10 @@ def __init__(self, yamlFile): # that must be the mega stack name self.name = self.stackDict.keys()[0] + # Override highlight-output with argument if present. + if highlight_arg is not None: + self.stackDict[self.name]['highlight-output'] = highlight_arg + # Find and set the mega stacks region. Exit if we can't find it if 'region' in self.stackDict[self.name]: self.region = self.stackDict[self.name]['region'] @@ -72,6 +77,12 @@ def __init__(self, yamlFile): # Array for holding CFStack objects once we create them self.stack_objs = [] + self.global_dict = self.stackDict[self.name].get('vars', {}) + if override_dict: + for override in override_dict: + if override in self.global_dict: + self.global_dict[override] = override_dict[override] + # Get the names of the sub stacks from the yaml file and sort in array self.cf_stacks = self.stackDict[self.name]['stacks'].keys() @@ -122,7 +133,9 @@ def __init__(self, yamlFile): region=self.region, sns_topic_arn=local_sns_arn, depends_on=the_stack.get('depends'), - tags=merged_tags + tags=merged_tags, + prefix=self.prefix, + global_dict=self.global_dict ) ) @@ -186,7 +199,7 @@ def check(self, stack_name=None): bool(stack.exists_in_cf( self.cf_desc_stacks)))) - def create(self, stack_name=None): + def create(self, stack_name=None, compact_body=False): """ Create all stacks in the yaml file. Any that already exist are skipped (no attempt to update) @@ -214,10 +227,15 @@ def create(self, stack_name=None): stack.read_template() self.logger.info("Creating: %s, %s" % ( stack.cf_stack_name, stack.get_params_tuples())) + stack_body = stack.template_body + if compact_body: + self.logger.info("Stack size: %u" % len(stack_body)) + stack_body = self.pack_body(stack.template_body) + self.logger.info("Packed size: %u" % len(stack_body)) try: self.cfconn.create_stack( stack_name=stack.cf_stack_name, - template_body=stack.template_body, + template_body=stack_body, parameters=stack.get_params_tuples(), capabilities=['CAPABILITY_IAM'], notification_arns=stack.sns_topic_arn, @@ -513,6 +531,12 @@ def watch_events(self, stack_name, while_status): time.sleep(5) return status + def pack_body(self, body): + """ + Pack the body by removing human readability chars like spaces and newlines + """ + return simplejson.dumps(simplejson.loads(body), separators=(',', ':')) + def _describe_all_stacks(self): """ Get all pages of stacks from describe_stacks API call. diff --git a/cumulus/__init__.py b/cumulus/__init__.py index 16ab438..19379b8 100755 --- a/cumulus/__init__.py +++ b/cumulus/__init__.py @@ -35,6 +35,26 @@ def main(): dest="stackname", required=False, help="The stack name, used with the watch action," " ignored for other actions") + conf_parser.add_argument( + "-c", "--compact", + action="store_true", dest="compactbody", required=False, + help="Compress each template body by removing spaces.") + conf_parser.add_argument( + "-p", "--prefix", metavar="PREFIX", + dest="cf_prefix", required=False, + help="The prefix of the created stacks. Default is the name of the mega stack.") + conf_parser.add_argument( + "-o", "--override-global", + action='append', dest="override_global", required=False, + help="Override a global variable. Example: --override-global \"globalVariable=newValue\"") + conf_parser.add_argument( + "--highlight", + action="store_true", dest="highlight", required=False, + help="Highlight output. This takes precedence over highlight-output in the yaml file.") + conf_parser.add_argument( + "--no-highlight", + action="store_false", dest="highlight", required=False, + help="Don't highlight output. This takes precedence over highlight-output in the yaml file.") args = conf_parser.parse_args() # Validate that action is something we know what to do with @@ -66,8 +86,17 @@ def main(): exit(1) logging.getLogger('boto').setLevel(boto_numeric_level) + if args.override_global: + try: + override_dict = dict(k for k in (x.split('=') for x in args.override_global)) + except ValueError as exception: + print "Illegal global override parameter in: %s" % args.override_global + exit(1) + else: + override_dict = None + # Create the mega_stack object and sort out dependencies - the_mega_stack = MegaStack(args.yamlfile) + the_mega_stack = MegaStack(args.yamlfile, args.cf_prefix, override_dict, args.highlight) the_mega_stack.sort_stacks_by_deps() # Print some info about what we found in the yaml and dependency order @@ -80,7 +109,7 @@ def main(): # Run the method of the mega stack object for the action provided if args.action == 'create': - the_mega_stack.create(args.stackname) + the_mega_stack.create(args.stackname, args.compactbody) if args.action == 'check': the_mega_stack.check(args.stackname) diff --git a/examples/legostack/legostack.yaml b/examples/legostack/legostack.yaml index c6423c6..43ca655 100644 --- a/examples/legostack/legostack.yaml +++ b/examples/legostack/legostack.yaml @@ -7,6 +7,9 @@ legostack: # Working on N.California. You can change it just remember to also update availablilty zones below and AMI ids region: us-west-1 highlight-output: true + # Add global variables + vars: + globalInstanceType: 't2.small' # Tags defined here are spread across every resource created by sub-stacks tags: globaltag: tagvalue @@ -125,7 +128,8 @@ legostack: - dmz-sub params: ParamInstanceType: - value: 't2.small' + type: global + variable: globalInstanceType ParamUseElasticIP: value: 'true' ParamInstanceAMI: @@ -198,7 +202,8 @@ legostack: - nat params: ParamInstanceType: - value: 't2.small' + type: global + variable: globalInstanceType ParamInstanceAMI: # Ubuntu 12.04 Precise HVM EBS value: 'ami-0dad4949'