From f936d8cfa542f58e14f3e7bff913af343d84f8e9 Mon Sep 17 00:00:00 2001 From: Eric Price Date: Thu, 18 Feb 2016 20:09:35 -0800 Subject: [PATCH 01/10] Replaced EnvConfig with get_loaded_template_subclasses() in EnvironmentBase --- src/environmentbase/environmentbase.py | 46 ++++++++------------------ src/environmentbase/resources.py | 7 ++-- src/examples/basic.py | 5 +-- 3 files changed, 19 insertions(+), 39 deletions(-) diff --git a/src/environmentbase/environmentbase.py b/src/environmentbase/environmentbase.py index 5a59c93..b864f03 100644 --- a/src/environmentbase/environmentbase.py +++ b/src/environmentbase/environmentbase.py @@ -24,14 +24,6 @@ class ValidationError(Exception): pass -class EnvConfig(object): - - def __init__(self, config_handlers=None): - self.config_handlers = config_handlers if config_handlers else [] - # self.stack_event_handlers = stack_event_handlers if stack_event_handlers else [] - # self.deploy_handlers = deploy_handlers if deploy_handlers else {} - - class EnvironmentBase(object): """ EnvironmentBase encapsulates functionality required to build and deploy a network and common resources for object storage within a specified region @@ -39,7 +31,6 @@ class EnvironmentBase(object): def __init__(self, view=None, - env_config=EnvConfig(), config_filename=res.R.CONFIG_FILENAME, config_file_override=None): """ @@ -53,7 +44,6 @@ def __init__(self, """ self.config_filename = config_filename - self.env_config = env_config self.config_file_override = config_file_override self.config = {} self.globals = {} @@ -62,7 +52,6 @@ def __init__(self, self.deploy_parameter_bindings = [] self.ignore_outputs = ['templateValidationHash', 'dateGenerated'] self.stack_outputs = {} - self._config_handlers = [] self.stack_monitor = None self._ami_cache = None self.cfn_connection = None @@ -70,10 +59,7 @@ def __init__(self, self.boto_session = None - # self.env_config = env_config - for config_handler in env_config.config_handlers: - self._add_config_handler(config_handler) - self.add_config_hook() + self.template_manifest = EnvironmentBase.get_loaded_template_subclasses() # Load the user interface self.view = view if view else cli.CLI() @@ -85,6 +71,15 @@ def __init__(self, # Allow the view to execute the user's requested action self.view.process_request(self) + @staticmethod + def get_loaded_template_subclasses(): + # Ensure this class is loaded + from template import Template + + template_subclasses = vars()['Template'].__subclasses__() + + return template_subclasses + def create_hook(self): """ Override in your subclass for custom resource creation. Called after config is loaded and template is @@ -148,8 +143,7 @@ def init_action(self, is_silent=False): Override in your subclass for custom initialization steps @param is_silent [boolean], supress console output (for testing) """ - config_handlers = self.env_config.config_handlers - res.R.generate_config(prompt=True, is_silent=is_silent, output_filename=self.config_filename, config_handlers=config_handlers) + res.R.generate_config(prompt=True, is_silent=is_silent, output_filename=self.config_filename, template_classes=self.template_manifest) def s3_prefix(self): """ @@ -461,27 +455,15 @@ def _validate_config(self, config, factory_schema=None): config_reqs_copy = copy.deepcopy(factory_schema) # Merge in any requirements provided by config handlers - for handler in self._config_handlers: - config_reqs_copy.update(handler.get_config_schema()) + for template_subclass in self.template_manifest: + config_schema = getattr(template_subclass, 'get_config_schema')() + config_reqs_copy.update(config_schema) self._validate_config_helper(config_reqs_copy, config, '') # Validate region self._validate_region(config) - def _add_config_handler(self, handler): - """ - Register classes that will augment the configuration defaults and/or validation logic here - """ - - if not hasattr(handler, 'get_factory_defaults') or not callable(getattr(handler, 'get_factory_defaults')): - raise ValidationError('Class %s cannot be a config handler, missing get_factory_defaults()' % type(handler).__name__ ) - - if not hasattr(handler, 'get_config_schema') or not callable(getattr(handler, 'get_config_schema')): - raise ValidationError('Class %s cannot be a config handler, missing get_config_schema()' % type(handler).__name__ ) - - self._config_handlers.append(handler) - @staticmethod def _config_env_override(config, path, print_debug=False): """ diff --git a/src/environmentbase/resources.py b/src/environmentbase/resources.py index 35758bc..d47ec3e 100644 --- a/src/environmentbase/resources.py +++ b/src/environmentbase/resources.py @@ -159,7 +159,7 @@ def _extract_config_section(self, config, config_key, filename, prompt=False): def generate_config(self, config_file=CONFIG_FILENAME, output_filename=None, - config_handlers=list(), + template_classes=list(), extract_map=_EXTRACTED_CONFIG_SECTIONS, prompt=False, is_silent=False): @@ -190,8 +190,9 @@ def get_config_schema(): config = self.parse_file(config_file, from_file=False) # Merge in any defaults provided by registered config handlers - for handler in config_handlers: - config.update(handler.get_factory_defaults()) + for template_subclass in template_classes: + factory_defaults = getattr(template_subclass, 'get_factory_defaults')() + config.update(factory_defaults) # Make changes to a new copy of the config config_copy = copy.deepcopy(config) diff --git a/src/examples/basic.py b/src/examples/basic.py index 6c29af6..f1e2d40 100755 --- a/src/examples/basic.py +++ b/src/examples/basic.py @@ -1,8 +1,6 @@ -from environmentbase.environmentbase import EnvConfig from environmentbase.networkbase import NetworkBase from environmentbase.patterns.bastion import Bastion from environmentbase.patterns.ha_cluster import HaCluster -from environmentbase.patterns.base_network import BaseNetwork class MyEnvClass(NetworkBase): @@ -26,5 +24,4 @@ def create_hook(self): suggested_instance_types=['t2.micro'])) if __name__ == '__main__': - env_config = EnvConfig(config_handlers=[BaseNetwork]) - MyEnvClass(env_config=env_config) + MyEnvClass() From 34b67299a83778a5182a1b7d6fb43ba0dce725f1 Mon Sep 17 00:00:00 2001 From: Eric Price Date: Thu, 18 Feb 2016 20:41:28 -0800 Subject: [PATCH 02/10] Changed get_loaded_template_subclasses() to not use import directive, removed add_config_hook(), added optional initilization prompt. --- src/environmentbase/environmentbase.py | 25 ++++++++++--------------- src/environmentbase/networkbase.py | 4 ---- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/src/environmentbase/environmentbase.py b/src/environmentbase/environmentbase.py index b864f03..4f161b8 100644 --- a/src/environmentbase/environmentbase.py +++ b/src/environmentbase/environmentbase.py @@ -32,7 +32,8 @@ class EnvironmentBase(object): def __init__(self, view=None, config_filename=res.R.CONFIG_FILENAME, - config_file_override=None): + config_file_override=None, + is_silent=False): """ Init method for environment base creates all common objects for a given environment within the CloudFormation template including a network, s3 bucket and requisite policies to allow ELB Access log aggregation and @@ -59,7 +60,8 @@ def __init__(self, self.boto_session = None - self.template_manifest = EnvironmentBase.get_loaded_template_subclasses() + # Get names of loaded subclasses of Template + self.template_manifest = EnvironmentBase.get_loaded_template_subclasses(is_silent) # Load the user interface self.view = view if view else cli.CLI() @@ -72,12 +74,13 @@ def __init__(self, self.view.process_request(self) @staticmethod - def get_loaded_template_subclasses(): + def get_loaded_template_subclasses(is_silent=False): # Ensure this class is loaded - from template import Template - - template_subclasses = vars()['Template'].__subclasses__() - + module = __import__('environmentbase.template', fromlist=['Template']) + klass = getattr(module, 'Template') + template_subclasses = klass.__subclasses__() + if not is_silent: + print "Initializing patterns: %s" % [cls.__name__ for cls in template_subclasses] return template_subclasses def create_hook(self): @@ -87,14 +90,6 @@ def create_hook(self): """ pass - def add_config_hook(self): - """ - Override in your subclass for adding custom config handlers. - Called after the other config handlers have been added. - After the hook completes the view is loaded and started. - """ - pass - def deploy_hook(self): """ Extension point for modifying behavior of deploy action. Called after config is loaded and before diff --git a/src/environmentbase/networkbase.py b/src/environmentbase/networkbase.py index 48162d0..03ad828 100755 --- a/src/environmentbase/networkbase.py +++ b/src/environmentbase/networkbase.py @@ -10,10 +10,6 @@ class NetworkBase(EnvironmentBase): for a common deployment within AWS. This is intended to be the 'base' stack for deploying child stacks """ - def add_config_hook(self): - super(NetworkBase, self).add_config_hook() - self._add_config_handler(BaseNetwork) - def create_hook(self): super(NetworkBase, self).create_hook() From 4085f23a4a38d78109b207d585f6623d9cd75478 Mon Sep 17 00:00:00 2001 From: Eric Price Date: Fri, 19 Feb 2016 12:26:06 -0800 Subject: [PATCH 03/10] Moved pattern loading to utility, added is_silent flag to EnvironmentBase constructor and fixed broken test cases --- src/environmentbase/environmentbase.py | 21 +++--------- src/environmentbase/resources.py | 17 ++-------- src/environmentbase/utility.py | 32 ++++++++++++++++++ src/tests/test_environmentbase.py | 47 ++++++++++++-------------- src/tests/test_resources.py | 6 ++-- 5 files changed, 65 insertions(+), 58 deletions(-) diff --git a/src/environmentbase/environmentbase.py b/src/environmentbase/environmentbase.py index 4f161b8..7a69441 100644 --- a/src/environmentbase/environmentbase.py +++ b/src/environmentbase/environmentbase.py @@ -60,8 +60,9 @@ def __init__(self, self.boto_session = None - # Get names of loaded subclasses of Template - self.template_manifest = EnvironmentBase.get_loaded_template_subclasses(is_silent) + # Show names of Template subclasses + if not is_silent: + print "Using patterns: %s" % [cls.__name__ for cls in utility.get_pattern_list()] # Load the user interface self.view = view if view else cli.CLI() @@ -73,16 +74,6 @@ def __init__(self, # Allow the view to execute the user's requested action self.view.process_request(self) - @staticmethod - def get_loaded_template_subclasses(is_silent=False): - # Ensure this class is loaded - module = __import__('environmentbase.template', fromlist=['Template']) - klass = getattr(module, 'Template') - template_subclasses = klass.__subclasses__() - if not is_silent: - print "Initializing patterns: %s" % [cls.__name__ for cls in template_subclasses] - return template_subclasses - def create_hook(self): """ Override in your subclass for custom resource creation. Called after config is loaded and template is @@ -138,7 +129,7 @@ def init_action(self, is_silent=False): Override in your subclass for custom initialization steps @param is_silent [boolean], supress console output (for testing) """ - res.R.generate_config(prompt=True, is_silent=is_silent, output_filename=self.config_filename, template_classes=self.template_manifest) + res.R.generate_config(prompt=True, is_silent=is_silent, output_filename=self.config_filename) def s3_prefix(self): """ @@ -450,9 +441,7 @@ def _validate_config(self, config, factory_schema=None): config_reqs_copy = copy.deepcopy(factory_schema) # Merge in any requirements provided by config handlers - for template_subclass in self.template_manifest: - config_schema = getattr(template_subclass, 'get_config_schema')() - config_reqs_copy.update(config_schema) + utility.update_schema_from_patterns(config_reqs_copy) self._validate_config_helper(config_reqs_copy, config, '') diff --git a/src/environmentbase/resources.py b/src/environmentbase/resources.py index d47ec3e..8ad50f0 100644 --- a/src/environmentbase/resources.py +++ b/src/environmentbase/resources.py @@ -4,6 +4,7 @@ import json import os import re +import utility # Declare R to be the singleton Resource instance @@ -159,28 +160,18 @@ def _extract_config_section(self, config, config_key, filename, prompt=False): def generate_config(self, config_file=CONFIG_FILENAME, output_filename=None, - template_classes=list(), extract_map=_EXTRACTED_CONFIG_SECTIONS, prompt=False, is_silent=False): """ Copies specified yaml/json file from the EGG resource to current directory, default is 'conifg.json'. Optionally - split out specific sections into separate files using extract_map. Additionally us config_handlers to add in - additional conifg content before serializing content to file. + split out specific sections into separate files using extract_map. @param config_file [string] Name of file within resource path to load. @param output_file [string] Name of generated config file (default is same as 'config_file') @param prompt [boolean] block for user input to abort file output if file already exists @param is_silent [boolena] supress console output (primarly for testing) @param extract_map [map] Specifies top-level sections of config to externalize to separate file. Where key=config section name, value=filename. - @param config_handlers [list(objects)] Config handlers should resemble the following: - class CustomHandler(object): - @staticmethod - def get_factory_defaults(): - return custom_config_addition - @staticmethod - def get_config_schema(): - return custom_config_validation """ # Output same file name as the input unless specified otherwise if not output_filename: @@ -190,9 +181,7 @@ def get_config_schema(): config = self.parse_file(config_file, from_file=False) # Merge in any defaults provided by registered config handlers - for template_subclass in template_classes: - factory_defaults = getattr(template_subclass, 'get_factory_defaults')() - config.update(factory_defaults) + utility.update_config_from_patterns(config) # Make changes to a new copy of the config config_copy = copy.deepcopy(config) diff --git a/src/environmentbase/utility.py b/src/environmentbase/utility.py index d0a24e3..63cb35d 100644 --- a/src/environmentbase/utility.py +++ b/src/environmentbase/utility.py @@ -119,6 +119,7 @@ def get_stack_depends_on_from_parent_template(parent_template_contents, stack_na # Otherwise return the DependsOn list that the stack was deployed with return stack_reference.get('DependsOn') + def get_template_s3_resource_path(prefix, template_name, include_timestamp=True): """ Constructs s3 resource path for provided template name @@ -143,3 +144,34 @@ def get_template_s3_url(bucket_name, resource_path): """ return 'https://%s.s3.amazonaws.com/%s' % (bucket_name, resource_path) + +def _get_subclasses_of(parent_import_path, parent_classname): + # Import environmentbase.template.Template (in case it's not already) + _module = __import__(parent_import_path, fromlist=[parent_classname]) + parent_class = getattr(_module, parent_classname) + subclasses = parent_class.__subclasses__() + return subclasses + + +def get_pattern_list(): + """ + Returns list of all imported subclasses of environmentbase.template.Template + """ + return _get_subclasses_of('environmentbase.template', 'Template') + + +def _update_from_patterns(_dict, fun_name): + class_list = get_pattern_list() + for template_subclass in class_list: + additional_dict = getattr(template_subclass, fun_name)() + _dict.update(additional_dict) + + return _dict + + +def update_schema_from_patterns(config_schema, class_list=None): + return _update_from_patterns(config_schema, 'get_config_schema') + + +def update_config_from_patterns(config, class_list=None): + return _update_from_patterns(config, 'get_factory_defaults') diff --git a/src/tests/test_environmentbase.py b/src/tests/test_environmentbase.py index a529f96..e36738c 100644 --- a/src/tests/test_environmentbase.py +++ b/src/tests/test_environmentbase.py @@ -8,7 +8,7 @@ import sys import copy from tempfile import mkdtemp -from environmentbase import cli, resources as res, environmentbase as eb +from environmentbase import cli, resources as res, environmentbase as eb, utility from environmentbase import networkbase import environmentbase.patterns.ha_nat from troposphere import ec2 @@ -36,17 +36,14 @@ def fake_cli(self, extra_args): return my_cli - def _create_dummy_config(self, env_base=None): + def _create_dummy_config(self): dummy_string = 'dummy' dummy_bool = False dummy_int = 3 dummy_list = ['A', 'B', 'C'] config_requirements = res.R.parse_file(res.Res.CONFIG_REQUIREMENTS_FILENAME, from_file=False) - - if env_base: - for handler in env_base.config_handlers: - config_requirements.update(handler.get_config_schema()) + utility.update_schema_from_patterns(config_requirements) config = {} for (section, keys) in config_requirements.iteritems(): @@ -76,7 +73,7 @@ def _create_local_file(self, name, content): def test_constructor(self): """Make sure EnvironmentBase passes control to view to process user requests""" fake_cli = self.fake_cli(['init']) - env_base = eb.EnvironmentBase(fake_cli) + env_base = eb.EnvironmentBase(fake_cli, is_silent=True) # Check that EnvironmentBase started the CLI fake_cli.process_request.assert_called_once_with(env_base) @@ -110,7 +107,7 @@ def process_request(self, controller): actions_called[action] += 1 - eb.EnvironmentBase(MyView()) + eb.EnvironmentBase(MyView(), is_silent=True) self.assertEqual(actions_called['init'], 1) self.assertEqual(actions_called['create'], 1) @@ -121,13 +118,14 @@ def test_config_yaml(self): """ Verify load_config can load non-default files """ alt_config_filename = 'config.yaml' config = res.R.parse_file(res.Res.CONFIG_FILENAME, from_file=False) + utility.update_config_from_patterns(config) with open(alt_config_filename, 'w') as f: f.write(yaml.dump(config, default_flow_style=False)) f.flush() fake_cli = self.fake_cli(['create', '--config-file', 'config.yaml']) - base = eb.EnvironmentBase(fake_cli, config_filename=alt_config_filename) + base = eb.EnvironmentBase(fake_cli, config_filename=alt_config_filename, is_silent=True) base.load_config() self.assertEqual(base.config['global']['environment_name'], 'environmentbase') @@ -147,7 +145,7 @@ def test_config_override(self): f.flush() fake_cli = self.fake_cli(['create']) - base = eb.EnvironmentBase(fake_cli) + base = eb.EnvironmentBase(fake_cli, is_silent=True) base.load_config() self.assertNotEqual(base.config['global']['environment_name'], original_value) @@ -157,7 +155,7 @@ def test_config_override(self): # existence check with self.assertRaises(Exception): - base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', config_filename])) + base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', config_filename]), is_silent=True) base.load_config() # remove config.json and create the alternate config file @@ -167,7 +165,7 @@ def test_config_override(self): with open(config_filename, 'w') as f: f.write(yaml.dump(config)) f.flush() - base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', config_filename])) + base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', config_filename]), is_silent=True) base.load_config() self.assertNotEqual(base.config['global']['environment_name'], original_value) @@ -177,7 +175,7 @@ def test_config_validation(self): environmentbase.TEMPLATE_REQUIREMENTS defines the required sections and keys for a valid input config file This test ensures that EnvironmentBase._validate_config() enforces the TEMPLATE_REQUIREMENTS contract """ - cntrl = eb.EnvironmentBase(self.fake_cli(['create'])) + cntrl = eb.EnvironmentBase(self.fake_cli(['create']), is_silent=True) valid_config = self._create_dummy_config() cntrl._validate_config(valid_config) @@ -240,9 +238,7 @@ def test_config_validation(self): }}}}) def test_extending_config(self): - - # Typically this would subclass eb.Template - class MyConfigHandler(object): + class MyTemplate(eb.Template): @staticmethod def get_factory_defaults(): return {'new_section': {'new_key': 'value'}} @@ -255,11 +251,11 @@ class MyEnvBase(eb.EnvironmentBase): pass view = self.fake_cli(['init']) - env_config = eb.EnvConfig(config_handlers=[MyConfigHandler]) controller = MyEnvBase( view=view, - env_config=env_config + is_silent=True ) + controller.init_action(is_silent=True) controller.load_config() @@ -274,17 +270,18 @@ class MyEnvBase(eb.EnvironmentBase): # recreate config file without 'new_section' and make sure it fails validation os.remove(res.Res.CONFIG_FILENAME) dummy_config = self._create_dummy_config() + del dummy_config['new_section'] self._create_local_file(res.Res.CONFIG_FILENAME, json.dumps(dummy_config, indent=4)) with self.assertRaises(eb.ValidationError): - base = MyEnvBase(view=view, env_config=env_config) + base = MyEnvBase(view=view, is_silent=True) base.load_config() def test_generate_config(self): """ Verify cli flags update config object """ # Verify that debug and output are set to the factory default - base = eb.EnvironmentBase(self.fake_cli(['init'])) + base = eb.EnvironmentBase(self.fake_cli(['init']), is_silent=True) res.R.generate_config(prompt=True, is_silent=True) base.load_config() @@ -298,20 +295,20 @@ def test_generate_config(self): def test_template_file_flag(self): # verify that the --template-file flag changes the config value dummy_value = 'dummy' - base = eb.EnvironmentBase(self.fake_cli(['create', '--template-file', dummy_value])) + base = eb.EnvironmentBase(self.fake_cli(['create', '--template-file', dummy_value]), is_silent=True) base.init_action(is_silent=True) base.load_config() self.assertEqual(base.config['global']['environment_name'], dummy_value) def test_config_file_flag(self): dummy_value = 'dummy' - base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', dummy_value])) + base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', dummy_value]), is_silent=True) base.init_action(is_silent=True) self.assertTrue(os.path.isfile(dummy_value)) def test_factory_default(self): with self.assertRaises(Exception): - base = eb.EnvironmentBase(self.fake_cli(['init'])) + base = eb.EnvironmentBase(self.fake_cli(['init']), is_silent=True) base.load_config() # Create refs to files that should be created and make sure they don't already exists @@ -321,14 +318,14 @@ def test_factory_default(self): self.assertFalse(os.path.isfile(ami_cache_file)) # Verify that create_missing_files works as intended - base = eb.EnvironmentBase(self.fake_cli(['init'])) + base = eb.EnvironmentBase(self.fake_cli(['init']), is_silent=True) base.init_action(is_silent=True) self.assertTrue(os.path.isfile(config_file)) # TODO: After ami_cache is updated change 'create_missing_files' to be singular # self.assertTrue(os.path.isfile(ami_cache_file)) # Verify that the previously created files are loaded up correctly - eb.EnvironmentBase(self.fake_cli(['create'])) + eb.EnvironmentBase(self.fake_cli(['create']), is_silent=True) # The following two tests use a create_action, which currently doesn't test correctly diff --git a/src/tests/test_resources.py b/src/tests/test_resources.py index e75519c..6801837 100644 --- a/src/tests/test_resources.py +++ b/src/tests/test_resources.py @@ -1,5 +1,6 @@ from unittest2 import TestCase from environmentbase.resources import Res +from environmentbase.template import Template from tempfile import mkdtemp import shutil import os @@ -101,7 +102,7 @@ def test_generate_config(self): } } - class CustomHandler(object): + class CustomHandler(Template): @staticmethod def get_factory_defaults(): return custom_config_addition @@ -120,8 +121,7 @@ def get_config_schema(): "Mappings": "mappings.json", "Resources": "resources.json", "Outputs": "output.json" - }, - config_handlers=[CustomHandler()] + } ) # Make sure all the extracted files exist From ec8aff4042b5ff114af85ebd8525f14566042a95 Mon Sep 17 00:00:00 2001 From: Eric Price Date: Fri, 19 Feb 2016 12:27:05 -0800 Subject: [PATCH 04/10] Removed EnvConfig from DEVELOPMENT.md --- DEVELOPMENT.md | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7e0b63f..0bfd8f2 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -9,9 +9,9 @@ python setup.py develop If you do not plan on modifying the code and will simply be using it, instead run: ``` python setup.py install -``` +``` -If you have the AWS CLI, you can run `aws configure` to generate the credentials files in the appropriate place. If you have already configured the AWS CLI, then no further steps are necessary. +If you have the AWS CLI, you can run `aws configure` to generate the credentials files in the appropriate place. If you have already configured the AWS CLI, then no further steps are necessary. You must ensure that the account you are authenticating with has at least the following permissions: @@ -20,7 +20,7 @@ You must ensure that the account you are authenticating with has at least the fo "ec2:DescribeRegions"], "Effect": "Allow", "Resource": "*" }]} ``` -This is required to perform the VPC lookups. +This is required to perform the VPC lookups. Run/Test/Clean @@ -40,7 +40,7 @@ python setup.py test python setup.py clean —-all ``` -Note: *.pyc files will be regenerated in src whenever you run the test suite but as long as they are git ignored it’s not a big deal. You can still remove them with `rm src/**/*.pyc` +Note: *.pyc files will be regenerated in src whenever you run the test suite but as long as they are git ignored it’s not a big deal. You can still remove them with `rm src/**/*.pyc` Getting started @@ -61,24 +61,23 @@ class MyEnv(NetworkBase): self.add_child_template(Bastion()) if __name__ == '__main__': - my_config = EnvConfig(config_handlers=[Bastion]) - MyEnv(env_config=my_config) + MyEnv() ``` To generate the cloudformation template for this python code, save the above snippet in a file called `my_env.py` and run `python my_env.py init`. -This will look at the patterns passed into the EnvConfig object and generate a config.json file with the relevant fields added. Fill this config file out, adding values for at least the following fields: +This will generate a config.json using the default config (and any loaded subclasses of Template which extend the default config) file with the relevant fields added. Fill this config file out, adding values for at least the following fields: -`template : ec2_key_default` - SSH key used to log into your EC2 instances -`template : s3_bucket` - S3 bucket used to upload the generated cloudformation templates +`template : ec2_key_default` - SSH key used to log into your EC2 instances +`template : s3_bucket` - S3 bucket used to upload the generated cloudformation templates Next run `python my_env.py create` to generate the cloudformation template using the updated config. Since we overrode environmentbase's `create_hook` function, this will hook into environmentbase's create action and add the bastion stack and any other resources you specified. -NOTE: You can also override config values using environment variables. You can create env variables using the format: -`
_` in all caps (e.g. `TEMPLATE_EC2_KEY_DEFAULT`) +NOTE: You can also override config values using environment variables. You can create env variables using the format: +`
_` in all caps (e.g. `TEMPLATE_EC2_KEY_DEFAULT`) -These are read in after the config file is loaded, so will override any values in your config.json +These are read in after the config file is loaded, so will override any values in your config.json Then run `python my_env.py deploy` to create the stack on [cloudformation](https://console.aws.amazon.com/cloudformation/) @@ -102,12 +101,12 @@ By default, networkbase will create one public and one private subnet for each a ], "subnet_config": [ { - "type": "public", + "type": "public", "size": "18", "name": "public" }, { - "type": "private", + "type": "private", "size": "22", "name": "private" }, @@ -136,18 +135,18 @@ Extension point for modifying behavior of delete action. Called after config is Extension point for reacting to the cloudformation stack event stream. If global.monitor_stack is enabled in config this function is used to react to stack events. Once a stack is created a notification topic will begin emitting events to a queue. Each event is passed to this call for further processing. The return value is used to indicate whether processing is complete (true indicates processing is complete, false indicates you are not yet done). Details about the event data can be read [here](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-listing-event-history.html) -The event_data hash provided the following mappings from the raw cloudformation event: -status = ResourceStatus -type = ResourceType -name = LogicalResourceId -reason = ResourceStatusReason -props = ResourceProperties +The event_data hash provided the following mappings from the raw cloudformation event: +status = ResourceStatus +type = ResourceType +name = LogicalResourceId +reason = ResourceStatusReason +props = ResourceProperties ## Dealing with versions 1. Basic commands - Reading tag information + Reading tag information - List all tags (apply across all branches): `git tag -l` @@ -197,7 +196,7 @@ props = ResourceProperties 2. Make code changes, test .. etc (merge changes from master into this branch) Remember to update: src/environmentbase/version.py to the same value as the branch name. 3. Merge into master - 4. Delete the version branch + 4. Delete the version branch 5. Create tag on master with same name as branch you just deleted. - + Note that the branch is deleted **before** the tag is created because they share the same name. Otherwise referring to things by the version name may be ambigious. From 0fc7b74ab3ed549a33fdb88708032780e8396c9d Mon Sep 17 00:00:00 2001 From: Eric Price Date: Fri, 19 Feb 2016 13:39:24 -0800 Subject: [PATCH 05/10] Added utility tests for Template loading --- src/tests/test_utility.py | 60 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/tests/test_utility.py diff --git a/src/tests/test_utility.py b/src/tests/test_utility.py new file mode 100644 index 0000000..db4e082 --- /dev/null +++ b/src/tests/test_utility.py @@ -0,0 +1,60 @@ +from unittest2 import TestCase +from environmentbase import utility +from environmentbase.template import Template + + +class Parent(object): + pass + + +class A(Parent): + pass + + +class B(Parent): + pass + + +class C(Parent): + pass + + +class MyTemplate(Template): + @staticmethod + def get_factory_defaults(): + return {'new_section': {'new_key': 'value'}} + + @staticmethod + def get_config_schema(): + return {'new_section': {'new_key': 'basestring'}} + + +class UtilityTestCase(TestCase): + + def test__get_subclasses_of(self): + actual_subclasses = [A, B, C] + retreived_subclasses = utility._get_subclasses_of('tests.test_utility', 'Parent') + self.assertEqual(actual_subclasses, retreived_subclasses) + + def test_get_pattern_list(self): + # Count patterns from previous test runs (no way to unload classes as far as I know) + num_patterns = len(utility.get_pattern_list()) + + # Verify that a loaded a pattern is identified + mod = __import__('environmentbase.patterns.bastion', fromlist=['Bastion']) + klazz = getattr(mod, 'Bastion') + patterns = utility.get_pattern_list() + + self.assertGreater(len(patterns), num_patterns) + self.assertIn(klazz, patterns) + + def test__update_from_patterns(self): + _dict = {} + utility._update_from_patterns(_dict, 'get_factory_defaults') + self.assertIn('new_section', _dict) + self.assertEqual('value', _dict['new_section']['new_key']) + + _dict = {} + utility._update_from_patterns(_dict, 'get_config_schema') + self.assertIn('new_section', _dict) + self.assertEqual('basestring', _dict['new_section']['new_key']) From c7df51adea9fa86b7f8bb948d212637a598b4f00 Mon Sep 17 00:00:00 2001 From: Eric Price Date: Fri, 19 Feb 2016 15:32:41 -0800 Subject: [PATCH 06/10] Fixed other example files --- src/examples/child_stack.py | 11 ++--------- src/examples/nested_child_stack.py | 18 +++++++----------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/examples/child_stack.py b/src/examples/child_stack.py index 3f4250f..d8451ae 100644 --- a/src/examples/child_stack.py +++ b/src/examples/child_stack.py @@ -1,6 +1,5 @@ from environmentbase.networkbase import NetworkBase from environmentbase.template import Template -from environmentbase.environmentbase import EnvConfig from troposphere import ec2 @@ -19,7 +18,7 @@ class MyChildTemplate(Template): # Called from add_child_template() after some common parameters are attached to this instance, see docs for details def build_hook(self): - self.add_resource(ec2.Instance("ec2instance", InstanceType="m3.medium", ImageId="ami-e7527ed7") ) + self.add_resource(ec2.Instance("ec2instance", InstanceType="m3.medium", ImageId="ami-e7527ed7")) # When no config.json file exists a new one is created using the 'factory default' file. This function # augments the factory default before it is written to file with the config values required @@ -35,10 +34,4 @@ def get_config_schema(): return {'my_child_template': {'favorite_color': 'str'}} if __name__ == '__main__': - - # EnvConfig holds references to handler classes used to extend certain functionality - # of EnvironmentBase. The config_handlers list takes any class that implements - # get_factory_defaults() and get_config_schema(). - env_config = EnvConfig(config_handlers=[MyChildTemplate]) - - MyRootTemplate(env_config=env_config) + MyRootTemplate() diff --git a/src/examples/nested_child_stack.py b/src/examples/nested_child_stack.py index 96f20df..c74cd37 100644 --- a/src/examples/nested_child_stack.py +++ b/src/examples/nested_child_stack.py @@ -1,6 +1,5 @@ from environmentbase.networkbase import NetworkBase from environmentbase.template import Template -from environmentbase.environmentbase import EnvConfig from environmentbase.patterns.bastion import Bastion from troposphere import ec2 @@ -10,6 +9,9 @@ class Controller(NetworkBase): Class creates a VPC and common network components for the environment """ def create_hook(self): + super(Controller, self).create_hook() + + # print self.template.subnets self.add_child_template(ChildTemplate('Child')) self.add_child_template(Bastion('Bastion')) @@ -26,10 +28,10 @@ def build_hook(self): InstanceType="m3.medium", ImageId="ami-e7527ed7", KeyName=self.ec2_key, - SubnetId=self.subnets['private'][0], + SubnetId=self.subnets['private']['private'][0], SecurityGroupIds=[self.common_security_group] )) - self.add_child_template(GrandchildTemplate('Grandchild')) + # self.add_child_template(GrandchildTemplate('Grandchild')) # When no config.json file exists a new one is created using the 'factory default' file. This function # augments the factory default before it is written to file with the config values required @@ -52,14 +54,8 @@ def build_hook(self): InstanceType="m3.medium", ImageId="ami-e7527ed7", KeyName=self.ec2_key, - SubnetId=self.subnets['private'][0], + SubnetId=self.subnets['private']['private'][0], SecurityGroupIds=[self.common_security_group])) if __name__ == '__main__': - - # EnvConfig holds references to handler classes used to extend certain functionality - # of EnvironmentBase. The config_handlers list takes any class that implements - # get_factory_defaults() and get_config_schema(). - env_config = EnvConfig(config_handlers=[ChildTemplate]) - - Controller(env_config=env_config) + Controller() From e9ce0e0ce1ba3a0896147aea87017cf3b2275563 Mon Sep 17 00:00:00 2001 From: Eric Price Date: Fri, 19 Feb 2016 19:11:58 -0800 Subject: [PATCH 07/10] initial autowire feature --- src/environmentbase/environmentbase.py | 17 +++++++++++++++++ src/environmentbase/template.py | 3 +++ 2 files changed, 20 insertions(+) diff --git a/src/environmentbase/environmentbase.py b/src/environmentbase/environmentbase.py index 7a69441..d5440be 100644 --- a/src/environmentbase/environmentbase.py +++ b/src/environmentbase/environmentbase.py @@ -230,6 +230,22 @@ def _root_template_url(self): bucket_name=self.template_args.get('s3_bucket'), resource_path=self._root_template_path()) + def load_runtime_config(self): + """ + For patterns defining custom config sections bind the loaded config values to associated class. + For class TestPattern whose get_factory_defaults() returns { "fav_color": "red" } will be able + to retreive the loaded value from the build_hook() (e.g. TestPattern.runtime_config['fav_color']) + """ + pattern_classes = utility.get_pattern_list() + for cls in pattern_classes: + runtime_config = {} + default_config = cls.get_factory_defaults() + for key in default_config.keys(): + value = self.config[key] + runtime_config[key] = value + + cls.runtime_config = runtime_config + def create_action(self): """ Default create_action invoked by the CLI @@ -237,6 +253,7 @@ def create_action(self): Override the create_hook in your environment to inject all of your cloudformation resources """ self.load_config() + self.load_runtime_config() self.initialize_template() # Do custom troposphere resource creation in your overridden copy of this method diff --git a/src/environmentbase/template.py b/src/environmentbase/template.py index 6812543..67487ac 100644 --- a/src/environmentbase/template.py +++ b/src/environmentbase/template.py @@ -47,6 +47,9 @@ class Template(t.Template): # Region to deploy to region = '' + # Custom config loaded by EnvironmentBase.load_pattern_configs() + runtime_config = {} + ARCH_MAP = "InstanceTypeToArch" IMAGE_MAP_PREFIX = "ImageMapFor" From 7d6ef895e6a939bc7243b45dbed4a7db310054c1 Mon Sep 17 00:00:00 2001 From: Eric Price Date: Sat, 20 Feb 2016 08:54:54 -0800 Subject: [PATCH 08/10] Added test_load_runtime_config() to test autowiring patterns --- src/tests/test_environmentbase.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/tests/test_environmentbase.py b/src/tests/test_environmentbase.py index e36738c..985a829 100644 --- a/src/tests/test_environmentbase.py +++ b/src/tests/test_environmentbase.py @@ -12,6 +12,17 @@ from environmentbase import networkbase import environmentbase.patterns.ha_nat from troposphere import ec2 +from environmentbase.template import Template + + +class MyTemplate(Template): + @staticmethod + def get_factory_defaults(): + return {'new_section': {'new_key': 'value'}} + + @staticmethod + def get_config_schema(): + return {'new_section': {'new_key': 'basestring'}} class EnvironmentBaseTestCase(TestCase): @@ -327,6 +338,16 @@ def test_factory_default(self): # Verify that the previously created files are loaded up correctly eb.EnvironmentBase(self.fake_cli(['create']), is_silent=True) + def test_load_runtime_config(self): + base = eb.EnvironmentBase(self.fake_cli(['create']), is_silent=True) + base.init_action(is_silent=True) + base.load_config() + + # verify that config section is attached to the class + base.config['new_section']['new_key'] = 'different_value' + base.load_runtime_config() + self.assertTrue(MyTemplate.runtime_config['new_section']['new_key'], 'different_value') + # The following two tests use a create_action, which currently doesn't test correctly # def test_controller_subclass(self): From 0dde126ff6328aef4a6c7653101983cc18f4ac7f Mon Sep 17 00:00:00 2001 From: Eric Price Date: Sat, 20 Feb 2016 18:23:50 -0800 Subject: [PATCH 09/10] Clean up network_base and bastion using runtime_config --- src/environmentbase/networkbase.py | 7 +--- src/environmentbase/patterns/base_network.py | 9 ++--- src/environmentbase/patterns/bastion.py | 41 +++++++++++--------- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/environmentbase/networkbase.py b/src/environmentbase/networkbase.py index 03ad828..ebbd5f1 100755 --- a/src/environmentbase/networkbase.py +++ b/src/environmentbase/networkbase.py @@ -13,12 +13,7 @@ class NetworkBase(EnvironmentBase): def create_hook(self): super(NetworkBase, self).create_hook() - network_config = self.config.get('network', {}) - boto_config = self.config.get('boto', {}) - nat_config = self.config.get('nat') - region_name = boto_config['region_name'] - - base_network_template = BaseNetwork('BaseNetwork', network_config, region_name, nat_config) + base_network_template = BaseNetwork('BaseNetwork') self.add_child_template(base_network_template) self.template._subnets = base_network_template._subnets.copy() diff --git a/src/environmentbase/patterns/base_network.py b/src/environmentbase/patterns/base_network.py index 2ef94d8..02ce153 100644 --- a/src/environmentbase/patterns/base_network.py +++ b/src/environmentbase/patterns/base_network.py @@ -73,13 +73,12 @@ def get_factory_defaults(): def get_config_schema(): return BaseNetwork.CONFIG_SCHEMA - def __init__(self, template_name, network_config, region_name, nat_config): + def __init__(self, template_name): super(BaseNetwork, self).__init__(template_name) - self.network_config = network_config - self.nat_config = nat_config - self.az_count = int(network_config.get('az_count', '2')) - self.region_name = region_name + self.network_config = self.runtime_config['network'] + self.nat_config = self.runtime_config['nat'] + self.az_count = int(self.network_config.get('az_count', '2')) self.stack_outputs = {} # Simple mapping of AZs to NATs, to prevent creating duplicates diff --git a/src/environmentbase/patterns/bastion.py b/src/environmentbase/patterns/bastion.py index b4c08dd..9ce6898 100644 --- a/src/environmentbase/patterns/bastion.py +++ b/src/environmentbase/patterns/bastion.py @@ -9,18 +9,12 @@ class Bastion(Template): Adds a bastion host within a given deployment based on environemntbase. """ - SUGGESTED_INSTANCE_TYPES = [ - "m1.small", "t2.micro", "t2.small", "t2.medium", - "m3.medium", - "c3.large", "c3.2xlarge" - ] - def __init__(self, name='bastion', - ingress_port='2222', - access_cidr='0.0.0.0/0', - default_instance_type='t2.micro', - suggested_instance_types=SUGGESTED_INSTANCE_TYPES, + ingress_port=None, + access_cidr=None, + default_instance_type=None, + suggested_instance_types=None, user_data=None): """ Method initializes bastion host in a given environment deployment @@ -29,15 +23,20 @@ def __init__(self, More info here: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-elb-listener.html @param access_cidr [string] - CIDR notation for external access to this tier. @param user_data [string] - User data to to initialize the bastion hosts. + @param default_instance_type [string] EC2 instance type name + @param suggested_instance_types [list] List of EC2 instance types available for selection in CloudFormation """ self.name = name - self.ingress_port = ingress_port - self.access_cidr = access_cidr - self.default_instance_type = default_instance_type - self.suggested_instance_types = suggested_instance_types self.user_data = user_data + # Let constructor parameters override runtime config settings + cfg = self.runtime_config['bastion'] + self.ingress_port = ingress_port or cfg['ingress_port'] + self.access_cidr = access_cidr or cfg['remote_access_cidr'] + self.default_instance_type = default_instance_type or cfg['default_instance_type'] + self.suggested_instance_types = suggested_instance_types or cfg['suggested_instance_types'] + super(Bastion, self).__init__(template_name=name) # Called after add_child_template() has attached common parameters and some instance attributes: @@ -72,7 +71,7 @@ def build_hook(self): utility_bucket=self.utility_bucket ) - bastion_asg = self.add_asg( + self.add_asg( layer_name=self.name, security_groups=[security_groups['bastion'], self.common_security_group], load_balancer=bastion_elb, @@ -88,7 +87,7 @@ def build_hook(self): self.add_output(Output( 'BastionELBDNSZoneId', - Value=GetAtt(bastion_elb, 'CanonicalHostedZoneNameID') + Value=GetAtt(bastion_elb, 'CanonicalHostedZoneNameID') )) self.add_output(Output( @@ -99,7 +98,12 @@ def build_hook(self): @staticmethod def get_factory_defaults(): return {"bastion": { - "instance_type": "t2.micro", + "default_instance_type": "t2.micro", + "suggested_instance_types": [ + "m1.small", "t2.micro", "t2.small", "t2.medium", + "m3.medium", + "c3.large", "c3.2xlarge" + ], "remote_access_cidr": "0.0.0.0/0", "ingress_port": 2222 }} @@ -107,7 +111,8 @@ def get_factory_defaults(): @staticmethod def get_config_schema(): return {"bastion": { - "instance_type": "str", + "default_instance_type": "str", + "suggested_instance_types": "list", "remote_access_cidr": "str", "ingress_port": "int" }} From 3b824b20d5d543048cc652b3dfcfedd616446e6f Mon Sep 17 00:00:00 2001 From: Eric Price Date: Sun, 21 Feb 2016 14:26:38 -0800 Subject: [PATCH 10/10] Updated RDS to use runtime_config --- src/environmentbase/patterns/rds.py | 95 ++++++++++++++--------------- 1 file changed, 46 insertions(+), 49 deletions(-) diff --git a/src/environmentbase/patterns/rds.py b/src/environmentbase/patterns/rds.py index 82c2a4f..2439eb8 100644 --- a/src/environmentbase/patterns/rds.py +++ b/src/environmentbase/patterns/rds.py @@ -1,7 +1,6 @@ from environmentbase.template import Template import environmentbase.resources as res from environmentbase.networkbase import NetworkBase -from environmentbase.environmentbase import EnvConfig from troposphere import Ref, Parameter, GetAtt, Output, Join, rds, ec2 @@ -59,7 +58,7 @@ def __init__(self, connect_from_cidr=None, connect_from_sg=None, subnet_set='private', - config_map=DEFAULT_CONFIG['db']): + config_map=None): """ Method initializes host in a given environment deployment @param tier_name: [string] - name of the tier to assign @@ -70,7 +69,7 @@ def __init__(self, """ self.tier_name = tier_name - self.config_map = config_map + self.config_map = config_map or self.runtime_config['db'] self.subnet_set = subnet_set if connect_from_cidr and connect_from_sg: @@ -247,64 +246,62 @@ class Controller(NetworkBase): > mysql -h -P -u -p """ + # For this example override the loaded config with the config below + db_config = { + "label1": { + "db_instance_type_default": "db.m1.small", + "rds_user_name": "defaultusername", + # Actual database name, cannot include non-alphanumeric characters (e.g. "-") + "master_db_name": "mydb", + "volume_size": 100, + "backup_retention_period": 30, + "rds_engine": "MySQL", + # 5.6.19 is no longer supported + "rds_engine_version": "5.6.22", + "preferred_backup_window": "02:00-02:30", + "preferred_maintenance_window": "sun:03:00-sun:04:00", + # Name of vm snapshot to use, empty string ("") means don't use an old snapshot + # Note: "master_db_name" value will be overridden if snapshot_id is non-empty + "snapshot_id": "", + "password": "changeme111111111111" + }, + "label2": { + "db_instance_type_default": "db.m1.small", + "rds_user_name": "defaultusername", + # Actual database name, cannot include non-alphanumeric characters (e.g. "-") + "master_db_name": "mydb2", + "volume_size": 100, + "backup_retention_period": 30, + "rds_engine": "MySQL", + # 5.6.19 is no longer supported + "rds_engine_version": "5.6.22", + "preferred_backup_window": "02:00-02:30", + "preferred_maintenance_window": "sun:03:00-sun:04:00", + # Name of vm snapshot to use, empty string ("") means don't use an old snapshot + # Note: "master_db_name" value will be overridden if snapshot_id is non-empty + "snapshot_id": "", + "password": "changeme1111111111111" + } + } + def create_hook(self): + super(Controller, self).create_hook() + # Create the rds instance pattern (includes standard standard parameters) my_db = RDS( 'dbTier', - subnet_set=self.template.subnets['private'].keys()[0], - config_map=db_config) + subnet_set='private', + config_map=self.db_config) # Attach pattern as a child template self.add_child_template(my_db) def deploy_hook(self): - for db_label, db_config in self.config['db'].iteritems(): + for db_label, db_config in self.db_config.iteritems(): db_resource_name = db_label.lower() + 'dbTier'.title() + 'RdsMasterUserPassword' print "adding ", db_resource_name self.add_parameter_binding(key=db_resource_name, value=db_config['password']) if __name__ == '__main__': - - db_config = { - 'label1': { - 'db_instance_type_default': 'db.m1.small', - 'rds_user_name': 'defaultusername', - # Actual database name, cannot include non-alphanumeric characters (e.g. '-') - 'master_db_name': 'mydb', - 'volume_size': 100, - 'backup_retention_period': 30, - 'rds_engine': 'MySQL', - # 5.6.19 is no longer supported - 'rds_engine_version': '5.6.22', - 'preferred_backup_window': '02:00-02:30', - 'preferred_maintenance_window': 'sun:03:00-sun:04:00', - # Name of vm snapshot to use, empty string ('') means don't use an old snapshot - # Note: 'master_db_name' value will be overridden if snapshot_id is non-empty - 'snapshot_id': '', - 'password': 'changeme111111111111' - }, - 'label2': { - 'db_instance_type_default': 'db.m1.small', - 'rds_user_name': 'defaultusername', - # Actual database name, cannot include non-alphanumeric characters (e.g. '-') - 'master_db_name': 'mydb2', - 'volume_size': 100, - 'backup_retention_period': 30, - 'rds_engine': 'MySQL', - # 5.6.19 is no longer supported - 'rds_engine_version': '5.6.22', - 'preferred_backup_window': '02:00-02:30', - 'preferred_maintenance_window': 'sun:03:00-sun:04:00', - # Name of vm snapshot to use, empty string ('') means don't use an old snapshot - # Note: 'master_db_name' value will be overridden if snapshot_id is non-empty - 'snapshot_id': '', - 'password': 'changeme1111111111111' - } - } - - my_config = res.FACTORY_DEFAULT_CONFIG - my_config['db'] = db_config - - env_config = EnvConfig(config_handlers=[RDS]) - Controller(env_config=env_config, config_file_override=my_config) + Controller()