From deeaec68a987fe362b2cc525ea2c52c5048b101f Mon Sep 17 00:00:00 2001 From: Ratnadeep Debnath Date: Tue, 15 Dec 2015 11:46:49 +0530 Subject: [PATCH 1/5] Implement atomic deployments for Nulecule application. Fixes #421 When there's an error during running a Nulecule application, rollback the changes made by stopping the application. --- atomicapp/nulecule/base.py | 8 +++++--- atomicapp/nulecule/main.py | 15 ++++++++++++--- atomicapp/plugin.py | 1 + atomicapp/providers/docker.py | 6 +++++- atomicapp/providers/kubernetes.py | 8 ++++++-- 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/atomicapp/nulecule/base.py b/atomicapp/nulecule/base.py index 3c0ace08..28bb6d87 100644 --- a/atomicapp/nulecule/base.py +++ b/atomicapp/nulecule/base.py @@ -160,7 +160,7 @@ def run(self, provider_key=None, dryrun=False): for component in self.components: component.run(provider_key, dryrun) - def stop(self, provider_key=None, dryrun=False): + def stop(self, provider_key=None, dryrun=False, ignore_errors=False): """ Stop the Nulecule application. @@ -168,6 +168,7 @@ def stop(self, provider_key=None, dryrun=False): provider_key (str): Provider to use for running Nulecule application dryrun (bool): Do not make changes to host when True + ignore_errors (bool): Ignore errors, if any, when True Returns: None @@ -175,7 +176,7 @@ def stop(self, provider_key=None, dryrun=False): provider_key, provider = self.get_provider(provider_key, dryrun) # stop the Nulecule application for component in self.components: - component.stop(provider_key, dryrun) + component.stop(provider_key, dryrun, ignore_errors) def load_config(self, config=None, ask=False, skip_asking=False): """ @@ -290,7 +291,7 @@ def run(self, provider_key, dryrun=False): provider.init() provider.run() - def stop(self, provider_key=None, dryrun=False): + def stop(self, provider_key=None, dryrun=False, ignore_errors=False): """ Stop the Nulecule component with the specified provider. """ @@ -300,6 +301,7 @@ def stop(self, provider_key=None, dryrun=False): provider_key, provider = self.get_provider(provider_key, dryrun) provider.artifacts = self.rendered_artifacts.get(provider_key, []) provider.init() + provider.ignore_errors = ignore_errors provider.stop() def load_config(self, config=None, ask=False, skip_asking=False): diff --git a/atomicapp/nulecule/main.py b/atomicapp/nulecule/main.py index b3c0586d..707a1419 100644 --- a/atomicapp/nulecule/main.py +++ b/atomicapp/nulecule/main.py @@ -231,7 +231,7 @@ def run(self, cli_provider, answers_output, ask, self.nulecule.load_config(config=self.nulecule.config, ask=ask) self.nulecule.render(cli_provider, dryrun) - self.nulecule.run(cli_provider, dryrun) + runtime_answers = self._get_runtime_answers( self.nulecule.config, cli_provider) self._write_answers( @@ -241,12 +241,21 @@ def run(self, cli_provider, answers_output, ask, self._write_answers(answers_output, runtime_answers, self.answers_format) - def stop(self, cli_provider, **kwargs): + try: + self.nulecule.run(cli_provider, dryrun) + except Exception as e: + logger.error('Application run error: %s' % e) + logger.debug('Nulecule run error: %s' % e, exc_info=True) + logger.info('Rolling back changes') + self.stop(cli_provider, ignore_errors=True, **kwargs) + + def stop(self, cli_provider, ignore_errors=False, **kwargs): """ Stops a running Nulecule application. Args: cli_provider (str): Provider running the Nulecule application + ignore_errors (bool): Ignore errors, if any, when True kwargs (dict): Extra keyword arguments """ # For stop we use the generated answer file from the run @@ -258,7 +267,7 @@ def stop(self, cli_provider, **kwargs): self.app_path, config=self.answers, dryrun=dryrun) self.nulecule.load_config(config=self.answers) self.nulecule.render(cli_provider, dryrun=dryrun) - self.nulecule.stop(cli_provider, dryrun) + self.nulecule.stop(cli_provider, dryrun, ignore_errors) def clean(self, force=False): # For future use diff --git a/atomicapp/plugin.py b/atomicapp/plugin.py index 0a245df9..344d5fda 100644 --- a/atomicapp/plugin.py +++ b/atomicapp/plugin.py @@ -53,6 +53,7 @@ def __init__(self, config, path, dryrun): self.config = config self.path = path self.dryrun = dryrun + self.ignore_errors = False if Utils.getRoot() == HOST_DIR: self.container = True diff --git a/atomicapp/providers/docker.py b/atomicapp/providers/docker.py index 6b116c8a..120404f9 100644 --- a/atomicapp/providers/docker.py +++ b/atomicapp/providers/docker.py @@ -138,4 +138,8 @@ def stop(self): if self.dryrun: logger.info("DRY-RUN: STOPPING CONTAINER %s", " ".join(cmd)) else: - subprocess.check_call(cmd) + try: + subprocess.check_call(cmd) + except Exception as e: + if not self.ignore_errors: + raise e diff --git a/atomicapp/providers/kubernetes.py b/atomicapp/providers/kubernetes.py index 600e28fa..9fda1cef 100644 --- a/atomicapp/providers/kubernetes.py +++ b/atomicapp/providers/kubernetes.py @@ -107,8 +107,12 @@ def _call(self, cmd): if self.dryrun: logger.info("DRY-RUN: %s", " ".join(cmd)) else: - ec, stdout, stderr = Utils.run_cmd(cmd, checkexitcode=True) - return stdout + try: + ec, stdout, stderr = Utils.run_cmd(cmd, checkexitcode=True) + return stdout + except Exception as e: + if not self.ignore_errors: + raise e def process_k8s_artifacts(self): """Processes Kubernetes manifests files and checks if manifest under From 0afcf1f3e5dcc2d62f8e8c1cc5cd81993c29718c Mon Sep 17 00:00:00 2001 From: Ratnadeep Debnath Date: Tue, 22 Dec 2015 20:13:22 +0530 Subject: [PATCH 2/5] Fixed unittests for refactored NuleculeComponent.stop() signature Fixes #421 --- tests/units/nulecule/test_nulecule.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/units/nulecule/test_nulecule.py b/tests/units/nulecule/test_nulecule.py index f35119e3..f8e283cb 100644 --- a/tests/units/nulecule/test_nulecule.py +++ b/tests/units/nulecule/test_nulecule.py @@ -28,6 +28,7 @@ class TestNuleculeStop(unittest.TestCase): def test_stop(self): provider = 'docker' dryrun = False + ignore_errors = False mock_component_1 = mock.Mock() mock_component_2 = mock.Mock() @@ -35,8 +36,10 @@ def test_stop(self): n.components = [mock_component_1, mock_component_2] n.stop(provider) - mock_component_1.stop.assert_called_once_with(provider, dryrun) - mock_component_2.stop.assert_called_once_with(provider, dryrun) + mock_component_1.stop.assert_called_once_with( + provider, dryrun, ignore_errors) + mock_component_2.stop.assert_called_once_with( + provider, dryrun, ignore_errors) class TestNuleculeLoadConfig(unittest.TestCase): From 1153b29ee1ce2303dde478e1c6ee1dded8a59dcd Mon Sep 17 00:00:00 2001 From: Ratnadeep Debnath Date: Sat, 9 Jan 2016 16:07:49 +0530 Subject: [PATCH 3/5] Implement atomic deployments for openshift provider. Fixes #421 --- atomicapp/providers/openshift.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/atomicapp/providers/openshift.py b/atomicapp/providers/openshift.py index 3695d12f..13fe4858 100644 --- a/atomicapp/providers/openshift.py +++ b/atomicapp/providers/openshift.py @@ -461,7 +461,11 @@ def stop(self): if self.dryrun: logger.info("DRY-RUN: DELETE %s", url) else: - self.oc.delete(url) + try: + self.oc.delete(url) + except Exception as e: + if not self.ignore_errors: + raise e def _process_artifacts(self): """ From 37b31c9b22dfb30b5110bd4fca0f2e1ad3267f11 Mon Sep 17 00:00:00 2001 From: Ratnadeep Debnath Date: Tue, 19 Jan 2016 15:13:23 +0530 Subject: [PATCH 4/5] Implemented atomic deployments for marathon provider. --- atomicapp/providers/marathon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/atomicapp/providers/marathon.py b/atomicapp/providers/marathon.py index 40d2d7f7..02162b25 100644 --- a/atomicapp/providers/marathon.py +++ b/atomicapp/providers/marathon.py @@ -97,7 +97,8 @@ def stop(self): msg = "Error deleting app: %s, Marathon API response %s - %s" % ( artifact["id"], status_code, return_data) logger.error(msg) - raise ProviderFailedException(msg) + if not self.ignore_errors: + raise ProviderFailedException(msg) def _process_artifacts(self): """ Parse and validate Marathon artifacts From 2ffb11c1487dede64b6993ff0d00c44534cab103 Mon Sep 17 00:00:00 2001 From: Ratnadeep Debnath Date: Tue, 19 Jan 2016 21:13:35 +0530 Subject: [PATCH 5/5] Raise an error when rolling back an application deployment. Fixes #421 --- atomicapp/nulecule/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/atomicapp/nulecule/main.py b/atomicapp/nulecule/main.py index 707a1419..f9d6ef19 100644 --- a/atomicapp/nulecule/main.py +++ b/atomicapp/nulecule/main.py @@ -248,6 +248,7 @@ def run(self, cli_provider, answers_output, ask, logger.debug('Nulecule run error: %s' % e, exc_info=True) logger.info('Rolling back changes') self.stop(cli_provider, ignore_errors=True, **kwargs) + raise NuleculeException('Rolled back changes.') def stop(self, cli_provider, ignore_errors=False, **kwargs): """