From 7e695d9217e7dad153986faa6aaf5231a5c9a308 Mon Sep 17 00:00:00 2001 From: Paul Pollack Date: Wed, 11 Jul 2018 14:05:34 -0400 Subject: [PATCH 1/4] fix up templates --- tf_from_asg.py | 49 +++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/tf_from_asg.py b/tf_from_asg.py index 6ba6d2b..c5bc1b4 100644 --- a/tf_from_asg.py +++ b/tf_from_asg.py @@ -8,23 +8,26 @@ module "{{ MODULE_NAME }}" { source = "../modules/autoscaling/qw_asg" - key_name = "${var.key_name}" + key_name = "{{ KEY_NAME }}" asg_cluster = "{{ ASG_CLUSTER }}" - zones = "${var.default_zones}" + zones = "${var.legacy_zones}" asg_queue = "{{ QUEUE_NAME }}" asg_name = "{{ ASG_NAME }}" worker_security_group = ["${var.worker_security_group}"] consumer_config = "{{ CONSUMER_CONFIG }}" r53_zone = "${var.r53_zone}" hosted_domain = "${var.hosted_domain}" - vpc_subnet_ids = ["${split(",", join(",", var.vpc_public_subnet_ids))}"] + vpc_subnet_ids = ["${aws_subnet.qw.*.id}"] name_tag = "${var.name_tag}" - zookeeper_dns = "${module.zookeeper.zookeeper_dns}" + zookeeper_dns = "${var.zookeeper_dns}" env = "${var.env}" app_branch = "${var.app_branch}" asg_desired = "{{ ASG_DESIRED }}" asg_max = "{{ ASG_MAX }}" asg_min = "{{ ASG_MIN }}" + instance_type = "{{ INSTANCE_TYPE }}" + qw_ami = "{{ AMI_ID }}" + iam_role_name = "AutoscalingEc2InstanceRole" deregistration_arn = "{{ DEREGISTRATION_ARN }}" lifecycle_hook_arn = "{{ LIFECYCLE_HOOK_ARN }}" } @@ -89,9 +92,11 @@ def get_autoscaling_information(autoscaling_client, asg_name_prefix): for response in asg_iterator: for asg_response in response['AutoScalingGroups']: if asg_response['AutoScalingGroupName'].startswith(asg_name_prefix): - lifecycle_hook_response = autoscaling_client.describe_lifecycle_hooks( - AutoScalingGroupName=asg_response['AutoScalingGroupName'] - ) + launch_config_response = autoscaling_client.describe_launch_configurations( + LaunchConfigurationNames=[ + asg_response['LaunchConfigurationName'] + ] + )['LaunchConfigurations'][0] asgs_to_process[asg_response['AutoScalingGroupName']] = { 'name' : asg_response['AutoScalingGroupName'], 'tags' : asg_response['Tags'], @@ -99,6 +104,9 @@ def get_autoscaling_information(autoscaling_client, asg_name_prefix): 'asg_min' : asg_response['MinSize'], 'asg_max' : asg_response['MaxSize'], 'asg_desired' : asg_response['DesiredCapacity'], + 'instance_type' : launch_config_response['InstanceType'], + 'key_name' : launch_config_response['KeyName'], + 'ami_id' : launch_config_response['ImageId'] } lc_to_asg_name = {asg['lc_name'] : asg['name'] for asg in asgs_to_process.values()} @@ -129,17 +137,22 @@ def generate_tf_for_asg(asg_info, template): # TODO: Think about whether or not we can make this more reusable - right now it's heavily tied to the current qw module implementation dns_cluster_name = get_dns_safe_cluster_name(asg_info) cluster_name = get_cluster_name(asg_info) - asg_context = {'MODULE_NAME' : cluster_name, - 'ASG_CLUSTER' : dns_cluster_name, - 'CONSUMER_CONFIG' : asg_info['lc_info'], - 'ASG_NAME' : asg_info['name'], - 'QUEUE_NAME' : get_queue_from_info(asg_info), - 'ASG_MIN' : asg_info['asg_min'], - 'ASG_MAX' : asg_info['asg_max'], - 'ASG_DESIRED' : asg_info['asg_desired'], - 'DEREGISTRATION_ARN' : DEREGISTRATION_ARN, - 'LIFECYCLE_HOOK_ARN' : LIFECYCLE_HOOK_ARN - } + asg_context = { + 'MODULE_NAME' : cluster_name, + 'ASG_CLUSTER' : dns_cluster_name, + 'CONSUMER_CONFIG' : asg_info['lc_info'], + 'ASG_NAME' : asg_info['name'], + 'QUEUE_NAME' : get_queue_from_info(asg_info), + 'ASG_MIN' : asg_info['asg_min'], + 'ASG_MAX' : asg_info['asg_max'], + 'ASG_DESIRED' : asg_info['asg_desired'], + 'DEREGISTRATION_ARN' : DEREGISTRATION_ARN, + 'LIFECYCLE_HOOK_ARN' : LIFECYCLE_HOOK_ARN, + 'INSTANCE_TYPE' : asg_info['instance_type'], + 'KEY_NAME' : asg_info['key_name'], + 'AMI_ID' : asg_info['ami_id'] + } + return template.render(**asg_context) def get_queue_from_info(asg_info): From 22e46aaa0f09e77a020cc0ec65d6612158192654 Mon Sep 17 00:00:00 2001 From: Paul Pollack Date: Thu, 12 Jul 2018 10:13:33 -0400 Subject: [PATCH 2/4] minor fixes --- tf_from_asg.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tf_from_asg.py b/tf_from_asg.py index c5bc1b4..e9b2a9c 100644 --- a/tf_from_asg.py +++ b/tf_from_asg.py @@ -6,18 +6,17 @@ DEFAULT_TEMPLATE = ''' module "{{ MODULE_NAME }}" { - source = "../modules/autoscaling/qw_asg" + source = "../../../../terraform/modules/autoscaling/qw_asg" key_name = "{{ KEY_NAME }}" asg_cluster = "{{ ASG_CLUSTER }}" - zones = "${var.legacy_zones}" asg_queue = "{{ QUEUE_NAME }}" asg_name = "{{ ASG_NAME }}" worker_security_group = ["${var.worker_security_group}"] consumer_config = "{{ CONSUMER_CONFIG }}" r53_zone = "${var.r53_zone}" hosted_domain = "${var.hosted_domain}" - vpc_subnet_ids = ["${aws_subnet.qw.*.id}"] + vpc_subnet_ids = ["${local.c3_supported_subnets}"] name_tag = "${var.name_tag}" zookeeper_dns = "${var.zookeeper_dns}" env = "${var.env}" @@ -34,8 +33,8 @@ ''' # TODO: They only differ in the final piece of the name, templatize better -LC_TEMPLATE = 'terraform import module.{}.aws_launch_configuration.qw-asg-launch-config {}' -ASG_TEMPLATE = 'terraform import module.{}.aws_autoscaling_group.qw-asg {}' +LC_TEMPLATE = 'terraform import module.{}.aws_launch_configuration.qw-asg-launch-config {}\n' +ASG_TEMPLATE = 'terraform import module.{}.aws_autoscaling_group.qw-asg {}\n' DEREGISTRATION_ARN = 'arn:aws:sns:us-east-1:368154587575:qw_autoscale_events' LIFECYCLE_HOOK_ARN = 'arn:aws:iam::368154587575:role/AutoScalingNotificationAccessRole' @@ -162,7 +161,7 @@ def get_queue_from_info(asg_info): def import_statements_from_asg(asg_name, asg_info): # asg_info is asg_name -> dictionary of collected infor - cluster_name = get_dns_safe_cluster_name(asg_info) + cluster_name = get_cluster_name(asg_info) return [LC_TEMPLATE.format(cluster_name, asg_info['lc_name']), ASG_TEMPLATE.format(cluster_name, asg_name)] From 2b4498cafcb6aa9a58bea419d32e9c73158b5ab6 Mon Sep 17 00:00:00 2001 From: Paul Pollack Date: Thu, 12 Jul 2018 10:29:55 -0400 Subject: [PATCH 3/4] no need to pass in ami id --- tf_from_asg.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tf_from_asg.py b/tf_from_asg.py index e9b2a9c..65aae72 100644 --- a/tf_from_asg.py +++ b/tf_from_asg.py @@ -25,7 +25,6 @@ asg_max = "{{ ASG_MAX }}" asg_min = "{{ ASG_MIN }}" instance_type = "{{ INSTANCE_TYPE }}" - qw_ami = "{{ AMI_ID }}" iam_role_name = "AutoscalingEc2InstanceRole" deregistration_arn = "{{ DEREGISTRATION_ARN }}" lifecycle_hook_arn = "{{ LIFECYCLE_HOOK_ARN }}" @@ -105,7 +104,6 @@ def get_autoscaling_information(autoscaling_client, asg_name_prefix): 'asg_desired' : asg_response['DesiredCapacity'], 'instance_type' : launch_config_response['InstanceType'], 'key_name' : launch_config_response['KeyName'], - 'ami_id' : launch_config_response['ImageId'] } lc_to_asg_name = {asg['lc_name'] : asg['name'] for asg in asgs_to_process.values()} @@ -149,7 +147,6 @@ def generate_tf_for_asg(asg_info, template): 'LIFECYCLE_HOOK_ARN' : LIFECYCLE_HOOK_ARN, 'INSTANCE_TYPE' : asg_info['instance_type'], 'KEY_NAME' : asg_info['key_name'], - 'AMI_ID' : asg_info['ami_id'] } return template.render(**asg_context) From 84181483397602b670af9aad0dc00255d2ecdc55 Mon Sep 17 00:00:00 2001 From: Paul Pollack Date: Thu, 12 Jul 2018 11:09:37 -0400 Subject: [PATCH 4/4] replace underscores with hypens --- tf_from_asg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf_from_asg.py b/tf_from_asg.py index 65aae72..6ef4054 100644 --- a/tf_from_asg.py +++ b/tf_from_asg.py @@ -163,7 +163,7 @@ def import_statements_from_asg(asg_name, asg_info): ASG_TEMPLATE.format(cluster_name, asg_name)] def get_dns_safe_cluster_name(asg_info): - return asg_info['name'].replace('/', '-').replace('.', '-') + return asg_info['name'].replace('/', '-').replace('.', '-').replace('_', '-') def get_cluster_name(asg_info): return asg_info['name'].replace('/', '_').replace('.', '_')