From 2453c624fe04214f25272b0e8e70a2f38af117f8 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 15:25:40 -0400 Subject: [PATCH 01/39] initial terraform setup --- Dockerfile | 23 +++++++-- terraform/envs/production.tfvars | 4 ++ terraform/envs/staging.tfvars | 4 ++ terraform/main.tf | 49 +++++++++++++++++++ terraform/modules/ecs_deployment/ecs.tf | 0 terraform/modules/ecs_deployment/outputs.tf | 0 terraform/modules/ecs_deployment/variables.tf | 0 7 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 terraform/envs/production.tfvars create mode 100644 terraform/envs/staging.tfvars create mode 100644 terraform/main.tf create mode 100644 terraform/modules/ecs_deployment/ecs.tf create mode 100644 terraform/modules/ecs_deployment/outputs.tf create mode 100644 terraform/modules/ecs_deployment/variables.tf diff --git a/Dockerfile b/Dockerfile index 0fe8706..9b91e86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,22 +3,33 @@ FROM python:3.11.4-slim-buster WORKDIR /app -ADD requirements.txt /app/requirements.txt - RUN set -ex \ && buildDeps=" \ build-essential \ libpq-dev \ " \ && deps=" \ + curl \ + vim \ + nano \ + procps \ postgresql-client \ " \ - && apt-get update && apt-get install -y $buildDeps $deps --no-install-recommends \ + && apt update && apt install -y $buildDeps $deps --no-install-recommends \ + + +# Install python dependencies +ADD requirements.txt /app/requirements.txt +RUN set -ex \ && pip install --no-cache-dir -r /app/requirements.txt \ - && apt-get purge -y --auto-remove $buildDeps \ + +# Cleanup installs +RUN set -ex \ + && apt purge -y --auto-remove $buildDeps \ $(! command -v gpg > /dev/null || echo 'gnupg dirmngr') \ && rm -rf /var/lib/apt/lists/* + ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH @@ -30,6 +41,8 @@ RUN npm install # END_FEATURE django_react COPY . /app/ + +# Add temporary copy of env file to allow running management commands COPY ./config/.env.example /app/config/.env # START_FEATURE django_react @@ -46,5 +59,5 @@ RUN rm /app/config/.env EXPOSE 8000 -CMD ["gunicorn", "--bind", ":8000", "--workers", "3", "config.wsgi:application"] +CMD ["gunicorn", "--bind", ":8000", "--workers", "3", "config.wsgi:application", "--access-logfile", "-", "--error-logfile", "-"] # END_FEATURE docker diff --git a/terraform/envs/production.tfvars b/terraform/envs/production.tfvars new file mode 100644 index 0000000..1b5a7c4 --- /dev/null +++ b/terraform/envs/production.tfvars @@ -0,0 +1,4 @@ +environment_name = "production" +aws_profile_name = "" # TODO: FILL ME IN +aws_region = "" # TODO: FILL ME IN +terraform_backend_bucket = "" # TODO: FILL ME IN diff --git a/terraform/envs/staging.tfvars b/terraform/envs/staging.tfvars new file mode 100644 index 0000000..baea3e4 --- /dev/null +++ b/terraform/envs/staging.tfvars @@ -0,0 +1,4 @@ +environment_name = "staging" +aws_profile_name = "" # TODO: FILL ME IN +aws_region = "" # TODO: FILL ME IN +terraform_backend_bucket = "" # TODO: FILL ME IN diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..14d9539 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,49 @@ +terraform { + backend "s3" { + bucket = var.terraform_backend_bucket + key = format("%.tfstate" % var.environment_name) + region = var.aws_region + profile = var.aws_profile_name + } + + required_providers { + aws = { + source = "hasicorp/aws" + verison = "~>5.59" + } + } +} + + +provider "aws" { + region = var.aws_region + profile = var.aws_profile_name +} + + +module "ecs_deployment" { + source = "./modules/ecs_deployment" + + environment = var.environment_name + var1 = "x" + var2 = "y" +} + + +variable "aws_profile_name" { + type = string +} + +variable "aws_region" { + type = string + default = "us-east-1" +} + +variable "environment_name" { + type = string +} + + +variable "terraform_backend_bucket" { + type = string +} diff --git a/terraform/modules/ecs_deployment/ecs.tf b/terraform/modules/ecs_deployment/ecs.tf new file mode 100644 index 0000000..e69de29 diff --git a/terraform/modules/ecs_deployment/outputs.tf b/terraform/modules/ecs_deployment/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf new file mode 100644 index 0000000..e69de29 From 75c45309e1ffe6d7c2157b4ca2bba97bb37e16a7 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 15:57:32 -0400 Subject: [PATCH 02/39] draft of ecs --- terraform/envs/production.tfvars | 3 ++ terraform/envs/staging.tfvars | 3 ++ terraform/main.tf | 19 +++++++-- terraform/modules/ecs_deployment/alb.tf | 0 terraform/modules/ecs_deployment/iam.tf | 0 terraform/modules/ecs_deployment/rds.tf | 0 terraform/modules/ecs_deployment/s3.tf | 36 +++++++++++++++++ .../modules/ecs_deployment/secrets_manager.tf | 27 +++++++++++++ .../modules/ecs_deployment/security_groups.tf | 40 +++++++++++++++++++ terraform/modules/ecs_deployment/variables.tf | 15 +++++++ terraform/modules/ecs_deployment/vpc.tf | 12 ++++++ 11 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 terraform/modules/ecs_deployment/alb.tf create mode 100644 terraform/modules/ecs_deployment/iam.tf create mode 100644 terraform/modules/ecs_deployment/rds.tf create mode 100644 terraform/modules/ecs_deployment/s3.tf create mode 100644 terraform/modules/ecs_deployment/secrets_manager.tf create mode 100644 terraform/modules/ecs_deployment/security_groups.tf create mode 100644 terraform/modules/ecs_deployment/vpc.tf diff --git a/terraform/envs/production.tfvars b/terraform/envs/production.tfvars index 1b5a7c4..d3a2af6 100644 --- a/terraform/envs/production.tfvars +++ b/terraform/envs/production.tfvars @@ -2,3 +2,6 @@ environment_name = "production" aws_profile_name = "" # TODO: FILL ME IN aws_region = "" # TODO: FILL ME IN terraform_backend_bucket = "" # TODO: FILL ME IN +vpc_id = "" # TODO: FILL ME IN +web_config_secret_name = "" # TODO: FILL ME IN +s3_bucket_prefix = "" # TODO: FILL ME IN diff --git a/terraform/envs/staging.tfvars b/terraform/envs/staging.tfvars index baea3e4..6788db5 100644 --- a/terraform/envs/staging.tfvars +++ b/terraform/envs/staging.tfvars @@ -2,3 +2,6 @@ environment_name = "staging" aws_profile_name = "" # TODO: FILL ME IN aws_region = "" # TODO: FILL ME IN terraform_backend_bucket = "" # TODO: FILL ME IN +vpc_id = "" # TODO: FILL ME IN +web_config_secret_name = "" # TODO: FILL ME IN +s3_bucket_prefix = "" # TODO: FILL ME IN diff --git a/terraform/main.tf b/terraform/main.tf index 14d9539..16bec1f 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -25,8 +25,9 @@ module "ecs_deployment" { source = "./modules/ecs_deployment" environment = var.environment_name - var1 = "x" - var2 = "y" + vpc_id = var.vpc_id + web_config_secret_name = var.web_config_secret_name + s3_bucket_prefix = var.s3_bucket_prefix } @@ -43,7 +44,19 @@ variable "environment_name" { type = string } - variable "terraform_backend_bucket" { type = string } + +variable "vpc_id" { + type string +} + +variable "web_config_secret_name" { + type string +} + + +variable "s3_bucket_prefix" { + type string +} \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/alb.tf b/terraform/modules/ecs_deployment/alb.tf new file mode 100644 index 0000000..e69de29 diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf new file mode 100644 index 0000000..e69de29 diff --git a/terraform/modules/ecs_deployment/rds.tf b/terraform/modules/ecs_deployment/rds.tf new file mode 100644 index 0000000..e69de29 diff --git a/terraform/modules/ecs_deployment/s3.tf b/terraform/modules/ecs_deployment/s3.tf new file mode 100644 index 0000000..7261709 --- /dev/null +++ b/terraform/modules/ecs_deployment/s3.tf @@ -0,0 +1,36 @@ +resource "aws_s3_bucket" "bucket" { + bucket = format("%s-%s", var.s3_bucket_prefix, var.environment_name) + + tags = { + Environment = var.environment_name + } +} + + +resource "aws_s3_bucket_public_access_block" "bucket" { + bucket = aws_s3_bucket.bucket.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + + +resource "aws_s3_bucket_versioning" "bucket" { + bucket = aws_s3_bucket.bucket.id + versioning_configuration { + status = "Enabled" + } +} + + +resource "aws_s3_bucket_server_side_encryption_configuration" "bucket" { + bucket = aws_s3_bucket.mybucket.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + } + } +} diff --git a/terraform/modules/ecs_deployment/secrets_manager.tf b/terraform/modules/ecs_deployment/secrets_manager.tf new file mode 100644 index 0000000..95ba2ad --- /dev/null +++ b/terraform/modules/ecs_deployment/secrets_manager.tf @@ -0,0 +1,27 @@ +resource "aws_secretsmanager_secret" "web_infrastructure" { + name = format("%s-infrastructure", var.environment_name) +} + +resource "aws_secretsmanager_secret_version" "web_infrastructure" { + secret_id = aws_secretsmanager_secret.web_infrastructure.id + secret_string = jsonencode({ + AWS_STORAGE_BUCKET_NAME = aws_s3_bucket.bucket.id + DATABASE_URL = format( + "postgres://dbuser:%s@%s:5432/%s?sslmode=require", + random_password.db_password.result, + aws_db_instance.database.address, + "database", + ) + DEFAULT_FROM_EMAIL = var.ses_from_email + SECRET_KEY = random_password.app_secret_key.result + }) +} + + +web_config_secret_name + + +resource "random_password" "app_secret_key" { + length = 32 + special = false +} \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/security_groups.tf b/terraform/modules/ecs_deployment/security_groups.tf new file mode 100644 index 0000000..e3393ee --- /dev/null +++ b/terraform/modules/ecs_deployment/security_groups.tf @@ -0,0 +1,40 @@ +resource "aws_securtiy_group" "load_balancer" { + name = format("%s load balancer", var.environment_name) + + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } +} + +resource "aws_securtiy_group" "web" { + name = format("%s web", var.environment_name) + + ingress { + from_port = 8080 + to_port = 8080 + protocol = "tcp" + security_groups = [aws_security_group.load_balancer.id] + } +} + +resource "aws_securtiy_group" "database" { + name = format("%s database", var.environment_name) + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.web.id] + } +} diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index e69de29..b1c5bf6 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -0,0 +1,15 @@ +variable "environment_name" { + type string +} + +variable "vpc_id" { + type string +} + +variable "web_config_secret_name" { + type string +} + +variable "s3_bucket_prefix" { + type string +} \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/vpc.tf b/terraform/modules/ecs_deployment/vpc.tf new file mode 100644 index 0000000..1875ec8 --- /dev/null +++ b/terraform/modules/ecs_deployment/vpc.tf @@ -0,0 +1,12 @@ +data "aws_vpc" "vpc" { + id = var.vpc_id +} + +data "aws_subnets" "subnets" { + filter { + name = "vpc-id" + values = [data.aws_vpc.vpc.id] + } +} + +data "aws_region" "current" {} From 5d919e4a3646dfac65890b0d3ca91345b6c903ad Mon Sep 17 00:00:00 2001 From: kyfantaz Date: Mon, 21 Oct 2024 16:03:15 -0400 Subject: [PATCH 03/39] fix --- terraform/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 16bec1f..cf8da39 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -49,14 +49,14 @@ variable "terraform_backend_bucket" { } variable "vpc_id" { - type string + type = string } variable "web_config_secret_name" { - type string + type = string } variable "s3_bucket_prefix" { - type string + type = string } \ No newline at end of file From 25388ed3864ebc9d744d09d1047756c5cea52bc4 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 16:18:38 -0400 Subject: [PATCH 04/39] adds rds --- terraform/envs/production.tfvars | 8 ++++ terraform/envs/staging.tfvars | 8 ++++ terraform/main.tf | 37 ++++++++++++++++++- terraform/modules/ecs_deployment/rds.tf | 25 +++++++++++++ .../modules/ecs_deployment/secrets_manager.tf | 3 +- terraform/modules/ecs_deployment/variables.tf | 22 ++++++++++- 6 files changed, 98 insertions(+), 5 deletions(-) diff --git a/terraform/envs/production.tfvars b/terraform/envs/production.tfvars index d3a2af6..7814f9b 100644 --- a/terraform/envs/production.tfvars +++ b/terraform/envs/production.tfvars @@ -1,3 +1,4 @@ +# Required Variables environment_name = "production" aws_profile_name = "" # TODO: FILL ME IN aws_region = "" # TODO: FILL ME IN @@ -5,3 +6,10 @@ terraform_backend_bucket = "" # TODO: FILL ME IN vpc_id = "" # TODO: FILL ME IN web_config_secret_name = "" # TODO: FILL ME IN s3_bucket_prefix = "" # TODO: FILL ME IN +rds_engine_version = "" # TODO: FILL ME IN + +# Optional Variables +rds_backup_retention_period = 30 +rds_deletion_protection = true +rds_instance_class = "db.m7g.large" +rds_multi_az = true diff --git a/terraform/envs/staging.tfvars b/terraform/envs/staging.tfvars index 6788db5..a6a8336 100644 --- a/terraform/envs/staging.tfvars +++ b/terraform/envs/staging.tfvars @@ -1,3 +1,4 @@ +# Required Variables environment_name = "staging" aws_profile_name = "" # TODO: FILL ME IN aws_region = "" # TODO: FILL ME IN @@ -5,3 +6,10 @@ terraform_backend_bucket = "" # TODO: FILL ME IN vpc_id = "" # TODO: FILL ME IN web_config_secret_name = "" # TODO: FILL ME IN s3_bucket_prefix = "" # TODO: FILL ME IN +rds_engine_version = "" # TODO: FILL ME IN + +# Optional Variables +rds_backup_retention_period = 10 +rds_deletion_protection = true +rds_instance_class = "db.t3.micro" +rds_multi_az = false diff --git a/terraform/main.tf b/terraform/main.tf index 16bec1f..b99ac5d 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -28,9 +28,16 @@ module "ecs_deployment" { vpc_id = var.vpc_id web_config_secret_name = var.web_config_secret_name s3_bucket_prefix = var.s3_bucket_prefix + rds_engine_version = var.rds_engine_version + rds_backup_retention_period = var.rds_backup_retention_period + rds_deletion_protection = var.rds_deletion_protection + rds_instance_class = var.rds_instance_class + rds_multi_az = var.rds_multi_az } +# Required Variables + variable "aws_profile_name" { type = string } @@ -56,7 +63,33 @@ variable "web_config_secret_name" { type string } - variable "s3_bucket_prefix" { type string -} \ No newline at end of file +} + +variable "rds_engine_version" { + type = string +} + + +# Optional Variables + +variable "rds_backup_retention_period" { + type = number + default = 30 +} + +variable "rds_deletion_protection" { + type = bool + default = true +} + +variable "rds_instance_class" { + type string + default = "db.t3.micro" +} + +variable "rds_multi_az" { + typr bool + default = false +} diff --git a/terraform/modules/ecs_deployment/rds.tf b/terraform/modules/ecs_deployment/rds.tf index e69de29..23ee827 100644 --- a/terraform/modules/ecs_deployment/rds.tf +++ b/terraform/modules/ecs_deployment/rds.tf @@ -0,0 +1,25 @@ +resource "random_password" "db_password" { + length = 50 + special = false +} + + +resource "aws_db_instance" "database" { + allocated_storage = 20 + max_allocated_storage = 10000 + allow_major_version_upgrade = true + apply_immediately = true + backup_retention_period = var.rds_backup_retention_period + db_name = "database" + deletion_protection = var.rds_deletion_protection + engine = "postgres" + engine_version = var.rds_engine_version + identifier = format("%s-database", var.environment_name) + instance_class = var.rds_instance_class + multi_az = var.rds_multi_az + password = random_password.db_password.result + storage_encrypted = true + storage_type = "gp2" + username = "dbuser" + vpc_security_group_ids = [aws_security_group.database.id] +} \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/secrets_manager.tf b/terraform/modules/ecs_deployment/secrets_manager.tf index 95ba2ad..1715698 100644 --- a/terraform/modules/ecs_deployment/secrets_manager.tf +++ b/terraform/modules/ecs_deployment/secrets_manager.tf @@ -7,10 +7,9 @@ resource "aws_secretsmanager_secret_version" "web_infrastructure" { secret_string = jsonencode({ AWS_STORAGE_BUCKET_NAME = aws_s3_bucket.bucket.id DATABASE_URL = format( - "postgres://dbuser:%s@%s:5432/%s?sslmode=require", + "postgres://dbuser:%s@%s:5432/database?sslmode=require", random_password.db_password.result, aws_db_instance.database.address, - "database", ) DEFAULT_FROM_EMAIL = var.ses_from_email SECRET_KEY = random_password.app_secret_key.result diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index b1c5bf6..0d802a1 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -12,4 +12,24 @@ variable "web_config_secret_name" { variable "s3_bucket_prefix" { type string -} \ No newline at end of file +} + +variable "rds_backup_retention_period" { + type = number +} + +variable "rds_deletion_protection" { + type = bool +} + +variable "rds_engine_version" { + type = string +} + +variable "rds_instance_class" { + type string +} + +variable "rds_multi_az" { + typr bool +} From 697a65b3cc080109919c566c116c7a42572b87ed Mon Sep 17 00:00:00 2001 From: kyfantaz Date: Mon, 21 Oct 2024 16:23:18 -0400 Subject: [PATCH 05/39] fix --- terraform/main.tf | 9 ++------- terraform/modules/ecs_deployment/variables.tf | 12 ++++++------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 16934aa..e66d8f6 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -14,13 +14,11 @@ terraform { } } - provider "aws" { region = var.aws_region profile = var.aws_profile_name } - module "ecs_deployment" { source = "./modules/ecs_deployment" @@ -35,9 +33,7 @@ module "ecs_deployment" { rds_multi_az = var.rds_multi_az } - # Required Variables - variable "aws_profile_name" { type = string } @@ -73,7 +69,6 @@ variable "rds_engine_version" { # Optional Variables - variable "rds_backup_retention_period" { type = number default = 30 @@ -85,11 +80,11 @@ variable "rds_deletion_protection" { } variable "rds_instance_class" { - type string + type = string default = "db.t3.micro" } variable "rds_multi_az" { - typr bool + typr = bool default = false } diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index 0d802a1..f52f535 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -1,17 +1,17 @@ variable "environment_name" { - type string + type = string } variable "vpc_id" { - type string + type = string } variable "web_config_secret_name" { - type string + type = string } variable "s3_bucket_prefix" { - type string + type = string } variable "rds_backup_retention_period" { @@ -27,9 +27,9 @@ variable "rds_engine_version" { } variable "rds_instance_class" { - type string + type = string } variable "rds_multi_az" { - typr bool + type = bool } From c6b2cd2beacc06eb0ba9540da4acd40b78fe4194 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 16:44:31 -0400 Subject: [PATCH 06/39] adds iam config --- terraform/envs/production.tfvars | 2 + terraform/envs/staging.tfvars | 2 + terraform/main.tf | 10 ++ terraform/modules/ecs_deployment/iam.tf | 136 ++++++++++++++++++ terraform/modules/ecs_deployment/rds.tf | 2 +- .../modules/ecs_deployment/secrets_manager.tf | 6 +- terraform/modules/ecs_deployment/variables.tf | 10 +- 7 files changed, 164 insertions(+), 4 deletions(-) diff --git a/terraform/envs/production.tfvars b/terraform/envs/production.tfvars index 7814f9b..b2b7548 100644 --- a/terraform/envs/production.tfvars +++ b/terraform/envs/production.tfvars @@ -7,6 +7,8 @@ vpc_id = "" # TODO: FILL ME IN web_config_secret_name = "" # TODO: FILL ME IN s3_bucket_prefix = "" # TODO: FILL ME IN rds_engine_version = "" # TODO: FILL ME IN +ses_identity = "" # TODO: FILL ME IN +ses_from_email = "" # TODO: FILL ME IN # Optional Variables rds_backup_retention_period = 30 diff --git a/terraform/envs/staging.tfvars b/terraform/envs/staging.tfvars index a6a8336..15f3681 100644 --- a/terraform/envs/staging.tfvars +++ b/terraform/envs/staging.tfvars @@ -7,6 +7,8 @@ vpc_id = "" # TODO: FILL ME IN web_config_secret_name = "" # TODO: FILL ME IN s3_bucket_prefix = "" # TODO: FILL ME IN rds_engine_version = "" # TODO: FILL ME IN +ses_identity = "" # TODO: FILL ME IN +ses_from_email = "" # TODO: FILL ME IN # Optional Variables rds_backup_retention_period = 10 diff --git a/terraform/main.tf b/terraform/main.tf index 16934aa..82b026c 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -33,6 +33,9 @@ module "ecs_deployment" { rds_deletion_protection = var.rds_deletion_protection rds_instance_class = var.rds_instance_class rds_multi_az = var.rds_multi_az + ses_identity = var.ses_identity + ses_from_email = var.ses_from_email +} } @@ -71,6 +74,13 @@ variable "rds_engine_version" { type = string } +variable "ses_identity" { + type string +} + +variable "ses_from_email" { + type string +} # Optional Variables diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf index e69de29..996559f 100644 --- a/terraform/modules/ecs_deployment/iam.tf +++ b/terraform/modules/ecs_deployment/iam.tf @@ -0,0 +1,136 @@ +data "aws_iam_policy_document" "ecs_assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + } +} + + +data "aws_iam_policy_document" "ecs_execution_role_policy" { + statement { + effect = "Allow" + actions = [ + "ecr:GetAuthorizationToken" + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage" + ] + resources = [ + "arn:aws:ecr:*:*:repository/${var.ecr_repository_name}" + ] + } + + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + resources = [ + "arn:aws:logs:*:*:log-group:${var.environment_name}:*", + ] + } + + statement { + effect = "Allow" + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + ] + resources = [ + "arn:aws:secretsmanager:*:*:secret:${aws_secretsmanager_secret.web_infrastructure.name}-*", + "arn:aws:secretsmanager:*:*:secret:${aws_secretsmanager_secret.web_config.name}-*" + ] + } + + statement { + effect = "Allow" + actions = [ + "kms:Decrypt" + ] + resources = [ + format("arn:aws:kms:*:*:aws/secretsmanager") + ] + } +} + + +data "aws_iam_policy_document" "ecs_task_role_policy" { + statement { + effect = "Allow" + actions = [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel" + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "ses:SendEmail", + "ses:SendRawEmail", + "ses:GetSendQuota" + ] + resources = [ + format("arn:aws:ses:*:*:identity/%s", var.ses_identity) + ] + condition { + test = "StringLike" + variable = "ses:FromAddress" + values = [var.ses_from_email] + } + } + + statement { + effect = "Allow" + actions = [ + "s3:*" + ] + resources = [ + format("arn:aws:s3:::%s", aws_s3_bucket.bucket.id), + format("arn:aws:s3:::%s/*", aws_s3_bucket.bucket.id) + ] + } + + statement { + effect = "Allow" + actions = [ + "kms:Decrypt" + ] + resources = [ + format("arn:aws:kms:*:*:aws/secretsmanager") + ] + } +} + +resource "aws_iam_role" "ecs_execution_role" { + name = format("%s-ecs-execution-role", var.environment_name) + assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json + inline_policy { + name = "ecs-execution-role-policy" + policy = data.aws_iam_policy_document.ecs_execution_role_policy.json + } +} + +resource "aws_iam_role" "ecs_task_role" { + name = format("%s-ecs-task-role", var.environment_name) + assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json + inline_policy { + name = "ecs-task-role-policy" + policy = data.aws_iam_policy_document.ecs_task_role_policy.json + } +} diff --git a/terraform/modules/ecs_deployment/rds.tf b/terraform/modules/ecs_deployment/rds.tf index 23ee827..03b0d37 100644 --- a/terraform/modules/ecs_deployment/rds.tf +++ b/terraform/modules/ecs_deployment/rds.tf @@ -22,4 +22,4 @@ resource "aws_db_instance" "database" { storage_type = "gp2" username = "dbuser" vpc_security_group_ids = [aws_security_group.database.id] -} \ No newline at end of file +} diff --git a/terraform/modules/ecs_deployment/secrets_manager.tf b/terraform/modules/ecs_deployment/secrets_manager.tf index 1715698..37c13b8 100644 --- a/terraform/modules/ecs_deployment/secrets_manager.tf +++ b/terraform/modules/ecs_deployment/secrets_manager.tf @@ -17,10 +17,12 @@ resource "aws_secretsmanager_secret_version" "web_infrastructure" { } -web_config_secret_name +data "aws_secretsmanager_secret" "web_config" { + name = var.web_config_secret_name +} resource "random_password" "app_secret_key" { length = 32 special = false -} \ No newline at end of file +} diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index 0d802a1..a896784 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -31,5 +31,13 @@ variable "rds_instance_class" { } variable "rds_multi_az" { - typr bool + type bool +} + +variable "ses_identity" { + type string +} + +variable "ses_from_email" { + type string } From 9d17f3620650bbc27975ccdd92a9553771f4c468 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 16:57:01 -0400 Subject: [PATCH 07/39] adds ecs config --- terraform/envs/production.tfvars | 3 + terraform/envs/staging.tfvars | 3 + terraform/main.tf | 19 +++- terraform/modules/ecs_deployment/ecs.tf | 105 ++++++++++++++++++ terraform/modules/ecs_deployment/variables.tf | 16 +++ 5 files changed, 145 insertions(+), 1 deletion(-) diff --git a/terraform/envs/production.tfvars b/terraform/envs/production.tfvars index b2b7548..b61850f 100644 --- a/terraform/envs/production.tfvars +++ b/terraform/envs/production.tfvars @@ -15,3 +15,6 @@ rds_backup_retention_period = 30 rds_deletion_protection = true rds_instance_class = "db.m7g.large" rds_multi_az = true +container_web_cpu = 1024 +container_web_memory = 1024 +container_count = 2 diff --git a/terraform/envs/staging.tfvars b/terraform/envs/staging.tfvars index 15f3681..812cc17 100644 --- a/terraform/envs/staging.tfvars +++ b/terraform/envs/staging.tfvars @@ -15,3 +15,6 @@ rds_backup_retention_period = 10 rds_deletion_protection = true rds_instance_class = "db.t3.micro" rds_multi_az = false +container_web_cpu = 256 +container_web_memory = 1024 +container_count = 1 diff --git a/terraform/main.tf b/terraform/main.tf index 82b026c..e754561 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -35,7 +35,9 @@ module "ecs_deployment" { rds_multi_az = var.rds_multi_az ses_identity = var.ses_identity ses_from_email = var.ses_from_email -} + container_web_cpu = var.container_web_cpu + container_web_memory = var.container_web_memory + container_count = var.container_web_memory } @@ -103,3 +105,18 @@ variable "rds_multi_az" { typr bool default = false } + +variable "container_web_cpu" { + type number + default 256 +} + +variable "container_web_memory" { + type number + default = 1024 +} + +variable "container_count" { + type number + default 1 +} \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/ecs.tf b/terraform/modules/ecs_deployment/ecs.tf index e69de29..77358d9 100644 --- a/terraform/modules/ecs_deployment/ecs.tf +++ b/terraform/modules/ecs_deployment/ecs.tf @@ -0,0 +1,105 @@ +resource "aws_ecs_cluster" "cluster" { + name = format("%-cluster", var.environment_name) + + setting { + name = "containerInsights" + value = "enabled" + } +} + + +resource "aws_ecs_cluster_capacity_providers" "fargate_provider" { + cluster_name = aws_ecs_cluster.cluster.name + + capacity_providers = ["FARGATE"] +} + + +locals { + ecs_secrets = [ + { + name : "DATABASE_URL", + valueFrom : "${aws_secretsmanager_secret.web_infrastructure.arn}:DATABASE_URL::" + }, + { + name : "SECRET_KEY", + valueFrom : "${aws_secretsmanager_secret.web_infrastructure.arn}:SECRET_KEY::" + }, + { + name : "AWS_STORAGE_BUCKET_NAME", + valueFrom : "${aws_secretsmanager_secret.web_infrastructure.arn}:AWS_STORAGE_BUCKET_NAME::" + }, + { + name : "DEFAULT_FROM_EMAIL", + valueFrom : "${aws_secretsmanager_secret.web_infrastructure.arn}:DEFAULT_FROM_EMAIL::" + }, + ] +} + + +resource "aws_ecs_task_definition" "web" { + family = "${var.environment_name}-web" + + container_definitions = jsonencode([ + { + name = "${var.environment_name}-web" + image = var.ecr_image_uri + essential = true + portMappings = [ + { + containerPort = 8080 + hostPort = 8080 + } + ], + logConfiguration = { + logDriver = "awslogs", + options = { + awslogs-group = aws_cloudwatch_log_group.log_group.name, + awslogs-region = data.aws_region.current.name, + awslogs-stream-prefix = "ecs" + } + } + secrets = local.ecs_secrets + } + ]) + + requires_compatibilities = ["FARGATE"] + cpu = var.container_web_cpu + memory = var.container_web_memory + execution_role_arn = aws_iam_role.ecs_execution_role.arn + network_mode = "awsvpc" + task_role_arn = aws_iam_role.ecs_task_role.arn +} + + +resource "aws_ecs_service" "web" { + name = "${var.environment_name}-web" + cluster = aws_ecs_cluster.application.id + task_definition = aws_ecs_task_definition.web.arn + desired_count = var.container_count + launch_type = "FARGATE" + enable_execute_command = true + + deployment_circuit_breaker { + enable = true + rollback = true + } + + load_balancer { + target_group_arn = aws_lb_target_group.application.arn + container_name = "${local.app_env_name}-web" + container_port = 8080 + } + + network_configuration { + subnets = data.aws_subnets.subnets.ids + security_groups = [aws_security_group.web.id] + assign_public_ip = true + } +} + + +resource "aws_cloudwatch_log_group" "log_group" { + name = var.environment_name + retention_in_days = 90 +} \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index a896784..7ee395c 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -41,3 +41,19 @@ variable "ses_identity" { variable "ses_from_email" { type string } + +variable "ecr_image_uri" { + type string +} + +variable "container_web_cpu" { + type number +} + +variable "container_web_memory" { + type number +} + +variable "container_count" { + type number +} \ No newline at end of file From 0cab76f3317b18fa19f6801c3855829b45a52a33 Mon Sep 17 00:00:00 2001 From: kyfantaz Date: Mon, 21 Oct 2024 17:00:56 -0400 Subject: [PATCH 08/39] fix --- terraform/main.tf | 4 ++-- terraform/modules/ecs_deployment/secrets_manager.tf | 4 ---- terraform/modules/ecs_deployment/variables.tf | 4 ++++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index e66d8f6..2a619ef 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -9,7 +9,7 @@ terraform { required_providers { aws = { source = "hasicorp/aws" - verison = "~>5.59" + version = "~>5.59" } } } @@ -85,6 +85,6 @@ variable "rds_instance_class" { } variable "rds_multi_az" { - typr = bool + type = bool default = false } diff --git a/terraform/modules/ecs_deployment/secrets_manager.tf b/terraform/modules/ecs_deployment/secrets_manager.tf index 1715698..5292505 100644 --- a/terraform/modules/ecs_deployment/secrets_manager.tf +++ b/terraform/modules/ecs_deployment/secrets_manager.tf @@ -16,10 +16,6 @@ resource "aws_secretsmanager_secret_version" "web_infrastructure" { }) } - -web_config_secret_name - - resource "random_password" "app_secret_key" { length = 32 special = false diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index f52f535..efdc5e8 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -2,6 +2,10 @@ variable "environment_name" { type = string } +variable "application_name" { + type = string +} + variable "vpc_id" { type = string } From c3755f106b6d2147e5ef16bce0ab4a2fea6898f0 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 17:03:45 -0400 Subject: [PATCH 09/39] adds alb config --- terraform/envs/production.tfvars | 2 + terraform/envs/staging.tfvars | 2 + terraform/main.tf | 14 ++++- terraform/modules/ecs_deployment/alb.tf | 52 +++++++++++++++++++ terraform/modules/ecs_deployment/variables.tf | 10 +++- 5 files changed, 78 insertions(+), 2 deletions(-) diff --git a/terraform/envs/production.tfvars b/terraform/envs/production.tfvars index b61850f..7fec925 100644 --- a/terraform/envs/production.tfvars +++ b/terraform/envs/production.tfvars @@ -9,6 +9,7 @@ s3_bucket_prefix = "" # TODO: FILL ME IN rds_engine_version = "" # TODO: FILL ME IN ses_identity = "" # TODO: FILL ME IN ses_from_email = "" # TODO: FILL ME IN +certificate_manager_arn = "" # TODO: FILL ME IN # Optional Variables rds_backup_retention_period = 30 @@ -18,3 +19,4 @@ rds_multi_az = true container_web_cpu = 1024 container_web_memory = 1024 container_count = 2 +ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" diff --git a/terraform/envs/staging.tfvars b/terraform/envs/staging.tfvars index 812cc17..dc4b541 100644 --- a/terraform/envs/staging.tfvars +++ b/terraform/envs/staging.tfvars @@ -9,6 +9,7 @@ s3_bucket_prefix = "" # TODO: FILL ME IN rds_engine_version = "" # TODO: FILL ME IN ses_identity = "" # TODO: FILL ME IN ses_from_email = "" # TODO: FILL ME IN +certificate_manager_arn = "" # TODO: FILL ME IN # Optional Variables rds_backup_retention_period = 10 @@ -18,3 +19,4 @@ rds_multi_az = false container_web_cpu = 256 container_web_memory = 1024 container_count = 1 +ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" diff --git a/terraform/main.tf b/terraform/main.tf index e754561..183d6d0 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -38,6 +38,8 @@ module "ecs_deployment" { container_web_cpu = var.container_web_cpu container_web_memory = var.container_web_memory container_count = var.container_web_memory + certificate_manager_arn = var.certificate_manager_arn + ssl_policy = var.ssl_policy } @@ -84,6 +86,11 @@ variable "ses_from_email" { type string } +variable "certificate_manager_arn" { + type string +} + + # Optional Variables variable "rds_backup_retention_period" { @@ -119,4 +126,9 @@ variable "container_web_memory" { variable "container_count" { type number default 1 -} \ No newline at end of file +} + +variable "ssl_policy" { + type string + default = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" +} diff --git a/terraform/modules/ecs_deployment/alb.tf b/terraform/modules/ecs_deployment/alb.tf index e69de29..23ea78a 100644 --- a/terraform/modules/ecs_deployment/alb.tf +++ b/terraform/modules/ecs_deployment/alb.tf @@ -0,0 +1,52 @@ +resource "aws_lb" "alb" { + name = var.environment_name + internal = false + load_balancer_type = "application" + security_groups = [aws_security_group.load_balancer.id] + subnets = data.aws_subnets.subnets.ids +} + + +resource "aws_lb_target_group" "target_group" { + name = var.environment_name + port = 8080 + protocol = "HTTP" + vpc_id = data.aws_vpc.vpc.id + target_type = "ip" + health_check { + path = "/health-check/" + } + lifecycle { + create_before_destroy = true + } +} + + +resource "aws_lb_listener" "http_redirect" { + load_balancer_arn = aws_lb.alb.arn + port = "80" + protocol = "HTTP" + + default_action { + type = "redirect" + + redirect { + port = "443" + protocol = "HTTPS" + status_code = "HTTP_301" + } + } +} + +resource "aws_lb_listener" "https" { + load_balancer_arn = aws_lb.alb.arn + port = "443" + protocol = "HTTPS" + ssl_policy = var.ssl_policy + certificate_arn = var.certificate_manager_arn + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.target_group.arn + } +} \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index 7ee395c..068bbb5 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -56,4 +56,12 @@ variable "container_web_memory" { variable "container_count" { type number -} \ No newline at end of file +} + +variable "certificate_manager_arn" { + type string +} + +variable "ssl_policy" { + type string +} From d83557a6c5f6895d74a0075ebdfedf469bd49de4 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 17:17:34 -0400 Subject: [PATCH 10/39] converts to per-env main.tf files --- terraform/.terraform/modules/modules.json | 1 + terraform/envs/production.tfvars | 22 --- terraform/envs/production/main.tf | 45 ++++++ terraform/envs/staging.tfvars | 22 --- terraform/envs/staging/main.tf | 45 ++++++ terraform/main.tf | 129 ------------------ terraform/modules/ecs_deployment/variables.tf | 52 ++++--- 7 files changed, 125 insertions(+), 191 deletions(-) create mode 100644 terraform/.terraform/modules/modules.json delete mode 100644 terraform/envs/production.tfvars create mode 100644 terraform/envs/production/main.tf delete mode 100644 terraform/envs/staging.tfvars create mode 100644 terraform/envs/staging/main.tf delete mode 100644 terraform/main.tf diff --git a/terraform/.terraform/modules/modules.json b/terraform/.terraform/modules/modules.json new file mode 100644 index 0000000..9a91210 --- /dev/null +++ b/terraform/.terraform/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"ecs_deployment","Source":"./modules/ecs_deployment","Dir":"modules/ecs_deployment"}]} \ No newline at end of file diff --git a/terraform/envs/production.tfvars b/terraform/envs/production.tfvars deleted file mode 100644 index 7fec925..0000000 --- a/terraform/envs/production.tfvars +++ /dev/null @@ -1,22 +0,0 @@ -# Required Variables -environment_name = "production" -aws_profile_name = "" # TODO: FILL ME IN -aws_region = "" # TODO: FILL ME IN -terraform_backend_bucket = "" # TODO: FILL ME IN -vpc_id = "" # TODO: FILL ME IN -web_config_secret_name = "" # TODO: FILL ME IN -s3_bucket_prefix = "" # TODO: FILL ME IN -rds_engine_version = "" # TODO: FILL ME IN -ses_identity = "" # TODO: FILL ME IN -ses_from_email = "" # TODO: FILL ME IN -certificate_manager_arn = "" # TODO: FILL ME IN - -# Optional Variables -rds_backup_retention_period = 30 -rds_deletion_protection = true -rds_instance_class = "db.m7g.large" -rds_multi_az = true -container_web_cpu = 1024 -container_web_memory = 1024 -container_count = 2 -ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" diff --git a/terraform/envs/production/main.tf b/terraform/envs/production/main.tf new file mode 100644 index 0000000..012d31b --- /dev/null +++ b/terraform/envs/production/main.tf @@ -0,0 +1,45 @@ +terraform { + backend "s3" { + bucket = "" # TODO: FILL ME IN + key = "production.tfstate" + region = "us-east-1" # TODO: FILL ME IN + profile = "" # TODO: FILL ME IN + } + + required_providers { + aws = { + source = "hasicorp/aws" + version = "~>5.59" + } + } +} + +provider "aws" { + region = "us-east-1" # TODO: FILL ME IN + profile = "" # TODO: FILL ME IN +} + +module "ecs_deployment" { + source = "./modules/ecs_deployment" + + # Required Variables + environment_name = "production" + terraform_backend_bucket = "" # TODO: FILL ME IN + vpc_id = "" # TODO: FILL ME IN + web_config_secret_name = "" # TODO: FILL ME IN + s3_bucket_prefix = "" # TODO: FILL ME IN + rds_engine_version = "" # TODO: FILL ME IN + ses_identity = "" # TODO: FILL ME IN + ses_from_email = "" # TODO: FILL ME IN + certificate_manager_arn = "" # TODO: FILL ME IN + + # Optional Variables + rds_backup_retention_period = 30 + rds_deletion_protection = true + rds_instance_class = "db.m7g.large" + rds_multi_az = true + container_web_cpu = 1024 + container_web_memory = 1024 + container_count = 2 + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" +} diff --git a/terraform/envs/staging.tfvars b/terraform/envs/staging.tfvars deleted file mode 100644 index dc4b541..0000000 --- a/terraform/envs/staging.tfvars +++ /dev/null @@ -1,22 +0,0 @@ -# Required Variables -environment_name = "staging" -aws_profile_name = "" # TODO: FILL ME IN -aws_region = "" # TODO: FILL ME IN -terraform_backend_bucket = "" # TODO: FILL ME IN -vpc_id = "" # TODO: FILL ME IN -web_config_secret_name = "" # TODO: FILL ME IN -s3_bucket_prefix = "" # TODO: FILL ME IN -rds_engine_version = "" # TODO: FILL ME IN -ses_identity = "" # TODO: FILL ME IN -ses_from_email = "" # TODO: FILL ME IN -certificate_manager_arn = "" # TODO: FILL ME IN - -# Optional Variables -rds_backup_retention_period = 10 -rds_deletion_protection = true -rds_instance_class = "db.t3.micro" -rds_multi_az = false -container_web_cpu = 256 -container_web_memory = 1024 -container_count = 1 -ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" diff --git a/terraform/envs/staging/main.tf b/terraform/envs/staging/main.tf new file mode 100644 index 0000000..195fddd --- /dev/null +++ b/terraform/envs/staging/main.tf @@ -0,0 +1,45 @@ +terraform { + backend "s3" { + bucket = "" # TODO: FILL ME IN + key = "staging.tfstate" + region = "us-east-1" # TODO: FILL ME IN + profile = "" # TODO: FILL ME IN + } + + required_providers { + aws = { + source = "hasicorp/aws" + version = "~>5.59" + } + } +} + +provider "aws" { + region = "us-east-1" # TODO: FILL ME IN + profile = "" # TODO: FILL ME IN +} + +module "ecs_deployment" { + source = "./modules/ecs_deployment" + + # Required Variables + environment_name = "staging" + terraform_backend_bucket = "" # TODO: FILL ME IN + vpc_id = "" # TODO: FILL ME IN + web_config_secret_name = "" # TODO: FILL ME IN + s3_bucket_prefix = "" # TODO: FILL ME IN + rds_engine_version = "" # TODO: FILL ME IN + ses_identity = "" # TODO: FILL ME IN + ses_from_email = "" # TODO: FILL ME IN + certificate_manager_arn = "" # TODO: FILL ME IN + + # Optional Variables + rds_backup_retention_period = 10 + rds_deletion_protection = true + rds_instance_class = "db.t3.micro" + rds_multi_az = false + container_web_cpu = 256 + container_web_memory = 1024 + container_count = 1 + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" +} diff --git a/terraform/main.tf b/terraform/main.tf deleted file mode 100644 index d62ff4c..0000000 --- a/terraform/main.tf +++ /dev/null @@ -1,129 +0,0 @@ -terraform { - backend "s3" { - bucket = var.terraform_backend_bucket - key = format("%.tfstate" % var.environment_name) - region = var.aws_region - profile = var.aws_profile_name - } - - required_providers { - aws = { - source = "hasicorp/aws" - version = "~>5.59" - } - } -} - -provider "aws" { - region = var.aws_region - profile = var.aws_profile_name -} - -module "ecs_deployment" { - source = "./modules/ecs_deployment" - - environment = var.environment_name - vpc_id = var.vpc_id - web_config_secret_name = var.web_config_secret_name - s3_bucket_prefix = var.s3_bucket_prefix - rds_engine_version = var.rds_engine_version - rds_backup_retention_period = var.rds_backup_retention_period - rds_deletion_protection = var.rds_deletion_protection - rds_instance_class = var.rds_instance_class - rds_multi_az = var.rds_multi_az - ses_identity = var.ses_identity - ses_from_email = var.ses_from_email - container_web_cpu = var.container_web_cpu - container_web_memory = var.container_web_memory - container_count = var.container_web_memory - certificate_manager_arn = var.certificate_manager_arn - ssl_policy = var.ssl_policy -} - -# Required Variables -variable "aws_profile_name" { - type = string -} - -variable "aws_region" { - type = string - default = "us-east-1" -} - -variable "environment_name" { - type = string -} - -variable "terraform_backend_bucket" { - type = string -} - -variable "vpc_id" { - type = string -} - -variable "web_config_secret_name" { - type = string -} - -variable "s3_bucket_prefix" { - type = string -} - -variable "rds_engine_version" { - type = string -} - -variable "ses_identity" { - type = string -} - -variable "ses_from_email" { - type = string -} - -variable "certificate_manager_arn" { - type = string -} - - -# Optional Variables -variable "rds_backup_retention_period" { - type = number - default = 30 -} - -variable "rds_deletion_protection" { - type = bool - default = true -} - -variable "rds_instance_class" { - type = string - default = "db.t3.micro" -} - -variable "rds_multi_az" { - type = bool - default = false -} - -variable "container_web_cpu" { - type = number - default = 256 -} - -variable "container_web_memory" { - type = number - default = 1024 -} - -variable "container_count" { - type = number - default = 1 -} - -variable "ssl_policy" { - type = string - default = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" -} diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index f2dcd78..0c60f6d 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -1,8 +1,18 @@ +# Required Variables +variable "aws_profile_name" { + type = string +} + +variable "aws_region" { + type = string + default = "us-east-1" +} + variable "environment_name" { type = string } -variable "application_name" { +variable "terraform_backend_bucket" { type = string } @@ -18,54 +28,60 @@ variable "s3_bucket_prefix" { type = string } -variable "rds_backup_retention_period" { - type = number +variable "rds_engine_version" { + type = string } -variable "rds_deletion_protection" { - type = bool +variable "ses_identity" { + type = string } -variable "rds_engine_version" { +variable "ses_from_email" { type = string } -variable "rds_instance_class" { +variable "certificate_manager_arn" { type = string } -variable "rds_multi_az" { - type = bool + +# Optional Variables +variable "rds_backup_retention_period" { + type = number + default = 30 } -variable "ses_identity" { - type = string +variable "rds_deletion_protection" { + type = bool + default = true } -variable "ses_from_email" { +variable "rds_instance_class" { type = string + default = "db.t3.micro" } -variable "ecr_image_uri" { - type = string +variable "rds_multi_az" { + type = bool + default = false } variable "container_web_cpu" { type = number + default = 256 } variable "container_web_memory" { type = number + default = 1024 } variable "container_count" { type = number -} - -variable "certificate_manager_arn" { - type = string + default = 1 } variable "ssl_policy" { type = string + default = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" } From ac3f51970bb40ece07db5007c6ba76ceb933ab41 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 17:31:27 -0400 Subject: [PATCH 11/39] fixes --- terraform/envs/production/main.tf | 6 ++++-- terraform/envs/staging/main.tf | 6 ++++-- terraform/modules/ecs_deployment/ecs.tf | 8 ++++---- terraform/modules/ecs_deployment/iam.tf | 2 +- terraform/modules/ecs_deployment/s3.tf | 2 +- .../modules/ecs_deployment/security_groups.tf | 6 +++--- terraform/modules/ecs_deployment/variables.tf | 17 ++++++++--------- 7 files changed, 25 insertions(+), 22 deletions(-) diff --git a/terraform/envs/production/main.tf b/terraform/envs/production/main.tf index 012d31b..2776baa 100644 --- a/terraform/envs/production/main.tf +++ b/terraform/envs/production/main.tf @@ -8,7 +8,7 @@ terraform { required_providers { aws = { - source = "hasicorp/aws" + source = "hashicorp/aws" version = "~>5.59" } } @@ -20,7 +20,7 @@ provider "aws" { } module "ecs_deployment" { - source = "./modules/ecs_deployment" + source = "../../modules/ecs_deployment" # Required Variables environment_name = "production" @@ -32,6 +32,8 @@ module "ecs_deployment" { ses_identity = "" # TODO: FILL ME IN ses_from_email = "" # TODO: FILL ME IN certificate_manager_arn = "" # TODO: FILL ME IN + ecr_repository_name = "" # TODO: FILL ME IN + ecr_image_uri = "" # TODO: FILL ME IN # Optional Variables rds_backup_retention_period = 30 diff --git a/terraform/envs/staging/main.tf b/terraform/envs/staging/main.tf index 195fddd..e499f3f 100644 --- a/terraform/envs/staging/main.tf +++ b/terraform/envs/staging/main.tf @@ -8,7 +8,7 @@ terraform { required_providers { aws = { - source = "hasicorp/aws" + source = "hashicorp/aws" version = "~>5.59" } } @@ -20,7 +20,7 @@ provider "aws" { } module "ecs_deployment" { - source = "./modules/ecs_deployment" + source = "../../modules/ecs_deployment" # Required Variables environment_name = "staging" @@ -32,6 +32,8 @@ module "ecs_deployment" { ses_identity = "" # TODO: FILL ME IN ses_from_email = "" # TODO: FILL ME IN certificate_manager_arn = "" # TODO: FILL ME IN + ecr_repository_name = "" # TODO: FILL ME IN + ecr_image_uri = "" # TODO: FILL ME IN # Optional Variables rds_backup_retention_period = 10 diff --git a/terraform/modules/ecs_deployment/ecs.tf b/terraform/modules/ecs_deployment/ecs.tf index 77358d9..e830ab8 100644 --- a/terraform/modules/ecs_deployment/ecs.tf +++ b/terraform/modules/ecs_deployment/ecs.tf @@ -1,5 +1,5 @@ resource "aws_ecs_cluster" "cluster" { - name = format("%-cluster", var.environment_name) + name = format("%s-cluster", var.environment_name) setting { name = "containerInsights" @@ -74,7 +74,7 @@ resource "aws_ecs_task_definition" "web" { resource "aws_ecs_service" "web" { name = "${var.environment_name}-web" - cluster = aws_ecs_cluster.application.id + cluster = aws_ecs_cluster.cluster.id task_definition = aws_ecs_task_definition.web.arn desired_count = var.container_count launch_type = "FARGATE" @@ -86,8 +86,8 @@ resource "aws_ecs_service" "web" { } load_balancer { - target_group_arn = aws_lb_target_group.application.arn - container_name = "${local.app_env_name}-web" + target_group_arn = aws_lb_target_group.target_group.arn + container_name = "${var.environment_name}-web" container_port = 8080 } diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf index 996559f..4125d2a 100644 --- a/terraform/modules/ecs_deployment/iam.tf +++ b/terraform/modules/ecs_deployment/iam.tf @@ -50,7 +50,7 @@ data "aws_iam_policy_document" "ecs_execution_role_policy" { ] resources = [ "arn:aws:secretsmanager:*:*:secret:${aws_secretsmanager_secret.web_infrastructure.name}-*", - "arn:aws:secretsmanager:*:*:secret:${aws_secretsmanager_secret.web_config.name}-*" + "arn:aws:secretsmanager:*:*:secret:${data.aws_secretsmanager_secret.web_config.name}-*" ] } diff --git a/terraform/modules/ecs_deployment/s3.tf b/terraform/modules/ecs_deployment/s3.tf index 7261709..5e69c2f 100644 --- a/terraform/modules/ecs_deployment/s3.tf +++ b/terraform/modules/ecs_deployment/s3.tf @@ -26,7 +26,7 @@ resource "aws_s3_bucket_versioning" "bucket" { resource "aws_s3_bucket_server_side_encryption_configuration" "bucket" { - bucket = aws_s3_bucket.mybucket.id + bucket = aws_s3_bucket.bucket.id rule { apply_server_side_encryption_by_default { diff --git a/terraform/modules/ecs_deployment/security_groups.tf b/terraform/modules/ecs_deployment/security_groups.tf index e3393ee..a6ebacb 100644 --- a/terraform/modules/ecs_deployment/security_groups.tf +++ b/terraform/modules/ecs_deployment/security_groups.tf @@ -1,4 +1,4 @@ -resource "aws_securtiy_group" "load_balancer" { +resource "aws_security_group" "load_balancer" { name = format("%s load balancer", var.environment_name) ingress { @@ -17,7 +17,7 @@ resource "aws_securtiy_group" "load_balancer" { } } -resource "aws_securtiy_group" "web" { +resource "aws_security_group" "web" { name = format("%s web", var.environment_name) ingress { @@ -28,7 +28,7 @@ resource "aws_securtiy_group" "web" { } } -resource "aws_securtiy_group" "database" { +resource "aws_security_group" "database" { name = format("%s database", var.environment_name) ingress { diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index 0c60f6d..0524818 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -1,13 +1,4 @@ # Required Variables -variable "aws_profile_name" { - type = string -} - -variable "aws_region" { - type = string - default = "us-east-1" -} - variable "environment_name" { type = string } @@ -44,6 +35,14 @@ variable "certificate_manager_arn" { type = string } +variable "ecr_repository_name" { + type = string +} + +variable "ecr_image_uri" { + type = string +} + # Optional Variables variable "rds_backup_retention_period" { From af1c80b5093543456e4a2d7c1e7b234d13ea7afa Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 17:39:00 -0400 Subject: [PATCH 12/39] adds readme entry --- readme.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 2c1d3cd..7688cf8 100644 --- a/readme.md +++ b/readme.md @@ -102,7 +102,7 @@ pip install eb-create-environment # For automating the creation of Elastic Bean pip install eb-ssm # For Elastic Beanstalk SSH functionality without requiring shared private keys ``` -## Creating a new environment +## Creating a new Elastic Beanstalk environment To do create a new Elastic Beanstalk environment, modify the contents of [.elasticbeanstalk/eb_create_environment.yml]([.elasticbeanstalk/eb_create_environment.yml]) and run `eb-create-environment -c .elasticbeanstalk/eb_create_environment.yml`. @@ -121,6 +121,28 @@ DEFAULT_FROM_EMAIL Following that, deploy your code to the environment (see below). + +## Creating a new ECS environment + +1. Create an ECR repository +2. Build and push the docker file to it +3. Create a bucket for holding terraform config +4. Create an SES identity and from email (if using SES) +5. Create an AWS certificate manager certificate for your domain +6. Create a secrets manager secret containing the config parameters needed by the application (do not include those managed by terraform in terraform/modules/ecs_deployment/secrets_manager.tf) +7. Ensure these config parameters have been added to the list in terraform/modules/ecs_deployment/ecs.tf +8. Fill in the missing values in terraform/envs//main.tf +9. Run terraform to set up that environment +``` +cd terraform/envs/ +terraform init +terraform plan +terraform apply +``` +10. Add a DNS entry from your domain name to the created load balancer + + + ## Deploying code To deploy new versions of your code to your environment, run `eb deploy ` using the EB CLI to deploy your code to that environment. From b9cb3065a60a30760c12a250b94b8f4729ff737d Mon Sep 17 00:00:00 2001 From: kyfantaz Date: Mon, 21 Oct 2024 17:41:35 -0400 Subject: [PATCH 13/39] fix Dockerfile --- Dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9b91e86..29f3427 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,13 +15,13 @@ RUN set -ex \ procps \ postgresql-client \ " \ - && apt update && apt install -y $buildDeps $deps --no-install-recommends \ + && apt update && apt install -y $buildDeps $deps --no-install-recommends # Install python dependencies ADD requirements.txt /app/requirements.txt RUN set -ex \ - && pip install --no-cache-dir -r /app/requirements.txt \ + && pip install --no-cache-dir -r /app/requirements.txt # Cleanup installs RUN set -ex \ @@ -34,6 +34,13 @@ ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH # START_FEATURE django_react +# LTS Version of Node +ARG NODE_VERSION=22 + +# install node +RUN curl -fsSL https://deb.nodesource.com/setup_{$NODE_VERSION}.x | bash - +RUN apt-get update && apt install nodejs -y + COPY ./nwb.config.js /app/nwb.config.js COPY ./package.json /app/package.json COPY ./package-lock.json /app/package-lock.json From 1cb50271c21937b44c18a7a9d3b50c4f9442cfc1 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 17:41:49 -0400 Subject: [PATCH 14/39] adds terraform to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1441253..825c31b 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ staticfiles/* static/webpack_bundles/ webpack-stats.json # END_FEATURE django_react + +.terraform.lock.hcl +.terraform/ From b5b1fd17af2e380e9854d59afb65d1409f763bc5 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 21 Oct 2024 17:42:50 -0400 Subject: [PATCH 15/39] cleanup --- terraform/envs/production/main.tf | 1 - terraform/envs/staging/main.tf | 1 - terraform/modules/ecs_deployment/variables.tf | 4 ---- 3 files changed, 6 deletions(-) diff --git a/terraform/envs/production/main.tf b/terraform/envs/production/main.tf index 2776baa..5febb9f 100644 --- a/terraform/envs/production/main.tf +++ b/terraform/envs/production/main.tf @@ -24,7 +24,6 @@ module "ecs_deployment" { # Required Variables environment_name = "production" - terraform_backend_bucket = "" # TODO: FILL ME IN vpc_id = "" # TODO: FILL ME IN web_config_secret_name = "" # TODO: FILL ME IN s3_bucket_prefix = "" # TODO: FILL ME IN diff --git a/terraform/envs/staging/main.tf b/terraform/envs/staging/main.tf index e499f3f..5c3aebb 100644 --- a/terraform/envs/staging/main.tf +++ b/terraform/envs/staging/main.tf @@ -24,7 +24,6 @@ module "ecs_deployment" { # Required Variables environment_name = "staging" - terraform_backend_bucket = "" # TODO: FILL ME IN vpc_id = "" # TODO: FILL ME IN web_config_secret_name = "" # TODO: FILL ME IN s3_bucket_prefix = "" # TODO: FILL ME IN diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index 0524818..3a029e7 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -3,10 +3,6 @@ variable "environment_name" { type = string } -variable "terraform_backend_bucket" { - type = string -} - variable "vpc_id" { type = string } From 3f012452c64b8c863524e83e0900630e70353f5f Mon Sep 17 00:00:00 2001 From: kyfantaz Date: Tue, 22 Oct 2024 13:14:19 -0400 Subject: [PATCH 16/39] fixes and refactor; adds outputs --- terraform/modules/ecs_deployment/ecs.tf | 26 ------------------- terraform/modules/ecs_deployment/iam.tf | 20 ++++++++------ terraform/modules/ecs_deployment/locals.tf | 17 ++++++++++++ terraform/modules/ecs_deployment/outputs.tf | 9 +++++++ terraform/modules/ecs_deployment/rds.tf | 5 ++-- .../modules/ecs_deployment/secrets_manager.tf | 2 +- terraform/modules/ecs_deployment/variables.tf | 5 +++- 7 files changed, 45 insertions(+), 39 deletions(-) create mode 100644 terraform/modules/ecs_deployment/locals.tf diff --git a/terraform/modules/ecs_deployment/ecs.tf b/terraform/modules/ecs_deployment/ecs.tf index e830ab8..52a720b 100644 --- a/terraform/modules/ecs_deployment/ecs.tf +++ b/terraform/modules/ecs_deployment/ecs.tf @@ -7,36 +7,12 @@ resource "aws_ecs_cluster" "cluster" { } } - resource "aws_ecs_cluster_capacity_providers" "fargate_provider" { cluster_name = aws_ecs_cluster.cluster.name capacity_providers = ["FARGATE"] } - -locals { - ecs_secrets = [ - { - name : "DATABASE_URL", - valueFrom : "${aws_secretsmanager_secret.web_infrastructure.arn}:DATABASE_URL::" - }, - { - name : "SECRET_KEY", - valueFrom : "${aws_secretsmanager_secret.web_infrastructure.arn}:SECRET_KEY::" - }, - { - name : "AWS_STORAGE_BUCKET_NAME", - valueFrom : "${aws_secretsmanager_secret.web_infrastructure.arn}:AWS_STORAGE_BUCKET_NAME::" - }, - { - name : "DEFAULT_FROM_EMAIL", - valueFrom : "${aws_secretsmanager_secret.web_infrastructure.arn}:DEFAULT_FROM_EMAIL::" - }, - ] -} - - resource "aws_ecs_task_definition" "web" { family = "${var.environment_name}-web" @@ -71,7 +47,6 @@ resource "aws_ecs_task_definition" "web" { task_role_arn = aws_iam_role.ecs_task_role.arn } - resource "aws_ecs_service" "web" { name = "${var.environment_name}-web" cluster = aws_ecs_cluster.cluster.id @@ -98,7 +73,6 @@ resource "aws_ecs_service" "web" { } } - resource "aws_cloudwatch_log_group" "log_group" { name = var.environment_name retention_in_days = 90 diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf index 4125d2a..c19cb85 100644 --- a/terraform/modules/ecs_deployment/iam.tf +++ b/terraform/modules/ecs_deployment/iam.tf @@ -120,17 +120,21 @@ data "aws_iam_policy_document" "ecs_task_role_policy" { resource "aws_iam_role" "ecs_execution_role" { name = format("%s-ecs-execution-role", var.environment_name) assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json - inline_policy { - name = "ecs-execution-role-policy" - policy = data.aws_iam_policy_document.ecs_execution_role_policy.json - } +} + +resource "aws_iam_role_policy" "ecs_execution_role_policy" { + name = "ecs-execution-role-policy" + role = aws_iam_role.ecs_execution_role.id + policy = data.aws_iam_policy_document.ecs_execution_role_policy.json } resource "aws_iam_role" "ecs_task_role" { name = format("%s-ecs-task-role", var.environment_name) assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json - inline_policy { - name = "ecs-task-role-policy" - policy = data.aws_iam_policy_document.ecs_task_role_policy.json - } +} + +resource "aws_iam_role_policy" "ecs_task_role_policy" { + name = "ecs-task-role-policy" + role = aws_iam_role.ecs_task_role.id + policy = data.aws_iam_policy_document.ecs_task_role_policy.json } diff --git a/terraform/modules/ecs_deployment/locals.tf b/terraform/modules/ecs_deployment/locals.tf new file mode 100644 index 0000000..25b741b --- /dev/null +++ b/terraform/modules/ecs_deployment/locals.tf @@ -0,0 +1,17 @@ +locals { + app_env_name = "${var.application_name}-${var.environment_name}" + + ecs_secrets_map = { + DATABASE_URL = "${aws_secretsmanager_secret.web_infrastructure.arn}:DATABASE_URL::" + SECRET_KEY = "${aws_secretsmanager_secret.web_infrastructure.arn}:SECRET_KEY::" + AWS_STORAGE_BUCKET_NAME = "${aws_secretsmanager_secret.web_infrastructure.arn}:AWS_STORAGE_BUCKET_NAME::" + DEFAULT_FROM_EMAIL = "${aws_secretsmanager_secret.web_infrastructure.arn}:DEFAULT_FROM_EMAIL::" + } + + ecs_secrets = [ + for key, value in local.ecs_secrets_map : { + name = key + valueFrom = value + } + ] +} \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/outputs.tf b/terraform/modules/ecs_deployment/outputs.tf index e69de29..8df6dde 100644 --- a/terraform/modules/ecs_deployment/outputs.tf +++ b/terraform/modules/ecs_deployment/outputs.tf @@ -0,0 +1,9 @@ +output "cluster_id" { + description = "The ID of the ECS cluster" + value = aws_ecs_cluster.cluster.id +} + +output "public_ip" { + description = "The public IP address of the load balancer for the web service" + value = aws_lb.alb.dns_name +} \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/rds.tf b/terraform/modules/ecs_deployment/rds.tf index 03b0d37..b645914 100644 --- a/terraform/modules/ecs_deployment/rds.tf +++ b/terraform/modules/ecs_deployment/rds.tf @@ -6,15 +6,14 @@ resource "random_password" "db_password" { resource "aws_db_instance" "database" { allocated_storage = 20 - max_allocated_storage = 10000 allow_major_version_upgrade = true apply_immediately = true backup_retention_period = var.rds_backup_retention_period - db_name = "database" + db_name = format("%s_db", replace(var.application_name, "-", "_")) deletion_protection = var.rds_deletion_protection engine = "postgres" engine_version = var.rds_engine_version - identifier = format("%s-database", var.environment_name) + identifier = format("%s-%s-db", var.application_name, var.environment_name) instance_class = var.rds_instance_class multi_az = var.rds_multi_az password = random_password.db_password.result diff --git a/terraform/modules/ecs_deployment/secrets_manager.tf b/terraform/modules/ecs_deployment/secrets_manager.tf index 37c13b8..4ae6c32 100644 --- a/terraform/modules/ecs_deployment/secrets_manager.tf +++ b/terraform/modules/ecs_deployment/secrets_manager.tf @@ -1,5 +1,5 @@ resource "aws_secretsmanager_secret" "web_infrastructure" { - name = format("%s-infrastructure", var.environment_name) + name = format("%s-%s-web-infrastructure", var.application_name, var.environment_name) } resource "aws_secretsmanager_secret_version" "web_infrastructure" { diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index 3a029e7..d3e5924 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -1,4 +1,8 @@ # Required Variables +variable "application_name" { + type = string +} + variable "environment_name" { type = string } @@ -39,7 +43,6 @@ variable "ecr_image_uri" { type = string } - # Optional Variables variable "rds_backup_retention_period" { type = number From cdd82b5073ea13a33d32c6e6ab45c02fe8422d11 Mon Sep 17 00:00:00 2001 From: kyfantaz Date: Wed, 23 Oct 2024 00:51:15 -0400 Subject: [PATCH 17/39] adds other secrets --- terraform/modules/ecs_deployment/iam.tf | 2 +- terraform/modules/ecs_deployment/locals.tf | 39 +++++++++++++------ .../modules/ecs_deployment/secrets_manager.tf | 1 + terraform/modules/ecs_deployment/variables.tf | 2 +- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf index c19cb85..d09ff24 100644 --- a/terraform/modules/ecs_deployment/iam.tf +++ b/terraform/modules/ecs_deployment/iam.tf @@ -60,7 +60,7 @@ data "aws_iam_policy_document" "ecs_execution_role_policy" { "kms:Decrypt" ] resources = [ - format("arn:aws:kms:*:*:aws/secretsmanager") + format("arn:aws:kms:*:*:aws/secretsmanager"), ] } } diff --git a/terraform/modules/ecs_deployment/locals.tf b/terraform/modules/ecs_deployment/locals.tf index 25b741b..928fe34 100644 --- a/terraform/modules/ecs_deployment/locals.tf +++ b/terraform/modules/ecs_deployment/locals.tf @@ -1,17 +1,34 @@ locals { app_env_name = "${var.application_name}-${var.environment_name}" - ecs_secrets_map = { - DATABASE_URL = "${aws_secretsmanager_secret.web_infrastructure.arn}:DATABASE_URL::" - SECRET_KEY = "${aws_secretsmanager_secret.web_infrastructure.arn}:SECRET_KEY::" - AWS_STORAGE_BUCKET_NAME = "${aws_secretsmanager_secret.web_infrastructure.arn}:AWS_STORAGE_BUCKET_NAME::" - DEFAULT_FROM_EMAIL = "${aws_secretsmanager_secret.web_infrastructure.arn}:DEFAULT_FROM_EMAIL::" - } - ecs_secrets = [ - for key, value in local.ecs_secrets_map : { - name = key - valueFrom = value - } + { + name : "ALLOWED_HOSTS", + valueFrom : format("%s:ALLOWED_HOSTS::", aws_secretsmanager_secret.web_infrastructure.arn) + }, + { + name : "DATABASE_URL", + valueFrom : format("%s:DATABASE_URL::", aws_secretsmanager_secret.web_infrastructure.arn) + }, + { + name : "SECRET_KEY", + valueFrom : format("%s:SECRET_KEY::", aws_secretsmanager_secret.web_infrastructure.arn) + }, + { + name : "AWS_STORAGE_BUCKET_NAME", + valueFrom : format("%s:AWS_STORAGE_BUCKET_NAME::", aws_secretsmanager_secret.web_infrastructure.arn) + }, + { + name : "DEFAULT_FROM_EMAIL", + valueFrom : format("%s:DEFAULT_FROM_EMAIL::", aws_secretsmanager_secret.web_infrastructure.arn) + }, + { + name : "GOOGLE_OAUTH2_KEY", + valueFrom : format("%s:GOOGLE_OAUTH2_KEY::", data.aws_secretsmanager_secret.web_config.arn) + }, + { + name : "GOOGLE_OAUTH2_SECRET", + valueFrom : format("%s:GOOGLE_OAUTH2_SECRET::", data.aws_secretsmanager_secret.web_config.arn) + }, ] } \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/secrets_manager.tf b/terraform/modules/ecs_deployment/secrets_manager.tf index 4ae6c32..f256307 100644 --- a/terraform/modules/ecs_deployment/secrets_manager.tf +++ b/terraform/modules/ecs_deployment/secrets_manager.tf @@ -13,6 +13,7 @@ resource "aws_secretsmanager_secret_version" "web_infrastructure" { ) DEFAULT_FROM_EMAIL = var.ses_from_email SECRET_KEY = random_password.app_secret_key.result + ALLOWED_HOSTS = "example.zagaran.com" }) } diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index d3e5924..f2551be 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -12,7 +12,7 @@ variable "vpc_id" { } variable "web_config_secret_name" { - type = string + type = string # key for secrets_manager secrets that are not terraform managed } variable "s3_bucket_prefix" { From 6bacba9aebf7db9f118a8553a637f2c394da7fea Mon Sep 17 00:00:00 2001 From: kyfantaz Date: Wed, 23 Oct 2024 16:41:06 -0400 Subject: [PATCH 18/39] fix --- terraform/modules/ecs_deployment/iam.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf index d09ff24..3c37eb7 100644 --- a/terraform/modules/ecs_deployment/iam.tf +++ b/terraform/modules/ecs_deployment/iam.tf @@ -38,7 +38,7 @@ data "aws_iam_policy_document" "ecs_execution_role_policy" { "logs:PutLogEvents" ] resources = [ - "arn:aws:logs:*:*:log-group:${var.environment_name}:*", + "arn:aws:logs:*:*:log-group:${local.app_env_name}:*", ] } @@ -49,8 +49,8 @@ data "aws_iam_policy_document" "ecs_execution_role_policy" { "secretsmanager:DescribeSecret" ] resources = [ - "arn:aws:secretsmanager:*:*:secret:${aws_secretsmanager_secret.web_infrastructure.name}-*", - "arn:aws:secretsmanager:*:*:secret:${data.aws_secretsmanager_secret.web_config.name}-*" + "arn:aws:secretsmanager:*:*:secret:${aws_secretsmanager_secret.web_infrastructure.name}", + "arn:aws:secretsmanager:*:*:secret:${data.aws_secretsmanager_secret.web_config.name}" ] } From ff92b92cb1787bc1dc38879dd26637d87ce57142 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Thu, 24 Oct 2024 14:00:41 -0400 Subject: [PATCH 19/39] adds application name to main.tf --- terraform/envs/production/main.tf | 1 + terraform/envs/staging/main.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/terraform/envs/production/main.tf b/terraform/envs/production/main.tf index 5febb9f..148bb7b 100644 --- a/terraform/envs/production/main.tf +++ b/terraform/envs/production/main.tf @@ -24,6 +24,7 @@ module "ecs_deployment" { # Required Variables environment_name = "production" + application_name = "" # TODO: FILL ME IN vpc_id = "" # TODO: FILL ME IN web_config_secret_name = "" # TODO: FILL ME IN s3_bucket_prefix = "" # TODO: FILL ME IN diff --git a/terraform/envs/staging/main.tf b/terraform/envs/staging/main.tf index 5c3aebb..d6bfca7 100644 --- a/terraform/envs/staging/main.tf +++ b/terraform/envs/staging/main.tf @@ -24,6 +24,7 @@ module "ecs_deployment" { # Required Variables environment_name = "staging" + application_name = "" # TODO: FILL ME IN vpc_id = "" # TODO: FILL ME IN web_config_secret_name = "" # TODO: FILL ME IN s3_bucket_prefix = "" # TODO: FILL ME IN From b7eb6dcd98654bd570f9c0388d115b7bd3354566 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Thu, 24 Oct 2024 16:47:08 -0400 Subject: [PATCH 20/39] fixes several issues with ECS deployment --- config/settings.py | 6 +- terraform/modules/ecs_deployment/ecs.tf | 16 +++--- terraform/modules/ecs_deployment/iam.tf | 6 +- terraform/modules/ecs_deployment/locals.tf | 55 ++++++++++--------- .../modules/ecs_deployment/secrets_manager.tf | 1 - .../modules/ecs_deployment/security_groups.tf | 51 ++++++++++++++++- 6 files changed, 92 insertions(+), 43 deletions(-) diff --git a/config/settings.py b/config/settings.py index e0ed80d..d3f96f5 100644 --- a/config/settings.py +++ b/config/settings.py @@ -48,6 +48,8 @@ # Set to True to enable the Django Debug Toolbar DEBUG_TOOLBAR=(bool, False), # END_FEATURE debug_toolbar + + EC2_METADATA=(bool, True) ) # If ALLWED_HOSTS has been configured, then we're running on a server and # can skip looking for a .env file (this assumes that .env files @@ -76,10 +78,12 @@ # that this is not the production site PRODUCTION = env("PRODUCTION") +EC2_METADATA = env("EC2_METADATA") + ALLOWED_HOSTS = env("ALLOWED_HOSTS") if LOCALHOST is True: ALLOWED_HOSTS = ["127.0.0.1", "localhost"] -else: +elif EC2_METADATA: # START_FEATURE elastic_beanstalk # if using AWS hosting from ec2_metadata import ec2_metadata diff --git a/terraform/modules/ecs_deployment/ecs.tf b/terraform/modules/ecs_deployment/ecs.tf index 52a720b..f7d39f3 100644 --- a/terraform/modules/ecs_deployment/ecs.tf +++ b/terraform/modules/ecs_deployment/ecs.tf @@ -1,5 +1,5 @@ resource "aws_ecs_cluster" "cluster" { - name = format("%s-cluster", var.environment_name) + name = "${var.application_name}-${var.environment_name}" setting { name = "containerInsights" @@ -14,11 +14,11 @@ resource "aws_ecs_cluster_capacity_providers" "fargate_provider" { } resource "aws_ecs_task_definition" "web" { - family = "${var.environment_name}-web" + family = "${var.application_name}-${var.environment_name}-web" container_definitions = jsonencode([ { - name = "${var.environment_name}-web" + name = "${var.application_name}-${var.environment_name}-web" image = var.ecr_image_uri essential = true portMappings = [ @@ -30,7 +30,7 @@ resource "aws_ecs_task_definition" "web" { logConfiguration = { logDriver = "awslogs", options = { - awslogs-group = aws_cloudwatch_log_group.log_group.name, + awslogs-group = aws_cloudwatch_log_group.web_log_group.name, awslogs-region = data.aws_region.current.name, awslogs-stream-prefix = "ecs" } @@ -48,7 +48,7 @@ resource "aws_ecs_task_definition" "web" { } resource "aws_ecs_service" "web" { - name = "${var.environment_name}-web" + name = "${var.application_name}-${var.environment_name}-web" cluster = aws_ecs_cluster.cluster.id task_definition = aws_ecs_task_definition.web.arn desired_count = var.container_count @@ -62,7 +62,7 @@ resource "aws_ecs_service" "web" { load_balancer { target_group_arn = aws_lb_target_group.target_group.arn - container_name = "${var.environment_name}-web" + container_name = "${var.application_name}-${var.environment_name}-web" container_port = 8080 } @@ -73,7 +73,7 @@ resource "aws_ecs_service" "web" { } } -resource "aws_cloudwatch_log_group" "log_group" { - name = var.environment_name +resource "aws_cloudwatch_log_group" "web_log_group" { + name = "${var.application_name}-${var.environment_name}-web" retention_in_days = 90 } \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf index 3c37eb7..a4b41d1 100644 --- a/terraform/modules/ecs_deployment/iam.tf +++ b/terraform/modules/ecs_deployment/iam.tf @@ -38,7 +38,7 @@ data "aws_iam_policy_document" "ecs_execution_role_policy" { "logs:PutLogEvents" ] resources = [ - "arn:aws:logs:*:*:log-group:${local.app_env_name}:*", + "${aws_cloudwatch_log_group.web_log_group.arn}:*" ] } @@ -49,8 +49,8 @@ data "aws_iam_policy_document" "ecs_execution_role_policy" { "secretsmanager:DescribeSecret" ] resources = [ - "arn:aws:secretsmanager:*:*:secret:${aws_secretsmanager_secret.web_infrastructure.name}", - "arn:aws:secretsmanager:*:*:secret:${data.aws_secretsmanager_secret.web_config.name}" + aws_secretsmanager_secret.web_infrastructure.arn, + data.aws_secretsmanager_secret.web_config.arn ] } diff --git a/terraform/modules/ecs_deployment/locals.tf b/terraform/modules/ecs_deployment/locals.tf index 928fe34..0b58e9e 100644 --- a/terraform/modules/ecs_deployment/locals.tf +++ b/terraform/modules/ecs_deployment/locals.tf @@ -1,34 +1,35 @@ locals { + infrastructure_secrets = [ + "DATABASE_URL", + "SECRET_KEY", + "AWS_STORAGE_BUCKET_NAME", + "DEFAULT_FROM_EMAIL" + ] + + config_secrets = [ + "ALLOWED_HOSTS", + "GOOGLE_OAUTH2_KEY", + "GOOGLE_OAUTH2_SECRET", + "EC2_METADATA", + ] + app_env_name = "${var.application_name}-${var.environment_name}" - ecs_secrets = [ - { - name : "ALLOWED_HOSTS", - valueFrom : format("%s:ALLOWED_HOSTS::", aws_secretsmanager_secret.web_infrastructure.arn) - }, - { - name : "DATABASE_URL", - valueFrom : format("%s:DATABASE_URL::", aws_secretsmanager_secret.web_infrastructure.arn) - }, - { - name : "SECRET_KEY", - valueFrom : format("%s:SECRET_KEY::", aws_secretsmanager_secret.web_infrastructure.arn) - }, + ecs_infrastructure_secrets = [ + for setting in local.infrastructure_secrets : { - name : "AWS_STORAGE_BUCKET_NAME", - valueFrom : format("%s:AWS_STORAGE_BUCKET_NAME::", aws_secretsmanager_secret.web_infrastructure.arn) - }, - { - name : "DEFAULT_FROM_EMAIL", - valueFrom : format("%s:DEFAULT_FROM_EMAIL::", aws_secretsmanager_secret.web_infrastructure.arn) - }, - { - name : "GOOGLE_OAUTH2_KEY", - valueFrom : format("%s:GOOGLE_OAUTH2_KEY::", data.aws_secretsmanager_secret.web_config.arn) - }, + name : setting + valueFrom : format("%s:%s::", aws_secretsmanager_secret.web_infrastructure.arn, setting) + } + ] + + ecs_config_secrets = [ + for setting in local.config_secrets : { - name : "GOOGLE_OAUTH2_SECRET", - valueFrom : format("%s:GOOGLE_OAUTH2_SECRET::", data.aws_secretsmanager_secret.web_config.arn) - }, + name : setting + valueFrom : format("%s:%s::", data.aws_secretsmanager_secret.web_config.arn, setting) + } ] + + ecs_secrets = concat(local.ecs_infrastructure_secrets, local.ecs_config_secrets) } \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/secrets_manager.tf b/terraform/modules/ecs_deployment/secrets_manager.tf index f256307..4ae6c32 100644 --- a/terraform/modules/ecs_deployment/secrets_manager.tf +++ b/terraform/modules/ecs_deployment/secrets_manager.tf @@ -13,7 +13,6 @@ resource "aws_secretsmanager_secret_version" "web_infrastructure" { ) DEFAULT_FROM_EMAIL = var.ses_from_email SECRET_KEY = random_password.app_secret_key.result - ALLOWED_HOSTS = "example.zagaran.com" }) } diff --git a/terraform/modules/ecs_deployment/security_groups.tf b/terraform/modules/ecs_deployment/security_groups.tf index a6ebacb..b907237 100644 --- a/terraform/modules/ecs_deployment/security_groups.tf +++ b/terraform/modules/ecs_deployment/security_groups.tf @@ -1,5 +1,5 @@ resource "aws_security_group" "load_balancer" { - name = format("%s load balancer", var.environment_name) + name = format("%s %s load balancer", var.application_name, var.environment_name) ingress { from_port = 80 @@ -15,10 +15,25 @@ resource "aws_security_group" "load_balancer" { cidr_blocks = ["0.0.0.0/0"] ipv6_cidr_blocks = ["::/0"] } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = format("%s-%s-lb", var.application_name, var.environment_name) + } + + lifecycle { + create_before_destroy = true + } } resource "aws_security_group" "web" { - name = format("%s web", var.environment_name) + name = format("%s %s web", var.application_name, var.environment_name) ingress { from_port = 8080 @@ -26,10 +41,25 @@ resource "aws_security_group" "web" { protocol = "tcp" security_groups = [aws_security_group.load_balancer.id] } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = format("%s-%s-web", var.application_name, var.environment_name) + } + + lifecycle { + create_before_destroy = true + } } resource "aws_security_group" "database" { - name = format("%s database", var.environment_name) + name = format("%s %s database", var.application_name, var.environment_name) ingress { from_port = 5432 @@ -37,4 +67,19 @@ resource "aws_security_group" "database" { protocol = "tcp" security_groups = [aws_security_group.web.id] } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = format("%s-%s-db", var.application_name, var.environment_name) + } + + lifecycle { + create_before_destroy = true + } } From cfc6a0d8b9bcfc9024aca7bc9de6d13deb5cb2b4 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Thu, 24 Oct 2024 16:57:50 -0400 Subject: [PATCH 21/39] readme --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 7688cf8..7f75762 100644 --- a/readme.md +++ b/readme.md @@ -129,8 +129,8 @@ Following that, deploy your code to the environment (see below). 3. Create a bucket for holding terraform config 4. Create an SES identity and from email (if using SES) 5. Create an AWS certificate manager certificate for your domain -6. Create a secrets manager secret containing the config parameters needed by the application (do not include those managed by terraform in terraform/modules/ecs_deployment/secrets_manager.tf) -7. Ensure these config parameters have been added to the list in terraform/modules/ecs_deployment/ecs.tf +6. Create a secrets manager secret containing the config parameters needed by the application (you do not need include "DATABASE_URL", "SECRET_KEY", "AWS_STORAGE_BUCKET_NAME", or "DEFAULT_FROM_EMAIL" as those are managed by terraform in terraform/modules/ecs_deployment/secrets_manager.tf) +7. Ensure your config parameters have been added to the list in terraform/modules/ecs_deployment/locals.tf 8. Fill in the missing values in terraform/envs//main.tf 9. Run terraform to set up that environment ``` From 55733725cfb3036d00fed502201e8e58bbf73bcf Mon Sep 17 00:00:00 2001 From: kyfantaz Date: Mon, 28 Oct 2024 17:19:56 -0400 Subject: [PATCH 22/39] Dockerfile fixes; adds .env for build purposes --- Dockerfile | 21 ++++++++++----------- config/.env.build | 11 +++++++++++ 2 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 config/.env.build diff --git a/Dockerfile b/Dockerfile index 29f3427..2a289b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,24 +33,23 @@ RUN set -ex \ ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH -# START_FEATURE django_react -# LTS Version of Node +COPY . /app/ +# Add temporary copy of env file to allow running management commands +COPY ./config/.env.build /app/config/.env + +# LTS Version of Node is 22 ARG NODE_VERSION=22 +# START_FEATURE django_react +# if using nwb, nwb requires Node 16. TODO remove nwb +ARG NODE_VERSION=16 +# END_FEATURE django_react + # install node RUN curl -fsSL https://deb.nodesource.com/setup_{$NODE_VERSION}.x | bash - RUN apt-get update && apt install nodejs -y -COPY ./nwb.config.js /app/nwb.config.js -COPY ./package.json /app/package.json -COPY ./package-lock.json /app/package-lock.json RUN npm install -# END_FEATURE django_react - -COPY . /app/ - -# Add temporary copy of env file to allow running management commands -COPY ./config/.env.example /app/config/.env # START_FEATURE django_react RUN ./node_modules/.bin/nwb build --no-vendor diff --git a/config/.env.build b/config/.env.build new file mode 100644 index 0000000..56eae5f --- /dev/null +++ b/config/.env.build @@ -0,0 +1,11 @@ +# Env file for the purposes of building the docker image +ALLOWED_HOSTS= +SECRET_KEY= +DATABASE_URL=sqlite:///db.sqlite3 +GOOGLE_OAUTH2_KEY= +GOOGLE_OAUTH2_SECRET= +DEFAULT_FROM_EMAIL= +EC2_METADATA=False +AWS_SES_REGION_NAME= +AWS_SES_REGION_ENDPOINT= +AWS_STORAGE_BUCKET_NAME= From a0a73466a61ef5a745a9c22ac309bd083c505806 Mon Sep 17 00:00:00 2001 From: kyfantaz Date: Wed, 30 Oct 2024 16:15:42 -0400 Subject: [PATCH 23/39] change 8000->8080 port, matching terraform --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2a289b0..fdd4707 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,7 +63,7 @@ RUN python manage.py collectstatic --noinput RUN rm /app/config/.env -EXPOSE 8000 +EXPOSE 8080 -CMD ["gunicorn", "--bind", ":8000", "--workers", "3", "config.wsgi:application", "--access-logfile", "-", "--error-logfile", "-"] +CMD ["gunicorn", "--bind", ":8080", "--workers", "3", "config.wsgi:application", "--access-logfile", "-", "--error-logfile", "-"] # END_FEATURE docker From f2f42e990164d9d59f9f1ff7da0df4e612cde452 Mon Sep 17 00:00:00 2001 From: kyfantaz Date: Thu, 31 Oct 2024 15:46:02 -0400 Subject: [PATCH 24/39] edits dockerfile; adds docker-compose.yml as option --- Dockerfile | 7 +++++++ docker-compose.yml | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index fdd4707..3c08fde 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,12 @@ # START_FEATURE docker FROM python:3.11.4-slim-buster +# reduces file creation +ENV PYTHONDONTWRITEBYTECODE=1 +# disables output buffering so logs are flushed to console +ENV PYTHONUNBUFFERED=1 + +# Set container working directory WORKDIR /app RUN set -ex \ @@ -33,6 +39,7 @@ RUN set -ex \ ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH +# Copy project files into the container COPY . /app/ # Add temporary copy of env file to allow running management commands COPY ./config/.env.build /app/config/.env diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a655e33 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,44 @@ +version: '3' + +services: + web: + build: . + command: > + /bin/bash -c " + python manage.py collectstatic --noinput && + sleep 2 && + python manage.py migrate --noinput && + python manage.py runserver 0.0.0.0:8001 + " + volumes: + - .:/app + ports: + - "8001:8001" + environment: + DEBUG: True + LOCALHOST: True + DATABASE_URL: postgres://dbuser:dbpassword@db:5432/dbname + SECRET_KEY: 'supersecret' + AWS_STORAGE_BUCKET_NAME: "" + DEFAULT_FROM_EMAIL: "example@example.com" + ALLOWED_HOSTS: 'localhost,127.0.0.1' + GOOGLE_OAUTH2_KEY: "" + GOOGLE_OAUTH2_SECRET: "" + EC2_METADATA: False + SENTRY_DSN: "" + depends_on: + - db + + db: + image: postgres:16 + environment: + POSTGRES_DB: dbname + POSTGRES_USER: dbuser + POSTGRES_PASSWORD: dbpassword + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + +volumes: + postgres_data: From e5d39c69c40a5bf250484ea62c064c7fc7b35895 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Thu, 31 Oct 2024 16:34:11 -0400 Subject: [PATCH 25/39] adds whitenoise --- config/settings.py | 12 ++++++++++-- requirements-dev.txt | 4 ++++ requirements.in | 1 + requirements.txt | 2 ++ terraform/modules/ecs_deployment/secrets_manager.tf | 4 ++-- 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/config/settings.py b/config/settings.py index d3f96f5..135b79b 100644 --- a/config/settings.py +++ b/config/settings.py @@ -14,7 +14,6 @@ import environ - env = environ.Env( # Sets Django's ALLOWED_HOSTS setting ALLOWED_HOSTS=(list, []), @@ -133,6 +132,9 @@ MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", + # START_FEATURE docker + "whitenoise.middleware.WhiteNoiseMiddleware", + # END_FEATURE docker "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "common.middleware.MaintenanceModeMiddleware", @@ -318,6 +320,12 @@ "default_acl": "private", } } + +STATIC_BACKEND = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage" +# START_FEATURE docker +STATIC_BACKEND = "whitenoise.storage.CompressedManifestStaticFilesStorage" +# END_FEATURE docker + # END_FEATURE django_storages STORAGES = { "default": DEFAULT_STORAGE, @@ -332,7 +340,7 @@ }, # END_FEATURE sass_bootstrap "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage", + "BACKEND": STATIC_BACKEND, }, } diff --git a/requirements-dev.txt b/requirements-dev.txt index c44efe1..757130e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -322,6 +322,10 @@ werkzeug==3.0.1 # via -r requirements-dev.in wheel==0.41.3 # via pip-tools +whitenoise==6.8.2 + # via + # -c requirements.txt + # -r requirements.in # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements.in b/requirements.in index 6fe690f..e7ad4e9 100644 --- a/requirements.in +++ b/requirements.in @@ -33,6 +33,7 @@ django-storages # START_FEATURE docker gunicorn +whitenoise # END_FEATURE docker # START_FEATURE django_ses diff --git a/requirements.txt b/requirements.txt index 5584a69..1d6925a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -157,3 +157,5 @@ urllib3==2.0.7 # sentry-sdk wcwidth==0.2.9 # via prompt-toolkit +whitenoise==6.8.2 + # via -r requirements.in diff --git a/terraform/modules/ecs_deployment/secrets_manager.tf b/terraform/modules/ecs_deployment/secrets_manager.tf index 4ae6c32..74a14bf 100644 --- a/terraform/modules/ecs_deployment/secrets_manager.tf +++ b/terraform/modules/ecs_deployment/secrets_manager.tf @@ -4,7 +4,7 @@ resource "aws_secretsmanager_secret" "web_infrastructure" { resource "aws_secretsmanager_secret_version" "web_infrastructure" { secret_id = aws_secretsmanager_secret.web_infrastructure.id - secret_string = jsonencode({ + secret_string = jsonencode({ AWS_STORAGE_BUCKET_NAME = aws_s3_bucket.bucket.id DATABASE_URL = format( "postgres://dbuser:%s@%s:5432/database?sslmode=require", @@ -13,7 +13,7 @@ resource "aws_secretsmanager_secret_version" "web_infrastructure" { ) DEFAULT_FROM_EMAIL = var.ses_from_email SECRET_KEY = random_password.app_secret_key.result - }) + }) } From 2753a6b185f11dbcb48a7361a8904e4a6839fb6a Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 4 Nov 2024 16:08:58 -0500 Subject: [PATCH 26/39] Changes secrets to be dynamic, updates names to {app}-{env} --- terraform/modules/ecs_deployment/ecs.tf | 12 ++++++------ terraform/modules/ecs_deployment/iam.tf | 6 +++--- terraform/modules/ecs_deployment/locals.tf | 18 ++---------------- terraform/modules/ecs_deployment/rds.tf | 2 +- terraform/modules/ecs_deployment/s3.tf | 2 +- .../modules/ecs_deployment/secrets_manager.tf | 7 ++++++- .../modules/ecs_deployment/security_groups.tf | 6 +++--- 7 files changed, 22 insertions(+), 31 deletions(-) diff --git a/terraform/modules/ecs_deployment/ecs.tf b/terraform/modules/ecs_deployment/ecs.tf index f7d39f3..1a0f246 100644 --- a/terraform/modules/ecs_deployment/ecs.tf +++ b/terraform/modules/ecs_deployment/ecs.tf @@ -1,5 +1,5 @@ resource "aws_ecs_cluster" "cluster" { - name = "${var.application_name}-${var.environment_name}" + name = "${local.app_env_name}" setting { name = "containerInsights" @@ -14,11 +14,11 @@ resource "aws_ecs_cluster_capacity_providers" "fargate_provider" { } resource "aws_ecs_task_definition" "web" { - family = "${var.application_name}-${var.environment_name}-web" + family = "${local.app_env_name}-web" container_definitions = jsonencode([ { - name = "${var.application_name}-${var.environment_name}-web" + name = "${local.app_env_name}-web" image = var.ecr_image_uri essential = true portMappings = [ @@ -48,7 +48,7 @@ resource "aws_ecs_task_definition" "web" { } resource "aws_ecs_service" "web" { - name = "${var.application_name}-${var.environment_name}-web" + name = "${local.app_env_name}-web" cluster = aws_ecs_cluster.cluster.id task_definition = aws_ecs_task_definition.web.arn desired_count = var.container_count @@ -62,7 +62,7 @@ resource "aws_ecs_service" "web" { load_balancer { target_group_arn = aws_lb_target_group.target_group.arn - container_name = "${var.application_name}-${var.environment_name}-web" + container_name = "${local.app_env_name}-web" container_port = 8080 } @@ -74,6 +74,6 @@ resource "aws_ecs_service" "web" { } resource "aws_cloudwatch_log_group" "web_log_group" { - name = "${var.application_name}-${var.environment_name}-web" + name = "${local.app_env_name}-web" retention_in_days = 90 } \ No newline at end of file diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf index a4b41d1..a04b3a7 100644 --- a/terraform/modules/ecs_deployment/iam.tf +++ b/terraform/modules/ecs_deployment/iam.tf @@ -112,13 +112,13 @@ data "aws_iam_policy_document" "ecs_task_role_policy" { "kms:Decrypt" ] resources = [ - format("arn:aws:kms:*:*:aws/secretsmanager") + "arn:aws:kms:*:*:aws/secretsmanager" ] } } resource "aws_iam_role" "ecs_execution_role" { - name = format("%s-ecs-execution-role", var.environment_name) + name = "${local.app_env_name}-ecs-execution-role" assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json } @@ -129,7 +129,7 @@ resource "aws_iam_role_policy" "ecs_execution_role_policy" { } resource "aws_iam_role" "ecs_task_role" { - name = format("%s-ecs-task-role", var.environment_name) + name = "${local.app_env_name}-ecs-task-role" assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json } diff --git a/terraform/modules/ecs_deployment/locals.tf b/terraform/modules/ecs_deployment/locals.tf index 0b58e9e..13b631e 100644 --- a/terraform/modules/ecs_deployment/locals.tf +++ b/terraform/modules/ecs_deployment/locals.tf @@ -1,22 +1,8 @@ locals { - infrastructure_secrets = [ - "DATABASE_URL", - "SECRET_KEY", - "AWS_STORAGE_BUCKET_NAME", - "DEFAULT_FROM_EMAIL" - ] - - config_secrets = [ - "ALLOWED_HOSTS", - "GOOGLE_OAUTH2_KEY", - "GOOGLE_OAUTH2_SECRET", - "EC2_METADATA", - ] - app_env_name = "${var.application_name}-${var.environment_name}" ecs_infrastructure_secrets = [ - for setting in local.infrastructure_secrets : + for setting in keys(jsondecode(nonsensitive(aws_secretsmanager_secret_version.web_infrastructure.secret_string))) : { name : setting valueFrom : format("%s:%s::", aws_secretsmanager_secret.web_infrastructure.arn, setting) @@ -24,7 +10,7 @@ locals { ] ecs_config_secrets = [ - for setting in local.config_secrets : + for setting in keys(jsondecode(nonsensitive(data.aws_secretsmanager_secret_version.web_config.secret_string))) : { name : setting valueFrom : format("%s:%s::", data.aws_secretsmanager_secret.web_config.arn, setting) diff --git a/terraform/modules/ecs_deployment/rds.tf b/terraform/modules/ecs_deployment/rds.tf index b645914..6de8653 100644 --- a/terraform/modules/ecs_deployment/rds.tf +++ b/terraform/modules/ecs_deployment/rds.tf @@ -13,7 +13,7 @@ resource "aws_db_instance" "database" { deletion_protection = var.rds_deletion_protection engine = "postgres" engine_version = var.rds_engine_version - identifier = format("%s-%s-db", var.application_name, var.environment_name) + identifier = "${local.app_env_name}-db" instance_class = var.rds_instance_class multi_az = var.rds_multi_az password = random_password.db_password.result diff --git a/terraform/modules/ecs_deployment/s3.tf b/terraform/modules/ecs_deployment/s3.tf index 5e69c2f..a983340 100644 --- a/terraform/modules/ecs_deployment/s3.tf +++ b/terraform/modules/ecs_deployment/s3.tf @@ -1,5 +1,5 @@ resource "aws_s3_bucket" "bucket" { - bucket = format("%s-%s", var.s3_bucket_prefix, var.environment_name) + bucket = "${var.s3_bucket_prefix}-${var.environment_name}" tags = { Environment = var.environment_name diff --git a/terraform/modules/ecs_deployment/secrets_manager.tf b/terraform/modules/ecs_deployment/secrets_manager.tf index 74a14bf..5704235 100644 --- a/terraform/modules/ecs_deployment/secrets_manager.tf +++ b/terraform/modules/ecs_deployment/secrets_manager.tf @@ -1,5 +1,5 @@ resource "aws_secretsmanager_secret" "web_infrastructure" { - name = format("%s-%s-web-infrastructure", var.application_name, var.environment_name) + name = "${local.app_env_name}-web-infrastructure" } resource "aws_secretsmanager_secret_version" "web_infrastructure" { @@ -22,6 +22,11 @@ data "aws_secretsmanager_secret" "web_config" { } +data "aws_secretsmanager_secret_version" "web_config" { + secret_id = data.aws_secretsmanager_secret.web_config.id +} + + resource "random_password" "app_secret_key" { length = 32 special = false diff --git a/terraform/modules/ecs_deployment/security_groups.tf b/terraform/modules/ecs_deployment/security_groups.tf index b907237..2a6ab85 100644 --- a/terraform/modules/ecs_deployment/security_groups.tf +++ b/terraform/modules/ecs_deployment/security_groups.tf @@ -24,7 +24,7 @@ resource "aws_security_group" "load_balancer" { } tags = { - Name = format("%s-%s-lb", var.application_name, var.environment_name) + Name = "${local.app_env_name}-lb" } lifecycle { @@ -50,7 +50,7 @@ resource "aws_security_group" "web" { } tags = { - Name = format("%s-%s-web", var.application_name, var.environment_name) + Name = "${local.app_env_name}-web" } lifecycle { @@ -76,7 +76,7 @@ resource "aws_security_group" "database" { } tags = { - Name = format("%s-%s-db", var.application_name, var.environment_name) + Name = "${local.app_env_name}-db" } lifecycle { From dd00f8956cc31225b82732861b0bd09da5d6cf9c Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Mon, 4 Nov 2024 16:29:15 -0500 Subject: [PATCH 27/39] updates readme --- readme.md | 12 ++++++------ terraform/modules/ecs_deployment/iam.tf | 2 +- terraform/modules/ecs_deployment/security_groups.tf | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/readme.md b/readme.md index 7f75762..934e1ca 100644 --- a/readme.md +++ b/readme.md @@ -125,21 +125,21 @@ Following that, deploy your code to the environment (see below). ## Creating a new ECS environment 1. Create an ECR repository -2. Build and push the docker file to it +2. Build and push the docker file to it (ECR provides docker commands for this) 3. Create a bucket for holding terraform config 4. Create an SES identity and from email (if using SES) 5. Create an AWS certificate manager certificate for your domain -6. Create a secrets manager secret containing the config parameters needed by the application (you do not need include "DATABASE_URL", "SECRET_KEY", "AWS_STORAGE_BUCKET_NAME", or "DEFAULT_FROM_EMAIL" as those are managed by terraform in terraform/modules/ecs_deployment/secrets_manager.tf) -7. Ensure your config parameters have been added to the list in terraform/modules/ecs_deployment/locals.tf -8. Fill in the missing values in terraform/envs//main.tf -9. Run terraform to set up that environment +6. Create a secrets manager secret containing the config parameters needed by the application (you do not need include "DATABASE_URL", "SECRET_KEY", "AWS_STORAGE_BUCKET_NAME", or "DEFAULT_FROM_EMAIL" as those are managed by terraform in `terraform/modules/ecs_deployment/secrets_manager.tf`) +7. Fill in the missing values in `terraform/envs//main.tf` +8. Run terraform to set up that environment ``` cd terraform/envs/ terraform init terraform plan terraform apply ``` -10. Add a DNS entry from your domain name to the created load balancer + +9. Add a DNS entry from your domain name to the created load balancer diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf index a04b3a7..54bb2cf 100644 --- a/terraform/modules/ecs_deployment/iam.tf +++ b/terraform/modules/ecs_deployment/iam.tf @@ -60,7 +60,7 @@ data "aws_iam_policy_document" "ecs_execution_role_policy" { "kms:Decrypt" ] resources = [ - format("arn:aws:kms:*:*:aws/secretsmanager"), + "arn:aws:kms:*:*:aws/secretsmanager" ] } } diff --git a/terraform/modules/ecs_deployment/security_groups.tf b/terraform/modules/ecs_deployment/security_groups.tf index 2a6ab85..fac1c38 100644 --- a/terraform/modules/ecs_deployment/security_groups.tf +++ b/terraform/modules/ecs_deployment/security_groups.tf @@ -1,5 +1,5 @@ resource "aws_security_group" "load_balancer" { - name = format("%s %s load balancer", var.application_name, var.environment_name) + name = "${local.app_env_name}-lb" ingress { from_port = 80 @@ -24,7 +24,7 @@ resource "aws_security_group" "load_balancer" { } tags = { - Name = "${local.app_env_name}-lb" + Name = "${var.application_name} ${var.environment_name} load balancer" } lifecycle { @@ -33,7 +33,7 @@ resource "aws_security_group" "load_balancer" { } resource "aws_security_group" "web" { - name = format("%s %s web", var.application_name, var.environment_name) + name = "${local.app_env_name}-web" ingress { from_port = 8080 @@ -50,7 +50,7 @@ resource "aws_security_group" "web" { } tags = { - Name = "${local.app_env_name}-web" + Name = "${var.application_name} ${var.environment_name} web" } lifecycle { @@ -59,7 +59,7 @@ resource "aws_security_group" "web" { } resource "aws_security_group" "database" { - name = format("%s %s database", var.application_name, var.environment_name) + name = "${local.app_env_name}-db" ingress { from_port = 5432 @@ -76,7 +76,7 @@ resource "aws_security_group" "database" { } tags = { - Name = "${local.app_env_name}-db" + Name = "${var.application_name} ${var.environment_name} database" } lifecycle { From 7536ef2739e20f12b8008559f1b1db63672b72e5 Mon Sep 17 00:00:00 2001 From: Grace Whitney Date: Thu, 7 Nov 2024 18:01:43 -0500 Subject: [PATCH 28/39] Add deployment script --- deploy.py | 178 ++++++++++++++++++++ terraform/envs/staging/main.tf | 40 ++++- terraform/modules/ecs_deployment/outputs.tf | 25 +++ 3 files changed, 241 insertions(+), 2 deletions(-) create mode 100644 deploy.py diff --git a/deploy.py b/deploy.py new file mode 100644 index 0000000..83a2a27 --- /dev/null +++ b/deploy.py @@ -0,0 +1,178 @@ +import argparse +import logging +import subprocess +import sys +import time + +import boto3.session + +AWS_REGION = "us-east-1" +AWS_PROFILE_NAME = "FILL ME IN" + +MIGRATION_TIMEOUT_SECONDS = 10 * 60 # Ten minutes + + +class MigrationFailed(Exception): + pass + + +class MigrationTimeOut(Exception): + pass + +parser = argparse.ArgumentParser() +parser.add_argument("--skip-build", action="store_true", help="Skips the build step and uses the existing ECR image.") +parser.add_argument("--use-image-from-env", help="If provided, skips the terraform build and instead uses the existing built image from the specified environment") +parser.add_argument("--skip-migration", action="store_true", help="Skips the migration step.") +parser.add_argument("-env", help="Terraform environment to deploy", required=True) + + +def deploy(args): + confirmation_message = f"Begin deploying to {args.env}? " + if args.use_image_from_env: + confirmation_message += f"This will use the current {args.use_image_from_env} deployment" + confirmation = input(confirmation_message) + if confirmation.lower() not in ["y", "yes"]: + logging.warning("Deployment canceled.") + return + + # Refresh terraform state + subprocess.run(["terraform", "refresh"], cwd=f"terraform/envs/{args.env}", check=True, capture_output=True) + + # ECR image setup + if args.use_image_from_env: + subprocess.run(["terraform", "refresh"], cwd=f"terraform/envs/{args.use_image_from_env}", check=True, capture_output=True) + copy_image_from_env(args.use_image_from_env, args.env) + elif not args.skip_build: + build_and_push_image(args.env) + + if not args.skip_migration: + # Run and wait for migrations + run_migrations(args.env) + + # Redeploy services + restart_web_service(args.env) + + logging.info("Deployment complete.") + + +def subprocess_output(command_args, **subprocess_kwargs): + output = subprocess.run(command_args, **subprocess_kwargs, capture_output=True, check=True) + return output.stdout.decode('utf-8').strip("\n").strip('"') + + +def get_terraform_output(output_key, env): + return subprocess_output(["terraform", "output", output_key], cwd=f"terraform/envs/{env}") + + +def build_and_push_image(env): + # Build and tag image + ecr_repository_name = get_terraform_output("ecr_repository_name", env) + ecr_image_uri = get_terraform_output("ecr_image_uri", env) + logging.info("Building docker image...") + build_command = ["docker", "build", "-t", ecr_repository_name, "."] + subprocess.run(build_command, check=True) + subprocess.run(["docker", "tag", f"{ecr_repository_name}:latest", ecr_image_uri], check=True) + + # Push image to ECR + logging.info("Logging in to ECR...") + password_command = ["aws", "ecr", "get-login-password", "--region", AWS_REGION, "--profile", AWS_PROFILE_NAME] + password = subprocess_output(password_command) + docker_login_command = ["docker", "login", "--username", "AWS", "--password-stdin", ecr_image_uri.split("/")[0]] + subprocess.run(docker_login_command, input=password, text=True, check=True) + logging.info("Pushing docker image to ECR...") + subprocess.run(["docker", "push", ecr_image_uri], check=True) + + # Remove unused docker images to preserve local disk space + subprocess.run(["docker", "image", "prune", "-f"]) + + +def copy_image_from_env(from_env, to_env): + # Retags image from from_env into to_env. + from_ecr_image_uri = get_terraform_output("ecr_image_uri", from_env) + from_ecr_repository_name, from_ecr_image_tag = from_ecr_image_uri.split("/") + ecr_client = boto3.session.Session(profile_name=AWS_PROFILE_NAME, region_name=AWS_REGION).client("ecr") + # Get image manifest + image_response = ecr_client.batch_get_image( + repositoryName=from_ecr_repository_name, + imageIds=[{ + "imageTag": from_ecr_image_tag + }], + acceptedMediaTypes=["string"] + ) + image_manifest = image_response["images"][0]["imageManifest"] + image_manifest_media_type = image_response["images"][0]["imageManifestMediaType"] + # Add new tag to manifest + to_ecr_image_uri = get_terraform_output("ecr_image_uri", to_env) + to_ecr_repository_name, to_ecr_image_tag = to_ecr_image_uri.split("/") + ecr_client.put_image( + repositoryName=to_ecr_repository_name, + imageManifest=image_manifest, + imageTag=to_ecr_image_tag, + imageManifestMediaType=image_manifest_media_type, + ) + + +def run_migrations(env): + # Runs a migration task using the web server task definition with an overridden command + cluster_id = get_terraform_output("cluster_id", env) + ecs_client = boto3.session.Session(profile_name=AWS_PROFILE_NAME, region_name=AWS_REGION).client("ecs") + logging.info("Starting migrations...") + run_task_response = ecs_client.run_task( + taskDefinition=get_terraform_output("web_task_definition_arn", env), + networkConfiguration={ + "awsvpcConfiguration" : { + "subnets": [get_terraform_output("web_network_configuration_subnet", env)], + "securityGroups": [get_terraform_output("web_network_configuration_security_group", env)], + "assignPublicIp": "ENABLED" + } + }, + cluster=cluster_id, + capacityProviderStrategy=[{'capacityProvider': 'FARGATE'}], + overrides={ + "containerOverrides": [{ + "name": get_terraform_output("web_service_name", env), + "command": ["python", "manage.py", "migrate", "--no-input"] + }] + } + ) + + # Wait for task to complete. Times out after MIGRATION_TIMEOUT_SECONDS + migration_task_id = run_task_response["tasks"][0]["taskArn"] + start = time.time() + status_check_interval = 10 # Check migration status every 10 seconds + + while time.time() - start < MIGRATION_TIMEOUT_SECONDS: + logging.info("Waiting for migrations to finish...") + describe_tasks_response = ecs_client.describe_tasks(cluster=cluster_id, tasks=[migration_task_id]) + task = describe_tasks_response["tasks"][0] + stop_code = task.get("stopCode") + if not stop_code: + time.sleep(status_check_interval) + continue + if stop_code == "EssentialContainerExited": + # The migration task has finished successfully + logging.info("Migration complete") + return + logging.error("Migration task failed.") + raise MigrationFailed() + logging.error("Migration timed out. It may still be running.") + raise MigrationTimeOut() + + +def restart_web_service(env): + # Restart ECS web service to deploy new code + ecs_client = boto3.session.Session(profile_name=AWS_PROFILE_NAME, region_name=AWS_REGION).client("ecs") + logging.info("Redeploying web service...") + ecs_client.update_service( + cluster=get_terraform_output("cluster_id", env), + service=get_terraform_output("web_service_name", env), + forceNewDeployment=True + ) + + +if __name__ == "__main__": + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(levelname)s - %(message)s") + deploy(args) + + diff --git a/terraform/envs/staging/main.tf b/terraform/envs/staging/main.tf index d6bfca7..d5f6277 100644 --- a/terraform/envs/staging/main.tf +++ b/terraform/envs/staging/main.tf @@ -5,7 +5,7 @@ terraform { region = "us-east-1" # TODO: FILL ME IN profile = "" # TODO: FILL ME IN } - + required_providers { aws = { source = "hashicorp/aws" @@ -21,7 +21,7 @@ provider "aws" { module "ecs_deployment" { source = "../../modules/ecs_deployment" - + # Required Variables environment_name = "staging" application_name = "" # TODO: FILL ME IN @@ -45,3 +45,39 @@ module "ecs_deployment" { container_count = 1 ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" } + +output "cluster_id" { + description = "The ID of the ECS cluster" + value = module.ecs_deployment.cluster_id +} + +output "ecr_repository_name" { + description = "The name of the ECR repository" + value = module.ecs_deployment.ecr_repository_name +} + +output "public_ip" { + description = "The public IP address of the load balancer for the web service" + value = module.ecs_deployment.public_ip +} + +output "web_service_name" { + description = "The name of the ECS container running the web service" + value = module.ecs_deployment.web_service_name +} + +output "web_network_configuration_security_group" { + description = "The security groups used by the ECS web task" + value = tolist(module.ecs_deployment.web_network_configuration_security_groups)[0] + +} + +output "web_network_configuration_subnet" { + description = "The ID of one the subnets used by the web task" + value = tolist(module.ecs_deployment.web_network_configuration_subnets)[0] +} + +output "web_task_definition_arn" { + description = "The ARN of the ECS web service task definition" + value = module.ecs_deployment.web_task_definition_arn +} diff --git a/terraform/modules/ecs_deployment/outputs.tf b/terraform/modules/ecs_deployment/outputs.tf index 8df6dde..d005c64 100644 --- a/terraform/modules/ecs_deployment/outputs.tf +++ b/terraform/modules/ecs_deployment/outputs.tf @@ -3,7 +3,32 @@ output "cluster_id" { value = aws_ecs_cluster.cluster.id } +output "ecr_repository_name" { + description = "The name of the ECR repository" + value = var.ecr_repository_name +} + output "public_ip" { description = "The public IP address of the load balancer for the web service" value = aws_lb.alb.dns_name +} + +output "web_service_name" { + description = "The name of the ECS web service. This is also the container name." + value = aws_ecs_service.web.name +} + +output "web_network_configuration_security_groups" { + description = "The security groups used by the ECS web task" + value = aws_ecs_service.web.network_configuration[0].security_groups +} + +output "web_network_configuration_subnets" { + description = "The ID of the subnets used by the web task" + value = aws_ecs_service.web.network_configuration[0].subnets +} + +output "web_task_definition_arn" { + description = "The ARN of the ECS web service task definition" + value = aws_ecs_task_definition.web.arn } \ No newline at end of file From f79deee9c1122f0c0a5674ec37cd62de23b97456 Mon Sep 17 00:00:00 2001 From: Grace Whitney Date: Fri, 8 Nov 2024 10:40:19 -0500 Subject: [PATCH 29/39] Autogenerate ECR URI and fixes to image tag copy --- deploy.py | 34 +++++++++++-------- terraform/envs/staging/main.tf | 3 +- terraform/modules/ecs_deployment/ecs.tf | 2 +- terraform/modules/ecs_deployment/locals.tf | 4 +++ terraform/modules/ecs_deployment/outputs.tf | 5 +++ terraform/modules/ecs_deployment/variables.tf | 4 --- 6 files changed, 31 insertions(+), 21 deletions(-) diff --git a/deploy.py b/deploy.py index 83a2a27..ac80ece 100644 --- a/deploy.py +++ b/deploy.py @@ -20,6 +20,7 @@ class MigrationTimeOut(Exception): pass parser = argparse.ArgumentParser() +parser.add_argument("--no-input", action="store_true", help="Skips request for confirmation before starting the deploy.") parser.add_argument("--skip-build", action="store_true", help="Skips the build step and uses the existing ECR image.") parser.add_argument("--use-image-from-env", help="If provided, skips the terraform build and instead uses the existing built image from the specified environment") parser.add_argument("--skip-migration", action="store_true", help="Skips the migration step.") @@ -27,15 +28,19 @@ class MigrationTimeOut(Exception): def deploy(args): - confirmation_message = f"Begin deploying to {args.env}? " - if args.use_image_from_env: - confirmation_message += f"This will use the current {args.use_image_from_env} deployment" - confirmation = input(confirmation_message) - if confirmation.lower() not in ["y", "yes"]: - logging.warning("Deployment canceled.") - return + if not args.no_input: + # Request user confirmation + confirmation_message = f"\nBegin deploying to {args.env}? " + if args.use_image_from_env: + confirmation_message += f"This will use the current {args.use_image_from_env} deployment. " + confirmation_message += "(y/n): " + confirmation = input(confirmation_message) + if confirmation.lower() not in ["y", "yes"]: + logging.warning("Deployment canceled.") + return # Refresh terraform state + logging.info("Refreshing terraform state...") subprocess.run(["terraform", "refresh"], cwd=f"terraform/envs/{args.env}", check=True, capture_output=True) # ECR image setup @@ -88,26 +93,26 @@ def build_and_push_image(env): def copy_image_from_env(from_env, to_env): # Retags image from from_env into to_env. - from_ecr_image_uri = get_terraform_output("ecr_image_uri", from_env) - from_ecr_repository_name, from_ecr_image_tag = from_ecr_image_uri.split("/") + logging.info(f"Copying image from {from_env} to {to_env}") + from_ecr_repository_name = get_terraform_output("ecr_repository_name", from_env) ecr_client = boto3.session.Session(profile_name=AWS_PROFILE_NAME, region_name=AWS_REGION).client("ecr") # Get image manifest image_response = ecr_client.batch_get_image( repositoryName=from_ecr_repository_name, imageIds=[{ - "imageTag": from_ecr_image_tag + "imageTag": from_env }], acceptedMediaTypes=["string"] ) + image_manifest = image_response["images"][0]["imageManifest"] image_manifest_media_type = image_response["images"][0]["imageManifestMediaType"] # Add new tag to manifest - to_ecr_image_uri = get_terraform_output("ecr_image_uri", to_env) - to_ecr_repository_name, to_ecr_image_tag = to_ecr_image_uri.split("/") + to_ecr_repository_name = get_terraform_output("ecr_repository_name", to_env) ecr_client.put_image( repositoryName=to_ecr_repository_name, imageManifest=image_manifest, - imageTag=to_ecr_image_tag, + imageTag=to_env, imageManifestMediaType=image_manifest_media_type, ) @@ -138,8 +143,9 @@ def run_migrations(env): # Wait for task to complete. Times out after MIGRATION_TIMEOUT_SECONDS migration_task_id = run_task_response["tasks"][0]["taskArn"] + logging.info(f"Migration task provisioned with ID {migration_task_id}") start = time.time() - status_check_interval = 10 # Check migration status every 10 seconds + status_check_interval = 30 # Check migration status at interval (in seconds) while time.time() - start < MIGRATION_TIMEOUT_SECONDS: logging.info("Waiting for migrations to finish...") diff --git a/terraform/envs/staging/main.tf b/terraform/envs/staging/main.tf index d5f6277..7678c1a 100644 --- a/terraform/envs/staging/main.tf +++ b/terraform/envs/staging/main.tf @@ -33,8 +33,7 @@ module "ecs_deployment" { ses_from_email = "" # TODO: FILL ME IN certificate_manager_arn = "" # TODO: FILL ME IN ecr_repository_name = "" # TODO: FILL ME IN - ecr_image_uri = "" # TODO: FILL ME IN - + # Optional Variables rds_backup_retention_period = 10 rds_deletion_protection = true diff --git a/terraform/modules/ecs_deployment/ecs.tf b/terraform/modules/ecs_deployment/ecs.tf index 1a0f246..653949c 100644 --- a/terraform/modules/ecs_deployment/ecs.tf +++ b/terraform/modules/ecs_deployment/ecs.tf @@ -19,7 +19,7 @@ resource "aws_ecs_task_definition" "web" { container_definitions = jsonencode([ { name = "${local.app_env_name}-web" - image = var.ecr_image_uri + image = local.ecr_image_uri essential = true portMappings = [ { diff --git a/terraform/modules/ecs_deployment/locals.tf b/terraform/modules/ecs_deployment/locals.tf index 13b631e..81d49a1 100644 --- a/terraform/modules/ecs_deployment/locals.tf +++ b/terraform/modules/ecs_deployment/locals.tf @@ -1,6 +1,10 @@ +data "aws_caller_identity" "current" {} + locals { app_env_name = "${var.application_name}-${var.environment_name}" + ecr_image_uri = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${data.aws_region.current.name}.amazonaws.com/${var.ecr_repository_name}:${var.environment_name}" + ecs_infrastructure_secrets = [ for setting in keys(jsondecode(nonsensitive(aws_secretsmanager_secret_version.web_infrastructure.secret_string))) : { diff --git a/terraform/modules/ecs_deployment/outputs.tf b/terraform/modules/ecs_deployment/outputs.tf index d005c64..f327af7 100644 --- a/terraform/modules/ecs_deployment/outputs.tf +++ b/terraform/modules/ecs_deployment/outputs.tf @@ -3,6 +3,11 @@ output "cluster_id" { value = aws_ecs_cluster.cluster.id } +output "ecr_image_uri" { + description = "The full URI of the ECR image" + value = local.ecr_image_uri +} + output "ecr_repository_name" { description = "The name of the ECR repository" value = var.ecr_repository_name diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index f2551be..0369bac 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -39,10 +39,6 @@ variable "ecr_repository_name" { type = string } -variable "ecr_image_uri" { - type = string -} - # Optional Variables variable "rds_backup_retention_period" { type = number From 69c90210996e853e640ac50d0354f6649117e3a0 Mon Sep 17 00:00:00 2001 From: Grace Whitney Date: Fri, 8 Nov 2024 10:47:54 -0500 Subject: [PATCH 30/39] Add skip logging --- deploy.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deploy.py b/deploy.py index ac80ece..bbf6857 100644 --- a/deploy.py +++ b/deploy.py @@ -49,8 +49,12 @@ def deploy(args): copy_image_from_env(args.use_image_from_env, args.env) elif not args.skip_build: build_and_push_image(args.env) + else: + logging.info("Skipping build step") - if not args.skip_migration: + if args.skip_migration: + logging.info("Skipping database migration") + else: # Run and wait for migrations run_migrations(args.env) From 6e012ecb7e8a221ed937490bd3f0b261c951d0ce Mon Sep 17 00:00:00 2001 From: Grace Whitney Date: Fri, 8 Nov 2024 11:21:18 -0500 Subject: [PATCH 31/39] Add use-latest option, update README --- .dockerignore | 9 +++++++++ deploy.py | 27 ++++++++++++++++++++------- readme.md | 24 ++++++++++++++++++++---- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/.dockerignore b/.dockerignore index 61bbb72..d6d3cea 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,3 +16,12 @@ # Operating system metafiles .DS_Store + +# Local build artifacts +node_modules/ +media/ +staticfiles/ + +# Deployment configuration +terraform/ +deploy.py diff --git a/deploy.py b/deploy.py index bbf6857..ba134ff 100644 --- a/deploy.py +++ b/deploy.py @@ -21,7 +21,8 @@ class MigrationTimeOut(Exception): parser = argparse.ArgumentParser() parser.add_argument("--no-input", action="store_true", help="Skips request for confirmation before starting the deploy.") -parser.add_argument("--skip-build", action="store_true", help="Skips the build step and uses the existing ECR image.") +parser.add_argument("--skip-build", action="store_true", help="Skips the build step and uses the existing ECR image for the environment.") +parser.add_argument("--use-latest", action="store_true", help="Skips the build step and uses the ECR image tagged `latest`.") parser.add_argument("--use-image-from-env", help="If provided, skips the terraform build and instead uses the existing built image from the specified environment") parser.add_argument("--skip-migration", action="store_true", help="Skips the migration step.") parser.add_argument("-env", help="Terraform environment to deploy", required=True) @@ -47,6 +48,8 @@ def deploy(args): if args.use_image_from_env: subprocess.run(["terraform", "refresh"], cwd=f"terraform/envs/{args.use_image_from_env}", check=True, capture_output=True) copy_image_from_env(args.use_image_from_env, args.env) + elif args.use_latest: + copy_latest_image(args.env) elif not args.skip_build: build_and_push_image(args.env) else: @@ -99,24 +102,34 @@ def copy_image_from_env(from_env, to_env): # Retags image from from_env into to_env. logging.info(f"Copying image from {from_env} to {to_env}") from_ecr_repository_name = get_terraform_output("ecr_repository_name", from_env) + to_ecr_repository_name = get_terraform_output("ecr_repository_name", to_env) + retag_image(from_ecr_repository_name, from_env, to_ecr_repository_name, to_env) + + +def copy_latest_image(env): + # Retags latest image in repository for use by env + logging.info(f"Copying latest image to {env}") + ecr_repository_name = get_terraform_output("ecr_repository_name", env) + retag_image(ecr_repository_name, "latest", ecr_repository_name, env) + + +def retag_image(from_repository, from_tag, to_repository, to_tag): ecr_client = boto3.session.Session(profile_name=AWS_PROFILE_NAME, region_name=AWS_REGION).client("ecr") # Get image manifest image_response = ecr_client.batch_get_image( - repositoryName=from_ecr_repository_name, + repositoryName=from_repository, imageIds=[{ - "imageTag": from_env + "imageTag": from_tag }], acceptedMediaTypes=["string"] ) - image_manifest = image_response["images"][0]["imageManifest"] image_manifest_media_type = image_response["images"][0]["imageManifestMediaType"] # Add new tag to manifest - to_ecr_repository_name = get_terraform_output("ecr_repository_name", to_env) ecr_client.put_image( - repositoryName=to_ecr_repository_name, + repositoryName=to_repository, imageManifest=image_manifest, - imageTag=to_env, + imageTag=to_tag, imageManifestMediaType=image_manifest_media_type, ) diff --git a/readme.md b/readme.md index 934e1ca..1fc6383 100644 --- a/readme.md +++ b/readme.md @@ -104,7 +104,7 @@ pip install eb-ssm # For Elastic Beanstalk SSH functionality without requiring ## Creating a new Elastic Beanstalk environment -To do create a new Elastic Beanstalk environment, modify the contents of [.elasticbeanstalk/eb_create_environment.yml]([.elasticbeanstalk/eb_create_environment.yml]) and run `eb-create-environment -c .elasticbeanstalk/eb_create_environment.yml`. +To create a new Elastic Beanstalk environment, modify the contents of [.elasticbeanstalk/eb_create_environment.yml]([.elasticbeanstalk/eb_create_environment.yml]) and run `eb-create-environment -c .elasticbeanstalk/eb_create_environment.yml`. See the docs for [eb-create-environment](https://github.com/zagaran/eb-create-environment/) for more details. @@ -125,7 +125,7 @@ Following that, deploy your code to the environment (see below). ## Creating a new ECS environment 1. Create an ECR repository -2. Build and push the docker file to it (ECR provides docker commands for this) +2. Build and push an initial docker file to it (ECR provides docker commands for this). 3. Create a bucket for holding terraform config 4. Create an SES identity and from email (if using SES) 5. Create an AWS certificate manager certificate for your domain @@ -139,16 +139,32 @@ terraform plan terraform apply ``` -9. Add a DNS entry from your domain name to the created load balancer +9. Redeploy your code using the steps described below (with the --use-latest option) to run initial migrations +10. Add a DNS entry from your domain name to the created load balancer ## Deploying code -To deploy new versions of your code to your environment, run `eb deploy ` using the EB CLI to deploy your code to that environment. +### Elastic Beanstalk +To deploy new versions of your code to an elastic beanstalk environment, run `eb deploy ` using the EB CLI to deploy your code to that environment. See the [eb-cli](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3.html) docs on general command line usage of Elastic Beanstalk. +### ECS +To deploy new versions of your code to an ECS environment, use the included `deploy.py` script. First fill in the +missing constants at the top of that file, and then run the script: +``` +python deploy.py -env +``` +This script will do the following: +1. Build the docker image using your local code version. +2. Push the docker image to the ECR location for the specified environment +3. Run database migrations +4. Deploy to the running web service + +Run `python deploy.py --help` to see available options. You may choose to use an existing ECR image or skip migrations. + ## SSH To SSH into an Elastic Beanstalk Environment, use [eb-ssm](https://github.com/zagaran/eb-ssm). \ No newline at end of file From 786b795212cebcd320ec02f28cf8d101d18c950f Mon Sep 17 00:00:00 2001 From: Grace Whitney Date: Fri, 8 Nov 2024 11:51:04 -0500 Subject: [PATCH 32/39] Update static storage setting --- config/settings.py | 4 ++-- readme.md | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/config/settings.py b/config/settings.py index 135b79b..e311c35 100644 --- a/config/settings.py +++ b/config/settings.py @@ -321,9 +321,9 @@ } } -STATIC_BACKEND = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage" +STATIC_BACKEND = "django.contrib.staticfiles.storage.StaticFilesStorage" if LOCALHOST else "django.contrib.staticfiles.storage.ManifestStaticFilesStorage" # START_FEATURE docker -STATIC_BACKEND = "whitenoise.storage.CompressedManifestStaticFilesStorage" +STATIC_BACKEND = "django.contrib.staticfiles.storage.StaticFilesStorage" if LOCALHOST else "whitenoise.storage.CompressedManifestStaticFilesStorage" # END_FEATURE docker # END_FEATURE django_storages diff --git a/readme.md b/readme.md index 1fc6383..533fa62 100644 --- a/readme.md +++ b/readme.md @@ -143,7 +143,6 @@ terraform apply 10. Add a DNS entry from your domain name to the created load balancer - ## Deploying code ### Elastic Beanstalk From 95d7310d1f12ea8b35854d5b8a1a00bbb6b2c8cb Mon Sep 17 00:00:00 2001 From: Grace Whitney Date: Fri, 8 Nov 2024 13:33:31 -0500 Subject: [PATCH 33/39] Add ssh script --- deploy.py | 46 +++++++++++++++++++++++++++++++++++----------- readme.md | 4 +++- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/deploy.py b/deploy.py index ba134ff..fa16cf0 100644 --- a/deploy.py +++ b/deploy.py @@ -19,14 +19,6 @@ class MigrationFailed(Exception): class MigrationTimeOut(Exception): pass -parser = argparse.ArgumentParser() -parser.add_argument("--no-input", action="store_true", help="Skips request for confirmation before starting the deploy.") -parser.add_argument("--skip-build", action="store_true", help="Skips the build step and uses the existing ECR image for the environment.") -parser.add_argument("--use-latest", action="store_true", help="Skips the build step and uses the ECR image tagged `latest`.") -parser.add_argument("--use-image-from-env", help="If provided, skips the terraform build and instead uses the existing built image from the specified environment") -parser.add_argument("--skip-migration", action="store_true", help="Skips the migration step.") -parser.add_argument("-env", help="Terraform environment to deploy", required=True) - def deploy(args): if not args.no_input: @@ -192,10 +184,42 @@ def restart_web_service(env): forceNewDeployment=True ) +def ssh(args): + # Runs a bash shell in a running task for the env. Note this may run in a short-lived task (e.g. migration task) + cluster_id = get_terraform_output("cluster_id", args.env) + service_name = get_terraform_output("web_service_name", args.env) + ecs_client = boto3.session.Session(profile_name=AWS_PROFILE_NAME, region_name=AWS_REGION).client("ecs") + list_tasks_resp = ecs_client.list_tasks(cluster=cluster_id, serviceName=service_name) + task_ids = list_tasks_resp["taskArns"] + + if task_ids: + task_id = task_ids[0] + bash_command = ["aws", "ecs", "execute-command", "--cluster", cluster_id, "--task", task_id, + "--region", AWS_REGION, "--profile", AWS_PROFILE_NAME, "--interactive", + "--command", "'/bin/bash'"] + subprocess.run(bash_command) + + + +parser = argparse.ArgumentParser(prog="python deploy.py") +parser.add_argument("--no-input", action="store_true", + help="Skips request for confirmation before starting the deploy.") +parser.add_argument("--skip-build", action="store_true", + help="Skips the build step and uses the existing ECR image for the environment.") +parser.add_argument("--use-latest", action="store_true", + help="Skips the build step and uses the ECR image tagged `latest`.") +parser.add_argument("--use-image-from-env", + help="If provided, skips the terraform build and instead uses the existing built image from the specified environment") +parser.add_argument("--skip-migration", action="store_true", help="Skips the migration step.") +parser.add_argument("-env", help="Terraform environment to deploy", required=True) +parser.set_defaults(func=deploy) + +subparsers = parser.add_subparsers(title="Extra utilities", prog="python deploy.py -env ") +ssh_parser = subparsers.add_parser("ssh", help="SSH into running container in env instead of deploying") +ssh_parser.set_defaults(func=ssh) + if __name__ == "__main__": args = parser.parse_args() logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(levelname)s - %(message)s") - deploy(args) - - + args.func(args) diff --git a/readme.md b/readme.md index 533fa62..bce8b85 100644 --- a/readme.md +++ b/readme.md @@ -166,4 +166,6 @@ Run `python deploy.py --help` to see available options. You may choose to use an ## SSH -To SSH into an Elastic Beanstalk Environment, use [eb-ssm](https://github.com/zagaran/eb-ssm). \ No newline at end of file +To SSH into an Elastic Beanstalk Environment, use [eb-ssm](https://github.com/zagaran/eb-ssm). + +To SSH into an ECS environment, use `python deploy.py -env ssh` From 016f20e9af8151d021c1c0c2d47bfecf72c17420 Mon Sep 17 00:00:00 2001 From: Grace Whitney Date: Fri, 8 Nov 2024 17:50:29 -0500 Subject: [PATCH 34/39] Wait for deployment to finish --- deploy.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/deploy.py b/deploy.py index fa16cf0..69dbdaf 100644 --- a/deploy.py +++ b/deploy.py @@ -56,8 +56,6 @@ def deploy(args): # Redeploy services restart_web_service(args.env) - logging.info("Deployment complete.") - def subprocess_output(command_args, **subprocess_kwargs): output = subprocess.run(command_args, **subprocess_kwargs, capture_output=True, check=True) @@ -178,12 +176,34 @@ def restart_web_service(env): # Restart ECS web service to deploy new code ecs_client = boto3.session.Session(profile_name=AWS_PROFILE_NAME, region_name=AWS_REGION).client("ecs") logging.info("Redeploying web service...") + cluster_id = get_terraform_output("cluster_id", env) + service_name = get_terraform_output("web_service_name", env) ecs_client.update_service( - cluster=get_terraform_output("cluster_id", env), - service=get_terraform_output("web_service_name", env), + cluster=cluster_id, + service=service_name, forceNewDeployment=True ) + status_check_interval = 30 + while True: + logging.info("Waiting for deployment to finish...") + services_response = ecs_client.describe_services(cluster=cluster_id, services=[service_name]) + deployments = services_response["services"][0]["deployments"] + new_deployment = next(deployment for deployment in deployments if deployment["status"] == "PRIMARY") + deployment_state = new_deployment["rolloutState"] + if deployment_state == "IN_PROGRESS": + time.sleep(status_check_interval) + continue + if deployment_state == "COMPLETED": + logging.info("\nSuccess! Deployment complete.") + elif deployment_state == "FAILED": + logging.error(f"\nDeployment failed! Reason: {new_deployment['rolloutStateReason']}") + else: + logging.warning(f"\nUnknown deployment state {deployment_state}. Please check the console.") + break + + + def ssh(args): # Runs a bash shell in a running task for the env. Note this may run in a short-lived task (e.g. migration task) cluster_id = get_terraform_output("cluster_id", args.env) From f0b862f2decff37c9fcdb2602a98fcb9fdf0d3cf Mon Sep 17 00:00:00 2001 From: Grace Whitney Date: Fri, 8 Nov 2024 18:26:36 -0500 Subject: [PATCH 35/39] Cleanup --- deploy.py | 69 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/deploy.py b/deploy.py index 69dbdaf..2951f2f 100644 --- a/deploy.py +++ b/deploy.py @@ -10,6 +10,7 @@ AWS_PROFILE_NAME = "FILL ME IN" MIGRATION_TIMEOUT_SECONDS = 10 * 60 # Ten minutes +STATUS_CHECK_INTERVAL = 30 class MigrationFailed(Exception): @@ -32,9 +33,11 @@ def deploy(args): logging.warning("Deployment canceled.") return - # Refresh terraform state - logging.info("Refreshing terraform state...") - subprocess.run(["terraform", "refresh"], cwd=f"terraform/envs/{args.env}", check=True, capture_output=True) + # Local state setup for relevant envs + terraform_envs = [args.env] + if args.use_image_from_env: + terraform_envs.append(args.use_image_from_env) + setup(terraform_envs) # ECR image setup if args.use_image_from_env: @@ -57,6 +60,13 @@ def deploy(args): restart_web_service(args.env) +def setup(envs): + # Refresh terraform state + logging.info("Refreshing terraform state...") + for env in envs: + subprocess.run(["terraform", "refresh"], cwd=f"terraform/envs/{env}", check=True, capture_output=True) + + def subprocess_output(command_args, **subprocess_kwargs): output = subprocess.run(command_args, **subprocess_kwargs, capture_output=True, check=True) return output.stdout.decode('utf-8').strip("\n").strip('"') @@ -152,7 +162,6 @@ def run_migrations(env): migration_task_id = run_task_response["tasks"][0]["taskArn"] logging.info(f"Migration task provisioned with ID {migration_task_id}") start = time.time() - status_check_interval = 30 # Check migration status at interval (in seconds) while time.time() - start < MIGRATION_TIMEOUT_SECONDS: logging.info("Waiting for migrations to finish...") @@ -160,13 +169,13 @@ def run_migrations(env): task = describe_tasks_response["tasks"][0] stop_code = task.get("stopCode") if not stop_code: - time.sleep(status_check_interval) + time.sleep(STATUS_CHECK_INTERVAL) continue if stop_code == "EssentialContainerExited": # The migration task has finished successfully logging.info("Migration complete") return - logging.error("Migration task failed.") + logging.error(f"Migration task failed with code {stop_code} and reason {task.get('stoppedReason')}.") raise MigrationFailed() logging.error("Migration timed out. It may still be running.") raise MigrationTimeOut() @@ -184,7 +193,6 @@ def restart_web_service(env): forceNewDeployment=True ) - status_check_interval = 30 while True: logging.info("Waiting for deployment to finish...") services_response = ecs_client.describe_services(cluster=cluster_id, services=[service_name]) @@ -192,20 +200,22 @@ def restart_web_service(env): new_deployment = next(deployment for deployment in deployments if deployment["status"] == "PRIMARY") deployment_state = new_deployment["rolloutState"] if deployment_state == "IN_PROGRESS": - time.sleep(status_check_interval) + time.sleep(STATUS_CHECK_INTERVAL) continue if deployment_state == "COMPLETED": - logging.info("\nSuccess! Deployment complete.") + logging.info("Success! Deployment complete.") elif deployment_state == "FAILED": - logging.error(f"\nDeployment failed! Reason: {new_deployment['rolloutStateReason']}") + logging.error(f"Deployment failed! Reason: {new_deployment['rolloutStateReason']}") else: - logging.warning(f"\nUnknown deployment state {deployment_state}. Please check the console.") + logging.warning(f"Unknown deployment state {deployment_state}. Please check the console.") break def ssh(args): # Runs a bash shell in a running task for the env. Note this may run in a short-lived task (e.g. migration task) + # Refresh terraform state + setup([args.env]) cluster_id = get_terraform_output("cluster_id", args.env) service_name = get_terraform_output("web_service_name", args.env) ecs_client = boto3.session.Session(profile_name=AWS_PROFILE_NAME, region_name=AWS_REGION).client("ecs") @@ -221,25 +231,28 @@ def ssh(args): -parser = argparse.ArgumentParser(prog="python deploy.py") -parser.add_argument("--no-input", action="store_true", - help="Skips request for confirmation before starting the deploy.") -parser.add_argument("--skip-build", action="store_true", - help="Skips the build step and uses the existing ECR image for the environment.") -parser.add_argument("--use-latest", action="store_true", - help="Skips the build step and uses the ECR image tagged `latest`.") -parser.add_argument("--use-image-from-env", - help="If provided, skips the terraform build and instead uses the existing built image from the specified environment") -parser.add_argument("--skip-migration", action="store_true", help="Skips the migration step.") -parser.add_argument("-env", help="Terraform environment to deploy", required=True) -parser.set_defaults(func=deploy) - -subparsers = parser.add_subparsers(title="Extra utilities", prog="python deploy.py -env ") -ssh_parser = subparsers.add_parser("ssh", help="SSH into running container in env instead of deploying") -ssh_parser.set_defaults(func=ssh) +def main(): + parser = argparse.ArgumentParser(prog="python deploy.py") + parser.add_argument("--no-input", action="store_true", + help="Skips request for confirmation before starting the deploy.") + parser.add_argument("--skip-build", action="store_true", + help="Skips the build step and uses the existing ECR image for the environment.") + parser.add_argument("--use-latest", action="store_true", + help="Skips the build step and uses the ECR image tagged `latest`.") + parser.add_argument("--use-image-from-env", + help="If provided, skips the terraform build and instead uses the existing built image from the specified environment") + parser.add_argument("--skip-migration", action="store_true", help="Skips the migration step.") + parser.add_argument("-env", help="Terraform environment to deploy", required=True) + parser.set_defaults(func=deploy) + subparsers = parser.add_subparsers(title="Extra utilities", prog="python deploy.py -env ") + ssh_parser = subparsers.add_parser("ssh", help="SSH into running container in env instead of deploying") + ssh_parser.set_defaults(func=ssh) -if __name__ == "__main__": args = parser.parse_args() logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(levelname)s - %(message)s") args.func(args) + + +if __name__ == "__main__": + main() From 5cf77fb3a299a6b1a646b13d6e8df35e97b67b06 Mon Sep 17 00:00:00 2001 From: Grace Whitney Date: Tue, 12 Nov 2024 13:55:21 -0500 Subject: [PATCH 36/39] Update prod main.tf and add cloudwatch url --- deploy.py | 20 ++++++++-- terraform/envs/production/main.tf | 42 ++++++++++++++++++++- terraform/envs/staging/main.tf | 5 +++ terraform/modules/ecs_deployment/outputs.tf | 5 +++ 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/deploy.py b/deploy.py index 2951f2f..4a73dff 100644 --- a/deploy.py +++ b/deploy.py @@ -175,9 +175,14 @@ def run_migrations(env): # The migration task has finished successfully logging.info("Migration complete") return - logging.error(f"Migration task failed with code {stop_code} and reason {task.get('stoppedReason')}.") + logging.error( + f"Migration task failed with code {stop_code} and reason {task.get('stoppedReason')}." + f"Check log stream for more info: {cloudwatch_log_url(env)}" + ) raise MigrationFailed() - logging.error("Migration timed out. It may still be running.") + logging.error( + f"Migration timed out. It may still be running. Check log stream for more info: {cloudwatch_log_url(env)}" + ) raise MigrationTimeOut() @@ -205,12 +210,19 @@ def restart_web_service(env): if deployment_state == "COMPLETED": logging.info("Success! Deployment complete.") elif deployment_state == "FAILED": - logging.error(f"Deployment failed! Reason: {new_deployment['rolloutStateReason']}") + logging.error( + f"Deployment failed! Reason: {new_deployment['rolloutStateReason']}. " + f"Check log stream for more info: {cloudwatch_log_url(env)}" + ) else: - logging.warning(f"Unknown deployment state {deployment_state}. Please check the console.") + logging.warning(f"Unknown deployment state {deployment_state}. Please check the ECS console.") break +def cloudwatch_log_url(env): + cloudwatch_log_group_name = get_terraform_output("cloudwatch_log_group_name", env) + return f"https://{AWS_REGION}.console.aws.amazon.com/cloudwatch/home?region={AWS_REGION}#logsV2:log-groups/log-group/{cloudwatch_log_group_name}" + def ssh(args): # Runs a bash shell in a running task for the env. Note this may run in a short-lived task (e.g. migration task) diff --git a/terraform/envs/production/main.tf b/terraform/envs/production/main.tf index 148bb7b..017113e 100644 --- a/terraform/envs/production/main.tf +++ b/terraform/envs/production/main.tf @@ -33,7 +33,6 @@ module "ecs_deployment" { ses_from_email = "" # TODO: FILL ME IN certificate_manager_arn = "" # TODO: FILL ME IN ecr_repository_name = "" # TODO: FILL ME IN - ecr_image_uri = "" # TODO: FILL ME IN # Optional Variables rds_backup_retention_period = 30 @@ -45,3 +44,44 @@ module "ecs_deployment" { container_count = 2 ssl_policy = "ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04" } + +output "cluster_id" { + description = "The ID of the ECS cluster" + value = module.ecs_deployment.cluster_id +} + +output "cloudwatch_log_group_name" { + description = "The name of the cloudwatch log group for the web service task" + value = module.ecs_deployment.cloudwatch_log_group_name +} + +output "ecr_repository_name" { + description = "The name of the ECR repository" + value = module.ecs_deployment.ecr_repository_name +} + +output "public_ip" { + description = "The public IP address of the load balancer for the web service" + value = module.ecs_deployment.public_ip +} + +output "web_service_name" { + description = "The name of the ECS container running the web service" + value = module.ecs_deployment.web_service_name +} + +output "web_network_configuration_security_group" { + description = "The security groups used by the ECS web task" + value = tolist(module.ecs_deployment.web_network_configuration_security_groups)[0] + +} + +output "web_network_configuration_subnet" { + description = "The ID of one the subnets used by the web task" + value = tolist(module.ecs_deployment.web_network_configuration_subnets)[0] +} + +output "web_task_definition_arn" { + description = "The ARN of the ECS web service task definition" + value = module.ecs_deployment.web_task_definition_arn +} diff --git a/terraform/envs/staging/main.tf b/terraform/envs/staging/main.tf index 7678c1a..ec8ba2b 100644 --- a/terraform/envs/staging/main.tf +++ b/terraform/envs/staging/main.tf @@ -50,6 +50,11 @@ output "cluster_id" { value = module.ecs_deployment.cluster_id } +output "cloudwatch_log_group_name" { + description = "The name of the cloudwatch log group for the web service task" + value = module.ecs_deployment.cloudwatch_log_group_name +} + output "ecr_repository_name" { description = "The name of the ECR repository" value = module.ecs_deployment.ecr_repository_name diff --git a/terraform/modules/ecs_deployment/outputs.tf b/terraform/modules/ecs_deployment/outputs.tf index f327af7..8d77caa 100644 --- a/terraform/modules/ecs_deployment/outputs.tf +++ b/terraform/modules/ecs_deployment/outputs.tf @@ -3,6 +3,11 @@ output "cluster_id" { value = aws_ecs_cluster.cluster.id } +output "cloudwatch_log_group_name" { + description = "The name of the cloudwatch log group for the web service task" + value = aws_cloudwatch_log_group.web_log_group.name +} + output "ecr_image_uri" { description = "The full URI of the ECR image" value = local.ecr_image_uri From 2a81a6d69d7d470b8ff83ae029aaacc09b102bf8 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Tue, 18 Feb 2025 17:52:14 -0500 Subject: [PATCH 37/39] updates SES permissions --- terraform/envs/production/main.tf | 1 - terraform/envs/staging/main.tf | 1 - terraform/modules/ecs_deployment/iam.tf | 14 ++++++++++---- terraform/modules/ecs_deployment/variables.tf | 4 ---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/terraform/envs/production/main.tf b/terraform/envs/production/main.tf index 017113e..0b02536 100644 --- a/terraform/envs/production/main.tf +++ b/terraform/envs/production/main.tf @@ -29,7 +29,6 @@ module "ecs_deployment" { web_config_secret_name = "" # TODO: FILL ME IN s3_bucket_prefix = "" # TODO: FILL ME IN rds_engine_version = "" # TODO: FILL ME IN - ses_identity = "" # TODO: FILL ME IN ses_from_email = "" # TODO: FILL ME IN certificate_manager_arn = "" # TODO: FILL ME IN ecr_repository_name = "" # TODO: FILL ME IN diff --git a/terraform/envs/staging/main.tf b/terraform/envs/staging/main.tf index ec8ba2b..c25f894 100644 --- a/terraform/envs/staging/main.tf +++ b/terraform/envs/staging/main.tf @@ -29,7 +29,6 @@ module "ecs_deployment" { web_config_secret_name = "" # TODO: FILL ME IN s3_bucket_prefix = "" # TODO: FILL ME IN rds_engine_version = "" # TODO: FILL ME IN - ses_identity = "" # TODO: FILL ME IN ses_from_email = "" # TODO: FILL ME IN certificate_manager_arn = "" # TODO: FILL ME IN ecr_repository_name = "" # TODO: FILL ME IN diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf index 54bb2cf..484d388 100644 --- a/terraform/modules/ecs_deployment/iam.tf +++ b/terraform/modules/ecs_deployment/iam.tf @@ -81,12 +81,18 @@ data "aws_iam_policy_document" "ecs_task_role_policy" { statement { effect = "Allow" actions = [ - "ses:SendEmail", - "ses:SendRawEmail", "ses:GetSendQuota" ] - resources = [ - format("arn:aws:ses:*:*:identity/%s", var.ses_identity) + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "ses:SendBulkTemplatedEmail", + "ses:SendEmail", + "ses:SendRawEmail", + "ses:SendTemplatedEmail" ] condition { test = "StringLike" diff --git a/terraform/modules/ecs_deployment/variables.tf b/terraform/modules/ecs_deployment/variables.tf index 0369bac..be4b90d 100644 --- a/terraform/modules/ecs_deployment/variables.tf +++ b/terraform/modules/ecs_deployment/variables.tf @@ -23,10 +23,6 @@ variable "rds_engine_version" { type = string } -variable "ses_identity" { - type = string -} - variable "ses_from_email" { type = string } From 70d1c8521f769d50ede7f168c366ddb941837394 Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Tue, 18 Feb 2025 17:52:49 -0500 Subject: [PATCH 38/39] fix --- terraform/modules/ecs_deployment/iam.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/modules/ecs_deployment/iam.tf b/terraform/modules/ecs_deployment/iam.tf index 484d388..f7e975b 100644 --- a/terraform/modules/ecs_deployment/iam.tf +++ b/terraform/modules/ecs_deployment/iam.tf @@ -94,6 +94,7 @@ data "aws_iam_policy_document" "ecs_task_role_policy" { "ses:SendRawEmail", "ses:SendTemplatedEmail" ] + resources = ["*"] condition { test = "StringLike" variable = "ses:FromAddress" From 800afaa50d07dccc4f284bd43dc0579bcac09d9e Mon Sep 17 00:00:00 2001 From: Benjamin Zagorsky Date: Thu, 13 Mar 2025 14:03:53 -0400 Subject: [PATCH 39/39] adds subnet group to database, vpc to security groups, and adds docs --- deploy.py | 2 +- readme.md | 24 ++++++++++++------- terraform/modules/ecs_deployment/rds.tf | 7 ++++++ .../modules/ecs_deployment/security_groups.tf | 3 +++ 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/deploy.py b/deploy.py index 4a73dff..9340902 100644 --- a/deploy.py +++ b/deploy.py @@ -254,7 +254,7 @@ def main(): parser.add_argument("--use-image-from-env", help="If provided, skips the terraform build and instead uses the existing built image from the specified environment") parser.add_argument("--skip-migration", action="store_true", help="Skips the migration step.") - parser.add_argument("-env", help="Terraform environment to deploy", required=True) + parser.add_argument("env", help="Terraform environment to deploy") parser.set_defaults(func=deploy) subparsers = parser.add_subparsers(title="Extra utilities", prog="python deploy.py -env ") diff --git a/readme.md b/readme.md index bce8b85..02fcffc 100644 --- a/readme.md +++ b/readme.md @@ -124,14 +124,20 @@ Following that, deploy your code to the environment (see below). ## Creating a new ECS environment -1. Create an ECR repository -2. Build and push an initial docker file to it (ECR provides docker commands for this). -3. Create a bucket for holding terraform config -4. Create an SES identity and from email (if using SES) -5. Create an AWS certificate manager certificate for your domain -6. Create a secrets manager secret containing the config parameters needed by the application (you do not need include "DATABASE_URL", "SECRET_KEY", "AWS_STORAGE_BUCKET_NAME", or "DEFAULT_FROM_EMAIL" as those are managed by terraform in `terraform/modules/ecs_deployment/secrets_manager.tf`) -7. Fill in the missing values in `terraform/envs//main.tf` -8. Run terraform to set up that environment +In the following steps, config variables go in `terraform/envs//main.tf`; most of them go in the definition of the `ecs_deployment` module. + +1. Create a VPC and subnets (or use the default VPC). This is config var `ecs_deployment.vpc_id`. +1. Create an ECR repository. This is config var `ecs_deployment.ecr_repository_name`. +2. Build and push an initial docker file to it (ECR provides docker commands for this) and tag it with a tag called ``. +3. Create a bucket for holding terraform config. This is config var `terraform.backend.bucket`. +4. Create an SES identity and from email (if using SES). The from email is config var `ecs_deployment.ses_from_email`. +5. Create an AWS certificate manager certificate for your domain. This is config var `ecs_deployment.certificate_manager_arn`. +6. Create a secrets manager secret containing the config parameters needed by the application (you do not need include "DATABASE_URL", "SECRET_KEY", "AWS_STORAGE_BUCKET_NAME", or "DEFAULT_FROM_EMAIL" as those are managed by terraform in `terraform/modules/ecs_deployment/secrets_manager.tf`). This is config var `ecs_deployment.web_config_secret_name`. +7. Fill in the remaining config vars: + * `ecs_deployment.application_name` with a name for your application. + * `ecs_deployment.rds_engine_version` with the version of Postgres you want to use. + * `ecs_deployment.s3_bucket_prefix` with a prefix for your s3 bucket so that it will have a globally unique name (the bucket will be named `_`). +8. Run terraform to set up that environment: ``` cd terraform/envs/ terraform init @@ -154,7 +160,7 @@ See the [eb-cli](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3. To deploy new versions of your code to an ECS environment, use the included `deploy.py` script. First fill in the missing constants at the top of that file, and then run the script: ``` -python deploy.py -env +python deploy.py ``` This script will do the following: 1. Build the docker image using your local code version. diff --git a/terraform/modules/ecs_deployment/rds.tf b/terraform/modules/ecs_deployment/rds.tf index 6de8653..bf5a66a 100644 --- a/terraform/modules/ecs_deployment/rds.tf +++ b/terraform/modules/ecs_deployment/rds.tf @@ -21,4 +21,11 @@ resource "aws_db_instance" "database" { storage_type = "gp2" username = "dbuser" vpc_security_group_ids = [aws_security_group.database.id] + db_subnet_group_name = aws_db_subnet_group.database.name +} + + +resource "aws_db_subnet_group" "database" { + name = "${local.app_env_name}-database-subnets" + subnet_ids = data.aws_subnets.subnets.ids } diff --git a/terraform/modules/ecs_deployment/security_groups.tf b/terraform/modules/ecs_deployment/security_groups.tf index fac1c38..318d79a 100644 --- a/terraform/modules/ecs_deployment/security_groups.tf +++ b/terraform/modules/ecs_deployment/security_groups.tf @@ -1,5 +1,6 @@ resource "aws_security_group" "load_balancer" { name = "${local.app_env_name}-lb" + vpc_id = var.vpc_id ingress { from_port = 80 @@ -34,6 +35,7 @@ resource "aws_security_group" "load_balancer" { resource "aws_security_group" "web" { name = "${local.app_env_name}-web" + vpc_id = var.vpc_id ingress { from_port = 8080 @@ -60,6 +62,7 @@ resource "aws_security_group" "web" { resource "aws_security_group" "database" { name = "${local.app_env_name}-db" + vpc_id = var.vpc_id ingress { from_port = 5432