11378 low scoring alerts

.gitignore

@@ -239,3 +239,6 @@ node_modules/
 
 # dotenv
 src/backend/app/.env
+src/backend/bedrock_alerts/lambda_build
+src/backend/bedrock_alerts/lambda_function.zip
+src/backend/bedrock_alerts/lambda_function.zip

infrastructure/terraform/components/notifyai/cloudwatch.tf

@@ -15,6 +15,11 @@ resource "aws_cloudwatch_log_group" "bedrock_lambda_evaluations" {
   name              = "/aws/lambda/${aws_lambda_function.bedrock_evaluations.function_name}"
   retention_in_days = var.log_retention_in_days
 
+  lifecycle {
+    prevent_destroy = true
+    ignore_changes  = [tags]
+  }
+
   tags = merge(
     local.default_tags,
     {

infrastructure/terraform/components/notifyai/eventbridge.tf

@@ -46,3 +46,54 @@ resource "aws_iam_role_policy" "eventbridge_scheduler_policy" {
     ]
   })
 }
+
+resource "aws_s3_bucket_notification" "bucket_notification" {
+  bucket = aws_s3_bucket.evaluation_programatic_results.id # Your S3 bucket ID
+
+  eventbridge = true # This flag enables sending notifications directly to EventBridge
+}
+
+resource "aws_cloudwatch_event_rule" "evaluation_results_uploaded" {
+  name        = "${local.csi}-evaluation-results-uploaded-rule"
+  description = "Triggers a Lambda when evaluation results are uploaded to S3"
+
+  event_pattern = jsonencode({
+    "source" : ["aws.s3"],
+    "detail-type" : ["Object Created"],
+    "detail" : {
+      "bucket" : {
+        "name" : [aws_s3_bucket.evaluation_programatic_results.id]
+      }
+    }
+  })
+
+  depends_on = [aws_s3_bucket_notification.bucket_notification]
+}
+
+resource "aws_cloudwatch_event_target" "invoke_alerts_lambda_on_results_upload" {
+  rule      = aws_cloudwatch_event_rule.evaluation_results_uploaded.name
+  target_id = "InvokeAlertsLambda"
+  arn       = aws_lambda_function.evaluations_alerts.arn
+
+  input_transformer {
+    input_paths = {
+      "bucket" = "$.detail.bucket.name",
+      "key"    = "$.detail.object.key"
+    }
+    input_template = <<EOF
+    {
+      "message": "Object created in bucket <bucket> with key <key>.",
+      "s3_bucket": <bucket>,
+      "s3_key": <key>
+    }
+    EOF
+  }
+}
+
+resource "aws_lambda_permission" "allow_eventbridge_to_invoke_alerts_lambda" {
+  statement_id  = "AllowEventBridgeInvoke"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.evaluations_alerts.function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.evaluation_results_uploaded.arn
+}

infrastructure/terraform/components/notifyai/lambda.tf

@@ -2,6 +2,7 @@ locals {
   lambda_name             = "${local.csi}-bedrock-messager"
   s3_lambda_logging_key   = "prompt-executions/"
   evaluations_lambda_name = "${local.csi}-bedrock-evaluations"
+  alerts_lambda_name      = "${local.csi}-bedrock-evaluations-alerts"
 }
 
 resource "aws_s3_bucket" "lambda_prompt_logging_s3_bucket" {
@@ -163,15 +164,14 @@ data "aws_iam_policy_document" "evaluations_lambda_policy_doc" {
     actions = [
       "logs:CreateLogGroup",
       "logs:CreateLogStream",
-      "logs:PutLogEvents"
+      "logs:PutLogEvents",
+      "bedrock:InvokeModel",
+      "bedrock:CreateEvaluationJob",
+      "s3:GetObject",
+      "s3:PutObject",
     ]
-    resources = ["arn:aws:logs:${var.region}:${var.aws_account_id}:log-group:/aws/lambda/${local.evaluations_lambda_name}:*"]
-  }
-
-  statement {
-    effect  = "Allow"
-    actions = ["bedrock:InvokeModel"]
     resources = [
+      "arn:aws:logs:${var.region}:${var.aws_account_id}:log-group:/aws/lambda/${local.evaluations_lambda_name}:*",
       "arn:aws:bedrock:${var.region}::foundation-model/${var.evaluation-evaluator-model-identifier}",
       "arn:aws:bedrock:${var.region}::foundation-model/${var.evaluation-inference-model-identifier}"
     ]
@@ -227,3 +227,79 @@ resource "aws_iam_role_policy_attachment" "evaluations_lambda_attachment" {
   role       = aws_iam_role.iam_for_evaluations_lambda.name
   policy_arn = aws_iam_policy.evaluations_lambda_policy.arn
 }
+
+resource "aws_lambda_function" "evaluations_alerts" {
+  function_name    = local.alerts_lambda_name
+  role             = aws_iam_role.iam_for_evaluations_alerts_lambda.arn
+  filename         = data.archive_file.evaluations_alerts_zip.output_path
+  source_code_hash = data.archive_file.evaluations_alerts_zip.output_base64sha256
+  handler          = "bedrock_alerts.evaluations_alert_lambda.lambda_handler"
+  runtime          = "python3.12"
+  timeout          = 30
+
+  environment {
+    variables = {
+      env_lambda_name        = local.alerts_lambda_name
+
+      env_results_bucket     = aws_s3_bucket.evaluation_programatic_results.bucket
+      env_results_bucket_key = aws_s3_object.results_object.key
+      env_sns_topic_arn      = aws_sns_topic.admail_eval_alerts_topic.arn
+    }
+  }
+}
+
+data "archive_file" "evaluations_alerts_zip" {
+  type        = "zip"
+  source_dir  = "../../../../src/backend/bedrock_alerts/lambda_build"
+  output_path = "${path.module}/evaluations_alerts.zip"
+}
+
+
+resource "aws_iam_role" "iam_for_evaluations_alerts_lambda" {
+  name = "${local.csi}-iam-for-evaluations-alerts-lambda"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect    = "Allow"
+        Principal = { Service = "lambda.amazonaws.com" }
+        Action    = "sts:AssumeRole"
+      }
+    ]
+  })
+}
+
+data "aws_iam_policy_document" "evaluations_lambda_alerts_policy_doc" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "bedrock:GetEvaluationJob",
+      "bedrock:DescribeEvaluationJob",
+      "bedrock:ListEvaluationJobs",
+      "s3:GetObject",
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "s3:ListBucket",
+      "sns:Publish"
+    ]
+    resources = [
+      "arn:aws:s3:::${aws_s3_object.results_object.bucket}/${aws_s3_object.results_object.key}*",
+      "arn:aws:bedrock:${var.region}:${var.aws_account_id}:evaluation-job/*",
+      "arn:aws:logs:${var.region}:${var.aws_account_id}:log-group:/aws/lambda/${local.alerts_lambda_name}:*",
+      "arn:aws:s3:::${aws_s3_object.results_object.bucket}",
+      "arn:aws:sns:${var.region}:${var.aws_account_id}:${aws_sns_topic.admail_eval_alerts_topic.name}",
+    ]
+  }
+}
+
+resource "aws_iam_policy" "evaluations_lambda_alerts_policy" {
+  name   = "${local.csi}-evaluations-lambda-alerts-policy"
+  policy = data.aws_iam_policy_document.evaluations_lambda_alerts_policy_doc.json
+}
+
+resource "aws_iam_role_policy_attachment" "evaluations_lambda_alerts_attachment" {
+  role       = aws_iam_role.iam_for_evaluations_alerts_lambda.name
+  policy_arn = aws_iam_policy.evaluations_lambda_alerts_policy.arn
+}

infrastructure/terraform/components/notifyai/sns.tf

@@ -0,0 +1,10 @@
+resource "aws_sns_topic" "admail_eval_alerts_topic" {
+  name = "${local.csi}-notifyai-eval-alerts"
+  tags = local.default_tags
+
+}
+resource "aws_sns_topic_subscription" "admail_eval_alerts_subscription" {
+  topic_arn = aws_sns_topic.admail_eval_alerts_topic.arn
+  protocol  = "email"
+  endpoint  = "admail-eval-alerts@example.com"
+}

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+pythonpath = src/backend
+testpaths = src/backend/app/tests
+python_files = test_*.py

src/backend/bedrock-prompt-messager/system_prompt.txt

@@ -62,7 +62,6 @@ System instructions Task: You are an expert specializing exclusively in the Roya
     }
 
     CRITICAL JSON formatting rules:
-    - Use lowercase keys: "description", "rating", "reason", "advice"
     - Use double quotes for all strings
     - Use \\n for line breaks and \\n- for bullet points
     - Escape any internal quotes with \\"

src/backend/bedrock_alerts/build_lambda.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Script to build an AWS Lambda deployment package for bedrock_alerts
+set -e
+
+
+BUILD_DIR="lambda_build"
+ZIP_FILE="evaluations_alerts.zip"
+
+
+echo "Creating build directory: $BUILD_DIR"
+mkdir -p "$BUILD_DIR"
+
+if [ -f "requirements.txt" ]; then
+    echo "Installing dependencies from requirements.txt to $BUILD_DIR"
+    pip install -r requirements.txt -t "$BUILD_DIR"
+else
+    echo "Error: requirements.txt not found"
+    exit 1
+fi
+
+echo "Creating bedrock_alerts package in $BUILD_DIR"
+mkdir -p "$BUILD_DIR/bedrock_alerts"
+
+for file in evaluations_alert_lambda.py evaluations_alert_service.py; do
+    if [ -f "$file" ]; then
+        echo "Copying $file to $BUILD_DIR/bedrock_alerts/"
+        cp "$file" "$BUILD_DIR/bedrock_alerts/"
+    else
+        echo "Error: $file not found"
+        exit 1
+    fi
+done
+
+if [ -f "bedrock_alerts/__init__.py" ]; then
+    echo "Copying bedrock_alerts/__init__.py to $BUILD_DIR/bedrock_alerts/"
+    cp "bedrock_alerts/__init__.py" "$BUILD_DIR/bedrock_alerts/"
+else
+    echo "Warning: bedrock_alerts/__init__.py not found, creating empty file"
+    touch "$BUILD_DIR/bedrock_alerts/__init__.py"
+fi
+
+echo "Creating zip file: $ZIP_FILE"
+cd "$BUILD_DIR"
+zip -r "../$ZIP_FILE" .
+cd ..
+
+echo "Lambda package created successfully: $ZIP_FILE"

src/backend/bedrock_alerts/evaluations_alert_lambda.py

@@ -0,0 +1,34 @@
+import json
+from bedrock_alerts.evaluations_alert_service import BedrockAlertsService
+import os
+import logging
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+def lambda_handler(event, context):
+    bucket = os.environ["env_results_bucket"]
+    bucket_key = os.environ["env_results_bucket_key"]
+    sns_topic_arn = os.environ["env_sns_topic_arn"]
+
+    alerts = BedrockAlertsService()
+    try:
+        results = alerts.find_results_file_in_s3(bucket, bucket_key)
+        rating_percentage = alerts.calculate_rating_percentage_from_list(results)
+        if rating_percentage < 75.0:
+            alerts.send_alert(sns_topic_arn)
+            logger.info(f"Alert sent successfully rating below 75 percent")
+            return {
+                'statusCode': 200,
+                'body': json.dumps({
+                    'message': 'Alert sent successfully',
+                })
+            }
+        else:
+            return
+    except Exception as e:
+        logger.error(f"Failed to send alert: {str(e)}")
+        raise

src/backend/bedrock_alerts/evaluations_alert_local_runner.py

@@ -0,0 +1,16 @@
+import os
+import json
+import logging
+from bedrock_alerts.evaluations_alert_lambda import lambda_handler
+from dotenv import load_dotenv
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+load_dotenv()
+
+response = lambda_handler({}, {})
+
+print(json.dumps(response, indent=2))

src/backend/bedrock_alerts/evaluations_alert_service.py

@@ -0,0 +1,77 @@
+import boto3
+import jsonlines
+import logging
+from botocore.exceptions import ClientError
+import io
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class BedrockAlertsService:
+    def __init__(self, sns_client=None, s3_client=None):
+        self.sns = sns_client if sns_client is not None else boto3.client("sns")
+        self.s3 = s3_client if s3_client is not None else boto3.client("s3")
+        self.success_percentage = 0.0
+
+    def find_results_file_in_s3(self, bucket, prefix):
+        try:
+            logger.info(f"Searching for files in bucket '{bucket}' with prefix '{prefix}'...")
+            paginator = self.s3.get_paginator('list_objects_v2')
+            pages = paginator.paginate(Bucket=bucket, Prefix=prefix)
+            for page in pages:
+                if "Contents" in page:
+                    for obj in page["Contents"]:
+                        if obj['Key'].endswith('_output.jsonl'):
+                            response = self.s3.get_object(Bucket=bucket, Key=obj['Key'])
+                            file_content_string = response['Body'].read().decode('utf-8')
+                            try:
+                                file_like_object = io.StringIO(file_content_string)
+                                with jsonlines.Reader(file_like_object) as reader:
+                                    return [obj for obj in reader]
+                            except Exception as e:
+                                print(f"Error occurred while reading the file: {e}")
+                                return None
+
+        except ClientError as e:
+            print(f"An AWS error occurred while listing objects: {e}")
+            return None
+
+    def calculate_rating_percentage_from_list(self, records_list: list) -> float:
+        total_rating_score = 0
+        rating_records_count = 0
+        for record in records_list:
+            try:
+                scores = record.get('automatedEvaluationResult', {}).get('scores', [])
+                for metric in scores:
+                    if metric.get('metricName') == 'Rating':
+                        total_rating_score += metric.get('result', 0.0)
+                        rating_records_count += 1
+                        break
+            except TypeError:
+                logger.warning(f"Skipping a record that is not in the expected format: {record}")
+                continue
+
+        if rating_records_count == 0:
+            logger.warning("No 'Rating' metrics were found in the list.")
+            return 0.0
+
+        success_percentage = round((total_rating_score / rating_records_count) * 100)
+        self.success_percentage = success_percentage
+        return success_percentage
+
+
+    def send_alert(self, topic_arn=None):
+        try:
+            subject = "Bedrock Model Evaluation Alert"
+            message = f"Alert Percentage: {self.success_percentage or 'N/A'}\n\nThis alert is below the threshold, please review bedrock model evaluations performance.\n\nThis is an automated alert from Bedrock."
+            response = self.sns.publish(TopicArn=topic_arn, Message=message, Subject=subject)
+            print(f"Alert sent successfully with response: {response}")
+            print("Message published")
+
+            return response
+        except Exception as e:
+            logger.error(
+                f"Failed to send alert: {str(e)}"
+            )
+            raise

src/backend/bedrock_alerts/requirements.txt

@@ -0,0 +1,5 @@
+boto3
+python-dotenv
+jsonlines
+pytest-mock
+moto