Published on

Setup Cloudwatch Alarms with Notifications through Teams Channel and Email

Authors
  • avatar
    Name
    Tamilarasu Gurusamy
    Twitter

Objective

Setup Cloudwatch Alarms for EC2 and RDS

  • The alarms will be setup for the following situations
    • When the instance cpu spikes above 90% for 1 minute
    • When any of the Instance Status Check fails
    • When max number of connections are active in RDS

Setup First Alarm

  • Create an EC2 instance with your required configurations
  • Before creating a cloudwatch alarm, lets proceed onto creating the sns topic where the alarm details will arrive
  • Navigate to SNS
  • Click on Topics, create a new topic with your desired name and choose the type as Standard
  • Navigate to Cloudwatch
  • Click on Alarms -> All Alarms -> Create Alarm
  • Click on Select Metric
  • Search for EC2 and click on it
  • Click on Per Instance Metrics
  • Search for the instance name that you want to be configured and Choose Select metric
  • A new page will appear where we can edit more settings for the alarm.
  • Make sure the instance id is correct and choose the Statistic to be Maximum, since we need immediate alerts and choose the period of 1 Minute
  • Leave the condition type to default and set the threshold value greater than the number you need, in my case it is 90.
  • Click on next
  • Next is the notification settings, where we need to define where the alarms details to be sent once they are triggered.
  • Click on choose a existing topic and select the one created earlier
  • Click on add notification and choose OK
  • Choose the same sns topic for this one
  • Create the alarm
  • Follow the same procedure for EC2 Status checks and RDS Connections

Setup a Lambda Function to Send a message to Teams

  • Navigate to Lambda
  • Click on create a new lambda
  • Give a meaningful name for the lambda
  • Keep everything as default
  • Choose the runtime as python3.13
  • Click on create function
  • Paste the following code in the lambda console
import json
import urllib.request
from urllib.parse import quote_plus
import os
import boto3
import smtplib
import threading
from datetime import datetime, timedelta
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

# Initialize EC2 client globally
region = boto3.session.Session().region_name
ec2_client = boto3.client('ec2', region_name=region)

# Load recipient emails from environment variable
recipient_emails = os.environ["RECIPIENT_EMAIL"]
recipient_list = [email.strip() for email in recipient_emails.split(',')]

def encode_alarm_name_for_url(alarm_name: str) -> str:
    encoded = alarm_name.replace('%', '$25')
    encoded = encoded.replace(' ', '+')
    return encoded

def get_cloudwatch_alarm_url(region: str, alarm_name: str) -> str:
    encoded_name = encode_alarm_name_for_url(alarm_name)
    return f"https://{region}.console.aws.amazon.com/cloudwatch/home?region={region}#alarmsV2:alarm/{encoded_name}"

def convert_utc_to_ist(utc_time_str):
    utc_time = datetime.fromisoformat(utc_time_str.replace("Z", "+00:00"))
    ist_time = utc_time + timedelta(hours=5, minutes=30)
    return ist_time.strftime('%d-%m-%Y %H:%M:%S IST')

def get_instance_name(instance_id):
    try:
        response = ec2_client.describe_tags(
            Filters=[
                {'Name': 'resource-id', 'Values': [instance_id]},
                {'Name': 'key', 'Values': ['Name']}
            ]
        )
        for tag in response['Tags']:
            if tag['Key'] == 'Name':
                return tag['Value']
    except Exception as e:
        print("Error fetching instance name:", e)
    return "Unknown"

def send_email(subject, body_html):
    try:
        msg = MIMEMultipart('alternative')
        msg['Subject'] = subject
        msg['From'] = os.environ['SENDER_EMAIL']
        msg['To'] = ', '.join(recipient_list)  # For email header only

        html_part = MIMEText(body_html, 'html')
        msg.attach(html_part)

        with smtplib.SMTP(os.environ['SMTP_HOST'], int(os.environ['SMTP_PORT'])) as server:
            server.starttls()
            server.login(os.environ['SMTP_USERNAME'], os.environ['SMTP_PASSWORD'])
            server.sendmail(msg['From'], recipient_list, msg.as_string())  # ✅ Fixed: use recipient_list directly

        print("Email sent via SMTP.")
    except Exception as e:
        print("Error sending email via SMTP:", e)

def send_teams_message(adaptive_card, webhook_url):
    try:
        req = urllib.request.Request(
            webhook_url,
            data=json.dumps(adaptive_card).encode("utf-8"),
            headers={"Content-Type": "application/json"},
            method="POST"
        )
        with urllib.request.urlopen(req) as res:
            print("Adaptive Card sent to Teams. Status:", res.status)
    except Exception as e:
        print("Error sending Adaptive Card to Teams:", e)

def lambda_handler(event, context):
    print("Received event:", json.dumps(event, indent=2))

    for record in event['Records']:
        sns_message = json.loads(record['Sns']['Message'])

        alarm_name = sns_message.get('AlarmName', 'N/A')
        new_state = sns_message.get('NewStateValue', 'N/A')
        reason = sns_message.get('NewStateReason', 'N/A')
        alarm_time_utc = sns_message.get('StateChangeTime', 'N/A')
        alarm_time_ist = convert_utc_to_ist(alarm_time_utc)
        namespace = sns_message.get('Trigger', {}).get('Namespace', 'Unknown')

        resource_id = 'Unknown'
        instance_name = 'Unknown'

        for dim in sns_message.get('Trigger', {}).get('Dimensions', []):
            if namespace == 'AWS/EC2' and dim.get('name') == 'InstanceId':
                resource_id = dim.get('value')
                instance_name = get_instance_name(resource_id)
            elif namespace == 'AWS/RDS' and dim.get('name') == 'DBInstanceIdentifier':
                resource_id = dim.get('value')

        final_alarm_url = get_cloudwatch_alarm_url(region, alarm_name)

        # Define status color & emoji
        state_color = "default"
        state_emoji = "ℹ️"
        state_color_code = "#444"

        if new_state == "ALARM":
            state_color = "attention"
            state_emoji = "🔴"
            state_color_code = "#D32F2F"
        elif new_state == "OK":
            state_color = "good"
            state_emoji = "🟢"
            state_color_code = "#388E3C"
        elif new_state == "INSUFFICIENT_DATA":
            state_color = "warning"
            state_emoji = "🟡"
            state_color_code = "#FBC02D"

        adaptive_card = {
            "type": "message",
            "attachments": [
                {
                    "contentType": "application/vnd.microsoft.card.adaptive",
                    "content": {
                        "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
                        "type": "AdaptiveCard",
                        "version": "1.5",
                        "body": [
                            {"type": "TextBlock", "text": f"{state_emoji} Alarm Triggered", "size": "Large", "weight": "Bolder", "color": state_color},
                            {"type": "TextBlock", "text": f"**Alarm Name:** {alarm_name}", "wrap": True},
                            {"type": "TextBlock", "text": f"**State:** {new_state}", "wrap": True, "color": state_color},
                            {"type": "TextBlock", "text": f"**Resource ID:** {resource_id}", "wrap": True},
                            {"type": "TextBlock", "text": f"**Instance Name:** {instance_name}", "wrap": True},
                            {"type": "TextBlock", "text": f"**Region:** {region}", "wrap": True},
                            {"type": "TextBlock", "text": f"**Namespace:** {namespace}", "wrap": True},
                            {"type": "TextBlock", "text": f"**Triggered At:** {alarm_time_ist}", "wrap": True}
                        ],
                        "actions": [
                            {"type": "Action.OpenUrl", "title": "View Alarm in CloudWatch", "url": final_alarm_url}
                        ]
                    }
                }
            ]
        }

        email_subject = f"{state_emoji} AWS Alarm: {alarm_name} - {new_state}"
        email_body_html = f"""
        <!DOCTYPE html>
        <html>
        <head>
          <meta charset="UTF-8">
          <title>AWS Alarm Notification</title>
          <style>
            body {{
              font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
              margin: 0;
              padding: 0;
              background-color: #f6f9fc;
            }}
            .container {{
              background-color: #ffffff;
              max-width: 600px;
              margin: 30px auto;
              padding: 30px;
              border-radius: 10px;
              box-shadow: 0 4px 14px rgba(0, 0, 0, 0.08);
            }}
            h2 {{
              color: {state_color_code};
            }}
            table {{
              width: 100%;
              border-collapse: collapse;
              margin-top: 20px;
            }}
            td {{
              padding: 10px 0;
            }}
            .label {{
              font-weight: bold;
              color: #333;
            }}
            .value {{
              color: #555;
            }}
            .footer {{
              margin-top: 30px;
              font-size: 13px;
              color: #888;
              text-align: center;
            }}
            .btn {{
              display: inline-block;
              margin-top: 20px;
              padding: 12px 20px;
              background-color: #0078D7;
              color: white !important;
              text-decoration: none;
              border-radius: 6px;
              font-weight: bold;
            }}
          </style>
        </head>
        <body>
          <div class="container">
            <h2>{state_emoji} Alarm Triggered</h2>
            <table>
              <tr><td class="label">Alarm Name:</td><td class="value">{alarm_name}</td></tr>
              <tr><td class="label">State:</td><td class="value">{new_state}</td></tr>
              <tr><td class="label">Resource ID:</td><td class="value">{resource_id}</td></tr>
              <tr><td class="label">Instance Name:</td><td class="value">{instance_name}</td></tr>
              <tr><td class="label">Region:</td><td class="value">{region}</td></tr>
              <tr><td class="label">Namespace:</td><td class="value">{namespace}</td></tr>
              <tr><td class="label">Triggered At:</td><td class="value">{alarm_time_ist}</td></tr>
            </table>
            <a class="btn" href="{final_alarm_url}" target="_blank">🔗 View Alarm in AWS Console</a>
            <div class="footer">This is an automated message from your AWS Monitoring System.</div>
          </div>
        </body>
        </html>
        """

        webhook_url = os.environ.get('TEAMS_WEBHOOK_URL')

        teams_thread = threading.Thread(target=send_teams_message, args=(adaptive_card, webhook_url))
        email_thread = threading.Thread(target=send_email, args=(email_subject, email_body_html))

        teams_thread.start()
        email_thread.start()

        teams_thread.join()
        email_thread.join()

        print("Teams and Email notification sent.")

    return {"statusCode": 200, "body": "Notification sent"}


  • We need to change some configuration to make sure that the lambda gets executed successfully
  • One is change the timeout of the function, by default the timeout of the lambda function is 3 seconds and we need to increase it to 10 secs.
  • For that, click on Configuration and edit the general configuration and increase the timeout to 10 secs.
  • Next select the role assigned to the lambda function and click on add permissions, attach policies and select the AmazonEC2ReadOnlyAccess, click on add permissions
  • Next we need to configure sns to trigger the lambda, so click on Add trigger on the function page and choose SNS and choose the appropriate sns and click on Add
  • Now we need to add the following environment variables
    • RECIPIENT_EMAIL
    • SENDER_EMAIL
    • SMTP_HOST
    • SMTP_PASSWORD
    • SMTP_USERNAME
    • SMTP_PORT
    • TEAMS_WEBHOOK_URL
  • The RECIPIENT_EMAIL env variable supports multiple emails seperated by a comma.