AWS CloudWatch 모니터링

1. Cloudwatch 로그 그룹을 생성한 후 info, error의 별도 Stream을 생성한다.

2. 스프링에 logback 파일을 설정한다.

3. Slack Webhook API 생성( https://api.slack.com ) 

=> Feature , Incoming Webhooks (Webhook URL channel 및 url 생성)

4. error 로그 람다 생성

트리거 - CloudWatch Logs , Log group , Filter name, Filter pattern 생성 : [error] =>패턴에 맞는 슬랙 알람)

// imports
const https = require("https");
const AWS = require('aws-sdk');
// get envarionment variables
const ENV = process.env;
const webhookUrl = ENV.slack_webhook_url;

exports.handler = async (event, context) => {
    const cloudwatchlogs = new AWS.CloudWatchLogs();
    const logGroupName = 'hhplus-nest-prod-logs'; // 여기에 로그 그룹 이름을 입력하세요
    const logStreamName = 'error'; // 여기에 로그 스트림 이름을 입력하세요

    const params = {
        logGroupName: logGroupName,
        logStreamName: logStreamName,
        startFromHead: false, // 처음부터 로그를 가져오려면 true로 설정
        limit: 1,
    };

    try {
        const data = await cloudwatchlogs.getLogEvents(params).promise();
        if (data && data.events && data.events.length > 0) {
            const latestLog = data.events[0].message;
            exports.handleLogMessage(latestLog);
        } else {
            console.log('No log events found.');
        }
    } catch (err) {
        console.error('Error fetching log events:', err);
    }
};

// set handle event func
exports.handleLogMessage = async (logMessage) => {
  const slackMessage = exports.createSlackMessage(logMessage);
  await exports.sendToSlack(slackMessage);
}
// create slack message
exports.createSlackMessage = (message) => {
  const time = exports.formatDate(message.StateChangeTime);
  return {
    attachments: [
      {
        title: `:warning: *[nest-prod-error]*`,
        fields: [
          {
            title: '발생시각',
            value: time
          },
          {
            title: '로그',
            value: message
          },
        ]
      }  
    ]
  }
}
// request slack webhook api
exports.sendToSlack = async (message) => {
  return await requestSlackWebhook(getSlackOptions(), message);
}
function getSlackOptions() {
  const { host, pathname } = new URL(webhookUrl);
  return {
    hostname: host,
    path: pathname,
    method: 'POST',
    headers: {
      'Content-Type': 'application/json'
    },
  }
}
function requestSlackWebhook(options, payload) {
  return new Promise((resolve, reject) => {
    const request = https.request(options, (response) => {
      response.setEncoding('utf8');
      let responseBody = '';
      response.on('data', (chunk) => { responseBody += chunk });
      response.on('end', () => { resolve(responseBody) });
    });
    
    request.on('error', (e) => {
      console.error(e);
      reject(e);
    });
    
    request.write(JSON.stringify(payload));
    request.end();
  })
}
// util - get alarm link
exports.getCloudWatchLink = (message) => {
  return `https://console.aws.amazon.com/cloudwatch/home?region=${exports.parseRegionCode(message.AlarmArn)}#alarm:alarmFilter=ANY;name=${encodeURIComponent(message.AlarmName)}`;
}
exports.parseRegionCode = (arn) => {
  return arn.replace("arn:aws:cloudwatch:", "").split(":")[0];
}
// util - current timestamp
exports.formatDate = () => {
  function zerofill(n) { return n < 10 ? '0' + n : n }
  
  const kstDate = new Date(new Date().getTime() + 9 * 60 * 60 * 1000);
  
  return kstDate.getFullYear().toString()
    + '-' + zerofill(kstDate.getMonth() + 1)
    + '-' + zerofill(kstDate.getDate())
    + '-' + zerofill(kstDate.getHours())
    + ':' + zerofill(kstDate.getMinutes())
    + ':' + zerofill(kstDate.getSeconds());
}

 

5. CloudWatch Alarm + SNS + Metric Lambda 생성

1) Amazon SNS 토믹 생성

2) CloudWatch -> Alarms 생성 ( ECS의 활성화된 서비스 선택 : CPU, Memory 지표설정)

3) SNS를 구독하는 새로운 Lambda Function을 만들고 SNS구독

-  코드 입력, 설정에서 환경변수와 트리거 설정

// imports
const https = require("https");

// get envarionment variables
const ENV = process.env;
const webhookUrl = ENV.webhook_url;

// set handler ( main )
exports.handler = async (event) => {
  await exports.handleEvent(event);
}
// set handle event func
exports.handleEvent = async (event) => {
  console.log("Event >>", JSON.stringify(event));
  const snsMessage = JSON.parse(event.Records[0].Sns.Message);
  const slackMessage = exports.createSlackMessage(snsMessage);
  await exports.sendToSlack(slackMessage);
}
// create slack message
exports.createSlackMessage = (message) => {
  const name = message.AlarmName;
  const oldState = message.OldStateValue;
  const newState = message.NewStateValue;
  const newStateReason = message.NewStateReason;
  const description = message.AlarmDescription;
  const time = exports.formatDate(message.StateChangeTime);
  
  return {
    attachments: [
      {
        title: `[$name]`,
        fields: [
          {
            title: '발생시각',
            value: time
          },
          {
            title: '설명',
            value: description
          },
          {
            title: '사유',
            value: newStateReason
          },
          { 
            title: '이전 상태',
            value: oldState,
            short: true
          },
          {
            title: '현재 상태',
            value: newState.message,
            short: true
          }
          {
            title: '링크',
            value: exports.getCloudWatchLink(message)
          }
        ]
      }  
    ]
  }
}
// request slack webhook api
exports.sendToSlack = async (message) => {
  return await requestSlackWebhook(getSlackOptions(), message);
}
function getSlackOptions() {
  const { host, pathname } = new URL(webhookUrl);
  return {
    hostname: host,
    path: pathname,
    method: 'POST',
    headers: {
      'Content-Type': 'application/json'
    },
  }
}
function requestSlackWebhook(options, payload) {
  return new Promise((resolve, reject) => {
    const request = https.request(options, (response) => {
      response.setEncoding('utf8');
      let responseBody = '';
      response.on('data', (chunk) => { responseBody += chunk });
      response.on('end', () => { resolve(responseBody) });
    });
    
    request.on('error', (e) => {
      console.error(e);
      reject(e);
    });
    
    request.write(JSON.stringify(payload));
    request.end();
  })
}
// util - get alarm link
exports.getCloudWatchLink = (message) => {
  return `https://console.aws.amazon.com/cloudwatch/home?region=${exports.parseRegionCode(message.AlarmArn)}#alarm:alarmFilter=ANY;name=${encodeURIComponent(message.AlarmName)}`;
}
exports.parseRegionCode = (arn) => {
  return arn.replace("arn:aws:cloudwatch:", "").split(":")[0];
}
// util - current timestamp
exports.formatDate = (timestring) => {
  if (!timestring) return '';
  
  function zerofill(n) { return n < 10 ? '0' + n : n }
  
  const kstDate = new Date(new Date(timestring) + 9 * 60 * 60 * 1000);
  
  return kstDate.getFullYear().toString()
    + '-' + zerofill(kstDate.getMonth() + 1)
    + '-' + zerofill(kstDate.getDate())
    + '-' + zerofill(kstDate.getHours())
    + ':' + zerofill(kstDate.getMinutes())
    + ':' + zerofill(kstDate.getSeconds());
}

 

클라우드 와치 로그그룹

클라우드 와치 메트릭스

CPU 알람  

메모리 알람