飞书告警通知 #53
Description
升级python
1.安装编译环境包（防止出现安装错误）
yum install gcc-c++ gcc make cmake zlib-devel bzip2-devel openssl-devel ncurse-devel libffi-devel -y
2.在线下载Python3.7源码包
#进入tmp目录
cd /tmp
#下载python3.7.3
wget https://www.python.org/ftp/python/3.7.3/Python-3.7.3.tar.xz
3.解压并配置
#解压
tar Jxvf Python-3.7.3.tar.xz
#进入python3.7.3目录
cd Python-3.7.3
#创建目录
mkdir -p /usr/local/python3
#配置（指定安装目录）
./configure --prefix=/usr/local/python3 --enable-optimizations
4.编译及安装
make && make install
5.更换系统默认Python版本
1).备份原系统旧版本python
mv /usr/bin/python /usr/bin/python.bak
mkdir /usr/bin/pip
mv /usr/bin/pip /usr/bin/pip.bak
2).配置环境变量：创建新版本Python和pip的软链接
ln -s /usr/local/python3/bin/python3.7 /usr/bin/python
ln -s /usr/local/python3/bin/pip3 /usr/bin/pip
3).查看python版本
python -V
6.修改yum功能
因为yum的功能依赖Pyhon2，现在更改默认Python版本后会导致yum无法正常工作，所以进行以下3处修复
第1处：
vim /usr/bin/yum
改成：#！ /usr/bin/python2.7
vim /usr/libexec/urlgrabber-ext-down
把最顶部的
改成：#！ /usr/bin/python2.7
vim /usr/sbin/firewalld
vim /usr/bin/firewall-cmd
修改N9E server.yaml文件
vim /opt/n9e/server/etc/server.yml
contactKeys:
- label: "Feishu Robot Token" ##新增加
key: feishu_robot_token ##新增加
notifyChannels:
- feishu ##新增加
修改N9E告警脚本
vim /opt/n9e/server/etc/script/notify.py
替换为以下内容
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import sys
import json
import os
import smtplib
import time
import requests
from email.mime.text import MIMEText
from email.header import Header
# 希望的demo实现效果：
# 1. 从stdin拿到告警信息之后，格式化为一个有缩进的json写入一个临时文件
# 2. 文件路径和名字是.alerts/${timestamp}_${ruleid}
# 3. 调用SMTP服务器发送告警，微信、钉钉、飞书、slack、jira、短信、电话等等留给社区实现
# 脚本二开指南
# 1. 可以根据下面的TEST_ALERT_JSON 中的结构修改脚本发送逻辑，定制化告警格式格式如下
"""
[告警类型：prometheus]
[规则名称：a]
[是否已恢复：已触发]
[告警级别：1]
[触发时间：2021-07-02 16:05:14]
[可读表达式：go_goroutines>0]
[当前值：[vector={__name__="go_goroutines", instance="localhost:9090", job="prometheus"}]: [value=33.000000]]
[标签组：instance=localhost:9090 job=prometheus]
"""
# 2. 每个告警会以json文件的格式存储在LOCAL_EVENT_FILE_DIR 下面，文件名为 filename = '%d_%d_%d' % (rule_id, event_id, trigger_time)
# 3. 告警通道需要自行定义Send类中的send_xxx同名方法，反射调用：举例 event.notify_channels = [qq dingding] 则需要Send类中 有 send_qq send_dingding方法
# 4. im发群信息，比如钉钉发群信息需要群的webhook机器人 token，这个信息可以在user的contacts map中，各个send_方法处理即可
# 5. 用户创建一个虚拟的用户保存上述im群 的机器人token信息 user的contacts map中
mail_host = "smtp.qq.com"
mail_port = 994
mail_user = "ulricqin"
mail_pass = "password"
mail_from = "751958224@qq.com"
# 本地告警event json存储目录
LOCAL_EVENT_FILE_DIR = ".alerts"
NOTIFY_CHANNELS_SPLIT_STR = " "
# 群机器人token 配置字段
FEISHU_ROBOT_TOKEN_NAME = "feishu_robot_token"
FEISHU_API = "修改为机器人webhook 地址"
# stdin 告警json实例
TEST_ALERT_JSON = {
"event": {
"alert_duration": 10,
"notify_channels": "feishu",
"res_classpaths": "",
"id": 4,
"notify_group_objs": None,
"rule_note": "",
"history_points": [
{
"metric": "go_goroutines",
"points": [
{
"t": 1625213114,
"v": 33.0
}
],
"tags": {
"instance": "localhost:9090",
"job": "prometheus"
}
}
],
"priority": 1,
"last_sent": True,
"tag_map": {
"instance": "localhost:9090",
"job": "prometheus"
},
"hash_id": "ecb258d2ca03454ee390a352913c461b",
"status": 0,
"tags": "instance=localhost:9090 job=prometheus",
"trigger_time": 1625213114,
"res_ident": "",
"rule_name": "a",
"is_prome_pull": 1,
"notify_users": "1",
"notify_groups": "",
"runbook_url": "",
"values": "[vector={__name__=\"go_goroutines\", instance=\"localhost:9090\", job=\"prometheus\"}]: [value=33.000000]",
"readable_expression": "go_goroutines>0",
"notify_user_objs": None,
"is_recovery": 0,
"rule_id": 1
},
"rule": {
"alert_duration": 10,
"notify_channels": "feishu",
"enable_stime": "00:00",
"id": 1,
"note": "",
"create_by": "root",
"append_tags": "",
"priority": 1,
"update_by": "root",
"type": 1,
"status": 0,
"recovery_notify": 0,
"enable_days_of_week": "1 2 3 4 5 6 7",
"callbacks": "localhost:10000",
"notify_users": "1",
"notify_groups": "",
"runbook_url": "",
"name": "a",
"update_at": 1625211576,
"create_at": 1625211576,
"enable_etime": "23:59",
"group_id": 1,
"expression": {
"evaluation_interval": 4,
"promql": "go_goroutines>0"
}
},
"users": [
{
"username": "root",
"status": 0,
"contacts": {
"feishu_robot_token": "xxxxxxx"
},
"create_by": "system",
"update_at": 1625211432,
"create_at": 1624871926,
"email": "",
"phone": "",
"role": "Admin",
"update_by": "root",
"portrait": "",
"nickname": "\u8d85\u7ba1",
"id": 1
}
]
}
def main():
payload = json.load(sys.stdin)
trigger_time = payload['event']['trigger_time']
event_id = payload['event']['id']
rule_id = payload['rule']['id']
notify_channels = payload['event'].get('notify_channels').strip().split(NOTIFY_CHANNELS_SPLIT_STR)
if len(notify_channels) == 0:
msg = "notify_channels_empty"
print(msg)
return
# 持久化到本地json文件
persist(payload, rule_id, event_id, trigger_time)
# 生成告警内容
alert_content = content_gen(payload)
for ch in notify_channels:
send_func_name = "send_{}".format(ch.strip())
has_func = hasattr(Send, send_func_name)
if not has_func:
msg = "[send_func_name_err][func_not_found_in_Send_class:{}]".format(send_func_name)
print(msg)
continue
send_func = getattr(Send, send_func_name)
send_func(alert_content, payload)
def content_gen(payload):
# 生成格式化告警内容
text = ""
event_obj = payload.get("event")
rule_type = event_obj.get("is_prome_pull")
type_str_m = {1: "prometheus", 0: "n9e"}
rule_type = type_str_m.get(rule_type)
text += "[告警类型：{}]\n".format(rule_type)
rule_name = event_obj.get("rule_name")
text += "[规则名称：{}]\n".format(rule_name)
is_recovery = event_obj.get("is_recovery")
is_recovery_str_m = {1: "已恢复", 0: "已触发"}
is_recovery = is_recovery_str_m.get(is_recovery)
text += "[是否已恢复：{}]\n".format(is_recovery)
priority = event_obj.get("priority")
text += "[告警级别：{}]\n".format(priority)
trigger_time = event_obj.get("trigger_time")
text += "[触发时间：{}]\n".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(trigger_time))))
readable_expression = event_obj.get("readable_expression")
text += "[可读表达式：{}]\n".format(readable_expression)
values = event_obj.get("values")
text += "[当前值：{}]\n".format(values)
tags = event_obj.get("tags")
text += "[标签组：{}]\n".format(tags)
print(text)
return text
def persist(payload, rule_id, event_id, trigger_time):
if not os.path.exists(LOCAL_EVENT_FILE_DIR):
os.makedirs(LOCAL_EVENT_FILE_DIR)
filename = '%d_%d_%d' % (rule_id, event_id, trigger_time)
filepath = os.path.join(LOCAL_EVENT_FILE_DIR, filename)
with open(filepath, 'w') as f:
f.write(json.dumps(payload, indent=4))
class Send(object):
@classmethod
def send_feishu(cls, alert_content, payload):
# 飞书发群信息需要群的webhook机器人 token，这个信息可以在user的contacts map中
users = payload.get("users")
for u in users:
contacts = u.get("contacts")
feishu_robot_token = contacts.get(FEISHU_ROBOT_TOKEN_NAME, "")
if feishu_robot_token == "":
print("feishu_robot_token_not_found")
continue
feishu_api_url = "修改为机器人webhook地址"
atMobiles = [u.get("phone")]
headers = {'Content-Type': 'application/json;charset=utf-8'}
pay_load = {
"msg_type": "text",
"content": {
"text": alert_content
},
"at": {
"atMobiles": atMobiles,
"isAtAll": False
}
}
res = requests.post(feishu_api_url, json.dumps(pay_load), headers=headers)
print(res.status_code)
print(res.text)
print("send_feishu")
def mail_test():
print("mail_test_todo")
recipients = ["ulricqin@qq.com", "ulric@163.com"]
message = MIMEText(mail_body, 'html', 'utf-8')
message['From'] = mail_from
message['To'] = ", ".join(recipients)
message["Subject"] = "n9e alert"
smtp = smtplib.SMTP_SSL(mail_host, mail_port)
smtp.login(mail_user, mail_pass)
smtp.sendmail(mail_from, recipients, message.as_string())
smtp.close()
print("mail_test_done")
if __name__ == "__main__":
if len(sys.argv) == 1:
main()
elif sys.argv[1] == "mail":
mail_test()
else:
print("I am confused")