From 03795bc67f1ddfdba33f3c396031cfae6e48a40c Mon Sep 17 00:00:00 2001 From: Mayuresh Gaitonde Date: Mon, 18 May 2020 16:47:48 -0700 Subject: [PATCH 1/2] Add a check to cancel out alert/recover in same pass --- modules/example_rules.yaml | 23 +++++++++++++++++++++++ modules/syslog.py | 26 ++++++++++++++++++-------- 2 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 modules/example_rules.yaml diff --git a/modules/example_rules.yaml b/modules/example_rules.yaml new file mode 100644 index 000000000..ef31950b4 --- /dev/null +++ b/modules/example_rules.yaml @@ -0,0 +1,23 @@ +#- id: 1 +# name: Neteng Bgp NLRI Mismatch +# regex: 'BGP_NLRI_MISMATCH: bgp_process_caps: mismatch NLRI with (?P.*?): .*' +# severity: info +# +#- id: 2 +# name: Neteng Bgp Prefix Threshold Exceeded +# regex: 'BGP_PREFIX_THRESH_EXCEEDED: (?P.*?): .*' +# +- id: 3 + name: Neteng Chassis Alert + regex: 'Alarm set: (?P\w+) color=RED.*' + status: alerting + +- id: 4 + name: Neteng External BGP Down + regex: '^RPD_BGP_NEIGHBOR_STATE_CHANGED: BGP peer (?P.*?) changed state from Established to .*' + status: alerting + +- id: 5 + name: Neteng External BGP Down + regex: '^RPD_BGP_NEIGHBOR_STATE_CHANGED: BGP peer (?P.*?) changed state from .* to Established.*' + status: recover diff --git a/modules/syslog.py b/modules/syslog.py index 5110adf50..33ef6d1a5 100644 --- a/modules/syslog.py +++ b/modules/syslog.py @@ -9,6 +9,10 @@ class SyslogCheckerRule(RuleType): required_options = set(['regex_file']) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.regex_rules = self.load_regex_rules() + def load_regex_rules(self): rules = None try: @@ -23,15 +27,11 @@ def to_localtz(self, ts): return ts.astimezone(tz.tzlocal()) def add_data(self, data): - regex_rules = self.load_regex_rules() - if not regex_rules: - return # check if any of the datapoints match any of the defined regexs - matched = [] for point in data: if 'syslog_message' not in point or 'syslog_severity' not in point or 'syslog_hostname' not in point: continue - for rule in regex_rules: + for rule in self.regex_rules: r = re.compile(rule['regex']) m = r.match(point['syslog_message']) if not m: @@ -53,6 +53,16 @@ def add_data(self, data): point['syslog_hostname'], point['entity'] ) - if match not in matched: - self.add_match(point) - matched.append(match) + if point['status'] == 'recover': + # dont alert on points that alert and clear in the same cycle + cleared = False + for pt in self.matches: + m = '{}:{}:{}'.format(pt['name'], pt['syslog_hostname'], pt['entity']) + if m == match and pt['status'] == 'alerting': + elastalert_logger.info('{} has recovered, removing'.format(m)) + self.matches.remove(pt) + cleared = True + if cleared: + continue + elastalert_logger.info('{} on {}'.format(point['status'], match)) + self.add_match(point) From 2e0614a8cda300ad674b3050790a17f3654f30b7 Mon Sep 17 00:00:00 2001 From: Mayuresh Gaitonde Date: Mon, 18 May 2020 16:50:15 -0700 Subject: [PATCH 2/2] remove --- modules/example_rules.yaml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 modules/example_rules.yaml diff --git a/modules/example_rules.yaml b/modules/example_rules.yaml deleted file mode 100644 index ef31950b4..000000000 --- a/modules/example_rules.yaml +++ /dev/null @@ -1,23 +0,0 @@ -#- id: 1 -# name: Neteng Bgp NLRI Mismatch -# regex: 'BGP_NLRI_MISMATCH: bgp_process_caps: mismatch NLRI with (?P.*?): .*' -# severity: info -# -#- id: 2 -# name: Neteng Bgp Prefix Threshold Exceeded -# regex: 'BGP_PREFIX_THRESH_EXCEEDED: (?P.*?): .*' -# -- id: 3 - name: Neteng Chassis Alert - regex: 'Alarm set: (?P\w+) color=RED.*' - status: alerting - -- id: 4 - name: Neteng External BGP Down - regex: '^RPD_BGP_NEIGHBOR_STATE_CHANGED: BGP peer (?P.*?) changed state from Established to .*' - status: alerting - -- id: 5 - name: Neteng External BGP Down - regex: '^RPD_BGP_NEIGHBOR_STATE_CHANGED: BGP peer (?P.*?) changed state from .* to Established.*' - status: recover