From da347c6f8791d228c2f62244a5d1c3ad6cc1b3fe Mon Sep 17 00:00:00 2001 From: Yiyang Tsui Date: Fri, 2 Feb 2018 15:29:32 -0800 Subject: [PATCH 1/7] provide a notification:varbinds view for elasticsearch index --- trapperkeeper/callbacks.py | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/trapperkeeper/callbacks.py b/trapperkeeper/callbacks.py index 22f667c..e6cf64d 100644 --- a/trapperkeeper/callbacks.py +++ b/trapperkeeper/callbacks.py @@ -164,3 +164,54 @@ def _call(self, transport_dispatcher, transport_domain, transport_address, whole logging.debug(err) self._send_mail(handler, trap, duplicate) + + # TODO: index_to_elastic_search + self._index_to_elasticsearch(trap) + + def _index_to_elasticsearch(self, trap): + subject = { + "trap_oid": trap.oid, + "trap_name": ObjectId(trap.oid).name, + "ipaddress": trap.host, + "hostname": self.resolver.hostname_or_ip(trap.host), + } + import pprint + pp = pprint.PrettyPrinter(indent=4) + print("="*150) + pp.pprint(trap.to_dict()) + print("vb"*25) + for vb in trap.varbinds: + pp.pprint(vb.to_dict(pretty=True)) + + trap_index = dict() + trap_index.update(trap.to_dict()) + + for vb in trap.varbinds: + trap_index.update(transform_varbind(vb)) + + trap_index["notification_id"] = trap_index["id"] + del trap_index["id"] + print("-"*25) + pp.pprint(trap_index) + + #pp.pprint(trap.varbinds) + #check utils.to_mibname + #trap.varbinds[0].pretty_value.... + #trap.varbinds[1].pretty_value.... + + +def transform_varbind(varbind): + d = varbind.to_dict(pretty=True) + # {'name': 'SNMPv2-MIB::sysLocation.0', + # 'notification_id': 40, + # 'oid': u'1.3.6.1.2.1.1.6.0', + # 'pretty_value': 'TrapperKeeper-Test', + # 'value': 'TrapperKeeper-Test', + # 'value_type': u'octet'} + result = dict() + result.update({ + d['name']: d['pretty_value'], + d['oid']: d['value'], + "%s:type" % (d['name']): d['value_type'] + }) + return result \ No newline at end of file From eaf4889a394a8db04331cb7b6c838a5171565423 Mon Sep 17 00:00:00 2001 From: Yiyang Tsui Date: Fri, 2 Feb 2018 15:41:01 -0800 Subject: [PATCH 2/7] provide mib name --- trapperkeeper/callbacks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trapperkeeper/callbacks.py b/trapperkeeper/callbacks.py index e6cf64d..76f51bb 100644 --- a/trapperkeeper/callbacks.py +++ b/trapperkeeper/callbacks.py @@ -188,11 +188,13 @@ def _index_to_elasticsearch(self, trap): for vb in trap.varbinds: trap_index.update(transform_varbind(vb)) - + trap_index["notification_id"] = trap_index["id"] + trap_index["mib_name"] = ObjectId(trap.oid).name del trap_index["id"] print("-"*25) pp.pprint(trap_index) + trap.pprint() #pp.pprint(trap.varbinds) #check utils.to_mibname From 852c162b417969d84240578b06f00fe3d546f87f Mon Sep 17 00:00:00 2001 From: Yiyang Tsui Date: Fri, 2 Feb 2018 16:13:59 -0800 Subject: [PATCH 3/7] connect to Elasticsearch and index the trap --- trapperkeeper/callbacks.py | 3 ++- trapperkeeper/utils.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/trapperkeeper/callbacks.py b/trapperkeeper/callbacks.py index 76f51bb..c4c67e1 100644 --- a/trapperkeeper/callbacks.py +++ b/trapperkeeper/callbacks.py @@ -14,7 +14,7 @@ from trapperkeeper.dde import DdeNotification from trapperkeeper.constants import SNMP_VERSIONS from trapperkeeper.models import Notification -from trapperkeeper.utils import parse_time_string, send_trap_email +from trapperkeeper.utils import parse_time_string, send_trap_email, index_trap_to_elasticsearch try: @@ -195,6 +195,7 @@ def _index_to_elasticsearch(self, trap): print("-"*25) pp.pprint(trap_index) trap.pprint() + index_trap_to_elasticsearch(trap_index) #pp.pprint(trap.varbinds) #check utils.to_mibname diff --git a/trapperkeeper/utils.py b/trapperkeeper/utils.py index 0ca5569..e4681f3 100644 --- a/trapperkeeper/utils.py +++ b/trapperkeeper/utils.py @@ -128,6 +128,14 @@ def send_trap_email(recipients, sender, subject, template_env, context): smtp.quit() +from elasticsearch import Elasticsearch +ES = Elasticsearch("http://localhost:9200") +#ES.indices.create(index='my-trap', ignore=400) + +def index_trap_to_elasticsearch(trap_index): + ES.index(index="my-trap", doc_type="trap", body=trap_index) + + def get_loglevel(args): verbose = args.verbose * 10 quiet = args.quiet * 10 From 3aa66ad7ebe89fcf875f94bc7984d84b017bde68 Mon Sep 17 00:00:00 2001 From: Yiyang Tsui Date: Fri, 2 Feb 2018 17:00:53 -0800 Subject: [PATCH 4/7] code refactoring: add es_doc property to both Notification and Varbind --- trapperkeeper/callbacks.py | 52 +++----------------------------------- trapperkeeper/models.py | 29 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 49 deletions(-) diff --git a/trapperkeeper/callbacks.py b/trapperkeeper/callbacks.py index c4c67e1..5840547 100644 --- a/trapperkeeper/callbacks.py +++ b/trapperkeeper/callbacks.py @@ -169,52 +169,6 @@ def _call(self, transport_dispatcher, transport_domain, transport_address, whole self._index_to_elasticsearch(trap) def _index_to_elasticsearch(self, trap): - subject = { - "trap_oid": trap.oid, - "trap_name": ObjectId(trap.oid).name, - "ipaddress": trap.host, - "hostname": self.resolver.hostname_or_ip(trap.host), - } - import pprint - pp = pprint.PrettyPrinter(indent=4) - print("="*150) - pp.pprint(trap.to_dict()) - print("vb"*25) - for vb in trap.varbinds: - pp.pprint(vb.to_dict(pretty=True)) - - trap_index = dict() - trap_index.update(trap.to_dict()) - - for vb in trap.varbinds: - trap_index.update(transform_varbind(vb)) - - trap_index["notification_id"] = trap_index["id"] - trap_index["mib_name"] = ObjectId(trap.oid).name - del trap_index["id"] - print("-"*25) - pp.pprint(trap_index) - trap.pprint() - index_trap_to_elasticsearch(trap_index) - - #pp.pprint(trap.varbinds) - #check utils.to_mibname - #trap.varbinds[0].pretty_value.... - #trap.varbinds[1].pretty_value.... - - -def transform_varbind(varbind): - d = varbind.to_dict(pretty=True) - # {'name': 'SNMPv2-MIB::sysLocation.0', - # 'notification_id': 40, - # 'oid': u'1.3.6.1.2.1.1.6.0', - # 'pretty_value': 'TrapperKeeper-Test', - # 'value': 'TrapperKeeper-Test', - # 'value_type': u'octet'} - result = dict() - result.update({ - d['name']: d['pretty_value'], - d['oid']: d['value'], - "%s:type" % (d['name']): d['value_type'] - }) - return result \ No newline at end of file + index_trap_to_elasticsearch(trap.es_doc) + + diff --git a/trapperkeeper/models.py b/trapperkeeper/models.py index 339518f..e495bf9 100644 --- a/trapperkeeper/models.py +++ b/trapperkeeper/models.py @@ -76,6 +76,24 @@ def pprint(self): for varbind in self.varbinds: varbind.pprint() + # for ElasticSearch index + @property + def es_doc(self): + doc = { + "notification_id": self.id, + "host": self.host, + "oid": self.oid, + "mib_name": ObjectId(self.oid).name, + "severity": self.severity, + "@timestamp": utcnow() + } + + for varbind in self.varbinds: + doc.update(varbind.es_doc) + + return doc + + @staticmethod def _from_pdu_v1(host, proto_module, version, pdu): trapoid = str(proto_module.apiTrapPDU.getEnterprise(pdu)) @@ -177,6 +195,17 @@ def to_dict(self, pretty=False): def pprint(self): print "\t", self.oid, "(%s)" % self.value_type, "=", self.value + @property + def es_doc(self): + d = self.to_dict(pretty=True) + doc = dict() + doc.update({ + d['name']: d['pretty_value'], + d['oid']: d['value'], + "%s:type" % (d['name']): d['value_type'] + }) + return doc + def __repr__(self): return "Varbind(oid=%s, value_type=%s, value=%s)" % ( repr(self.oid), repr(self.value_type), repr(self.value) From c110dd57ccde9960ae88d006019b505d1989f709 Mon Sep 17 00:00:00 2001 From: Yiyang Tsui Date: Fri, 2 Feb 2018 17:11:15 -0800 Subject: [PATCH 5/7] try to loggin indexing --- trapperkeeper/callbacks.py | 3 ++- trapperkeeper/utils.py | 9 ++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/trapperkeeper/callbacks.py b/trapperkeeper/callbacks.py index 5840547..f2fb5b8 100644 --- a/trapperkeeper/callbacks.py +++ b/trapperkeeper/callbacks.py @@ -169,6 +169,7 @@ def _call(self, transport_dispatcher, transport_domain, transport_address, whole self._index_to_elasticsearch(trap) def _index_to_elasticsearch(self, trap): - index_trap_to_elasticsearch(trap.es_doc) + info = index_trap_to_elasticsearch(trap.es_doc) + logging.info("ES index created %s" % info) diff --git a/trapperkeeper/utils.py b/trapperkeeper/utils.py index e4681f3..c1b6675 100644 --- a/trapperkeeper/utils.py +++ b/trapperkeeper/utils.py @@ -12,6 +12,9 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +from elasticsearch import Elasticsearch +ES = Elasticsearch("http://localhost:9200") + _TIME_STRING_RE = re.compile( r"(?:(?P\d+)d)?" @@ -128,12 +131,8 @@ def send_trap_email(recipients, sender, subject, template_env, context): smtp.quit() -from elasticsearch import Elasticsearch -ES = Elasticsearch("http://localhost:9200") -#ES.indices.create(index='my-trap', ignore=400) - def index_trap_to_elasticsearch(trap_index): - ES.index(index="my-trap", doc_type="trap", body=trap_index) + return ES.index(index="trap", doc_type="trap", body=trap_index) def get_loglevel(args): From 93ba30566441c9be77e63aad59260e807513bd1a Mon Sep 17 00:00:00 2001 From: Yiyang Tsui Date: Fri, 2 Feb 2018 17:49:19 -0800 Subject: [PATCH 6/7] append Python Elasticsearch Client in requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index d499269..62c4643 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ pysnmp==4.2.5 pytz==2014.1 tornado==3.2 wsgiref==0.1.2 +elasticsearch==6.1.1 From b54d96a673f2bccf3da01566f62338311ca2d41f Mon Sep 17 00:00:00 2001 From: Yiyang Tsui Date: Thu, 8 Feb 2018 13:09:40 -0800 Subject: [PATCH 7/7] make elasticsearch configurable and some code refactroing --- bin/trapperkeeper | 9 +++++++-- conf/trapperkeeper.yaml | 12 ++++++++++++ trapperkeeper/callbacks.py | 13 ++++++------- trapperkeeper/es_client.py | 9 +++++++++ trapperkeeper/utils.py | 6 +----- 5 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 trapperkeeper/es_client.py diff --git a/bin/trapperkeeper b/bin/trapperkeeper index e5c81d1..68d2082 100755 --- a/bin/trapperkeeper +++ b/bin/trapperkeeper @@ -15,7 +15,7 @@ from trapperkeeper.config import Config from trapperkeeper.models import get_db_engine, Session from trapperkeeper.utils import get_template_env, get_loglevel, CachingResolver from trapperkeeper import __version__ - +from trapperkeeper.es_client import get_elasticsearch_client def stats_server(port): class Stats(tornado.web.RequestHandler): @@ -49,6 +49,11 @@ def main(): config = Config.from_file(args.config) db_engine = get_db_engine(config["database"]) + es_host = config.get("elasticsearch_host") + if es_host: + es_client = get_elasticsearch_client(es_host) + else: + es_client = None community = config["community"] if not community: community = None @@ -62,7 +67,7 @@ def main(): template_env = get_template_env( hostname_or_ip=resolver.hostname_or_ip ) - cb = TrapperCallback(conn, template_env, config, resolver, community) + cb = TrapperCallback(conn, template_env, config, resolver, community, es_client) logging.basicConfig( level=get_loglevel(args), diff --git a/conf/trapperkeeper.yaml b/conf/trapperkeeper.yaml index 0b6012d..c38d4ee 100644 --- a/conf/trapperkeeper.yaml +++ b/conf/trapperkeeper.yaml @@ -121,3 +121,15 @@ config: # # Type: bool ipv6: false + + # Elastic Search Host + # + # Type: string + elasticsearch_host: "http://localhost:9200" + + # Elastic Search Index + # Since starting from ES ver 6.0, there is no type, + # index name will be used as type name for ES ver < 6.0 + # + # Type: string + elasticsearch_index: "trap" diff --git a/trapperkeeper/callbacks.py b/trapperkeeper/callbacks.py index f2fb5b8..1467605 100644 --- a/trapperkeeper/callbacks.py +++ b/trapperkeeper/callbacks.py @@ -14,7 +14,8 @@ from trapperkeeper.dde import DdeNotification from trapperkeeper.constants import SNMP_VERSIONS from trapperkeeper.models import Notification -from trapperkeeper.utils import parse_time_string, send_trap_email, index_trap_to_elasticsearch +from trapperkeeper.utils import parse_time_string, send_trap_email +from trapperkeeper.es_client import index_trap_to_elasticsearch try: @@ -26,13 +27,14 @@ def dde_run(notification): class TrapperCallback(object): - def __init__(self, conn, template_env, config, resolver, community): + def __init__(self, conn, template_env, config, resolver, community, es_client): self.conn = conn self.template_env = template_env self.config = config self.hostname = socket.gethostname() self.resolver = resolver self.community = community + self.es_client = es_client def __call__(self, *args, **kwargs): try: @@ -165,11 +167,8 @@ def _call(self, transport_dispatcher, transport_domain, transport_address, whole self._send_mail(handler, trap, duplicate) - # TODO: index_to_elastic_search - self._index_to_elasticsearch(trap) + if self.es_client is not None: + index_trap_to_elasticsearch(self.es_client, self.config.get("elasticsearch_index"), trap.es_doc) - def _index_to_elasticsearch(self, trap): - info = index_trap_to_elasticsearch(trap.es_doc) - logging.info("ES index created %s" % info) diff --git a/trapperkeeper/es_client.py b/trapperkeeper/es_client.py new file mode 100644 index 0000000..5f91f69 --- /dev/null +++ b/trapperkeeper/es_client.py @@ -0,0 +1,9 @@ +from elasticsearch import Elasticsearch + + +def get_elasticsearch_client(host): + return Elasticsearch(host) + + +def index_trap_to_elasticsearch(es, index_name, doc): + return es.index(index=index_name, doc_type=index_name, body=doc) \ No newline at end of file diff --git a/trapperkeeper/utils.py b/trapperkeeper/utils.py index c1b6675..ff68926 100644 --- a/trapperkeeper/utils.py +++ b/trapperkeeper/utils.py @@ -12,8 +12,7 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from elasticsearch import Elasticsearch -ES = Elasticsearch("http://localhost:9200") + _TIME_STRING_RE = re.compile( @@ -131,9 +130,6 @@ def send_trap_email(recipients, sender, subject, template_env, context): smtp.quit() -def index_trap_to_elasticsearch(trap_index): - return ES.index(index="trap", doc_type="trap", body=trap_index) - def get_loglevel(args): verbose = args.verbose * 10