Skip to content

[WIP] Internal Performance monitoring #121

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dgarros wants to merge 9 commits into
base: master
Choose a base branch
from
Open

[WIP] Internal Performance monitoring #121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5d126ec
Add support for Alias for Graphs
Jan 7, 2017
6a2c399
Add telegraf to input-jti container for internal monitoring
Jan 7, 2017
9517f83
Update data streaming collector dashboard with Alias
Jan 7, 2017
31c7fc0
Update telegraf version and configuration
Jan 7, 2017
788e12e
Create Dashboard for internal performance monitoring
Jan 7, 2017
a534181
Add telegraf conf for input-jti
Jan 7, 2017
8602f89
Add telegraf in input-syslog container for perf-monitoring
Jan 10, 2017
76d9a90
Enable network monitoring for eth0 only
Jan 10, 2017
1dd281e
Change DB name for internal perf monitoring and refactor dashboards
Jan 11, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ RUN /usr/sbin/enable_insecure_key
# Latest version
ENV GRAFANA_VERSION 3.1.1-1470047149
ENV INFLUXDB_VERSION 1.0.2
ENV TELEGRAF_VERSION 1.0.1
ENV TELEGRAF_VERSION 1.1.2

RUN apt-get -y update && \
apt-get -y install \
Expand Down Expand Up @@ -90,7 +90,7 @@ RUN curl -s -o /tmp/telegraf_latest_amd64.deb https://dl.influxdata.com/tele
dpkg -i /tmp/telegraf_latest_amd64.deb && \
rm /tmp/telegraf_latest_amd64.deb

ADD docker/telegraf/telegraf.conf /etc/telegraf/telegraf.conf
ADD docker/telegraf/telegraf.toml /etc/telegraf/telegraf.conf

RUN mkdir /etc/service/telegraf
ADD docker/telegraf/telegraf.launcher.sh /etc/service/telegraf/run
Expand Down
31 changes: 31 additions & 0 deletions dashboards/data_steaming_collector_dashboard.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT mean(\"value\") FROM \"jnpr.analyticsd\" WHERE (\"type\" = 'traffic-stats.rxpps' OR \"type\" = 'traffic-stats.txpps') AND \"device\" =~ /$host_regex/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", time($GroupBy)",
Expand All @@ -106,6 +107,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s) FROM \"jnpr.jvision\" WHERE (\"type\" = 'egress_stats.if_packets' OR \"type\" = 'ingress_stats.if_packets') AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", time($GroupBy)",
Expand All @@ -116,6 +118,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s) FROM \"jnpr.jvision\" WHERE (\"type\" = 'egress_stats.if_pkts' OR \"type\" = 'ingress_stats.if_pkts') AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", time($GroupBy)",
Expand All @@ -126,6 +129,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"counters/out-packets\"), 1s) AS \"OUT\", derivative(mean(\"counters/in-packets\"), 1s) AS \"IN\" FROM \"openconfig-ifd\" WHERE \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY interface, device, time($GroupBy)",
Expand All @@ -136,6 +140,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s) FROM \"jnpr.jvision\" WHERE (\"type\" = 'egress_stats.if_fc_stats.if_packets' OR \"type\" = 'ingress_stats.if_fc_stats.if_packets') AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", \"family\", \"forwarding_class\", time($GroupBy)",
Expand Down Expand Up @@ -237,6 +242,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT mean(\"value\") FROM \"jnpr.analyticsd\" WHERE \"type\" = 'traffic-stats.txbps' AND \"device\" =~ /$host_regex/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", time($GroupBy)",
Expand All @@ -247,6 +253,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s)*8 FROM \"jnpr.jvision\" WHERE (\"type\" = 'egress_stats.if_octets' OR \"type\" = 'ingress_stats.if_octets') AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", time($GroupBy)",
Expand All @@ -257,6 +264,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT mean(\"value\") FROM \"jnpr.analyticsd\" WHERE \"type\" = 'traffic-stats.rxbps' AND \"device\" =~ /$host_regex/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", time($GroupBy)",
Expand All @@ -267,6 +275,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"counters/out-octets\"), 1s)*8 AS \"OUT\", derivative(mean(\"counters/in-octets\"), 1s)*8 AS \"IN\" FROM \"openconfig-ifd\" WHERE \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY interface, device, time($GroupBy)",
Expand All @@ -277,6 +286,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s)*8 FROM \"jnpr.jvision\" WHERE (\"type\" = 'egress_stats.if_fc_stats.if_octets' OR \"type\" = 'ingress_stats.if_fc_stats.if_octets') AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", \"family\", \"forwarding_class\", time($GroupBy)",
Expand Down Expand Up @@ -363,6 +373,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s) FROM \"jnpr.jvision\" WHERE \"type\" = 'egress_queue_info.packets' AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"egress_queue\", time($GroupBy)",
Expand All @@ -373,6 +384,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"counters/out-discards\"), 1s) AS \"Discard out\" FROM \"openconfig-ifd\" WHERE \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY interface, device, time($GroupBy)",
Expand Down Expand Up @@ -474,6 +486,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(\"value\", 1s) FROM \"jnpr.analyticsd\" WHERE ( \"type\" = 'traffic-stats.txmcpkt' OR \"type\" = 'traffic-stats.txucpkt' OR \"type\" = 'traffic-stats.txmcpkt' ) AND \"device\" =~ /$host_regex/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\"",
Expand All @@ -484,6 +497,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s) FROM \"jnpr.jvision\" WHERE (\"type\" = 'egress_stats.if_uc_pkts' OR \"type\" = 'egress_stats.if_bc_pkts' OR \"type\" = 'ingress_stats.if_mc_pkts') AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", time($GroupBy)",
Expand All @@ -494,6 +508,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(\"value\", 1s) FROM \"jnpr.analyticsd\" WHERE ( \"type\" = 'traffic-stats.rxmcpkt' OR \"type\" = 'traffic-stats.rxbcpkt' OR \"type\" = 'traffic-stats.rxucpkt' ) AND \"device\" =~ /$host_regex/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\"",
Expand All @@ -504,6 +519,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"counters/out-unicast-pkts\"), 1s) AS \"Ucast out\", derivative(mean(\"counters/in-unicast-pkts\"), 1s) AS \"Ucast in\" FROM \"openconfig-ifd\" WHERE \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY interface, device, time($GroupBy)",
Expand All @@ -514,6 +530,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"counters/out-broadcast-pkts\"), 1s) AS \"Bcast out\", derivative(mean(\"counters/in-broadcast-pkts\"), 1s) AS \"Bcast in\" FROM \"openconfig-ifd\" WHERE \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY interface, device, time($GroupBy)",
Expand All @@ -524,6 +541,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"counters/out-multicast-pkts\"), 1s) AS \"Mcast out\", derivative(mean(\"counters/in-multicast-pkts\"), 1s) AS \"Mcast in\" FROM \"openconfig-ifd\" WHERE \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY interface, device, time($GroupBy)",
Expand Down Expand Up @@ -623,6 +641,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s) FROM \"jnpr.analyticsd\" WHERE (\"type\" = 'traffic-stats.rxcrcerr' OR \"type\" = 'traffic-stats.rxdroppkt') AND \"device\" =~ /$host_regex/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", time($GroupBy)",
Expand All @@ -633,6 +652,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s) FROM \"jnpr.jvision\" WHERE (\"type\" = 'ingress_errors.if_in_discards' OR \"type\" = 'ingress_errors.if_in_errors' OR \"type\" = 'ingress_errors.if_in_frame_errors' OR \"type\" = 'ingress_errors.if_in_l2_mismatch_timeouts' OR \"type\" = 'ingress_errors.if_in_l2chan_errors' OR \"type\" = 'ingress_errors.if_in_l3_incompletes' OR \"type\" = 'ingress_errors.if_in_qdrops' OR \"type\" = 'ingress_errors.if_in_resource_errors' OR \"type\" = 'ingress_errors.if_in_runts') AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"type\", time($GroupBy)",
Expand Down Expand Up @@ -734,6 +754,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT \"value\" FROM \"jnpr.analyticsd\" WHERE \"type\" = 'queue-stats.latency' AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"interface\", \"device\"",
Expand Down Expand Up @@ -822,6 +843,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT mean(\"value\") FROM \"jnpr.analyticsd\" WHERE \"type\" = 'queue-stats.queue-depth' AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY time($interval), \"device\", \"interface\"",
Expand All @@ -832,6 +854,7 @@
}
,{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT \"value\" FROM \"jnpr.jvision\" WHERE \"type\" = 'egress_queue_info.cur_buffer_occupancy' AND \"device\" =~ /$host_regex$/ AND \"interface\" =~ /$interface$/ AND $timeFilter GROUP BY \"device\", \"interface\"",
Expand Down Expand Up @@ -933,6 +956,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT \"value\"*8 FROM \"jnpr.jvision\" WHERE \"type\" = 'lspstata.byte_rate' AND \"device\" =~ /$host_regex$/ AND $timeFilter GROUP BY \"device\", \"lspname\", \"type\"",
Expand Down Expand Up @@ -1019,6 +1043,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT \"value\" FROM \"jnpr.jvision\" WHERE \"type\" = 'lspstata.packet_rate' AND \"device\" =~ /$host_regex$/ AND $timeFilter GROUP BY \"device\", \"lspname\"",
Expand Down Expand Up @@ -1118,6 +1143,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s) FROM \"jnpr.jvision\" WHERE \"type\" = 'filter_counter.bytes' AND \"device\" =~ /$host_regex$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"filter_counter_name\", time($GroupBy)",
Expand Down Expand Up @@ -1206,6 +1232,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT derivative(mean(\"value\"), 1s) FROM \"jnpr.jvision\" WHERE \"type\" = 'filter_counter.packets' AND \"device\" =~ /$host_regex$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"filter_counter_name\", time($GroupBy)",
Expand Down Expand Up @@ -1294,6 +1321,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT mean(\"value\") FROM \"jnpr.jvision\" WHERE \"type\" = 'memory_usage.HEAP' AND \"device\" =~ /$host_regex$/ AND $timeFilter GROUP BY \"device\", \"interface\", \"filter_name\", time($GroupBy)",
Expand Down Expand Up @@ -1395,6 +1423,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT mean(\"value\") FROM \"jnpr.jvision\" WHERE \"type\" = 'cpu_mem.bytes_allocated' AND \"device\" =~ /$host_regex$/ AND $timeFilter GROUP BY \"device\", \"interface\", time($GroupBy)",
Expand Down Expand Up @@ -1483,6 +1512,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT mean(\"value\") FROM \"jnpr.jvision\" WHERE \"type\" = 'cpu_mem.size' AND \"device\" =~ /$host_regex$/ AND $timeFilter GROUP BY \"device\", \"interface\", time($GroupBy)",
Expand Down Expand Up @@ -1571,6 +1601,7 @@
"targets": [
{
"column": "value",
"alias": "",
"dsType": "influxdb",
"interval": "",
"query": "SELECT mean(\"value\") FROM \"jnpr.jvision\" WHERE \"type\" = 'cpu_mem.utilization' AND \"device\" =~ /$host_regex$/ AND $timeFilter GROUP BY \"device\", \"interface\", time($GroupBy)",
Expand Down
Loading