From 598d9a3233e5f62df6ae28ebcd2c9d06c021d59b Mon Sep 17 00:00:00 2001 From: Miguel Nistal Date: Mon, 4 May 2020 11:46:26 -0400 Subject: [PATCH 01/11] Cleaned up app factory a bit, removed unecesssary comments, conformed the config variable to the common practices with SQLALCHEMY --- taskobra/web/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/taskobra/web/__init__.py b/taskobra/web/__init__.py index 33e3d51..c8154cf 100644 --- a/taskobra/web/__init__.py +++ b/taskobra/web/__init__.py @@ -20,10 +20,8 @@ def create_app(): static_folder='static') # Root Path for url_for('static') # Load config from ENV - app.config['DATABASE_URI'] = os.environ.get('DATABASE_URI', 'sqlite:///taskobra.sqlite.db') - # TODO: OAuth Key - # TODO: ??? - print(f" * Using Database URI {app.config['DATABASE_URI']}") + app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URI', 'sqlite:///taskobra.sqlite.db') + # Bind Route Blueprints Packages to the base App app.register_blueprint(api.blueprint) app.register_blueprint(ui.blueprint) From 8c953e947fd397372150d830f6df0d1b32110f4a Mon Sep 17 00:00:00 2001 From: Miguel Nistal Date: Mon, 4 May 2020 12:56:00 -0400 Subject: [PATCH 02/11] Updated javsacript and routes to handle host names, need more info about queries to make more progress --- taskobra/web/static/js/taskobra.js | 55 +++++++++++++++++++++--------- taskobra/web/views/api.py | 23 ++++++------- 2 files changed, 49 insertions(+), 29 deletions(-) diff --git a/taskobra/web/static/js/taskobra.js b/taskobra/web/static/js/taskobra.js index 98ba281..4fd48ec 100644 --- a/taskobra/web/static/js/taskobra.js +++ b/taskobra/web/static/js/taskobra.js @@ -23,6 +23,9 @@ function render_systems(system_list) { // Add it to the content section document.querySelector("#taskobra-hostlist-entries").appendChild(instance); }); + + // Always ensure at least one hostname is checked + $('input:checkbox:first').each(function () { this.checked = true }) } /* @@ -32,24 +35,44 @@ function render_systems(system_list) { */ function render_charts() { document.querySelectorAll(".taskobra-chart").forEach(chart => { - if ($( chart ).parent('.active').length == 0) { return } + // Query the UI for information about what the user wants rendered var metric_type = chart.getAttribute('data-metric-type') - $.ajax({url: "/api/metrics/" + metric_type, chart: chart, success: function(chart_data) { - var labels = [ [ {label: 'Time', id: 'time'}, {label: 'Utilization', id: 'utilization', type: 'number'} ] ]; - var data = google.visualization.arrayToDataTable( - labels.concat(chart_data) - ); + var selected_hostnames = $("tr input:checked").map(function () { return this.value }).get() + + // Ensure the chart is visible and a set of data sets are selected before rendering + if ($( chart ).parent('.active').length == 0) { return } + if (selected_hostnames.length == 0) { return } + + // Asynchronously fetch data and draw the chart + $.ajax({ + url: "/api/metrics/" + metric_type, + data: {'hostnames': selected_hostnames.join(',')}, + chart: chart, hostnames: selected_hostnames, + success: function(chart_data) { + // Generate the labels for the legend based on the selected hosts + var labels = [ {label: 'Time', id: 'time'} ]; + this.hostnames.forEach(function (hostname) { + var hostname_id = hostname.toLowerCase().split(' ').join('') + labels.push({label: hostname, id: hostname_id, type: 'number'}) + }) - var options = { - curveType: 'function', - width: $(window).width()*0.80, - height: $(window).height()*0.50, - chartArea: {'width': '90%', 'height': '80%'}, - legend: {position: 'none'} - }; + // Google requires a 'DataTable' object for it series + // This is of the shape [ [{ column info }] [x1, y1, z1] [x2, y2, z2] ] + var data = google.visualization.arrayToDataTable( + [ labels ].concat(chart_data) + ); - var chart = new google.visualization.LineChart(this.chart); - chart.draw(data, options); + // Use window information to make the chart responsive to page sizing + var options = { + curveType: 'function', + width: $(window).width()*0.80, + height: $(window).height()*0.50, + chartArea: {'width': '90%', 'height': '80%'}, + legend: {position: 'none'} + }; + + var chart = new google.visualization.LineChart(this.chart); + chart.draw(data, options); }}) }); } @@ -68,7 +91,6 @@ window.onload = (event) => { // Load the Visualization API and the corechart package. google.charts.load('current', {'packages':['corechart']}); google.charts.setOnLoadCallback(render_charts); - setInterval(render_charts, 1000); }; /* @@ -77,4 +99,5 @@ window.onload = (event) => { */ $( document ).ready(function () { $('#v-pills-cpu-tab').tab('show') + setInterval(render_charts, 1000); }) \ No newline at end of file diff --git a/taskobra/web/views/api.py b/taskobra/web/views/api.py index d323e2f..42cfa3d 100644 --- a/taskobra/web/views/api.py +++ b/taskobra/web/views/api.py @@ -1,4 +1,4 @@ -from flask import Blueprint, jsonify +from flask import Blueprint, jsonify, request import json import random import statistics @@ -26,18 +26,15 @@ def systems(): def metrics_cpu(): # [ [x, y], [x2, y2] ... ] #CPUPercent.join(Systems).query(system.system_name == "") - percent_list = [] - snapshots = Snapshot.query.all() - for snapshot in snapshots: - cpu_percent = [] - for metric in snapshot.metrics: - if isinstance(metric, CpuPercent): - cpu_percent.append(metric.mean) - #total_cpu = statistics.mean( - # metric.mean for metric in snapshot.metrics if isinstance(metric, CpuPercent) - #) - total_cpu = statistics.mean(cpu_percent) - percent_list.append([snapshot.timestamp, total_cpu]) + hostnames = request.args.get('hostnames').split(',') + for hostname in hostnames: + percent_list = [] + snapshots = Snapshot.query.all() + for snapshot in snapshots: + total_cpu = statistics.mean( + metric.mean for metric in snapshot.metrics if isinstance(metric, CpuPercent) + ) + percent_list.append([snapshot.timestamp, total_cpu]) return jsonify(percent_list) From f6eb9b781842f847015e0cdeaebf8871772b1016 Mon Sep 17 00:00:00 2001 From: Tom Manner Date: Mon, 4 May 2020 18:47:48 -0400 Subject: [PATCH 03/11] Started working on the system to snapshot relationship. --- taskobra/orm/relationships/__init__.py | 1 + taskobra/orm/relationships/system_snapshot.py | 11 +++++++++++ taskobra/orm/snapshot.py | 1 + taskobra/orm/system.py | 3 ++- tests/orm/relationship/test_SystemComponent.py | 2 +- tests/orm/relationship/test_SystemSnapshot.py | 14 ++++++++++++++ 6 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 taskobra/orm/relationships/system_snapshot.py create mode 100644 tests/orm/relationship/test_SystemSnapshot.py diff --git a/taskobra/orm/relationships/__init__.py b/taskobra/orm/relationships/__init__.py index 18db15c..abe640c 100644 --- a/taskobra/orm/relationships/__init__.py +++ b/taskobra/orm/relationships/__init__.py @@ -1,3 +1,4 @@ from .system_component import SystemComponent +from .system_snapshot import system_snapshot_table from .user_role import user_role_table from .user_system_role import UserSystemRole diff --git a/taskobra/orm/relationships/system_snapshot.py b/taskobra/orm/relationships/system_snapshot.py new file mode 100644 index 0000000..dd363b9 --- /dev/null +++ b/taskobra/orm/relationships/system_snapshot.py @@ -0,0 +1,11 @@ +# Libraries +from sqlalchemy import Column, ForeignKey, Integer, Table +# Taskobra +from taskobra.orm.base import ORMBase + + +system_snapshot_table = Table( + "SystemSnapshot", ORMBase.metadata, + Column("system_id", Integer, ForeignKey("System.unique_id")), + Column("snapshot_id", Integer, ForeignKey("Snapshot.unique_id")), +) diff --git a/taskobra/orm/snapshot.py b/taskobra/orm/snapshot.py index f95ca6e..b1fca53 100644 --- a/taskobra/orm/snapshot.py +++ b/taskobra/orm/snapshot.py @@ -20,6 +20,7 @@ class UnmergableException(Exception): pass __tablename__ = "Snapshot" unique_id = Column(Integer, primary_key=True) timestamp = Column(DateTime) + system = relationship("System", secondary=system_snapshot_table) metrics = relationship("Metric", secondary=snapshot_metric_table, lazy="joined") sample_count = Column(Integer, default=1) sample_rate = Column(Float, default=1.0) diff --git a/taskobra/orm/system.py b/taskobra/orm/system.py index be3ba31..73e5381 100644 --- a/taskobra/orm/system.py +++ b/taskobra/orm/system.py @@ -4,7 +4,7 @@ from sqlalchemy.orm import relationship # Taskobra from taskobra.orm.base import ORMBase -from taskobra.orm.relationships import SystemComponent +from taskobra.orm.relationships import SystemComponent, system_snapshot_table class System(ORMBase): @@ -13,6 +13,7 @@ class System(ORMBase): name = Column(String) user_roles = relationship("UserSystemRole") system_components = relationship("SystemComponent") + snapshots = relationship("Snapshot", secondary=system_snapshot_table) @property def components(self): diff --git a/tests/orm/relationship/test_SystemComponent.py b/tests/orm/relationship/test_SystemComponent.py index c2c7bdc..cc82d54 100644 --- a/tests/orm/relationship/test_SystemComponent.py +++ b/tests/orm/relationship/test_SystemComponent.py @@ -3,7 +3,7 @@ class TestSystemComponent(ORMTestCase): - def test_SystemComponent_user_property(self): + def test_SystemComponent_component_property(self): system = System(name="Fred's Computer") component = Component() system_component = SystemComponent(system=system) diff --git a/tests/orm/relationship/test_SystemSnapshot.py b/tests/orm/relationship/test_SystemSnapshot.py new file mode 100644 index 0000000..ca57084 --- /dev/null +++ b/tests/orm/relationship/test_SystemSnapshot.py @@ -0,0 +1,14 @@ +from ..ORMTestCase import ORMTestCase +from taskobra.orm import get_engine, get_session, Snapshot, System + + +class TestSystemSnapshot(ORMTestCase): + def test_SystemSnapshot_creation(self): + system = System( + snapshots=[ + Snapshot(timestamp=datetime(2020, 3, 9, 9, 53, 53)), + Snapshot(timestamp=datetime(2020, 3, 9, 9, 53, 52)), + ] + ) + + [self.assertIs(system, snapshot.system) for snapshot in system.snapshots] From 53d097341606038045aebdb6b37616ab319420ad Mon Sep 17 00:00:00 2001 From: Miguel Nistal Date: Wed, 6 May 2020 16:43:44 -0400 Subject: [PATCH 04/11] Implemented serialization of hostname-delimited snapshots for api routes --- taskobra/orm/snapshot.py | 2 +- taskobra/web/static/js/taskobra.js | 6 ++++-- taskobra/web/views/api.py | 33 ++++++++++++++++++------------ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/taskobra/orm/snapshot.py b/taskobra/orm/snapshot.py index b1fca53..ae2ee4f 100644 --- a/taskobra/orm/snapshot.py +++ b/taskobra/orm/snapshot.py @@ -11,7 +11,7 @@ # Taskobra from taskobra.orm.base import ORMBase from taskobra.orm.relationships.snapshot_metric import snapshot_metric_table - +from taskobra.orm.relationships.system_snapshot import system_snapshot_table class Snapshot(ORMBase): diff --git a/taskobra/web/static/js/taskobra.js b/taskobra/web/static/js/taskobra.js index 4fd48ec..f83ee05 100644 --- a/taskobra/web/static/js/taskobra.js +++ b/taskobra/web/static/js/taskobra.js @@ -10,7 +10,7 @@ function render_systems(system_list) { system_list.forEach(host => { // Fill in the attrs of the instance var instance = template.content.cloneNode(true); - instance.querySelector(".hostlist-checkbox").value = host.hostname; + instance.querySelector(".hostlist-checkbox").value = host.unique_id; instance.querySelector(".hostlist-name").textContent = host.hostname; instance.querySelector(".hostlist-cores").textContent = host.cores; instance.querySelector(".hostlist-memory").textContent = host.memory; @@ -37,6 +37,8 @@ function render_charts() { document.querySelectorAll(".taskobra-chart").forEach(chart => { // Query the UI for information about what the user wants rendered var metric_type = chart.getAttribute('data-metric-type') + var selected_host_ids = $("tr input:checked").map(function () { return this.value }).get() + // TODO: Change value -> getClass var selected_hostnames = $("tr input:checked").map(function () { return this.value }).get() // Ensure the chart is visible and a set of data sets are selected before rendering @@ -46,7 +48,7 @@ function render_charts() { // Asynchronously fetch data and draw the chart $.ajax({ url: "/api/metrics/" + metric_type, - data: {'hostnames': selected_hostnames.join(',')}, + data: {'host_ids': selected_host_ids.join(',')}, chart: chart, hostnames: selected_hostnames, success: function(chart_data) { // Generate the labels for the legend based on the selected hosts diff --git a/taskobra/web/views/api.py b/taskobra/web/views/api.py index 42cfa3d..11c201c 100644 --- a/taskobra/web/views/api.py +++ b/taskobra/web/views/api.py @@ -8,6 +8,22 @@ blueprint = Blueprint('api', __name__, url_prefix='/api') +def serialize_metrics(host_ids, metric_type): + percent_list = [] + systems = System.query.filter(System.unique_id.in_(host_ids)).all() + print(systems) + for idx, system in enumerate(systems): + for snapshot in system.snapshots: + print(snapshot) + total_cpu = statistics.mean( + metric.mean for metric in snapshot.metrics if isinstance(metric, metric_type) + ) + snapshot_row = [timestamp] + [None] * len(hostnames) + snapshot_row[idx+1] = total_cpu + percent_list.append(snapshot_row) + print(percent_list) + return percent_list + @blueprint.route('/') def base(): return jsonify({}) @@ -15,7 +31,8 @@ def base(): @blueprint.route('/systems') def systems(): system_list = [ - {'hostname': system.name, + {'unique_id' : system.unique_id, + 'hostname': system.name, 'cores' : sum([component.core_count for _, component in system.components if isinstance(component, CPU)]), 'memory': '16GB', 'storage': '500GB' } for system in System.query.all() @@ -24,18 +41,8 @@ def systems(): @blueprint.route('/metrics/cpu') def metrics_cpu(): - # [ [x, y], [x2, y2] ... ] - #CPUPercent.join(Systems).query(system.system_name == "") - hostnames = request.args.get('hostnames').split(',') - for hostname in hostnames: - percent_list = [] - snapshots = Snapshot.query.all() - for snapshot in snapshots: - total_cpu = statistics.mean( - metric.mean for metric in snapshot.metrics if isinstance(metric, CpuPercent) - ) - percent_list.append([snapshot.timestamp, total_cpu]) - + host_ids = request.args.get('host_ids').split(',') + percent_list = serialize_metrics(host_ids, CpuPercent) return jsonify(percent_list) @blueprint.route('/metrics/gpu') From 0e3200e49b9952e44f23c48718bb85078370c85f Mon Sep 17 00:00:00 2001 From: Tom Manner Date: Wed, 6 May 2020 16:56:03 -0400 Subject: [PATCH 05/11] Added System to Snapshot relationship with back population and a default order by Skip for route tests for @manistal to take a look at. --- taskobra/orm/__init__.py | 2 +- taskobra/orm/snapshot.py | 6 ++++-- taskobra/orm/system.py | 2 +- tests/orm/relationship/test_SystemSnapshot.py | 9 +++------ tests/web/test_routes.py | 10 ++++++++++ 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/taskobra/orm/__init__.py b/taskobra/orm/__init__.py index 07cc3ab..a6c4a31 100644 --- a/taskobra/orm/__init__.py +++ b/taskobra/orm/__init__.py @@ -6,4 +6,4 @@ from .snapshot import Snapshot from .snapshot_control import SnapshotControl from .system import System -from .relationships import SystemComponent, user_role_table, UserSystemRole +from .relationships import SystemComponent, user_role_table, UserSystemRole, system_snapshot_table diff --git a/taskobra/orm/snapshot.py b/taskobra/orm/snapshot.py index b1fca53..a780c40 100644 --- a/taskobra/orm/snapshot.py +++ b/taskobra/orm/snapshot.py @@ -4,13 +4,14 @@ from functools import reduce from itertools import chain from math import ceil, log -from sqlalchemy import DateTime, Column, Float, Integer +from sqlalchemy import DateTime, Column, Float, ForeignKey, Integer from sqlalchemy.orm import relationship from typing import Generator from typing import Iterable # Taskobra from taskobra.orm.base import ORMBase from taskobra.orm.relationships.snapshot_metric import snapshot_metric_table +from taskobra.orm.relationships.system_snapshot import system_snapshot_table class Snapshot(ORMBase): @@ -20,7 +21,8 @@ class UnmergableException(Exception): pass __tablename__ = "Snapshot" unique_id = Column(Integer, primary_key=True) timestamp = Column(DateTime) - system = relationship("System", secondary=system_snapshot_table) + system_id = Column(Integer, ForeignKey("System.unique_id")) + system = relationship("System", back_populates="snapshots") metrics = relationship("Metric", secondary=snapshot_metric_table, lazy="joined") sample_count = Column(Integer, default=1) sample_rate = Column(Float, default=1.0) diff --git a/taskobra/orm/system.py b/taskobra/orm/system.py index 73e5381..3ea67f3 100644 --- a/taskobra/orm/system.py +++ b/taskobra/orm/system.py @@ -13,7 +13,7 @@ class System(ORMBase): name = Column(String) user_roles = relationship("UserSystemRole") system_components = relationship("SystemComponent") - snapshots = relationship("Snapshot", secondary=system_snapshot_table) + snapshots = relationship("Snapshot", back_populates="system", order_by="desc(Snapshot.timestamp)") @property def components(self): diff --git a/tests/orm/relationship/test_SystemSnapshot.py b/tests/orm/relationship/test_SystemSnapshot.py index ca57084..127182a 100644 --- a/tests/orm/relationship/test_SystemSnapshot.py +++ b/tests/orm/relationship/test_SystemSnapshot.py @@ -1,14 +1,11 @@ +from datetime import datetime from ..ORMTestCase import ORMTestCase from taskobra.orm import get_engine, get_session, Snapshot, System class TestSystemSnapshot(ORMTestCase): def test_SystemSnapshot_creation(self): - system = System( - snapshots=[ - Snapshot(timestamp=datetime(2020, 3, 9, 9, 53, 53)), - Snapshot(timestamp=datetime(2020, 3, 9, 9, 53, 52)), - ] - ) + system = System() + system.snapshots.append(Snapshot(timestamp=datetime(2020, 3, 9, 9, 53, 53))) [self.assertIs(system, snapshot.system) for snapshot in system.snapshots] diff --git a/tests/web/test_routes.py b/tests/web/test_routes.py index 34c3067..402cfc8 100644 --- a/tests/web/test_routes.py +++ b/tests/web/test_routes.py @@ -1,47 +1,57 @@ +from unittest import skip + from .WebTestCase import WebTestCase import flask class TestRoutes(WebTestCase): + @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_home_route_context(self): with self.app.test_request_context('/'): assert flask.request.path == '/' + @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_home_route(self): with self.app.test_client() as client: rsp = client.get('/') assert rsp.content_type == 'text/html; charset=utf-8' assert rsp.status_code == 200 + @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_base_route(self): with self.app.test_client() as client: rsp = client.get('/api/') assert rsp.content_type == 'application/json' assert rsp.status_code == 200 + @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_systems_route(self): with self.app.test_client() as client: rsp = client.get('/api/systems') assert rsp.content_type == 'application/json' assert rsp.status_code == 200 + @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_metrics_cpu_route(self): with self.app.test_client() as client: rsp = client.get('/api/metrics/cpu') assert rsp.content_type == 'application/json' assert rsp.status_code == 200 + @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_metrics_gpu_route(self): with self.app.test_client() as client: rsp = client.get('/api/metrics/gpu') assert rsp.content_type == 'application/json' assert rsp.status_code == 200 + @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_metrics_memory_route(self): with self.app.test_client() as client: rsp = client.get('/api/metrics/memory') assert rsp.content_type == 'application/json' assert rsp.status_code == 200 + @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_metrics_storage_route(self): with self.app.test_client() as client: rsp = client.get('/api/metrics/storage') From 891c78971ad9c148c542cd53abc250cb97f99e1e Mon Sep 17 00:00:00 2001 From: Miguel Nistal Date: Wed, 6 May 2020 17:17:18 -0400 Subject: [PATCH 06/11] Added infrastructure to pull in host specific data --- taskobra/monitor/__main__.py | 3 ++- taskobra/monitor/system_info.py | 3 ++- taskobra/orm/system.py | 2 +- taskobra/web/static/js/taskobra.js | 8 ++++++-- taskobra/web/views/api.py | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/taskobra/monitor/__main__.py b/taskobra/monitor/__main__.py index b4a987c..ff556ad 100644 --- a/taskobra/monitor/__main__.py +++ b/taskobra/monitor/__main__.py @@ -53,11 +53,12 @@ def create_database_engine(args): def main(args): database_engine = create_database_engine(args) - system_info.create_system(args, database_engine) + current_system = system_info.create_system(args, database_engine) while True: time.sleep(args.sample_rate) snapshot = create_snapshot(args) snapshot.sample_rate = args.sample_rate + snapshot.system = current_system with get_session(bind=database_engine) as session: print(snapshot) session.add(snapshot) diff --git a/taskobra/monitor/system_info.py b/taskobra/monitor/system_info.py index 79c67d9..0fa9c72 100644 --- a/taskobra/monitor/system_info.py +++ b/taskobra/monitor/system_info.py @@ -29,7 +29,8 @@ def create_system(args, database_engine): if current_system is None: session.add(system) session.commit() - + + return system #gpu = GPU( # manufacturer="NVIDIA", # model="1070", diff --git a/taskobra/orm/system.py b/taskobra/orm/system.py index 3ea67f3..2e12919 100644 --- a/taskobra/orm/system.py +++ b/taskobra/orm/system.py @@ -13,7 +13,7 @@ class System(ORMBase): name = Column(String) user_roles = relationship("UserSystemRole") system_components = relationship("SystemComponent") - snapshots = relationship("Snapshot", back_populates="system", order_by="desc(Snapshot.timestamp)") + snapshots = relationship("Snapshot", back_populates="system", order_by="desc(Snapshot.timestamp)", lazy="joined") @property def components(self): diff --git a/taskobra/web/static/js/taskobra.js b/taskobra/web/static/js/taskobra.js index f83ee05..811ca57 100644 --- a/taskobra/web/static/js/taskobra.js +++ b/taskobra/web/static/js/taskobra.js @@ -57,12 +57,15 @@ function render_charts() { var hostname_id = hostname.toLowerCase().split(' ').join('') labels.push({label: hostname, id: hostname_id, type: 'number'}) }) + console.log("AHHH THIS IS THE CHARTS AJAX") + console.log(labels) // Google requires a 'DataTable' object for it series // This is of the shape [ [{ column info }] [x1, y1, z1] [x2, y2, z2] ] var data = google.visualization.arrayToDataTable( [ labels ].concat(chart_data) ); + console.log([ labels ].concat(chart_data)) // Use window information to make the chart responsive to page sizing var options = { @@ -70,7 +73,8 @@ function render_charts() { width: $(window).width()*0.80, height: $(window).height()*0.50, chartArea: {'width': '90%', 'height': '80%'}, - legend: {position: 'none'} + legend: {position: 'none'}, + interpolateNulls: true }; var chart = new google.visualization.LineChart(this.chart); @@ -101,5 +105,5 @@ window.onload = (event) => { */ $( document ).ready(function () { $('#v-pills-cpu-tab').tab('show') - setInterval(render_charts, 1000); + setInterval(render_charts, 10000); }) \ No newline at end of file diff --git a/taskobra/web/views/api.py b/taskobra/web/views/api.py index 11c201c..bfad315 100644 --- a/taskobra/web/views/api.py +++ b/taskobra/web/views/api.py @@ -18,7 +18,7 @@ def serialize_metrics(host_ids, metric_type): total_cpu = statistics.mean( metric.mean for metric in snapshot.metrics if isinstance(metric, metric_type) ) - snapshot_row = [timestamp] + [None] * len(hostnames) + snapshot_row = [snapshot.timestamp] + [None] * len(systems) snapshot_row[idx+1] = total_cpu percent_list.append(snapshot_row) print(percent_list) From eeaffb259bbfa3d701ec03f9643e225bf60980c0 Mon Sep 17 00:00:00 2001 From: Tom Manner Date: Wed, 6 May 2020 17:25:29 -0400 Subject: [PATCH 07/11] Implemented virtual memory usage metric type. --- taskobra/monitor/__main__.py | 6 +++-- taskobra/orm/__init__.py | 2 +- taskobra/orm/metrics/__init__.py | 1 + taskobra/orm/metrics/metric.py | 1 + taskobra/orm/metrics/virtual_memory.py | 28 ++++++++++++++++++++ tests/orm/metrics/test_VirtualMemoryUsage.py | 15 +++++++++++ 6 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 taskobra/orm/metrics/virtual_memory.py create mode 100644 tests/orm/metrics/test_VirtualMemoryUsage.py diff --git a/taskobra/monitor/__main__.py b/taskobra/monitor/__main__.py index b4a987c..576fae3 100644 --- a/taskobra/monitor/__main__.py +++ b/taskobra/monitor/__main__.py @@ -5,9 +5,9 @@ import psutil import sys import time -import platform +import platform -from taskobra.orm import CpuPercent, get_engine, get_session, Snapshot +from taskobra.orm import CpuPercent, VirtualMemoryUsage, get_engine, get_session, Snapshot from taskobra.monitor import system_info @@ -27,6 +27,8 @@ def create_snapshot(args): # print(f"Disk : {psutil.disk_usage('/')}") # print(f"VMem : {psutil.virtual_memory()}") # print(f"SwapMem : {psutil.swap_memory()}") + vmem = psutil.virtual_memory() + snapshot.metrics.append(VirtualMemoryUsage(total=vmem.total, mean=vmem.used)) snapshot.metrics.append(CpuPercent(core_id=0, mean=psutil.cpu_percent())) # snapshot = for each metric snapshot.add(metric) return snapshot diff --git a/taskobra/orm/__init__.py b/taskobra/orm/__init__.py index 07cc3ab..1e11dfe 100644 --- a/taskobra/orm/__init__.py +++ b/taskobra/orm/__init__.py @@ -1,6 +1,6 @@ from .base import get_engine, get_session, ORMBase from .components import Component, CPU, GPU, Memory, OperatingSystem, Storage -from .metrics import Metric, CpuPercent +from .metrics import Metric, CpuPercent, VirtualMemoryUsage from .role import Role from .user import User from .snapshot import Snapshot diff --git a/taskobra/orm/metrics/__init__.py b/taskobra/orm/metrics/__init__.py index fca228b..b3e0baf 100644 --- a/taskobra/orm/metrics/__init__.py +++ b/taskobra/orm/metrics/__init__.py @@ -1,2 +1,3 @@ from .metric import Metric from .cpu_percent import CpuPercent +from .virtual_memory import VirtualMemoryUsage diff --git a/taskobra/orm/metrics/metric.py b/taskobra/orm/metrics/metric.py index e8f7002..8b584c5 100644 --- a/taskobra/orm/metrics/metric.py +++ b/taskobra/orm/metrics/metric.py @@ -16,6 +16,7 @@ class Metric(ORMBase): variance = Column(Float, default=0.0) metric_type = Column(Enum( "CpuPercent", + "VirtualMemoryUsage", "TestMetricMetric", "TestSnapshotMetric", name="MetricType" diff --git a/taskobra/orm/metrics/virtual_memory.py b/taskobra/orm/metrics/virtual_memory.py new file mode 100644 index 0000000..5c1877a --- /dev/null +++ b/taskobra/orm/metrics/virtual_memory.py @@ -0,0 +1,28 @@ +from collections import defaultdict +from sqlalchemy import Column, Float, ForeignKey, Integer +from typing import Collection +from taskobra.orm.metrics.metric import Metric + + +class VirtualMemoryUsage(Metric): + __tablename__ = "VirtualMemoryUsage" + unique_id = Column(Integer, ForeignKey("Metric.unique_id"), primary_key=True) + total = Column(Float) + __mapper_args__ = { + "polymorphic_identity": __tablename__ + } + + @property + def used(self): + return self.mean + + @property + def percent(self): + return self.used / self.total + + def __repr__(self): + s = f" 1: + s += f" sd:{self.standard_deviation:.3} {self.sample_count})" + s += ")>" + return s diff --git a/tests/orm/metrics/test_VirtualMemoryUsage.py b/tests/orm/metrics/test_VirtualMemoryUsage.py new file mode 100644 index 0000000..ce04e79 --- /dev/null +++ b/tests/orm/metrics/test_VirtualMemoryUsage.py @@ -0,0 +1,15 @@ +from ..ORMTestCase import ORMTestCase +from sqlalchemy import Column, ForeignKey, Integer +import statistics +from taskobra.orm import get_engine, get_session +from taskobra.orm.metrics import VirtualMemoryUsage + + +class TestCpuPercent(ORMTestCase): + def test_prune(self): + with get_session(bind=get_engine("sqlite:///:memory:")) as session: + session.add(VirtualMemoryUsage(total=10, mean=2)) + session.add(VirtualMemoryUsage(total=10, mean=3)) + session.add(VirtualMemoryUsage(total=10, mean=4)) + session.add(VirtualMemoryUsage(total=10, mean=4)) + session.add(VirtualMemoryUsage(total=10, mean=5)) From 6ff947ca602c8a26c365712888ef82edc785ade9 Mon Sep 17 00:00:00 2001 From: Miguel Nistal Date: Wed, 6 May 2020 17:40:17 -0400 Subject: [PATCH 08/11] Fixed javascript and serialization for the graph drawing --- taskobra/web/static/js/taskobra.js | 6 ++---- taskobra/web/views/api.py | 5 +---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/taskobra/web/static/js/taskobra.js b/taskobra/web/static/js/taskobra.js index 811ca57..6724386 100644 --- a/taskobra/web/static/js/taskobra.js +++ b/taskobra/web/static/js/taskobra.js @@ -18,6 +18,7 @@ function render_systems(system_list) { instance.querySelector("tr").addEventListener("click", function(event){ var hostlist_checkbox = event.currentTarget.querySelector(".hostlist-checkbox"); hostlist_checkbox.checked = !hostlist_checkbox.checked; + render_charts() }, false); // Add it to the content section @@ -57,15 +58,12 @@ function render_charts() { var hostname_id = hostname.toLowerCase().split(' ').join('') labels.push({label: hostname, id: hostname_id, type: 'number'}) }) - console.log("AHHH THIS IS THE CHARTS AJAX") - console.log(labels) // Google requires a 'DataTable' object for it series // This is of the shape [ [{ column info }] [x1, y1, z1] [x2, y2, z2] ] var data = google.visualization.arrayToDataTable( [ labels ].concat(chart_data) ); - console.log([ labels ].concat(chart_data)) // Use window information to make the chart responsive to page sizing var options = { @@ -105,5 +103,5 @@ window.onload = (event) => { */ $( document ).ready(function () { $('#v-pills-cpu-tab').tab('show') - setInterval(render_charts, 10000); + setInterval(render_charts, 1000); }) \ No newline at end of file diff --git a/taskobra/web/views/api.py b/taskobra/web/views/api.py index bfad315..c0d1a6c 100644 --- a/taskobra/web/views/api.py +++ b/taskobra/web/views/api.py @@ -11,18 +11,15 @@ def serialize_metrics(host_ids, metric_type): percent_list = [] systems = System.query.filter(System.unique_id.in_(host_ids)).all() - print(systems) for idx, system in enumerate(systems): for snapshot in system.snapshots: - print(snapshot) total_cpu = statistics.mean( metric.mean for metric in snapshot.metrics if isinstance(metric, metric_type) ) snapshot_row = [snapshot.timestamp] + [None] * len(systems) snapshot_row[idx+1] = total_cpu percent_list.append(snapshot_row) - print(percent_list) - return percent_list + return sorted(percent_list, key=lambda row: row[0]) @blueprint.route('/') def base(): From 528941f3fcc92591751801e045476604d8ae3dcd Mon Sep 17 00:00:00 2001 From: Miguel Nistal Date: Wed, 6 May 2020 17:46:40 -0400 Subject: [PATCH 09/11] Updated to include memory charts --- taskobra/web/static/js/taskobra.js | 2 +- taskobra/web/views/api.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/taskobra/web/static/js/taskobra.js b/taskobra/web/static/js/taskobra.js index 6724386..182a9b2 100644 --- a/taskobra/web/static/js/taskobra.js +++ b/taskobra/web/static/js/taskobra.js @@ -40,7 +40,7 @@ function render_charts() { var metric_type = chart.getAttribute('data-metric-type') var selected_host_ids = $("tr input:checked").map(function () { return this.value }).get() // TODO: Change value -> getClass - var selected_hostnames = $("tr input:checked").map(function () { return this.value }).get() + var selected_hostnames = $("tr input:checked").map(function () { return this.querySelector(".hostlist-name").textContent }).get() // Ensure the chart is visible and a set of data sets are selected before rendering if ($( chart ).parent('.active').length == 0) { return } diff --git a/taskobra/web/views/api.py b/taskobra/web/views/api.py index c0d1a6c..acc7c7a 100644 --- a/taskobra/web/views/api.py +++ b/taskobra/web/views/api.py @@ -51,9 +51,8 @@ def metrics_gpu(): @blueprint.route('/metrics/memory') def metrics_memory(): - percent_list = [ - [idx, random.uniform(0, 100)] for idx in range(0, 1000) - ] + host_ids = request.args.get('host_ids').split(',') + percent_list = serialize_metrics(host_ids, VirtualMemoryUsage) return jsonify(percent_list) @blueprint.route('/metrics/storage') From ae5f07474860f46a9bc813d0cccc5b1095841e98 Mon Sep 17 00:00:00 2001 From: Miguel Nistal Date: Wed, 6 May 2020 17:59:05 -0400 Subject: [PATCH 10/11] Working copy of the memory and multihost configuration stuff, WOO --- taskobra/web/static/html/home.html | 2 +- taskobra/web/static/js/taskobra.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/taskobra/web/static/html/home.html b/taskobra/web/static/html/home.html index b0633f9..7cd0fca 100644 --- a/taskobra/web/static/html/home.html +++ b/taskobra/web/static/html/home.html @@ -68,7 +68,7 @@

Storage Utilization

- +
diff --git a/taskobra/web/static/js/taskobra.js b/taskobra/web/static/js/taskobra.js index 182a9b2..65ef83d 100644 --- a/taskobra/web/static/js/taskobra.js +++ b/taskobra/web/static/js/taskobra.js @@ -11,6 +11,7 @@ function render_systems(system_list) { // Fill in the attrs of the instance var instance = template.content.cloneNode(true); instance.querySelector(".hostlist-checkbox").value = host.unique_id; + instance.querySelector(".hostlist-checkbox").setAttribute('data-hostname', host.hostname); instance.querySelector(".hostlist-name").textContent = host.hostname; instance.querySelector(".hostlist-cores").textContent = host.cores; instance.querySelector(".hostlist-memory").textContent = host.memory; @@ -39,8 +40,7 @@ function render_charts() { // Query the UI for information about what the user wants rendered var metric_type = chart.getAttribute('data-metric-type') var selected_host_ids = $("tr input:checked").map(function () { return this.value }).get() - // TODO: Change value -> getClass - var selected_hostnames = $("tr input:checked").map(function () { return this.querySelector(".hostlist-name").textContent }).get() + var selected_hostnames = $("tr input:checked").map(function () { return this.getAttribute('data-hostname') }).get() // Ensure the chart is visible and a set of data sets are selected before rendering if ($( chart ).parent('.active').length == 0) { return } From dd0b2b39899f046c18175d233fa1a50efc42fd48 Mon Sep 17 00:00:00 2001 From: Miguel Nistal Date: Wed, 6 May 2020 18:52:30 -0400 Subject: [PATCH 11/11] Working on fixing the route unit tests, updated to handle the no args case more grasefull --- taskobra/web/views/api.py | 4 ++-- tests/web/test_routes.py | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/taskobra/web/views/api.py b/taskobra/web/views/api.py index acc7c7a..2c85b21 100644 --- a/taskobra/web/views/api.py +++ b/taskobra/web/views/api.py @@ -38,7 +38,7 @@ def systems(): @blueprint.route('/metrics/cpu') def metrics_cpu(): - host_ids = request.args.get('host_ids').split(',') + host_ids = request.args.get('host_ids', '').split(',') percent_list = serialize_metrics(host_ids, CpuPercent) return jsonify(percent_list) @@ -51,7 +51,7 @@ def metrics_gpu(): @blueprint.route('/metrics/memory') def metrics_memory(): - host_ids = request.args.get('host_ids').split(',') + host_ids = request.args.get('host_ids', '').split(',') percent_list = serialize_metrics(host_ids, VirtualMemoryUsage) return jsonify(percent_list) diff --git a/tests/web/test_routes.py b/tests/web/test_routes.py index 402cfc8..e6e3a43 100644 --- a/tests/web/test_routes.py +++ b/tests/web/test_routes.py @@ -4,54 +4,46 @@ import flask class TestRoutes(WebTestCase): - @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_home_route_context(self): with self.app.test_request_context('/'): assert flask.request.path == '/' - @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_home_route(self): with self.app.test_client() as client: rsp = client.get('/') assert rsp.content_type == 'text/html; charset=utf-8' assert rsp.status_code == 200 - @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_base_route(self): with self.app.test_client() as client: rsp = client.get('/api/') assert rsp.content_type == 'application/json' assert rsp.status_code == 200 - @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_systems_route(self): with self.app.test_client() as client: rsp = client.get('/api/systems') assert rsp.content_type == 'application/json' assert rsp.status_code == 200 - @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_metrics_cpu_route(self): with self.app.test_client() as client: rsp = client.get('/api/metrics/cpu') assert rsp.content_type == 'application/json' assert rsp.status_code == 200 - @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_metrics_gpu_route(self): with self.app.test_client() as client: rsp = client.get('/api/metrics/gpu') assert rsp.content_type == 'application/json' assert rsp.status_code == 200 - @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_metrics_memory_route(self): with self.app.test_client() as client: rsp = client.get('/api/metrics/memory') assert rsp.content_type == 'application/json' assert rsp.status_code == 200 - @skip("Routes broken by system_id foreign key column added to Snapshot class") def test_api_metrics_storage_route(self): with self.app.test_client() as client: rsp = client.get('/api/metrics/storage')