diff --git a/taskobra/monitor/__main__.py b/taskobra/monitor/__main__.py index b4a987c..3526567 100644 --- a/taskobra/monitor/__main__.py +++ b/taskobra/monitor/__main__.py @@ -5,9 +5,9 @@ import psutil import sys import time -import platform +import platform -from taskobra.orm import CpuPercent, get_engine, get_session, Snapshot +from taskobra.orm import CpuPercent, VirtualMemoryUsage, get_engine, get_session, Snapshot from taskobra.monitor import system_info @@ -27,6 +27,8 @@ def create_snapshot(args): # print(f"Disk : {psutil.disk_usage('/')}") # print(f"VMem : {psutil.virtual_memory()}") # print(f"SwapMem : {psutil.swap_memory()}") + vmem = psutil.virtual_memory() + snapshot.metrics.append(VirtualMemoryUsage(total=vmem.total, mean=vmem.used)) snapshot.metrics.append(CpuPercent(core_id=0, mean=psutil.cpu_percent())) # snapshot = for each metric snapshot.add(metric) return snapshot @@ -53,11 +55,12 @@ def create_database_engine(args): def main(args): database_engine = create_database_engine(args) - system_info.create_system(args, database_engine) + current_system = system_info.create_system(args, database_engine) while True: time.sleep(args.sample_rate) snapshot = create_snapshot(args) snapshot.sample_rate = args.sample_rate + snapshot.system = current_system with get_session(bind=database_engine) as session: print(snapshot) session.add(snapshot) diff --git a/taskobra/monitor/system_info.py b/taskobra/monitor/system_info.py index 79c67d9..0fa9c72 100644 --- a/taskobra/monitor/system_info.py +++ b/taskobra/monitor/system_info.py @@ -29,7 +29,8 @@ def create_system(args, database_engine): if current_system is None: session.add(system) session.commit() - + + return system #gpu = GPU( # manufacturer="NVIDIA", # model="1070", diff --git a/taskobra/orm/__init__.py b/taskobra/orm/__init__.py index 07cc3ab..76c96e4 100644 --- a/taskobra/orm/__init__.py +++ b/taskobra/orm/__init__.py @@ -1,9 +1,9 @@ from .base import get_engine, get_session, ORMBase from .components import Component, CPU, GPU, Memory, OperatingSystem, Storage -from .metrics import Metric, CpuPercent +from .metrics import Metric, CpuPercent, VirtualMemoryUsage from .role import Role from .user import User from .snapshot import Snapshot from .snapshot_control import SnapshotControl from .system import System -from .relationships import SystemComponent, user_role_table, UserSystemRole +from .relationships import SystemComponent, user_role_table, UserSystemRole, system_snapshot_table diff --git a/taskobra/orm/metrics/__init__.py b/taskobra/orm/metrics/__init__.py index fca228b..b3e0baf 100644 --- a/taskobra/orm/metrics/__init__.py +++ b/taskobra/orm/metrics/__init__.py @@ -1,2 +1,3 @@ from .metric import Metric from .cpu_percent import CpuPercent +from .virtual_memory import VirtualMemoryUsage diff --git a/taskobra/orm/metrics/metric.py b/taskobra/orm/metrics/metric.py index e8f7002..8b584c5 100644 --- a/taskobra/orm/metrics/metric.py +++ b/taskobra/orm/metrics/metric.py @@ -16,6 +16,7 @@ class Metric(ORMBase): variance = Column(Float, default=0.0) metric_type = Column(Enum( "CpuPercent", + "VirtualMemoryUsage", "TestMetricMetric", "TestSnapshotMetric", name="MetricType" diff --git a/taskobra/orm/metrics/virtual_memory.py b/taskobra/orm/metrics/virtual_memory.py new file mode 100644 index 0000000..5c1877a --- /dev/null +++ b/taskobra/orm/metrics/virtual_memory.py @@ -0,0 +1,28 @@ +from collections import defaultdict +from sqlalchemy import Column, Float, ForeignKey, Integer +from typing import Collection +from taskobra.orm.metrics.metric import Metric + + +class VirtualMemoryUsage(Metric): + __tablename__ = "VirtualMemoryUsage" + unique_id = Column(Integer, ForeignKey("Metric.unique_id"), primary_key=True) + total = Column(Float) + __mapper_args__ = { + "polymorphic_identity": __tablename__ + } + + @property + def used(self): + return self.mean + + @property + def percent(self): + return self.used / self.total + + def __repr__(self): + s = f" 1: + s += f" sd:{self.standard_deviation:.3} {self.sample_count})" + s += ")>" + return s diff --git a/taskobra/orm/relationships/__init__.py b/taskobra/orm/relationships/__init__.py index 18db15c..abe640c 100644 --- a/taskobra/orm/relationships/__init__.py +++ b/taskobra/orm/relationships/__init__.py @@ -1,3 +1,4 @@ from .system_component import SystemComponent +from .system_snapshot import system_snapshot_table from .user_role import user_role_table from .user_system_role import UserSystemRole diff --git a/taskobra/orm/relationships/system_snapshot.py b/taskobra/orm/relationships/system_snapshot.py new file mode 100644 index 0000000..dd363b9 --- /dev/null +++ b/taskobra/orm/relationships/system_snapshot.py @@ -0,0 +1,11 @@ +# Libraries +from sqlalchemy import Column, ForeignKey, Integer, Table +# Taskobra +from taskobra.orm.base import ORMBase + + +system_snapshot_table = Table( + "SystemSnapshot", ORMBase.metadata, + Column("system_id", Integer, ForeignKey("System.unique_id")), + Column("snapshot_id", Integer, ForeignKey("Snapshot.unique_id")), +) diff --git a/taskobra/orm/snapshot.py b/taskobra/orm/snapshot.py index f95ca6e..a780c40 100644 --- a/taskobra/orm/snapshot.py +++ b/taskobra/orm/snapshot.py @@ -4,13 +4,14 @@ from functools import reduce from itertools import chain from math import ceil, log -from sqlalchemy import DateTime, Column, Float, Integer +from sqlalchemy import DateTime, Column, Float, ForeignKey, Integer from sqlalchemy.orm import relationship from typing import Generator from typing import Iterable # Taskobra from taskobra.orm.base import ORMBase from taskobra.orm.relationships.snapshot_metric import snapshot_metric_table +from taskobra.orm.relationships.system_snapshot import system_snapshot_table class Snapshot(ORMBase): @@ -20,6 +21,8 @@ class UnmergableException(Exception): pass __tablename__ = "Snapshot" unique_id = Column(Integer, primary_key=True) timestamp = Column(DateTime) + system_id = Column(Integer, ForeignKey("System.unique_id")) + system = relationship("System", back_populates="snapshots") metrics = relationship("Metric", secondary=snapshot_metric_table, lazy="joined") sample_count = Column(Integer, default=1) sample_rate = Column(Float, default=1.0) diff --git a/taskobra/orm/system.py b/taskobra/orm/system.py index be3ba31..2e12919 100644 --- a/taskobra/orm/system.py +++ b/taskobra/orm/system.py @@ -4,7 +4,7 @@ from sqlalchemy.orm import relationship # Taskobra from taskobra.orm.base import ORMBase -from taskobra.orm.relationships import SystemComponent +from taskobra.orm.relationships import SystemComponent, system_snapshot_table class System(ORMBase): @@ -13,6 +13,7 @@ class System(ORMBase): name = Column(String) user_roles = relationship("UserSystemRole") system_components = relationship("SystemComponent") + snapshots = relationship("Snapshot", back_populates="system", order_by="desc(Snapshot.timestamp)", lazy="joined") @property def components(self): diff --git a/taskobra/web/__init__.py b/taskobra/web/__init__.py index 33e3d51..c8154cf 100644 --- a/taskobra/web/__init__.py +++ b/taskobra/web/__init__.py @@ -20,10 +20,8 @@ def create_app(): static_folder='static') # Root Path for url_for('static') # Load config from ENV - app.config['DATABASE_URI'] = os.environ.get('DATABASE_URI', 'sqlite:///taskobra.sqlite.db') - # TODO: OAuth Key - # TODO: ??? - print(f" * Using Database URI {app.config['DATABASE_URI']}") + app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URI', 'sqlite:///taskobra.sqlite.db') + # Bind Route Blueprints Packages to the base App app.register_blueprint(api.blueprint) app.register_blueprint(ui.blueprint) diff --git a/taskobra/web/static/html/home.html b/taskobra/web/static/html/home.html index b0633f9..7cd0fca 100644 --- a/taskobra/web/static/html/home.html +++ b/taskobra/web/static/html/home.html @@ -68,7 +68,7 @@

Storage Utilization

- +
diff --git a/taskobra/web/static/js/taskobra.js b/taskobra/web/static/js/taskobra.js index 98ba281..65ef83d 100644 --- a/taskobra/web/static/js/taskobra.js +++ b/taskobra/web/static/js/taskobra.js @@ -10,7 +10,8 @@ function render_systems(system_list) { system_list.forEach(host => { // Fill in the attrs of the instance var instance = template.content.cloneNode(true); - instance.querySelector(".hostlist-checkbox").value = host.hostname; + instance.querySelector(".hostlist-checkbox").value = host.unique_id; + instance.querySelector(".hostlist-checkbox").setAttribute('data-hostname', host.hostname); instance.querySelector(".hostlist-name").textContent = host.hostname; instance.querySelector(".hostlist-cores").textContent = host.cores; instance.querySelector(".hostlist-memory").textContent = host.memory; @@ -18,11 +19,15 @@ function render_systems(system_list) { instance.querySelector("tr").addEventListener("click", function(event){ var hostlist_checkbox = event.currentTarget.querySelector(".hostlist-checkbox"); hostlist_checkbox.checked = !hostlist_checkbox.checked; + render_charts() }, false); // Add it to the content section document.querySelector("#taskobra-hostlist-entries").appendChild(instance); }); + + // Always ensure at least one hostname is checked + $('input:checkbox:first').each(function () { this.checked = true }) } /* @@ -32,24 +37,46 @@ function render_systems(system_list) { */ function render_charts() { document.querySelectorAll(".taskobra-chart").forEach(chart => { - if ($( chart ).parent('.active').length == 0) { return } + // Query the UI for information about what the user wants rendered var metric_type = chart.getAttribute('data-metric-type') - $.ajax({url: "/api/metrics/" + metric_type, chart: chart, success: function(chart_data) { - var labels = [ [ {label: 'Time', id: 'time'}, {label: 'Utilization', id: 'utilization', type: 'number'} ] ]; - var data = google.visualization.arrayToDataTable( - labels.concat(chart_data) - ); + var selected_host_ids = $("tr input:checked").map(function () { return this.value }).get() + var selected_hostnames = $("tr input:checked").map(function () { return this.getAttribute('data-hostname') }).get() + + // Ensure the chart is visible and a set of data sets are selected before rendering + if ($( chart ).parent('.active').length == 0) { return } + if (selected_hostnames.length == 0) { return } + + // Asynchronously fetch data and draw the chart + $.ajax({ + url: "/api/metrics/" + metric_type, + data: {'host_ids': selected_host_ids.join(',')}, + chart: chart, hostnames: selected_hostnames, + success: function(chart_data) { + // Generate the labels for the legend based on the selected hosts + var labels = [ {label: 'Time', id: 'time'} ]; + this.hostnames.forEach(function (hostname) { + var hostname_id = hostname.toLowerCase().split(' ').join('') + labels.push({label: hostname, id: hostname_id, type: 'number'}) + }) - var options = { - curveType: 'function', - width: $(window).width()*0.80, - height: $(window).height()*0.50, - chartArea: {'width': '90%', 'height': '80%'}, - legend: {position: 'none'} - }; + // Google requires a 'DataTable' object for it series + // This is of the shape [ [{ column info }] [x1, y1, z1] [x2, y2, z2] ] + var data = google.visualization.arrayToDataTable( + [ labels ].concat(chart_data) + ); - var chart = new google.visualization.LineChart(this.chart); - chart.draw(data, options); + // Use window information to make the chart responsive to page sizing + var options = { + curveType: 'function', + width: $(window).width()*0.80, + height: $(window).height()*0.50, + chartArea: {'width': '90%', 'height': '80%'}, + legend: {position: 'none'}, + interpolateNulls: true + }; + + var chart = new google.visualization.LineChart(this.chart); + chart.draw(data, options); }}) }); } @@ -68,7 +95,6 @@ window.onload = (event) => { // Load the Visualization API and the corechart package. google.charts.load('current', {'packages':['corechart']}); google.charts.setOnLoadCallback(render_charts); - setInterval(render_charts, 1000); }; /* @@ -77,4 +103,5 @@ window.onload = (event) => { */ $( document ).ready(function () { $('#v-pills-cpu-tab').tab('show') + setInterval(render_charts, 1000); }) \ No newline at end of file diff --git a/taskobra/web/views/api.py b/taskobra/web/views/api.py index d323e2f..2c85b21 100644 --- a/taskobra/web/views/api.py +++ b/taskobra/web/views/api.py @@ -1,4 +1,4 @@ -from flask import Blueprint, jsonify +from flask import Blueprint, jsonify, request import json import random import statistics @@ -8,6 +8,19 @@ blueprint = Blueprint('api', __name__, url_prefix='/api') +def serialize_metrics(host_ids, metric_type): + percent_list = [] + systems = System.query.filter(System.unique_id.in_(host_ids)).all() + for idx, system in enumerate(systems): + for snapshot in system.snapshots: + total_cpu = statistics.mean( + metric.mean for metric in snapshot.metrics if isinstance(metric, metric_type) + ) + snapshot_row = [snapshot.timestamp] + [None] * len(systems) + snapshot_row[idx+1] = total_cpu + percent_list.append(snapshot_row) + return sorted(percent_list, key=lambda row: row[0]) + @blueprint.route('/') def base(): return jsonify({}) @@ -15,7 +28,8 @@ def base(): @blueprint.route('/systems') def systems(): system_list = [ - {'hostname': system.name, + {'unique_id' : system.unique_id, + 'hostname': system.name, 'cores' : sum([component.core_count for _, component in system.components if isinstance(component, CPU)]), 'memory': '16GB', 'storage': '500GB' } for system in System.query.all() @@ -24,21 +38,8 @@ def systems(): @blueprint.route('/metrics/cpu') def metrics_cpu(): - # [ [x, y], [x2, y2] ... ] - #CPUPercent.join(Systems).query(system.system_name == "") - percent_list = [] - snapshots = Snapshot.query.all() - for snapshot in snapshots: - cpu_percent = [] - for metric in snapshot.metrics: - if isinstance(metric, CpuPercent): - cpu_percent.append(metric.mean) - #total_cpu = statistics.mean( - # metric.mean for metric in snapshot.metrics if isinstance(metric, CpuPercent) - #) - total_cpu = statistics.mean(cpu_percent) - percent_list.append([snapshot.timestamp, total_cpu]) - + host_ids = request.args.get('host_ids', '').split(',') + percent_list = serialize_metrics(host_ids, CpuPercent) return jsonify(percent_list) @blueprint.route('/metrics/gpu') @@ -50,9 +51,8 @@ def metrics_gpu(): @blueprint.route('/metrics/memory') def metrics_memory(): - percent_list = [ - [idx, random.uniform(0, 100)] for idx in range(0, 1000) - ] + host_ids = request.args.get('host_ids', '').split(',') + percent_list = serialize_metrics(host_ids, VirtualMemoryUsage) return jsonify(percent_list) @blueprint.route('/metrics/storage') diff --git a/tests/orm/metrics/test_VirtualMemoryUsage.py b/tests/orm/metrics/test_VirtualMemoryUsage.py new file mode 100644 index 0000000..ce04e79 --- /dev/null +++ b/tests/orm/metrics/test_VirtualMemoryUsage.py @@ -0,0 +1,15 @@ +from ..ORMTestCase import ORMTestCase +from sqlalchemy import Column, ForeignKey, Integer +import statistics +from taskobra.orm import get_engine, get_session +from taskobra.orm.metrics import VirtualMemoryUsage + + +class TestCpuPercent(ORMTestCase): + def test_prune(self): + with get_session(bind=get_engine("sqlite:///:memory:")) as session: + session.add(VirtualMemoryUsage(total=10, mean=2)) + session.add(VirtualMemoryUsage(total=10, mean=3)) + session.add(VirtualMemoryUsage(total=10, mean=4)) + session.add(VirtualMemoryUsage(total=10, mean=4)) + session.add(VirtualMemoryUsage(total=10, mean=5)) diff --git a/tests/orm/relationship/test_SystemComponent.py b/tests/orm/relationship/test_SystemComponent.py index c2c7bdc..cc82d54 100644 --- a/tests/orm/relationship/test_SystemComponent.py +++ b/tests/orm/relationship/test_SystemComponent.py @@ -3,7 +3,7 @@ class TestSystemComponent(ORMTestCase): - def test_SystemComponent_user_property(self): + def test_SystemComponent_component_property(self): system = System(name="Fred's Computer") component = Component() system_component = SystemComponent(system=system) diff --git a/tests/orm/relationship/test_SystemSnapshot.py b/tests/orm/relationship/test_SystemSnapshot.py new file mode 100644 index 0000000..127182a --- /dev/null +++ b/tests/orm/relationship/test_SystemSnapshot.py @@ -0,0 +1,11 @@ +from datetime import datetime +from ..ORMTestCase import ORMTestCase +from taskobra.orm import get_engine, get_session, Snapshot, System + + +class TestSystemSnapshot(ORMTestCase): + def test_SystemSnapshot_creation(self): + system = System() + system.snapshots.append(Snapshot(timestamp=datetime(2020, 3, 9, 9, 53, 53))) + + [self.assertIs(system, snapshot.system) for snapshot in system.snapshots] diff --git a/tests/web/test_routes.py b/tests/web/test_routes.py index 34c3067..e6e3a43 100644 --- a/tests/web/test_routes.py +++ b/tests/web/test_routes.py @@ -1,3 +1,5 @@ +from unittest import skip + from .WebTestCase import WebTestCase import flask