JS for Graphing by Hostname

taskobra/monitor/__main__.py

@@ -5,9 +5,9 @@
 import psutil
 import sys
 import time
-import platform 
+import platform
 
-from taskobra.orm import CpuPercent, get_engine, get_session, Snapshot
+from taskobra.orm import CpuPercent, VirtualMemoryUsage, get_engine, get_session, Snapshot
 from taskobra.monitor import system_info
 
 
@@ -27,6 +27,8 @@ def create_snapshot(args):
     # print(f"Disk    : {psutil.disk_usage('/')}")
     # print(f"VMem    : {psutil.virtual_memory()}")
     # print(f"SwapMem : {psutil.swap_memory()}")
+    vmem = psutil.virtual_memory()
+    snapshot.metrics.append(VirtualMemoryUsage(total=vmem.total, mean=vmem.used))
     snapshot.metrics.append(CpuPercent(core_id=0, mean=psutil.cpu_percent()))
     # snapshot = for each metric snapshot.add(metric)
     return snapshot
@@ -53,11 +55,12 @@ def create_database_engine(args):
 
 def main(args):
     database_engine = create_database_engine(args)
-    system_info.create_system(args, database_engine)
+    current_system = system_info.create_system(args, database_engine)
     while True:
         time.sleep(args.sample_rate)
         snapshot = create_snapshot(args)
         snapshot.sample_rate = args.sample_rate
+        snapshot.system = current_system
         with get_session(bind=database_engine) as session:
             print(snapshot)
             session.add(snapshot)

taskobra/monitor/system_info.py

@@ -29,7 +29,8 @@ def create_system(args, database_engine):
         if current_system is None:
             session.add(system)
             session.commit()
-
+
+    return system 
     #gpu = GPU(
     #    manufacturer="NVIDIA",
     #    model="1070",

taskobra/orm/__init__.py

@@ -1,9 +1,9 @@
 from .base import get_engine, get_session, ORMBase
 from .components import Component, CPU, GPU, Memory, OperatingSystem, Storage
-from .metrics import Metric, CpuPercent
+from .metrics import Metric, CpuPercent, VirtualMemoryUsage
 from .role import Role
 from .user import User
 from .snapshot import Snapshot
 from .snapshot_control import SnapshotControl
 from .system import System
-from .relationships import SystemComponent, user_role_table, UserSystemRole
+from .relationships import SystemComponent, user_role_table, UserSystemRole, system_snapshot_table

taskobra/orm/metrics/__init__.py

@@ -1,2 +1,3 @@
 from .metric import Metric
 from .cpu_percent import CpuPercent
+from .virtual_memory import VirtualMemoryUsage

taskobra/orm/metrics/metric.py

@@ -16,6 +16,7 @@ class Metric(ORMBase):
     variance = Column(Float, default=0.0)
     metric_type = Column(Enum(
         "CpuPercent",
+        "VirtualMemoryUsage",
         "TestMetricMetric",
         "TestSnapshotMetric",
         name="MetricType"

taskobra/orm/metrics/virtual_memory.py

@@ -0,0 +1,28 @@
+from collections import defaultdict
+from sqlalchemy import Column, Float, ForeignKey, Integer
+from typing import Collection
+from taskobra.orm.metrics.metric import Metric
+
+
+class VirtualMemoryUsage(Metric):
+    __tablename__ = "VirtualMemoryUsage"
+    unique_id = Column(Integer, ForeignKey("Metric.unique_id"), primary_key=True)
+    total = Column(Float)
+    __mapper_args__ = {
+        "polymorphic_identity": __tablename__
+    }
+
+    @property
+    def used(self):
+        return self.mean
+
+    @property
+    def percent(self):
+        return self.used / self.total
+
+    def __repr__(self):
+        s = f"<VirtualMemoryUsage({int(self.used)}/{int(self.total)}: {100*self.percent:.1f}"
+        if self.sample_count > 1:
+            s += f" sd:{self.standard_deviation:.3} {self.sample_count})"
+        s += ")>"
+        return s

taskobra/orm/relationships/__init__.py

@@ -1,3 +1,4 @@
 from .system_component import SystemComponent
+from .system_snapshot import system_snapshot_table
 from .user_role import user_role_table
 from .user_system_role import UserSystemRole

taskobra/orm/relationships/system_snapshot.py

@@ -0,0 +1,11 @@
+# Libraries
+from sqlalchemy import Column, ForeignKey, Integer, Table
+# Taskobra
+from taskobra.orm.base import ORMBase
+
+
+system_snapshot_table = Table(
+    "SystemSnapshot", ORMBase.metadata,
+    Column("system_id", Integer, ForeignKey("System.unique_id")),
+    Column("snapshot_id", Integer, ForeignKey("Snapshot.unique_id")),
+)

taskobra/orm/snapshot.py

@@ -4,13 +4,14 @@
 from functools import reduce
 from itertools import chain
 from math import ceil, log
-from sqlalchemy import DateTime, Column, Float, Integer
+from sqlalchemy import DateTime, Column, Float, ForeignKey, Integer
 from sqlalchemy.orm import relationship
 from typing import Generator
 from typing import Iterable
 # Taskobra
 from taskobra.orm.base import ORMBase
 from taskobra.orm.relationships.snapshot_metric import snapshot_metric_table
+from taskobra.orm.relationships.system_snapshot import system_snapshot_table
 
 
 class Snapshot(ORMBase):
@@ -20,6 +21,8 @@ class UnmergableException(Exception): pass
     __tablename__ = "Snapshot"
     unique_id = Column(Integer, primary_key=True)
     timestamp = Column(DateTime)
+    system_id = Column(Integer, ForeignKey("System.unique_id"))
+    system = relationship("System", back_populates="snapshots")
     metrics = relationship("Metric", secondary=snapshot_metric_table, lazy="joined")
     sample_count = Column(Integer, default=1)
     sample_rate = Column(Float, default=1.0)

taskobra/orm/system.py

@@ -4,7 +4,7 @@
 from sqlalchemy.orm import relationship
 # Taskobra
 from taskobra.orm.base import ORMBase
-from taskobra.orm.relationships import SystemComponent
+from taskobra.orm.relationships import SystemComponent, system_snapshot_table
 
 
 class System(ORMBase):
@@ -13,6 +13,7 @@ class System(ORMBase):
     name = Column(String)
     user_roles = relationship("UserSystemRole")
     system_components = relationship("SystemComponent")
+    snapshots = relationship("Snapshot", back_populates="system", order_by="desc(Snapshot.timestamp)", lazy="joined")
 
     @property
     def components(self):

taskobra/web/__init__.py

@@ -20,10 +20,8 @@ def create_app():
             static_folder='static')         # Root Path for url_for('static')
 
     # Load config from ENV
-    app.config['DATABASE_URI'] = os.environ.get('DATABASE_URI', 'sqlite:///taskobra.sqlite.db')
-    # TODO: OAuth Key
-    # TODO: ???
-    print(f" * Using Database URI {app.config['DATABASE_URI']}")
+    app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URI', 'sqlite:///taskobra.sqlite.db')
+
     # Bind Route Blueprints Packages to the base App
     app.register_blueprint(api.blueprint)
     app.register_blueprint(ui.blueprint)

taskobra/web/static/html/home.html

@@ -68,7 +68,7 @@ <h2> Storage Utilization </h2>
         <tr>
           <th scope="row">
             <div class="form-check d-flex align-items-center justify-content-center">
-              <input class="form-check-input position-static hostlist-checkbox" type="checkbox" value="" aria-label="Select host">
+              <input class="form-check-input position-static hostlist-checkbox" type="checkbox" value="" data-hostname="" aria-label="Select host">
             </div>
           </th>
           <td class="hostlist-name"></td>

taskobra/web/static/js/taskobra.js

@@ -10,19 +10,24 @@ function render_systems(system_list) {
   system_list.forEach(host => {
     // Fill in the attrs of the instance
     var instance = template.content.cloneNode(true);
-    instance.querySelector(".hostlist-checkbox").value = host.hostname;
+    instance.querySelector(".hostlist-checkbox").value = host.unique_id;
+    instance.querySelector(".hostlist-checkbox").setAttribute('data-hostname', host.hostname);
     instance.querySelector(".hostlist-name").textContent = host.hostname;
     instance.querySelector(".hostlist-cores").textContent = host.cores;
     instance.querySelector(".hostlist-memory").textContent = host.memory;
     instance.querySelector(".hostlist-storage").textContent = host.storage;
     instance.querySelector("tr").addEventListener("click", function(event){
       var hostlist_checkbox = event.currentTarget.querySelector(".hostlist-checkbox");
       hostlist_checkbox.checked = !hostlist_checkbox.checked;
+      render_charts()
     }, false);
 
     // Add it to the content section 
     document.querySelector("#taskobra-hostlist-entries").appendChild(instance);
   });
+
+  // Always ensure at least one hostname is checked 
+  $('input:checkbox:first').each(function () { this.checked = true })
 }
 
 /*
@@ -32,24 +37,46 @@ function render_systems(system_list) {
 */
 function render_charts() {
   document.querySelectorAll(".taskobra-chart").forEach(chart => {
-    if ($( chart ).parent('.active').length == 0) { return }
+    // Query the UI for information about what the user wants rendered
     var metric_type = chart.getAttribute('data-metric-type')
-    $.ajax({url: "/api/metrics/" + metric_type, chart: chart, success: function(chart_data) {
-      var labels = [ [ {label: 'Time', id: 'time'}, {label: 'Utilization', id: 'utilization', type: 'number'} ] ];
-      var data = google.visualization.arrayToDataTable(
-        labels.concat(chart_data)
-      );
+    var selected_host_ids  = $("tr input:checked").map(function () { return this.value }).get()
+    var selected_hostnames = $("tr input:checked").map(function () { return this.getAttribute('data-hostname') }).get()
+
+    // Ensure the chart is visible and a set of data sets are selected before rendering
+    if ($( chart ).parent('.active').length == 0) { return }
+    if (selected_hostnames.length == 0) { return }
+
+    // Asynchronously fetch data and draw the chart 
+    $.ajax({
+      url: "/api/metrics/" + metric_type, 
+      data: {'host_ids': selected_host_ids.join(',')},
+      chart: chart, hostnames: selected_hostnames, 
+      success: function(chart_data) {
+        // Generate the labels for the legend based on the selected hosts 
+        var labels = [ {label: 'Time', id: 'time'} ]; 
+        this.hostnames.forEach(function (hostname) { 
+          var hostname_id = hostname.toLowerCase().split(' ').join('')
+          labels.push({label: hostname, id: hostname_id, type: 'number'}) 
+        })
 
-      var options = {
-        curveType: 'function',
-        width: $(window).width()*0.80, 
-        height: $(window).height()*0.50, 
-        chartArea: {'width': '90%', 'height': '80%'},
-        legend: {position: 'none'}
-      };
+        // Google requires a 'DataTable' object for it series
+        // This is of the shape [ [{ column info }] [x1, y1, z1] [x2, y2, z2] ]
+        var data = google.visualization.arrayToDataTable(
+          [ labels ].concat(chart_data)
+        );
 
-      var chart = new google.visualization.LineChart(this.chart);
-      chart.draw(data, options);
+        // Use window information to make the chart responsive to page sizing
+        var options = {
+          curveType: 'function',
+          width: $(window).width()*0.80, 
+          height: $(window).height()*0.50, 
+          chartArea: {'width': '90%', 'height': '80%'},
+          legend: {position: 'none'}, 
+          interpolateNulls: true
+        };
+
+        var chart = new google.visualization.LineChart(this.chart);
+        chart.draw(data, options);
     }})
   });
 }
@@ -68,7 +95,6 @@ window.onload = (event) => {
   // Load the Visualization API and the corechart package.
   google.charts.load('current', {'packages':['corechart']});
   google.charts.setOnLoadCallback(render_charts);
-  setInterval(render_charts, 1000);
 };
 
 /* 
@@ -77,4 +103,5 @@ window.onload = (event) => {
 */
 $( document ).ready(function () {
   $('#v-pills-cpu-tab').tab('show')
+  setInterval(render_charts, 1000);
 })

taskobra/web/views/api.py

@@ -1,4 +1,4 @@
-from flask import Blueprint, jsonify
+from flask import Blueprint, jsonify, request
 import json
 import random
 import statistics
@@ -8,14 +8,28 @@
 
 blueprint = Blueprint('api', __name__, url_prefix='/api')
 
+def serialize_metrics(host_ids, metric_type):
+    percent_list = []
+    systems = System.query.filter(System.unique_id.in_(host_ids)).all()
+    for idx, system in enumerate(systems):
+        for snapshot in system.snapshots:
+            total_cpu = statistics.mean(
+                metric.mean for metric in snapshot.metrics if isinstance(metric, metric_type)
+            )
+            snapshot_row = [snapshot.timestamp] + [None] * len(systems)
+            snapshot_row[idx+1] = total_cpu
+            percent_list.append(snapshot_row)
+    return sorted(percent_list, key=lambda row: row[0])
+
 @blueprint.route('/')
 def base():
     return jsonify({})
 
 @blueprint.route('/systems')
 def systems():
     system_list = [
-        {'hostname': system.name,
+        {'unique_id' : system.unique_id, 
+        'hostname': system.name,
         'cores' : sum([component.core_count for _, component in system.components if isinstance(component, CPU)]),
         'memory': '16GB', 'storage': '500GB' }
         for system in System.query.all()
@@ -24,21 +38,8 @@ def systems():
 
 @blueprint.route('/metrics/cpu')
 def metrics_cpu():
-    # [ [x, y], [x2, y2] ... ]
-    #CPUPercent.join(Systems).query(system.system_name == "")
-    percent_list = []
-    snapshots = Snapshot.query.all()
-    for snapshot in snapshots:
-        cpu_percent = []
-        for metric in snapshot.metrics:
-            if isinstance(metric, CpuPercent):
-                cpu_percent.append(metric.mean)
-        #total_cpu = statistics.mean(
-        #    metric.mean for metric in snapshot.metrics if isinstance(metric, CpuPercent)
-        #)
-        total_cpu = statistics.mean(cpu_percent)
-        percent_list.append([snapshot.timestamp, total_cpu])
-
+    host_ids = request.args.get('host_ids', '').split(',')
+    percent_list = serialize_metrics(host_ids, CpuPercent)
     return jsonify(percent_list)
 
 @blueprint.route('/metrics/gpu')
@@ -50,9 +51,8 @@ def metrics_gpu():
 
 @blueprint.route('/metrics/memory')
 def metrics_memory():
-    percent_list = [
-        [idx, random.uniform(0, 100)] for idx in range(0, 1000)
-    ]
+    host_ids = request.args.get('host_ids', '').split(',')
+    percent_list = serialize_metrics(host_ids, VirtualMemoryUsage)
     return jsonify(percent_list)
 
 @blueprint.route('/metrics/storage')

tests/orm/metrics/test_VirtualMemoryUsage.py

@@ -0,0 +1,15 @@
+from ..ORMTestCase import ORMTestCase
+from sqlalchemy import Column, ForeignKey, Integer
+import statistics
+from taskobra.orm import get_engine, get_session
+from taskobra.orm.metrics import VirtualMemoryUsage
+
+
+class TestCpuPercent(ORMTestCase):
+    def test_prune(self):
+        with get_session(bind=get_engine("sqlite:///:memory:")) as session:
+            session.add(VirtualMemoryUsage(total=10, mean=2))
+            session.add(VirtualMemoryUsage(total=10, mean=3))
+            session.add(VirtualMemoryUsage(total=10, mean=4))
+            session.add(VirtualMemoryUsage(total=10, mean=4))
+            session.add(VirtualMemoryUsage(total=10, mean=5))

tests/orm/relationship/test_SystemComponent.py

@@ -3,7 +3,7 @@
 
 
 class TestSystemComponent(ORMTestCase):
-    def test_SystemComponent_user_property(self):
+    def test_SystemComponent_component_property(self):
         system = System(name="Fred's Computer")
         component = Component()
         system_component = SystemComponent(system=system)

tests/orm/relationship/test_SystemSnapshot.py

@@ -0,0 +1,11 @@
+from datetime import datetime
+from ..ORMTestCase import ORMTestCase
+from taskobra.orm import get_engine, get_session, Snapshot, System
+
+
+class TestSystemSnapshot(ORMTestCase):
+    def test_SystemSnapshot_creation(self):
+        system = System()
+        system.snapshots.append(Snapshot(timestamp=datetime(2020, 3, 9, 9, 53, 53)))
+
+        [self.assertIs(system, snapshot.system) for snapshot in system.snapshots]

tests/web/test_routes.py

@@ -1,3 +1,5 @@
+from unittest import skip
+
 from .WebTestCase import WebTestCase
 import flask