Moving GMT to microjoules (#932)

* Splitting measurements table in metrics and values; Moving sampling_resolution to phase_stats

* Typos and textual additions

* (Tests): Updated tests and demo data to new measurement table split

* Renamed sampling resolution to sampling rate

* Added sampling rate also to comparison views

* (Tests): Aligned display order of phases stats and tests

* (rework): Moved all providers from mJ to uJ

* (rework): Moved calibrate to uJ

* (feature/rework): Fronted can now display values as Wh instead of J. Reworked transformation of values and units

* (rework): Moving sampling rate to INT; Using uJ as default energy now in phase_stats

* Clarifications and comments and removed legacy

* (Tests): Updated all test data to uJ; Updated all tests for new uJ energy; added new tests for SR

* (fix): tests

* Added ee

* (style): Typos [skip ci]

* (Fix): Ordering was incorrect for getting compare data.

* (Fix): Merge markers

* (fix): New uJ demo data got lost

* (fix): Some fixes while refactoring

* (fix): Tests

* (Tests): Network I/O

* (Style): Some typos etc.

* (rework): Moved phase_stats totally to Decimal

* (Fix): Tests - NetworkIO sample data is actually sampled at 1 s interval

* (Tests): Added NetworkIOProvider Tests

api/api_helpers.py

@@ -58,12 +58,6 @@ def store_artifact(artifact_type: Enum, key:str, data, ex=2592000):
 # The same unit for proper comparison!
 #
 def rescale_metric_value(value, unit):
-    if unit == 'mJ':
-        value = value * 1_000
-        unit = 'uJ'
-
-    # We only expect values to be uJ for energy in the future. Changing values now temporarily.
-    # TODO: Refactor this once all data in the DB is uJ
     if unit not in ('uJ', 'ug') and not unit.startswith('ugCO2e/'):
         raise ValueError('Unexpected unit occured for metric rescaling: ', unit)
 
@@ -249,7 +243,7 @@ def get_comparison_details(user, ids, comparison_db_key):
         WHERE
             (TRUE = %s OR user_id = ANY(%s::int[]))
             AND id = ANY(%s::uuid[])
-        ORDER BY created_at DESC -- must be same order as get_phase_stats so that the order in the comparison bar charts aligns with the comparsion_details array
+        ORDER BY created_at ASC -- must be same order as get_phase_stats so that the order in the comparison bar charts aligns with the comparsion_details array
     ''').format(sql.Identifier(comparison_db_key))
 
     params = (user.is_super_user(), user.visible_users(), ids)
@@ -398,7 +392,7 @@ def get_phase_stats(user, ids):
                 (TRUE = %s OR b.user_id = ANY(%s::int[]))
                 AND a.run_id = ANY(%s::uuid[])
             ORDER BY
-                -- at least the run_ids must be same order as get_comparsion_details so that the order in the comparison bar charts aligns with the comparsion_details array
+                -- at least the run_ids must be same order as get_comparison_details so that the order in the comparison bar charts aligns with the comparsion_details array
                 b.created_at ASC,
                 a.id ASC
             """

api/main.py

@@ -491,6 +491,7 @@ async def get_measurements_single(run_id: str, user: User = Depends(authenticate
     params = (user.is_super_user(), user.visible_users(), run_id)
 
     # extremely important to order here, cause the charting library in JS cannot do that automatically!
+    # Furthermore we do time-lag caclulations and need the order of metric first and then time in stats.js:179... . Please do not change
     query = f"{query} ORDER BY mm.metric ASC, mm.detail_name ASC, mv.time ASC"
 
     data = DB().fetch_all(query, params=params)

docker/structure.sql

@@ -172,9 +172,9 @@ CREATE TABLE phase_stats (
     type text NOT NULL,
     max_value bigint,
     min_value bigint,
-    sampling_rate_avg DOUBLE PRECISION NOT NULL,
-    sampling_rate_max DOUBLE PRECISION NOT NULL,
-    sampling_rate_95p DOUBLE PRECISION NOT NULL,
+    sampling_rate_avg int NOT NULL,
+    sampling_rate_max int NOT NULL,
+    sampling_rate_95p int NOT NULL,
     unit text NOT NULL,
     created_at timestamp with time zone DEFAULT now(),
     updated_at timestamp with time zone

frontend/js/authentication.js

@@ -1,7 +1,6 @@
 (() => {
     $(window).on('load', function() {
         const authentication_token = localStorage.getItem('authentication_token');
-
         if (authentication_token != null) {
             $("#authentication-token").val(authentication_token);
         }

frontend/js/ci.js

@@ -46,7 +46,7 @@ const getEChartsOptions = () => {
 
 const getChartOptions = (measurements) => {
     const options = getEChartsOptions();
-    options.title.text = `Workflow energy cost per run [mJ]`;
+    options.title.text = `Workflow energy cost per run [J]`;
 
     const legend = new Set()
     const labels = []

frontend/js/helpers/charts.js

@@ -475,14 +475,14 @@ const removeKeyMetricsRadarChart = (phase) => {
     document.querySelector(`.ui.tab[data-tab='${phase}'] .radar-chart`).remove()
 }
 
-const displayKeyMetricsBarChart = (legend, labels, data, phase) => {
+const displayKeyMetricsBarChart = (legend, labels, data, phase, unit) => {
 
     let series = data.map((el, idx) => { return {type: "bar", name: legend[idx], data: el}})
     let chartDom = document.querySelector(`.ui.tab[data-tab='${phase}'] .bar-chart .statistics-chart`);
     document.querySelector(`.ui.tab[data-tab='${phase}'] .bar-chart .chart-title`).innerText = TOP_BAR_CHART_TITLE;
 
     let myChart = echarts.init(chartDom);
-    let options = getLineBarChartOptions(null, labels, series, null, TOP_BAR_CHART_UNIT, 'category', null, true);
+    let options = getLineBarChartOptions(null, labels, series, null, unit, 'category', null, true);
     myChart.setOption(options);
 
     // set callback when ever the user changes the viewport
@@ -546,7 +546,7 @@ const displayKeyMetricsEmbodiedCarbonChart = (phase) => {
 
 }
 
-const displayTotalChart = (legend, labels, data) => {
+const displayTotalChart = (legend, labels, data, unit) => {
     const chartDom = document.querySelector(`#total-phases-data .bar-chart .statistics-chart`);
     document.querySelector(`#total-phases-data .bar-chart .chart-title`).innerText = TOTAL_CHART_BOTTOM_TITLE;
 
@@ -573,7 +573,7 @@ const displayTotalChart = (legend, labels, data) => {
     }
 
 
-    const options = getLineBarChartOptions(null, labels, series, null, TOTAL_CHART_UNIT, 'category', null, true)
+    const options = getLineBarChartOptions(null, labels, series, null, unit, 'category', null, true)
 
     myChart.setOption(options);
         // set callback when ever the user changes the viewport

frontend/js/helpers/config.js.example

@@ -261,12 +261,13 @@ METRIC_MAPPINGS = {
     'disk_io_bytesread_powermetrics_vm': {
         'clean_name': 'Disk I/O (Read)',
         'source': 'powermetrics',
-        'explanation': 'Disk I/O - Bytes read per second for SDD/HDD',
+        'explanation': 'Disk I/O - Bytes read per second for SSD/HDD',
     },
     'disk_io_byteswritten_powermetrics_vm': {
         'clean_name': 'Disk I/O (Write)',
         'source': 'powermetrics',
-        'explanation': 'Disk I/O - Bytes written per second for SDD/HDD',
+        'explanation': 'Disk I/O - Bytes written per second for SSD/HDD',
+
     },
     'energy_impact_powermetrics_vm': {
         'clean_name': 'Energy impact',
@@ -389,7 +390,6 @@ const DEFAULT_ENERGY_TIMELINE_BADGE_METRICS = [
 // title and filter function for the top left most chart in the Detailed Metrics / Compare view
 const TOTAL_CHART_BOTTOM_TITLE = 'Total energy consumption';
 const TOTAL_CHART_BOTTOM_LABEL = 'Machine Energy';
-const TOTAL_CHART_UNIT = 'mJ';
 // function must return boolean
 const total_chart_bottom_condition = (metric) => {
     if(metric.match(/^.*_energy.*_machine$/) !== null) return true;
@@ -398,7 +398,6 @@ const total_chart_bottom_condition = (metric) => {
 
 // title and filter function for the top left most chart in the Detailed Metrics / Compare view
 const TOP_BAR_CHART_TITLE = 'Energy metrics'
-const TOP_BAR_CHART_UNIT = 'mJ'
 const top_bar_chart_condition = (metric) => {
     if(metric.indexOf('_energy_') !== -1) return true;
     return false;

frontend/js/helpers/converters.js

@@ -1,45 +1,52 @@
-const convertValue = (value, unit) => {
-    // we do not allow a dynamic rescaling here, as we need all the units we feed into
-    // to be on the same order of magnitude
+let display_in_watthours = localStorage.getItem('display_in_watthours');
+if(display_in_watthours == 'true') display_in_watthours = true;
+else display_in_watthours = false;
 
-    if (value == null) return [value, unit];
+const transformIfNotNull = (value, divide_by) => {
+    if (value == null) return null;
+    return (value / divide_by);
+}
 
-    if(unit.startsWith('ugCO2e/')) return [(value/(10**6)).toFixed(2), unit.substr(1)]
+// we do not allow a dynamic rescaling here, as we need all the units we feed into
+// to be on the same order of magnitude for comparisons and calcuations
+//
+// Function furthemore uses .substr instead of just replacing the unit, as some units have demominators like Bytes/s or
+// ugCO2e/ page request which we want to retain
+const convertValue = (value, unit) => {
 
-    switch (unit) {
-    case 'mJ':
-        return [(value / 1_000).toFixed(2), 'J'];
-        break;
-    case 'mW':
-        return [(value / 1_000).toFixed(2), 'W'];
-        break;
-    case 'Ratio':
-        return [(value / 100).toFixed(2), '%'];
-        break;
-    case 'centi°C':
-        return [(value / 100).toFixed(2), '°C'];
-        break;
-    case 'Hz':
-        return [(value / 1000000).toFixed(2), 'GHz'];
-        break;
-    case 'ns':
-        return [(value / 1_000_000_000).toFixed(2), 's'];
-        break;
-    case 'us':
-        return [(value / 1_000_000).toFixed(2), 's'];
-        break;
-    case 'ug':
-        return [(value/(10**6)).toFixed(2), 'g']
-        break;
-    case 'Bytes':
-        return [(value / 1_000_000).toFixed(2), 'MB'];
-        break;
-    case 'Bytes/s':
-        return [(value / 1_000_000).toFixed(2), 'MB/s'];
-        break;
+    const compare_unit = unit.split('/', 2)[0]
 
-    default:
-        return [value, unit];        // no conversion in default calse
+    switch (compare_unit) {
+        case 'ugCO2e':
+            return [transformIfNotNull(value, 1_000_000), unit.substr(1)]
+        case 'mJ':
+            if (display_in_watthours)
+                return [transformIfNotNull(value, 1_000 * 3_600), `Wh${unit.substr(2)}`];
+            else
+                return [transformIfNotNull(value, 1_000), unit.substr(1)];
+        case 'uJ':
+            if (display_in_watthours)
+                return [transformIfNotNull(value, 1_000_000 * 3_600), `Wh${unit.substr(2)}`];
+            else
+                return [transformIfNotNull(value, 1_000_000), unit.substr(1)];
+        case 'mW':
+            return [transformIfNotNull(value, 1_000), unit.substr(1)];
+        case 'Ratio':
+            return [transformIfNotNull(value, 100), `%${unit.substr(5)}`];
+        case 'centi°C':
+            return [transformIfNotNull(value, 100), unit.substr(5)];
+        case 'Hz':
+            return [transformIfNotNull(value, 1_000_000_000), `G${unit}`];
+        case 'ns':
+            return [transformIfNotNull(value, 1_000_000_000), unit.substr(1)];
+        case 'us':
+            return [transformIfNotNull(value, 1_000_000), unit.substr(1)];
+        case 'ug':
+            return [transformIfNotNull(value, 1_000_000), unit.substr(1)]
+        case 'Bytes':
+            return [transformIfNotNull(value, 1_000_000), `MB${unit.substr(5)}`];
+        default:
+            return [value, unit];        // no conversion in default case
     }
 }