From 87e9db6e8afb7af52b7e4464974f9fcb521aaa84 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Fri, 20 Dec 2024 11:39:48 +0100 Subject: [PATCH 01/10] Added average function to badges --- api/eco_ci.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/api/eco_ci.py b/api/eco_ci.py index 91109be1..2698e244 100644 --- a/api/eco_ci.py +++ b/api/eco_ci.py @@ -243,35 +243,48 @@ async def get_ci_badge_get(repo: str, branch: str, workflow:str, mode: str = 'la if metric == 'energy': metric = 'energy_uj' metric_unit = 'uJ' - label = 'Energy used' + label = 'energy used' default_color = 'orange' elif metric == 'carbon': metric = 'carbon_ug' metric_unit = 'ug' - label = 'Carbon emitted' + label = 'carbon emitted' default_color = 'black' else: raise RequestValidationError('Unsupported metric requested') + if duration_days and (duration_days < 1 or duration_days > 365): + raise RequestValidationError('Duration days must be between 1 and 365 days') + params = [repo, branch, workflow] - query = f""" - SELECT SUM({metric}) + query = ''' + SELECT {}({}) FROM ci_measurements WHERE repo = %s AND branch = %s AND workflow_id = %s - """ + {} + ''' - if mode == 'last': - query = f"""{query} - GROUP BY run_id - ORDER BY MAX(created_at) DESC - LIMIT 1 - """ + if mode == 'avg': + if not duration_days: + raise RequestValidationError('Duration days must be set for average') + query = query.format('AVG', metric, 'AND created_at > NOW() - make_interval(days => %s)') + params.append(duration_days) + label = f"Per run moving average ({duration_days} days) {label}" + elif mode == 'last': + query = query.format('SUM', metric, 'GROUP BY run_id ORDER BY MAX(created_at) DESC LIMIT 1') + label = f"Last run {label}" elif mode == 'totals' and duration_days: - query = f"{query} AND created_at > NOW() - make_interval(days => %s)" + query = query.format('SUM', metric, 'AND created_at > NOW() - make_interval(days => %s)') params.append(duration_days) + label = f"Last {duration_days} days total {label}" + elif mode == 'totals': + query = query.format('SUM', metric, '') + label = f"All runs total {label}" + else: + raise RuntimeError('Unknown mode') data = DB().fetch_one(query, params=params) From 4c0589fe9dbd8b36c898aedc843d3cf155240133 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Fri, 20 Dec 2024 11:56:27 +0100 Subject: [PATCH 02/10] Added tests --- tests/api/test_api_eco_ci.py | 56 ++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/api/test_api_eco_ci.py b/tests/api/test_api_eco_ci.py index 54bdc7fe..1fa3bad5 100644 --- a/tests/api/test_api_eco_ci.py +++ b/tests/api/test_api_eco_ci.py @@ -172,6 +172,62 @@ def test_ci_measurement_add_filters(): data = fetch_data_from_db(measurement_model['run_id']) compare_carbondb_data(measurement_model, data) +def test_ci_badge_duration_error(): + response = requests.get(f"{API_URL}/v1/ci/badge/get?repo=green-coding-solutions/ci-carbon-testing&branch=main&workflow=48163287&mode=avg&duration_days=900", timeout=15) + assert response.status_code == 422 + assert response.text == '{"success":false,"err":"Duration days must be between 1 and 365 days","body":null}' + + +def test_ci_badge_get_last(): + Tests.import_demo_data() + + response = requests.get(f"{API_URL}/v1/ci/badge/get?repo=green-coding-solutions/ci-carbon-testing&branch=main&workflow=48163287&mode=last", timeout=15) + assert response.status_code == 200, Tests.assertion_info('success', response.text) + assert 'Last run energy used' in response.text, Tests.assertion_info('success', response.text) + assert '28.95 J' in response.text, Tests.assertion_info('success', response.text) + + response = requests.get(f"{API_URL}/v1/ci/badge/get?repo=green-coding-solutions/ci-carbon-testing&branch=main&workflow=48163287&mode=last&metric=carbon", timeout=15) + assert response.status_code == 200, Tests.assertion_info('success', response.text) + assert 'Last run carbon emitted' in response.text, Tests.assertion_info('success', response.text) + assert '17.32 mg' in response.text, Tests.assertion_info('success', response.text) + + +def test_ci_badge_get_totals(): + Tests.import_demo_data() + + response = requests.get(f"{API_URL}/v1/ci/badge/get?repo=green-coding-solutions/ci-carbon-testing&branch=main&workflow=48163287&mode=totals", timeout=15) + assert response.status_code == 200, Tests.assertion_info('success', response.text) + assert 'All runs total energy used' in response.text, Tests.assertion_info('success', response.text) + assert '49.02 kJ' in response.text, Tests.assertion_info('success', response.text) + + response = requests.get(f"{API_URL}/v1/ci/badge/get?repo=green-coding-solutions/ci-carbon-testing&branch=main&workflow=48163287&mode=totals&metric=carbon", timeout=15) + assert response.status_code == 200, Tests.assertion_info('success', response.text) + assert 'All runs total carbon emitted' in response.text, Tests.assertion_info('success', response.text) + assert '15.56 g' in response.text, Tests.assertion_info('success', response.text) + +def test_ci_badge_get_average(): + + for i in range(1,3): + measurement_model = MEASUREMENT_MODEL_NEW.copy() + measurement_model['energy_uj'] += i*1000 + measurement_model['carbon_ug'] = i*1000 + + response = requests.post(f"{API_URL}/v2/ci/measurement/add", json=measurement_model, timeout=15) + assert response.status_code == 204, Tests.assertion_info('success', response.text) + + + response = requests.get(f"{API_URL}/v1/ci/badge/get?repo={MEASUREMENT_MODEL_NEW['repo']}&branch={MEASUREMENT_MODEL_NEW['branch']}&workflow={MEASUREMENT_MODEL_NEW['workflow']}&mode=avg&duration_days=5", timeout=15) + assert response.status_code == 200, Tests.assertion_info('success', response.text) + assert 'Per run moving average (5 days) energy used' in response.text, Tests.assertion_info('success', response.text) + assert '124.50 mJ' in response.text, Tests.assertion_info('success', response.text) + + response = requests.get(f"{API_URL}/v1/ci/badge/get?repo={MEASUREMENT_MODEL_NEW['repo']}&branch={MEASUREMENT_MODEL_NEW['branch']}&workflow={MEASUREMENT_MODEL_NEW['workflow']}&mode=avg&duration_days=5&metric=carbon", timeout=15) + assert response.status_code == 200, Tests.assertion_info('success', response.text) + + assert 'Per run moving average (5 days) carbon emitted' in response.text, Tests.assertion_info('success', response.text) + assert '1.50 mg' in response.text, Tests.assertion_info('success', response.text) + + ## helpers def fetch_data_from_db(run_id): From 0d4e1974ecf7e1d9e8d10f2c1b2e918633166317 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Fri, 20 Dec 2024 13:57:05 +0100 Subject: [PATCH 03/10] Average now done correctly grouped per run_id --- api/eco_ci.py | 21 +++++++++++++-------- tests/api/test_api_eco_ci.py | 15 +++++++++++---- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/api/eco_ci.py b/api/eco_ci.py index 2698e244..fac507d1 100644 --- a/api/eco_ci.py +++ b/api/eco_ci.py @@ -260,28 +260,33 @@ async def get_ci_badge_get(repo: str, branch: str, workflow:str, mode: str = 'la params = [repo, branch, workflow] - query = ''' - SELECT {}({}) + query = f""" + SELECT SUM({metric}) FROM ci_measurements WHERE repo = %s AND branch = %s AND workflow_id = %s - {} - ''' + """ if mode == 'avg': if not duration_days: raise RequestValidationError('Duration days must be set for average') - query = query.format('AVG', metric, 'AND created_at > NOW() - make_interval(days => %s)') + query = f""" + WITH my_table as ( + SELECT SUM({metric}) my_sum + FROM ci_measurements + WHERE repo = %s AND branch = %s AND workflow_id = %s AND created_at > NOW() - make_interval(days => %s) + GROUP BY run_id + ) SELECT AVG(my_sum) FROM my_table; + """ params.append(duration_days) label = f"Per run moving average ({duration_days} days) {label}" elif mode == 'last': - query = query.format('SUM', metric, 'GROUP BY run_id ORDER BY MAX(created_at) DESC LIMIT 1') + query = f"{query} GROUP BY run_id ORDER BY MAX(created_at) DESC LIMIT 1" label = f"Last run {label}" elif mode == 'totals' and duration_days: - query = query.format('SUM', metric, 'AND created_at > NOW() - make_interval(days => %s)') + query = f"{query} AND created_at > NOW() - make_interval(days => %s)" params.append(duration_days) label = f"Last {duration_days} days total {label}" elif mode == 'totals': - query = query.format('SUM', metric, '') label = f"All runs total {label}" else: raise RuntimeError('Unknown mode') diff --git a/tests/api/test_api_eco_ci.py b/tests/api/test_api_eco_ci.py index 1fa3bad5..ae69277a 100644 --- a/tests/api/test_api_eco_ci.py +++ b/tests/api/test_api_eco_ci.py @@ -209,23 +209,30 @@ def test_ci_badge_get_average(): for i in range(1,3): measurement_model = MEASUREMENT_MODEL_NEW.copy() - measurement_model['energy_uj'] += i*1000 - measurement_model['carbon_ug'] = i*1000 + measurement_model['carbon_ug'] = 1000 + response = requests.post(f"{API_URL}/v2/ci/measurement/add", json=measurement_model, timeout=15) + assert response.status_code == 204, Tests.assertion_info('success', response.text) + for i in range(1,6): + measurement_model = MEASUREMENT_MODEL_NEW.copy() + measurement_model['energy_uj'] *= 1000 + measurement_model['carbon_ug'] = i*100000 + measurement_model['run_id'] = 'Other run' response = requests.post(f"{API_URL}/v2/ci/measurement/add", json=measurement_model, timeout=15) assert response.status_code == 204, Tests.assertion_info('success', response.text) + response = requests.get(f"{API_URL}/v1/ci/badge/get?repo={MEASUREMENT_MODEL_NEW['repo']}&branch={MEASUREMENT_MODEL_NEW['branch']}&workflow={MEASUREMENT_MODEL_NEW['workflow']}&mode=avg&duration_days=5", timeout=15) assert response.status_code == 200, Tests.assertion_info('success', response.text) assert 'Per run moving average (5 days) energy used' in response.text, Tests.assertion_info('success', response.text) - assert '124.50 mJ' in response.text, Tests.assertion_info('success', response.text) + assert '307.62 J' in response.text, Tests.assertion_info('success', response.text) response = requests.get(f"{API_URL}/v1/ci/badge/get?repo={MEASUREMENT_MODEL_NEW['repo']}&branch={MEASUREMENT_MODEL_NEW['branch']}&workflow={MEASUREMENT_MODEL_NEW['workflow']}&mode=avg&duration_days=5&metric=carbon", timeout=15) assert response.status_code == 200, Tests.assertion_info('success', response.text) assert 'Per run moving average (5 days) carbon emitted' in response.text, Tests.assertion_info('success', response.text) - assert '1.50 mg' in response.text, Tests.assertion_info('success', response.text) + assert '751.00 mg' in response.text, Tests.assertion_info('success', response.text) ## helpers From 950ac8d8ed92930c2758ad5cf8feed603e6ad3e7 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Fri, 20 Dec 2024 13:59:30 +0100 Subject: [PATCH 04/10] Tab init must happen very early so tab menu is not broken if return happens --- frontend/js/ci.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/js/ci.js b/frontend/js/ci.js index bfd2bb74..71daff14 100644 --- a/frontend/js/ci.js +++ b/frontend/js/ci.js @@ -473,6 +473,9 @@ const bindRefreshButton = (repo, branch, workflow_id, chart_instance) => { $(document).ready((e) => { (async () => { + + $('.ui.secondary.menu .item').tab(); // must happen very early so tab menu is not broken if return happens + const query_string = window.location.search; const url_params = (new URLSearchParams(query_string)) @@ -550,8 +553,6 @@ $(document).ready((e) => { displayStatsTable(filteredMeasurements); }); - $('.ui.secondary.menu .item').tab(); - setTimeout(function(){console.log("Resize"); window.dispatchEvent(new Event('resize'))}, 500); })(); }); From 0c7f6da918eaca5458f2f8efaf1b10670a6a6a26 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Fri, 20 Dec 2024 14:13:43 +0100 Subject: [PATCH 05/10] Making totals more visible --- frontend/css/green-coding.css | 4 ++++ frontend/js/ci.js | 30 ++++++++++++++++-------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/frontend/css/green-coding.css b/frontend/css/green-coding.css index c6a962b3..8d9313b1 100644 --- a/frontend/css/green-coding.css +++ b/frontend/css/green-coding.css @@ -310,6 +310,10 @@ a { transition: opacity .1s ease; } +.bold { + font-weight: bold; +} + /* PRINT STYLESHEETS */ @media print { diff --git a/frontend/js/ci.js b/frontend/js/ci.js index 71daff14..830388ff 100644 --- a/frontend/js/ci.js +++ b/frontend/js/ci.js @@ -74,6 +74,8 @@ const createStatsArrays = (measurements) => { // iterates 2n times (1 full, 1 b measurements.forEach(measurement => { let [energy_uj, run_id, created_at, label, cpu, commit_hash, duration_us, source, cpu_util, workflow_name, lat, lon, city, carbon_intensity_g, carbon_ug] = measurement; + label = escapeString(label) // we print it later and it is user submitted string + if (!measurementsByLabel[label]) { measurementsByLabel[label] = { energy_uj: [], @@ -267,30 +269,30 @@ const displayStatsTable = (measurements) => { const full_run_stats_avg_node = document.createElement("tr") full_run_stats_avg_node.innerHTML += ` - All steps - ${numberFormatter.format(full_run_stats.energy_uj.avg/1000000)} J (± ${numberFormatter.format(full_run_stats.energy_uj.stddev_rel)}%) - ${numberFormatter.format(full_run_stats.duration_us.avg/1000000)} s (± ${numberFormatter.format(full_run_stats.duration_us.stddev_rel)}%) - ${numberFormatter.format(full_run_stats.cpu_util.avg)}% (± ${numberFormatter.format(full_run_stats.cpu_util.stddev_rel)}%%) - ${numberFormatter.format(full_run_stats.carbon_intensity_g.avg)} gCO2/kWh (± ${numberFormatter.format(full_run_stats.carbon_intensity_g.stddev_rel)}%) - ${numberFormatterLong.format(full_run_stats.carbon_ug.avg/1000000)} gCO2e (± ${numberFormatter.format(full_run_stats.carbon_ug.stddev_rel)}%) - ${fullRunArray.count}`; + Total run + ${numberFormatter.format(full_run_stats.energy_uj.avg/1000000)} J (± ${numberFormatter.format(full_run_stats.energy_uj.stddev_rel)}%) + ${numberFormatter.format(full_run_stats.duration_us.avg/1000000)} s (± ${numberFormatter.format(full_run_stats.duration_us.stddev_rel)}%) + ${numberFormatter.format(full_run_stats.cpu_util.avg)}% (± ${numberFormatter.format(full_run_stats.cpu_util.stddev_rel)}%%) + ${numberFormatter.format(full_run_stats.carbon_intensity_g.avg)} gCO2/kWh (± ${numberFormatter.format(full_run_stats.carbon_intensity_g.stddev_rel)}%) + ${numberFormatterLong.format(full_run_stats.carbon_ug.avg/1000000)} gCO2e (± ${numberFormatter.format(full_run_stats.carbon_ug.stddev_rel)}%) + ${fullRunArray.count}`; avg_table.appendChild(full_run_stats_avg_node); const full_run_stats_total_node = document.createElement("tr") full_run_stats_total_node.innerHTML += ` - All steps - ${numberFormatter.format(full_run_stats.energy_uj.total/1000000)} J - ${numberFormatter.format(full_run_stats.duration_us.total/1000000)} s - ${numberFormatterLong.format(full_run_stats.carbon_ug.total/1000000)} gCO2e - ${fullRunArray.count}`; + Per total run + ${numberFormatter.format(full_run_stats.energy_uj.total/1000000)} J + ${numberFormatter.format(full_run_stats.duration_us.total/1000000)} s + ${numberFormatterLong.format(full_run_stats.carbon_ug.total/1000000)} gCO2e + ${fullRunArray.count}`; total_table.appendChild(full_run_stats_total_node) for (const label in labelsArray) { const label_stats = calculateStats(labelsArray[label].energy_uj, labelsArray[label].carbon_ug, labelsArray[label].carbon_intensity_g, labelsArray[label].duration_us, labelsArray[label].cpu_util) const label_stats_avg_node = document.createElement("tr") label_stats_avg_node.innerHTML += ` - ${label} + ${label} ${numberFormatter.format(label_stats.energy_uj.avg/1000000)} J (± ${numberFormatter.format(label_stats.energy_uj.stddev_rel)}%) ${numberFormatter.format(label_stats.duration_us.avg/1000000)} s (± ${numberFormatter.format(label_stats.duration_us.stddev_rel)}%) ${numberFormatter.format(label_stats.cpu_util.avg)}% (± ${numberFormatter.format(label_stats.cpu_util.stddev_rel)}%%) @@ -302,7 +304,7 @@ const displayStatsTable = (measurements) => { const label_stats_total_node = document.createElement("tr") label_stats_total_node.innerHTML += ` - ${label} + ${label} ${numberFormatter.format(label_stats.energy_uj.total/1000000)} J ${numberFormatter.format(label_stats.duration_us.total/1000000)} s ${numberFormatterLong.format(label_stats.carbon_ug.total/1000000)} gCO2e From e915f16d4b41653350c39b7265300aed52416bcd Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Sat, 21 Dec 2024 22:59:57 +0100 Subject: [PATCH 06/10] Full rework of CI view. Stats are now fetched via API; Stats table only on request; URL state is now updated on new fetch data --- api/eco_ci.py | 57 ++++++++ frontend/ci.html | 28 +++- frontend/js/ci.js | 271 +++++++++++------------------------- frontend/js/compare.js | 7 - frontend/js/helpers/main.js | 5 + frontend/js/stats.js | 13 +- frontend/js/timeline.js | 6 - 7 files changed, 173 insertions(+), 214 deletions(-) diff --git a/api/eco_ci.py b/api/eco_ci.py index fac507d1..c83d4aea 100644 --- a/api/eco_ci.py +++ b/api/eco_ci.py @@ -179,6 +179,63 @@ async def get_ci_measurements(repo: str, branch: str, workflow: str, start_date: return ORJSONResponse({'success': True, 'data': data}) +@router.get('/v1/ci/stats') +async def get_ci_stats(repo: str, branch: str, workflow: str, start_date: date, end_date: date): + + + query = ''' + WITH my_table as ( + SELECT + SUM(energy_uj) as a, + SUM(duration_us) as b, + SUM(cpu_util_avg) as c, + SUM(carbon_intensity_g) as d, + SUM(carbon_ug) as e + FROM ci_measurements + WHERE + repo = %s AND branch = %s AND workflow_id = %s + AND DATE(created_at) >= TO_DATE(%s, 'YYYY-MM-DD') AND DATE(created_at) <= TO_DATE(%s, 'YYYY-MM-DD') + GROUP BY run_id + ) SELECT + -- Cast is to avoid DECIMAL which ORJJSON cannot handle + AVG(a)::float, SUM(a)::float, STDDEV(a)::float, (STDDEV(a) / NULLIF(AVG(a), 0))::float * 100, + AVG(b)::float, SUM(b)::float, STDDEV(b)::float, (STDDEV(b) / NULLIF(AVG(b), 0))::float * 100, + AVG(c)::float, SUM(c)::float, STDDEV(c)::float, (STDDEV(c) / NULLIF(AVG(c), 0))::float * 100, + AVG(d)::float, SUM(d)::float, STDDEV(d)::float, (STDDEV(d) / NULLIF(AVG(d), 0))::float * 100, + AVG(e)::float, SUM(e)::float, STDDEV(e)::float, (STDDEV(e) / NULLIF(AVG(e), 0))::float * 100, + COUNT(*) + FROM my_table; + ''' + params = (repo, branch, workflow, str(start_date), str(end_date)) + totals_data = DB().fetch_one(query, params=params) + + if totals_data is None or totals_data[0] is None: # aggregate query always returns row + return Response(status_code=204) # No-Content + + query = ''' + SELECT + -- Cast is to avoid DECIMAL which ORJJSON cannot handle + AVG(energy_uj)::float, SUM(energy_uj)::float, STDDEV(energy_uj)::float, (STDDEV(energy_uj) / NULLIF(AVG(energy_uj), 0))::float * 100, + AVG(duration_us)::float, SUM(duration_us)::float, STDDEV(duration_us)::float, (STDDEV(duration_us) / NULLIF(AVG(duration_us), 0))::float * 100, + AVG(cpu_util_avg)::float, SUM(cpu_util_avg)::float, STDDEV(cpu_util_avg)::float, (STDDEV(cpu_util_avg) / NULLIF(AVG(cpu_util_avg), 0))::float * 100, + AVG(carbon_intensity_g)::float, SUM(carbon_intensity_g)::float, STDDEV(carbon_intensity_g)::float, (STDDEV(carbon_intensity_g) / NULLIF(AVG(carbon_intensity_g), 0))::float * 100, + AVG(carbon_ug)::float, SUM(carbon_ug)::float, STDDEV(carbon_ug)::float, (STDDEV(carbon_ug) / NULLIF(AVG(carbon_ug), 0))::float * 100, + COUNT(*), label + FROM ci_measurements + WHERE + repo = %s AND branch = %s AND workflow_id = %s + AND DATE(created_at) >= TO_DATE(%s, 'YYYY-MM-DD') AND DATE(created_at) <= TO_DATE(%s, 'YYYY-MM-DD') + GROUP BY label + ''' + params = (repo, branch, workflow, str(start_date), str(end_date)) + per_label_data = DB().fetch_all(query, params=params) + + if per_label_data is None or per_label_data[0] is None: + return Response(status_code=204) # No-Content + + return ORJSONResponse({'success': True, 'data': {'totals': totals_data, 'per_label': per_label_data}}) + + @router.get('/v1/ci/repositories') async def get_ci_repositories(repo: str | None = None, sort_by: str = 'name'): diff --git a/frontend/ci.html b/frontend/ci.html index 0bf113ad..3e78f8ff 100644 --- a/frontend/ci.html +++ b/frontend/ci.html @@ -158,7 +158,33 @@

Pipeline stats

-
+ +
+ + +
+
+ Runs detail table is not displayed automatically +
+

Please click the button below to fetch data.

+

You can change the default behaviour under Settings

+ +
+
    +
    + + +
    + + +