add moving average / percentiles to charge level

CHANGELOG.md

@@ -31,6 +31,7 @@
 - Improve rounding to month / weeks / days in Updates "Since Previous Update" column (#4164 - @swiffer)
 - feat: Improve cost filter on Charges dashboard to show charges with negative cost as well (#4179 - @jheredianet)
 - feat: display vehicle VIN as a fallback for vehicle name on grafana dashboards (#4198 - @arcastro)
+- feat: Add Moving Average / Percentiles to Charge Level dashboard & bucket data to support longer periods (#4200 - @swiffer)
 
 #### Translations
 

grafana/dashboards/charge-level.json

@@ -171,12 +171,13 @@
             "type": "grafana-postgresql-datasource",
             "uid": "TeslaMate"
           },
+          "editorMode": "code",
           "format": "time_series",
           "group": [],
           "hide": false,
           "metricColumn": "none",
           "rawQuery": true,
-          "rawSql": "SELECT\n\t$__time(date),\n\tbattery_level AS \"Battery Level\",\n\tusable_battery_level AS \"Usable Battery Level\"\nfrom positions\n\tWHERE $__timeFilter(date) AND car_id = $car_id\n\tORDER BY Time ASC\n;",
+          "rawSql": "SELECT\n\tdate_bin('2 minutes'::interval, date, to_timestamp(${__from:date:seconds})) as time,\n\tavg(battery_level) AS \"Battery Level\",\n\tavg(usable_battery_level) AS \"Usable Battery Level\"\nfrom positions\n\tWHERE $__timeFilter(date) AND car_id = $car_id\n\tgroup by time\n\tORDER BY time ASC\n;",
           "refId": "A",
           "select": [
             [
@@ -188,6 +189,23 @@
               }
             ]
           ],
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          },
           "timeColumn": "time",
           "where": [
             {
@@ -225,6 +243,35 @@
             ],
             "limit": 50
           }
+        },
+        {
+          "datasource": {
+            "type": "grafana-postgresql-datasource",
+            "uid": "TeslaMate"
+          },
+          "editorMode": "code",
+          "format": "table",
+          "hide": false,
+          "rawQuery": true,
+          "rawSql": "-- To be able to calucate percentiles for unevenly sampled values we are bucketing & gapfilling values before running calcuations\r\nwith positions_filtered as (\r\n    select\r\n        date,\r\n        battery_level\r\n    from\r\n        positions p\r\n    where\r\n        p.car_id = $car_id\r\n        -- p.ideal_battery_range_km condition is added to reduce overall amount of data and avoid data biases while driving (unevenly sampled data)\r\n        and p.ideal_battery_range_km is not null\r\n        and 1 = $include_average_percentiles\r\n),\r\ngen_date_series as (\r\n    select\r\n        -- series is used to bucket data and avoid gaps in series used to determine percentiles\r\n        generate_series(to_timestamp(${__from:date:seconds}), to_timestamp(${__to:date:seconds}), concat($bucket_width, ' seconds')::INTERVAL) as series_id\r\n),\r\ndate_series as (\r\n    select\r\n        series_id,\r\n        -- before joining, get beginning of next series to be able to left join `positions_filtered`\r\n        lead(series_id) over (order by series_id asc) as next_series_id\r\n    from\r\n        gen_date_series\r\n),\r\npositions_bucketed as (\r\n    select\r\n        series_id,\r\n        -- simple average can result in loss of accuracy, see https://www.timescale.com/blog/what-time-weighted-averages-are-and-why-you-should-care/ for details\r\n        avg(battery_level) as battery_level\r\n    from\r\n        date_series\r\n    left join positions_filtered on\r\n        positions_filtered.date >= date_series.series_id\r\n        and positions_filtered.date < date_series.next_series_id\r\n    group by\r\n        series_id\r\n),\r\n-- PostgreSQL cannot IGNORE NULLS via Window Functions LAST_VALUE - therefore use natural behavior of COUNT & MAX, see https://www.reddit.com/r/SQL/comments/wb949v/comment/ii5mmmi/ for details\r\npositions_bucketed_gapfilling_locf_intermediate as (\r\n    select\r\n        series_id,\r\n        battery_level,\r\n        count(battery_level) over (order by series_id) as i\r\n    from\r\n        positions_bucketed\r\n\r\n),\r\npositions_bucketed_gapfilled_locf as (\r\n    select\r\n        series_id,\r\n        max(battery_level) over (partition by i) as battery_level\r\n    from\r\n        positions_bucketed_gapfilling_locf_intermediate\r\n),\r\n-- PostgreSQL cannot use PERCENTILE_DISC as Window Function - therefore use ARRAY_AGG and UNNEST, see https://stackoverflow.com/a/72718604 for details\r\npositions_bucketed_gapfilled_locf_percentile_intermediate as (\r\n    select\r\n        series_id,\r\n        battery_level,\r\n        array_agg(battery_level) over w as arr,\r\n        avg(battery_level) over w as battery_level_avg\r\n    from\r\n        positions_bucketed_gapfilled_locf\r\n    window w as (rows between (86400 / $bucket_width) * ($days_moving_average_percentiles / 2) preceding and (86400 / $bucket_width) * ($days_moving_average_percentiles / 2) following)\r\n)\r\n\r\nselect\r\n    series_id,\r\n    case when battery_level is null then null else (select percentile_cont(0.075) within group (order by s) from unnest(arr) trick(s)) end as \"$days_moving_average_percentiles Day Moving 7,5% Percentile (${bucket_width:text} buckets)\",\r\n    case when battery_level is null then null else (battery_level_avg) end as \"$days_moving_average_percentiles Day Moving Average (${bucket_width:text} buckets)\",\r\n    case when battery_level is null then null else (select percentile_cont(0.5) within group (order by s) from unnest(arr) trick(s)) end as \"$days_moving_average_percentiles Day Moving Median (${bucket_width:text} buckets)\",\r\n    case when battery_level is null then null else (select percentile_cont(0.925) within group (order by s) from unnest(arr) trick(s)) end as \"$days_moving_average_percentiles Day Moving 92,5% Percentile (${bucket_width:text} buckets)\"\r\nfrom\r\n    positions_bucketed_gapfilled_locf_percentile_intermediate;",
+          "refId": "C",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          }
         }
       ],
       "title": "Charge Level",
@@ -263,7 +310,7 @@
       "type": "timeseries"
     }
   ],
-  "refresh": false,
+  "refresh": "",
   "schemaVersion": 39,
   "tags": [
     "tesla"
@@ -315,11 +362,121 @@
         "tagsQuery": "",
         "type": "query",
         "useTags": false
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": "2h",
+          "value": "7200"
+        },
+        "description": "Data used to calculate Moving Average / Percentiles is unevenly sampled in TeslaMate. To avoid biases towards more frequently sampled values, the data is bucketed. For buckets without sampled values, the last observed value is carried forward. Bucketing is not time-weighted but is a simple average. Increasing the bucket width results in a loss of accuracy.",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Bucket Width",
+        "multi": false,
+        "name": "bucket_width",
+        "options": [
+          {
+            "selected": false,
+            "text": "1h",
+            "value": "3600"
+          },
+          {
+            "selected": true,
+            "text": "2h",
+            "value": "7200"
+          },
+          {
+            "selected": false,
+            "text": "4h",
+            "value": "14400"
+          }
+        ],
+        "query": "1h : 3600, 2h : 7200, 4h : 14400",
+        "queryValue": "",
+        "skipUrlSync": false,
+        "type": "custom"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": "yes",
+          "value": "1"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Include Moving Average / Percentiles",
+        "multi": false,
+        "name": "include_average_percentiles",
+        "options": [
+          {
+            "selected": false,
+            "text": "no",
+            "value": "0"
+          },
+          {
+            "selected": true,
+            "text": "yes",
+            "value": "1"
+          }
+        ],
+        "query": "no : 0, yes : 1",
+        "queryValue": "",
+        "skipUrlSync": false,
+        "type": "custom"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": "1/6 of interval",
+          "value": "6"
+        },
+        "description": "",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Moving Average / Percentiles Width",
+        "multi": false,
+        "name": "intervals_moving_average_percentiles",
+        "options": [
+          {
+            "selected": true,
+            "text": "1/6 of interval",
+            "value": "6"
+          },
+          {
+            "selected": false,
+            "text": "1/12 of interval",
+            "value": "12"
+          }
+        ],
+        "query": "1/6 of interval : 6, 1/12 of interval : 12",
+        "queryValue": "",
+        "skipUrlSync": false,
+        "type": "custom"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "type": "grafana-postgresql-datasource",
+          "uid": "TeslaMate"
+        },
+        "definition": "select ((${__to:date:seconds} - ${__from:date:seconds}) / 86400 / $intervals_moving_average_percentiles)",
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "name": "days_moving_average_percentiles",
+        "options": [],
+        "query": "select ((${__to:date:seconds} - ${__from:date:seconds}) / 86400 / $intervals_moving_average_percentiles)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
       }
     ]
   },
   "time": {
-    "from": "now-7d",
+    "from": "now-6M",
     "to": "now"
   },
   "timepicker": {
@@ -350,6 +507,6 @@
   "timezone": "",
   "title": "Charge Level",
   "uid": "WopVO_mgz",
-  "version": 5,
+  "version": 16,
   "weekStart": ""
 }