From 52a659d5b540f4a9b66a2da5d52f61c43ad11a09 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Thu, 24 Oct 2024 10:41:19 +0200 Subject: [PATCH 1/8] Make sure our recent fix to #1196 does not ignore windows that are not related to the last night, fixes #1196 --- R/g.part5.savetimeseries.R | 19 ++++++++++++++++--- R/g.report.part5.R | 36 ++++++++++++++++++++---------------- man/GGIR.Rd | 10 ++++++---- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/R/g.part5.savetimeseries.R b/R/g.part5.savetimeseries.R index 38712f66d..dd1e5182c 100644 --- a/R/g.part5.savetimeseries.R +++ b/R/g.part5.savetimeseries.R @@ -19,6 +19,7 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname, } # Create numeric time to faciltiate merging ts$timenum = as.numeric(ts$timestamp) + epochSize = ts$timenum[2] - ts$timenum[1] ms5rawlevels$timenum = as.numeric(ms5rawlevels$date_time) mdat = merge(ts, ms5rawlevels, by = "timenum") rm(ts, ms5rawlevels) @@ -27,11 +28,23 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname, mdat = mdat[,-which(names(mdat) == "date_time")] if ("require_complete_lastnight_part5" %in% names(params_output) && params_output[["require_complete_lastnight_part5"]] == TRUE) { - last_timestamp = as.numeric(format(mdat$timestamp[length(mdat$timestamp)], "%H")) - if ((timewindow == "MM" || timewindow == "OO") && last_timestamp < 9) { + N_window0_at_end = which(rev(mdat$window) != 0)[1] + N_hours_window0_at_end = ((N_window0_at_end * epochSize) / 3600) + lastHour = as.numeric(format(mdat$timestamp[length(mdat$timestamp)], "%H")) + # If last hour is less than 9 then we know recording ended between midnight and 9am + # which means that sleep onset may not be reliably detected + # If last window with a non-zero number ends after 6pm in the before last day + # then that confirms that sleep was estimated fromt his incomplete night + if ((timewindow == "MM" || timewindow == "OO") && lastHour < 9 && + N_hours_window0_at_end < 9 + 6) { mdat$window[which(mdat$window == max(mdat$window))] = 0 } - if (timewindow == "WW" && last_timestamp < 15) { + # If last hour is less than 15 then we know recording ended between midnight and 3pm + # which means that wakeup may not be reliably detected + # If last window with a non-zero number ends after 6pm in the before last day + # then that confirms that sleep was estimated from this incomplete night + if (timewindow == "WW" && lastHour < 15 && + N_hours_window0_at_end < 15 + 6) { mdat$window[which(mdat$window == max(mdat$window))] = 0 } } diff --git a/R/g.report.part5.R b/R/g.report.part5.R index 956162cd8..3b7844039 100644 --- a/R/g.report.part5.R +++ b/R/g.report.part5.R @@ -47,32 +47,34 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c x$wear_min_day = (1 - (x$nonwear_perc_day / 100)) * x$dur_day_min #valid minute during waking hours x$wear_perc_day = 100 - x$nonwear_perc_day #wear percentage during waking hours - x$lasttimestamp = as.numeric(x$lasttimestamp) - + x$lastHour = as.numeric(x$lastHour) + x$calendar_date = as.Date(x$calendar_date) minimumValidMinutesMM = 0 # default if (length(params_cleaning[["includedaycrit"]]) == 2) { minimumValidMinutesMM = params_cleaning[["includedaycrit"]][2] * 60 } if (params_output[["require_complete_lastnight_part5"]] == FALSE) { - x$lastnight = FALSE + x$lastWindow = FALSE + x$lastDate = x$calendar_date } else { - x$lastnight = x$window_number == max(x$window_number) + x$lastWindow = x$window_number == max(x$window_number) + x$lastDate = as.Date(x$lastDate) } if (window == "WW" | window == "OO") { indices = which(x$wear_perc_day >= includeday_wearPercentage & x$wear_min_day >= includeday_absolute & x$dur_spt_min > 0 & x$dur_day_min > 0 & - ((x$lastnight == TRUE & x$lasttimestamp >= 15 & window == "WW") | - (x$lastnight == TRUE & x$lasttimestamp >= 9 & window == "OO") | - x$lastnight == FALSE) & + ((x$lastWindow == TRUE & x$lastHour >= 15 & (x$lastDate - x$calendar_date) >= -1 & window == "WW") | + (x$lastWindow == TRUE & x$lastHour >= 9 & (x$lastDate - x$calendar_date) >= -1 & window == "OO") | + x$lastWindow == FALSE) & include_window == TRUE & x$wear_min_day_spt >= minimumValidMinutesMM) } else if (window == "MM") { indices = which(x$wear_perc_day >= includeday_wearPercentage & x$wear_min_day >= includeday_absolute & x$dur_spt_min > 0 & x$dur_day_min > 0 & - ((x$lastnight == TRUE & x$lasttimestamp > 9) | - x$lastnight == FALSE) & + ((x$lastWindow == TRUE & x$lastHour > 9 & (x$lastDate - x$calendar_date) >= -1) | + x$lastWindow == FALSE) & x$dur_day_spt_min >= (params_cleaning[["minimum_MM_length.part5"]] * 60) & include_window == TRUE & x$wear_min_day_spt >= minimumValidMinutesMM) @@ -136,9 +138,11 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c output = output[-cut, which(colnames(output) != "")] } if (exists("last_timestamp") == TRUE) { - output$lasttimestamp = as.numeric(format(last_timestamp, "%H")) + output$lastHour = as.numeric(format(last_timestamp, "%H")) + output$lastDate = as.Date(last_timestamp) } else { - output$lasttimestamp = Inf # use dummy value + output$lastHour = Inf # use dummy value + output$lastDate = Inf # use dummy value } out = as.matrix(output) if (length(expectedCols) > 0) { @@ -163,9 +167,9 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c "daysleeper|sleeplog_used|_spt_sleep|_spt_wake"), x = names(out), value = FALSE) window_number = as.numeric(out[,"window_number"]) - lastwindow = which(window_number == max(window_number, na.rm = TRUE)) - if (length(col2na) > 0 & length(lastwindow) > 0) { - out[lastwindow, col2na] = "" # set last row to NA for all sleep related variables + lastwindow_indices = which(window_number == max(window_number, na.rm = TRUE)) + if (length(col2na) > 0 & length(lastwindow_indices) > 0) { + out[lastwindow_indices, col2na] = "" # set last row to NA for all sleep related variables } } return(out) @@ -290,8 +294,8 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c # store all summaries in csv files with cleaning criteria validdaysi = getValidDayIndices(x = OF3_clean, window = uwi[j], params_cleaning = params_cleaning) - if ("lasttimestamp" %in% colnames(OF3_clean)) { - OF3_clean = OF3_clean[, -which(colnames(OF3_clean) == "lasttimestamp")] + if ("lastHour" %in% colnames(OF3_clean)) { + OF3_clean = OF3_clean[, -which(colnames(OF3_clean) %in% c("lastHour", "lasteDate"))] } if (length(validdaysi) > 0) { data.table::fwrite( diff --git a/man/GGIR.Rd b/man/GGIR.Rd index e01bf42e2..c563edbf1 100755 --- a/man/GGIR.Rd +++ b/man/GGIR.Rd @@ -1541,10 +1541,12 @@ GGIR(mode = 1:5, when viewingwindow = 1 or \code{includenightcrit} when viewingwindow = 2} \item{require_complete_lastnight_part5}{ Boolean (default = FALSE). - When set to TRUE: The last WW window is excluded if recording ends - between midnight and 3pm; The last OO and MM window are excluded if - recording ends between midnight and 9am. This to avoid risk that recording - end biases the sleep estimates for the last night. + When set to TRUE: The last WW window is excluded if the recording ends + between midnight and 3pm, and starts on a date that is on or one day before the + recording end date; The last OO and MM window are excluded if recording + ends between midnight and 9am, and starts on a date that is on or one + day before the recording end date. This to avoid risk that recording end + biases the sleep estimates for the last night. } } } From e44eb1a65fa85d1304785085e92ae2387f17fbb4 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Thu, 24 Oct 2024 16:44:42 +0200 Subject: [PATCH 2/8] update release number --- DESCRIPTION | 4 ++-- man/GGIR-package.Rd | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index eee3a6d29..75cbe9166 100755 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: GGIR Type: Package Title: Raw Accelerometer Data Analysis -Version: 3.1-4 -Date: 2024-09-03 +Version: 3.1-5 +Date: 2024-11-07 Authors@R: c(person("Vincent T","van Hees",role=c("aut","cre"), email="v.vanhees@accelting.com"), person("Jairo H","Migueles",role="aut", diff --git a/man/GGIR-package.Rd b/man/GGIR-package.Rd index ba17d58ea..d4d584b51 100755 --- a/man/GGIR-package.Rd +++ b/man/GGIR-package.Rd @@ -21,8 +21,8 @@ \tabular{ll}{ Package: \tab GGIR\cr Type: \tab Package\cr - Version: \tab 3.1-2\cr - Date: \tab 2024-07-02\cr + Version: \tab 3.1-5\cr + Date: \tab 2024-11-07\cr License: \tab Apache License (== 2.0)\cr Discussion group: \tab https://groups.google.com/forum/#!forum/rpackageggir\cr } From 1f807a66b646bd72ee322eb3d21c06ef0a0f1bac Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Thu, 24 Oct 2024 16:44:59 +0200 Subject: [PATCH 3/8] replace link to documentation in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bd7f72f77..2fb3a2a0a 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![](https://cranlogs.r-pkg.org/badges/last-month/GGIR)](https://cran.r-project.org/package=GGIR) ## Getting started: -The package [vignette](https://CRAN.R-project.org/package=GGIR/vignettes/GGIR.html) and [this](https://youtu.be/S8YPTrYNWdU) short tutorial video provide an introduction to GGIR, including: How it can be installed, Key software features, and where to get help. +The package [documentation](https://wadpac.github.io/GGIR/) provides extensive documentation on GGIR and its functionalities. ## Contribution guidelines: We always welcome contributions to the package. From eba0a35b5f77c028925f79a80190e7be86766cb6 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Thu, 24 Oct 2024 17:11:33 +0200 Subject: [PATCH 4/8] improve phrasing of change log to better match the updates --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index bd3017f51..0fc4ea813 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,9 +4,9 @@ - Add parameters require_complete_lastnight_part5 to control whether last window is included if last night is incomplete. #1196 - - Adjust sleeplog times to recording length in part 5 when classifying a night that was not detected in part 4. #1206 + - Handle sleeplog times beyond the end of the recording. #1206 - - Fixed minor bug in g.part5.addfirstwake causing the first wake is not correctly added when no SIBs are detected from the beginning of the recording until the first detected night. #1198 + - Fixed minor bug in g.part5.addfirstwake causing the first wake is not correctly added when no SIBs are detected in between the start of the recorded and the algorithm or diary based estimate of wakeup time. #1198 - Revise MM window definition in daylight saving time days, as it assumed fixed day duration of 24 hours #1211 From 48184aa8957ba1701037e5c482b49e5772ab2fa2 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Thu, 24 Oct 2024 17:52:29 +0200 Subject: [PATCH 5/8] clarify csv reports --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 0fc4ea813..03e74e20d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,7 +12,7 @@ - General: GGIR version look-up in .onattach() no longer crashes when computer is offline, fixes #1203. -- Reports: The calendar_date and filename columns in reports have been standardized, as %Y-%m-%d and the input accelerometer file name, respectively. #1197 +- Reports: The calendar_date and filename columns in csv reports have been standardized, as %Y-%m-%d and the input accelerometer file name, respectively. #1197 - Part 1: Reverse default value for nonwear_range_threshold as changed in 3.1-3 back to 150 as more research needed to support the change. #1172 From 6f31ad3f2aa3ba585f1d000181c37eb2d18251b7 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Mon, 4 Nov 2024 17:56:58 +0100 Subject: [PATCH 6/8] undo commit 3b9e2b7, fixes #1221 --- NEWS.md | 2 ++ R/g.part4.R | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index bd3017f51..571d74d00 100644 --- a/NEWS.md +++ b/NEWS.md @@ -18,6 +18,8 @@ - Part 1: Tidied up code in internal function detect_nonwear_clipping.R to ease future reviews. #1212 +- Part 4: Undo commit 3b9e2b7 (in between 3.1-1 and 3.1-2 release in June 2024) which broke functionality for handling sleep that starts before the double DST hour in the autumn and ends inside it. #1221 + # CHANGES IN GGIR VERSION 3.1-4 - Part 3: Update threshold used for HorAngle to 60 degree, and auto-setting HASPT.ignore.invalid to NA when NotWorn guider is used. #1186 diff --git a/R/g.part4.R b/R/g.part4.R index e54482bb9..24fcbbfe8 100644 --- a/R/g.part4.R +++ b/R/g.part4.R @@ -814,9 +814,10 @@ g.part4 = function(datadir = c(), metadatadir = c(), f0 = f0, f1 = f1, # in the autumn and ends inside the dst hour negval = which(nocs < 0) if (length(negval) > 0) { - kk0 = as.numeric(spocum.t$start[negval]) # episode onsets - kk1 = as.numeric(spocum.t$end[negval]) + 1 # episode endings - nocs[negval] = kk1 - kk0 + kk0 = as.numeric(spocum.t$start[which(spocum.t$overlapGuider == 1)]) # episode onsets + kk1 = as.numeric(spocum.t$end[which(spocum.t$overlapGuider == 1)]) # episode endings + kk1[negval] = kk1[negval] + 1 + nocs = kk1 - kk0 } if (length(nocs) > 0) { spocum.t.dur.noc = sum(nocs) From 9638f28999df36e3b1f0840627b70152b77fbcc8 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Mon, 4 Nov 2024 19:39:49 +0100 Subject: [PATCH 7/8] Update NEWS.md --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 571d74d00..ddc00b02e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -18,7 +18,7 @@ - Part 1: Tidied up code in internal function detect_nonwear_clipping.R to ease future reviews. #1212 -- Part 4: Undo commit 3b9e2b7 (in between 3.1-1 and 3.1-2 release in June 2024) which broke functionality for handling sleep that starts before the double DST hour in the autumn and ends inside it. #1221 +- Part 4: Undo 3 line changes from commit 3b9e2b7 (in between 3.1-1 and 3.1-2 release in June 2024) which broke functionality for handling sleep that starts before the double DST hour in the autumn and ends inside it. #1221 # CHANGES IN GGIR VERSION 3.1-4 From 5b79cdb8ec2c5d09a7544f5d11764772b34a13fb Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Mon, 4 Nov 2024 19:42:02 +0100 Subject: [PATCH 8/8] Update NEWS.md --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index ddc00b02e..6995aea0a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -18,7 +18,7 @@ - Part 1: Tidied up code in internal function detect_nonwear_clipping.R to ease future reviews. #1212 -- Part 4: Undo 3 line changes from commit 3b9e2b7 (in between 3.1-1 and 3.1-2 release in June 2024) which broke functionality for handling sleep that starts before the double DST hour in the autumn and ends inside it. #1221 +- Part 4: Undo changes to 3 line in g.part4 function from commit 3b9e2b7 (in between 3.1-1 and 3.1-2 release in June 2024) which broke functionality for handling sleep that starts before the double DST hour in the autumn and ends inside it. #1221 # CHANGES IN GGIR VERSION 3.1-4