Merge branch 'master' into work839_955_1028_1129

DESCRIPTION

@@ -1,8 +1,8 @@
 Package: GGIR
 Type: Package
 Title: Raw Accelerometer Data Analysis
-Version: 3.1-4
-Date: 2024-09-03
+Version: 3.1-5
+Date: 2024-11-07
 Authors@R: c(person("Vincent T","van Hees",role=c("aut","cre"),
                     email="[email protected]"),
              person("Jairo H","Migueles",role="aut",

NEWS.md

@@ -31,20 +31,22 @@
 
   - Add parameters require_complete_lastnight_part5 to control whether last window is included if last night is incomplete. #1196
 
-  - Adjust sleeplog times to recording length in part 5 when classifying a night that was not detected in part 4. #1206
+  - Handle sleeplog times beyond the end of the recording. #1206
 
-  - Fixed minor bug in g.part5.addfirstwake causing the first wake is not correctly added when no SIBs are detected from the beginning of the recording until the first detected night. #1198
+  - Fixed minor bug in g.part5.addfirstwake causing the first wake is not correctly added when no SIBs are detected in between the start of the recorded and the algorithm or diary based estimate of wakeup time. #1198
 
   - Revise MM window definition in daylight saving time days, as it assumed fixed day duration of 24 hours #1211 
 
 - General: GGIR version look-up in .onattach() no longer crashes when computer is offline, fixes #1203.
 
-- Reports: The calendar_date and filename columns in reports have been standardized, as %Y-%m-%d and the input accelerometer file name, respectively. #1197
+- Reports: The calendar_date and filename columns in csv reports have been standardized, as %Y-%m-%d and the input accelerometer file name, respectively. #1197
 
 - Part 1: Reverse default value for nonwear_range_threshold as changed in 3.1-3 back to 150 as more research needed to support the change. #1172
 
 - Part 1: Tidied up code in internal function detect_nonwear_clipping.R to ease future reviews. #1212
 
+- Part 4: Undo changes to 3 line in g.part4 function from commit 3b9e2b7 (in between 3.1-1 and 3.1-2 release in June 2024) which broke functionality for handling sleep that starts before the double DST hour in the autumn and ends inside it. #1221
+
 # CHANGES IN GGIR VERSION 3.1-4
 
 - Part 3: Update threshold used for HorAngle to 60 degree, and auto-setting HASPT.ignore.invalid to NA when NotWorn guider is used. #1186

R/g.part4.R

@@ -814,9 +814,10 @@ g.part4 = function(datadir = c(), metadatadir = c(), f0 = f0, f1 = f1,
                   # in the autumn and ends inside the dst hour
                   negval = which(nocs < 0)
                   if (length(negval) > 0) {
-                    kk0 = as.numeric(spocum.t$start[negval])  # episode onsets
-                    kk1 = as.numeric(spocum.t$end[negval]) + 1 # episode endings
-                    nocs[negval] = kk1 - kk0
+                    kk0 = as.numeric(spocum.t$start[which(spocum.t$overlapGuider == 1)])  # episode onsets
+                    kk1 = as.numeric(spocum.t$end[which(spocum.t$overlapGuider == 1)])  # episode endings
+                    kk1[negval] = kk1[negval] + 1
+                    nocs = kk1 - kk0
                   }
                   if (length(nocs) > 0) {
                     spocum.t.dur.noc = sum(nocs)

R/g.part5.savetimeseries.R

@@ -21,6 +21,7 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
   }
   # Create numeric time to faciltiate merging
   ts$timenum = as.numeric(ts$timestamp)
+  epochSize = ts$timenum[2] - ts$timenum[1]
   ms5rawlevels$timenum = as.numeric(ms5rawlevels$date_time)
   mdat = merge(ts, ms5rawlevels, by = "timenum")
   rm(ts, ms5rawlevels)
@@ -29,11 +30,23 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
   mdat = mdat[,-which(names(mdat) == "date_time")]
   if ("require_complete_lastnight_part5" %in% names(params_output) &&
       params_output[["require_complete_lastnight_part5"]] == TRUE) {
-    last_timestamp = as.numeric(format(mdat$timestamp[length(mdat$timestamp)], "%H"))
-    if ((timewindow == "MM" || timewindow == "OO") && last_timestamp < 9) {
+    N_window0_at_end = which(rev(mdat$window) != 0)[1]
+    N_hours_window0_at_end = ((N_window0_at_end * epochSize) / 3600)
+    lastHour = as.numeric(format(mdat$timestamp[length(mdat$timestamp)], "%H"))
+    # If last hour is less than 9 then we know recording ended between midnight and 9am
+    # which means that sleep onset may not be reliably detected
+    # If last window with a non-zero number ends after 6pm in the before last day
+    # then that confirms that sleep was estimated fromt his incomplete night
+    if ((timewindow == "MM" || timewindow == "OO") && lastHour < 9 &&
+        N_hours_window0_at_end < 9 + 6) {
       mdat$window[which(mdat$window == max(mdat$window))] = 0 
     }
-    if (timewindow == "WW" && last_timestamp < 15) {
+    # If last hour is less than 15 then we know recording ended between midnight and 3pm
+    # which means that wakeup may not be reliably detected
+    # If last window with a non-zero number ends after 6pm in the before last day
+    # then that confirms that sleep was estimated from this incomplete night
+    if (timewindow == "WW" && lastHour < 15 &&
+        N_hours_window0_at_end < 15 + 6) {
       mdat$window[which(mdat$window == max(mdat$window))] = 0 
     }
   }

R/g.report.part5.R

@@ -58,25 +58,30 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
       x$wear_perc_day = 100 - x$nonwear_perc_day #wear percentage during waking hours
       x$wear_min_spt = (1 - (x$nonwear_perc_spt / 100)) * x$dur_spt_min #valid minute during waking hours
       x$wear_perc_spt = 100 - x$nonwear_perc_spt #wear percentage during waking hours
-      x$lasttimestamp = as.numeric(x$lasttimestamp)
+
+      x$lastHour = as.numeric(x$lastHour)
+      x$calendar_date = as.Date(x$calendar_date)
+
       minimumValidMinutesMM = 0 # default
       if (length(params_cleaning[["includedaycrit"]]) == 2) {
         minimumValidMinutesMM = params_cleaning[["includedaycrit"]][2] * 60
       }
       if (params_output[["require_complete_lastnight_part5"]] == FALSE) {
-        x$lastnight = FALSE
+        x$lastWindow = FALSE
+        x$lastDate = x$calendar_date
       } else {
-        x$lastnight = x$window_number == max(x$window_number)
+        x$lastWindow = x$window_number == max(x$window_number)
+        x$lastDate = as.Date(x$lastDate)
       }
       if (window == "WW" | window == "OO") {
         indices = which(x$wear_perc_day >= includeday_wearPercentage &
                           x$wear_min_day >= includeday_absolute &
                           x$wear_perc_spt >= includenight_wearPercentage &
                           x$wear_min_spt >= includenight_absolute &
                           x$dur_spt_min > 0 & x$dur_day_min > 0 &
-                          ((x$lastnight == TRUE & x$lasttimestamp >= 15 & window == "WW") |
-                             (x$lastnight == TRUE & x$lasttimestamp >= 9 & window == "OO") |
-                             x$lastnight == FALSE) &
+                          ((x$lastWindow == TRUE & x$lastHour >= 15 & (x$lastDate - x$calendar_date) >= -1 & window == "WW") |
+                             (x$lastWindow == TRUE & x$lastHour >= 9 & (x$lastDate - x$calendar_date) >= -1 & window == "OO") |
+                             x$lastWindow == FALSE) &
                           include_window == TRUE &
                           x$wear_min_day_spt >= minimumValidMinutesMM)
       } else if (window == "MM") {
@@ -85,8 +90,8 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
                           x$wear_perc_spt >= includenight_wearPercentage &
                           x$wear_min_spt >= includenight_absolute &
                           x$dur_spt_min > 0 & x$dur_day_min > 0 &
-                          ((x$lastnight == TRUE & x$lasttimestamp > 9) |
-                             x$lastnight == FALSE) &
+                          ((x$lastWindow == TRUE & x$lastHour > 9 & (x$lastDate - x$calendar_date) >= -1) |
+                             x$lastWindow == FALSE) &
                           x$dur_day_spt_min >= (params_cleaning[["minimum_MM_length.part5"]] * 60) &
                           include_window == TRUE &
                           x$wear_min_day_spt >= minimumValidMinutesMM)
@@ -150,9 +155,11 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
         output = output[-cut, which(colnames(output) != "")]
       }
       if (exists("last_timestamp") == TRUE) {
-        output$lasttimestamp = as.numeric(format(last_timestamp, "%H"))
+        output$lastHour = as.numeric(format(last_timestamp, "%H"))
+        output$lastDate = as.Date(last_timestamp)
       } else {
-        output$lasttimestamp = Inf # use dummy value
+        output$lastHour = Inf # use dummy value
+        output$lastDate = Inf # use dummy value
       }
       out = as.matrix(output)
       if (length(expectedCols) > 0) {
@@ -177,9 +184,9 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
                                        "daysleeper|sleeplog_used|_spt_sleep|_spt_wake"),
                       x = names(out), value = FALSE)
         window_number = as.numeric(out[,"window_number"])
-        lastwindow = which(window_number == max(window_number, na.rm = TRUE))
-        if (length(col2na) > 0 & length(lastwindow) > 0) {
-          out[lastwindow, col2na] = "" # set last row to NA for all sleep related variables
+        lastwindow_indices = which(window_number == max(window_number, na.rm = TRUE))
+        if (length(col2na) > 0 & length(lastwindow_indices) > 0) {
+          out[lastwindow_indices, col2na] = "" # set last row to NA for all sleep related variables
         }
       }
       return(out)
@@ -304,8 +311,8 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
                 # store all summaries in csv files with cleaning criteria
                 validdaysi = getValidDayIndices(x = OF3_clean, window = uwi[j],
                                                 params_cleaning = params_cleaning)
-                if ("lasttimestamp" %in% colnames(OF3_clean)) {
-                  OF3_clean = OF3_clean[, -which(colnames(OF3_clean) == "lasttimestamp")]
+                if ("lastHour" %in% colnames(OF3_clean)) {
+                  OF3_clean = OF3_clean[, -which(colnames(OF3_clean) %in% c("lastHour", "lasteDate"))]
                 }
                 if (length(validdaysi) > 0) {
                   data.table::fwrite(

README.md

@@ -5,7 +5,7 @@
 [![](https://cranlogs.r-pkg.org/badges/last-month/GGIR)](https://cran.r-project.org/package=GGIR)
 
 ## Getting started:
-The package [vignette](https://CRAN.R-project.org/package=GGIR/vignettes/GGIR.html) and [this](https://youtu.be/S8YPTrYNWdU) short tutorial video provide an introduction to GGIR, including: How it can be installed, Key software features, and where to get help.
+The package [documentation](https://wadpac.github.io/GGIR/) provides extensive documentation on GGIR and its functionalities.
 
 ## Contribution guidelines:
 We always welcome contributions to the package.

man/GGIR-package.Rd

@@ -21,8 +21,8 @@
   \tabular{ll}{
   Package: \tab GGIR\cr
   Type: \tab Package\cr
-  Version: \tab 3.1-2\cr
-  Date: \tab 2024-07-02\cr
+  Version: \tab 3.1-5\cr
+  Date: \tab 2024-11-07\cr
   License: \tab Apache License (== 2.0)\cr
   Discussion group: \tab https://groups.google.com/forum/#!forum/rpackageggir\cr
   }

man/GGIR.Rd

@@ -1557,10 +1557,12 @@ GGIR(mode = 1:5,
           when viewingwindow = 1 or \code{includenightcrit} when viewingwindow = 2}
       \item{require_complete_lastnight_part5}{    
           Boolean (default = FALSE).
-          When set to TRUE: The last WW window is excluded if recording ends 
-          between midnight and 3pm; The last OO and MM window are excluded if 
-          recording ends between midnight and 9am. This to avoid risk that recording
-          end biases the sleep estimates for the last night.
+          When set to TRUE: The last WW window is excluded if the recording ends 
+          between midnight and 3pm, and starts on a date that is on or one day before the 
+          recording end date; The last OO and MM window are excluded if recording 
+          ends between midnight and 9am, and starts on a date that is on or one
+          day before the recording end date. This to avoid risk that recording end 
+          biases the sleep estimates for the last night.
       }
     }
   }