Merge pull request #956 from wadpac/issue906_create_part6

Add part 6 for household co-analysis and Circadian Rhythm analysis

I am merging this now following our offline correspondence about this PR

.Rbuildignore

@@ -11,4 +11,4 @@ RELEASE_CYCLE.md
 ^docs$
 ^pkgdown$
 ^vignettes/chapter*.Rmd
-^data-raw$
+^dev-functions$

DESCRIPTION

@@ -30,6 +30,6 @@ URL: https://github.com/wadpac/GGIR/, https://groups.google.com/forum/#!forum/Rp
 BugReports: https://github.com/wadpac/GGIR/issues
 License: Apache License (== 2.0) | file LICENSE
 Suggests: testthat, covr, knitr, rmarkdown, actilifecounts, ActCR, GGIRread, read.gt3x
-Imports: data.table, foreach, doParallel, signal, zoo, unisensR, ineq, methods
+Imports: data.table, foreach, doParallel, signal, zoo, unisensR, ineq, methods, psych, irr
 Depends: stats, utils, R (>= 3.5)
 VignetteBuilder: knitr

NAMESPACE

@@ -40,9 +40,12 @@ export(g.analyse, g.calibrate,
        correctOlderMilestoneData,
        convertEpochData, appendRecords, extractID,
        g.part5_analyseSegment, g.part5_initialise_ts,
-       g.part5.analyseRest, check_log, g.report.part5_dictionary)
+       g.part5.analyseRest, part6AlignIndividuals,
+       part6PairwiseAggregation, g.part6, g.report.part6, check_log, g.report.part5_dictionary)
 
-importFrom("grDevices", "colors", "dev.off", "pdf","rainbow","rgb")
+
+importFrom("grDevices", "colors", "dev.off", "pdf",
+           "rgb", "rainbow", "palette")
 importFrom("graphics", "abline", "axis", "par", "plot", "plot.new",
            "rect","axis.POSIXct", "barplot", "box", "legend",
            "lines", "mtext", "text", "arrows", "segments")
@@ -53,7 +56,7 @@ importFrom("utils", "read.csv", "sessionInfo", "write.csv",
 importFrom("stats", "aggregate.data.frame", "weighted.mean",
            "rnorm","median","aggregate","C", "lm.wfit",
            "quantile", "sd","coef", "lm", "embed", "na.pass",
-            "residuals", "fitted", "cor")
+            "residuals", "fitted", "reshape", "cor")
 import(data.table)
 importFrom("methods", "is")
 importFrom("utils", "available.packages", "packageVersion", "help", "read.table")

NEWS.md

@@ -6,6 +6,11 @@
 
 - Part 2: Bug fix in csv report generation when appending part 2 milestone data with differing number of columns #951
 
+- Part 6: Added as a new part
+
+- Part 6: Expanded with Household co-analysis
+
+- Part 6: Expanded with Cosinor analysis
 
 # CHANGES IN GGIR VERSION 3.0-0
 
@@ -20,6 +25,8 @@ relative to start and end of the most active time window as identified.  #905
 
 - Part 5: Change default for segmentDAYSPTcrit.part5 from c(0,0) to c(0, 0.9) and now prohibiting the use of c(0, 0) as it gives biased estimates. We knew this, but some users started using the default without attempting to understand it, by which it seems necessary to force a sensible default. #940
 
+- Add contributing guidelines for github.io documentation #923
+
 - Part 5: Added optioned "OO" to argument timewindow, which defines windows from sleep Onset to sleep Onset #931
 
 # CHANGES IN GGIR VERSION 2.10-4
@@ -40,8 +47,6 @@ relative to start and end of the most active time window as identified.  #905
 
 - Fix recently introduced bug where GGIR environment was not exported to cluster in GGIR part 1, 2, 3, and 5 #910
 
-- Add contributing guidelines for github.io documentation #923
-
 # CHANGES IN GGIR VERSION 2.10-3
 
 - Part 1: Fixed minor bug in ismovisens that failed when datadir started with "./" #897

R/GGIR.R

@@ -1,6 +1,6 @@
 GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
                 studyname = c(), f0 = 1, f1 = 0,
-                do.report = c(2, 4, 5), configfile = c(),
+                do.report = c(2, 4, 5, 6), configfile = c(),
                 myfun = c(), verbose = TRUE,
                 ...) {
   #get input variables
@@ -20,23 +20,23 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
       }
     }
   }
-
+  
   if (length(datadir) == 0) {
     stop('\nVariable datadir is not specified')
   }
-
+  
   if (length(outputdir) == 0) {
     stop('\nVariable outputdir is not specified')
   }
-
+  
   # Convert paths from Windows specific slashed to generic slashes
   outputdir = gsub(pattern = "\\\\", replacement = "/", x = outputdir)
   datadir = gsub(pattern = "\\\\", replacement = "/", x = datadir)
-
+  
   #===========================
   # Establish default start / end file index to process
   filelist = isfilelist(datadir)
-
+  
   if (filelist == FALSE) {
     if (dir.exists(datadir) == FALSE && 1 %in% mode) {
       # Note: The check whether datadir exists is only relevant when running part 1
@@ -81,25 +81,26 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
     f1 = 0
   }
   # Establish which parts need to be processed:
-  dopart1 = dopart2 = dopart3 = dopart4 = dopart5 = FALSE
+  dopart1 = dopart2 = dopart3 = dopart4 = dopart5 = dopart6 = FALSE
   if (length(which(mode == 0)) > 0) {
-    dopart1 = dopart2 = dopart3 = dopart4 = dopart5 = TRUE
+    dopart1 = dopart2 = dopart3 = dopart4 = dopart5 = TRUE # intentionally not including dopart6
   } else {
     # if (length(which(mode == 0)) > 0) dopart0 = TRUE
     if (length(which(mode == 1)) > 0) dopart1 = TRUE
     if (length(which(mode == 2)) > 0) dopart2 = TRUE
     if (length(which(mode == 3)) > 0) dopart3 = TRUE
     if (length(which(mode == 4)) > 0) dopart4 = TRUE
     if (length(which(mode == 5)) > 0) dopart5 = TRUE
+    if (length(which(mode == 6)) > 0) dopart6 = TRUE
   }
-
+  
   if (filelist == TRUE) {
     metadatadir = paste0(outputdir, "/output_", studyname)
   } else {
     outputfoldername = unlist(strsplit(datadir, "/"))[length(unlist(strsplit(datadir, "/")))]
     metadatadir = paste0(outputdir, "/output_", outputfoldername)
   }
-
+  
   # Configuration file - check whether it exists or auto-load
   configfile_csv = c()
   ex = "csv"
@@ -128,7 +129,7 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
       }
     }
   }
-
+  
   #----------------------------------------------------------
   # Extract parameters from user input, configfile and/or defaults.
   params = extract_params(input = input, configfile_csv = configfile_csv) # load default parameters here in g.shell.GGIR
@@ -146,21 +147,21 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
     dopart3 = dopart4 = dopart5 = FALSE
     mode = mode[which(mode <= 2)]
   }
-
+  
   if (dopart3 == TRUE & params_metrics[["do.anglez"]] == FALSE & params_general[["dataFormat"]] == "raw") {
     params_metrics[["do.anglez"]] = TRUE
   }
-
+  
   if (length(myfun) != 0) { # Run check on myfun object, if provided
     check_myfun(myfun, params_general[["windowsizes"]])
   }
-
+  
   if (params_output[["visualreport"]] == TRUE & params_general[["dataFormat"]] != "raw") {
     params_output[["visualreport"]] == FALSE
     warning(paste0("Turning off visualreport generation because",
                    " dataFormat is not raw."), call. = FALSE)
   }
-
+  
   # check package dependencies
   if (params_metrics$do.neishabouricounts == TRUE) {
     is_actilifecounts_installed = is.element('actilifecounts', installed.packages()[,1])
@@ -172,7 +173,7 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
       }
     }
   }
-
+  
   if (params_247$cosinor == TRUE) {
     is_ActCR_installed = is.element('ActCR', installed.packages()[,1])
     if (is_ActCR_installed == FALSE) {
@@ -214,7 +215,7 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
       }
     }
   }
-
+  
   #-----------------------------------------------------------
   # Print GGIR header to console
   GGIRversion = "could not extract version"
@@ -329,43 +330,55 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
             params_phyact = params_phyact,
             verbose = verbose)
   }
+  if (dopart6 == TRUE) {
+    if (verbose == TRUE) print_console_header("Part 6")
+    if (f1 == 0) {
+      f1 = length(dir(paste0(metadatadir, "/meta/ms5.outraw/", 
+                             params_phyact[["part6_threshold_combi"]])))
+    }
+    g.part6(datadir = datadir, metadatadir = metadatadir, f0 = f0, f1 = f1,
+            params_general = params_general, params_phyact = params_phyact,
+            params_247 = params_247,
+            verbose = verbose)
+  }
+
   #--------------------------------------------------
   # Store configuration parameters in config file
   LS = ls()
   LS = LS[which(LS %in% c("input", "txt", "dopart1", "dopart2", "dopart3", "LS",
-                          "dopart4", "dopart5", "metadatadir", "ci", "config",
+                          "dopart4", "dopart5", "dopart6", "metadatadir", "ci", "config",
                           "configfile", "filelist", "outputfoldername", "numi", "logi",
                           "conv2logical", "conv2num", "SI", "params", "argNames", "dupArgNames",
                           "print_console_header", "configfile_csv", "myfun", "ex",
                           "GGIRversion",  "dupArgValues", "verbose", "is_GGIRread_installed", 
                           "is_read.gt3x_installed", "is_ActCR_installed", 
                           "is_actilifecounts_installed", "rawaccfiles", 
                           "checkFormat", "getExt") == FALSE)]
-
+  
   config.parameters = mget(LS)
   config.matrix = as.data.frame(createConfigFile(config.parameters, GGIRversion))
   config.matrix$context[which(config.matrix$context == "")] = "not applicable"
   if (dir.exists(metadatadir)) {
     data.table::fwrite(config.matrix, file = paste0(metadatadir, "/config.csv"),
                        row.names = FALSE, sep = params_output[["sep_config"]])
   } else {
-      warning("\nCould not write config file.")
+    warning("\nCould not write config file.")
   }
   #==========================
   # Report generation:
   # -----
   # check a few basic assumptions before continuing
   if (length(which(do.report == 2)) > 0) {
-    if (verbose == TRUE) print_console_header("Report part 2")
     N.files.ms2.out = length(dir(paste0(metadatadir, "/meta/ms2.out")))
     if (N.files.ms2.out > 0) {
+      if (verbose == TRUE) print_console_header("Report part 2")
       # if (N.files.ms2.out < f0) f0 = 1
       # if (N.files.ms2.out < f1) f1 = N.files.ms2.out
       if (length(f0) == 0) f0 = 1
       if (f1 == 0) f1 = N.files.ms2.out
       if (length(params_247[["qwindow"]]) > 2 |
           is.character(params_247[["qwindow"]]) |
-        (length(params_247[["qwindow"]]) == 2 & !all(c(0, 24) %in% params_247[["qwindow"]]))) {
+          (length(params_247[["qwindow"]]) == 2 & !all(c(0, 24) %in% params_247[["qwindow"]]))) {
         store.long = TRUE
       } else {
         store.long = FALSE
@@ -374,14 +387,12 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
                      maxdur = params_cleaning[["maxdur"]],
                      store.long = store.long, do.part2.pdf = params_output[["do.part2.pdf"]],
                      verbose = verbose, sep_reports = params_output[["sep_reports"]])
-    } else {
-      if (verbose == TRUE) cat("\nSkipped because no milestone data available")
     }
   }
   if (length(which(do.report == 4)) > 0) {
-    if (verbose == TRUE) print_console_header("Report part 4")
     N.files.ms4.out = length(dir(paste0(metadatadir, "/meta/ms4.out")))
     if (N.files.ms4.out > 0) {
+      if (verbose == TRUE) print_console_header("Report part 4")
       if (N.files.ms4.out < f0) f0 = 1
       if (N.files.ms4.out < f1) f1 = N.files.ms4.out
       if (f1 == 0) f1 = N.files.ms4.out
@@ -391,14 +402,12 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
                      data_cleaning_file = params_cleaning[["data_cleaning_file"]],
                      sleepwindowType = params_sleep[["sleepwindowType"]],
                      verbose = verbose, sep_reports = params_output[["sep_reports"]])
-    } else {
-      if (verbose == TRUE) cat("\nSkipped because no milestone data available")
     }
   }
   if (length(which(do.report == 5)) > 0) {
     N.files.ms5.out = length(dir(paste0(metadatadir, "/meta/ms5.out")))
-    if (verbose == TRUE) print_console_header("Report part 5")
     if (N.files.ms5.out > 0) {
+      if (verbose == TRUE) print_console_header("Report part 5")
       if (N.files.ms5.out < f0) f0 = 1
       if (N.files.ms5.out < f1) f1 = N.files.ms5.out
       if (f1 == 0) f1 = N.files.ms5.out
@@ -409,8 +418,18 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
                      LUX_day_segments = params_247[["LUX_day_segments"]],
                      verbose = verbose, sep_reports = params_output[["sep_reports"]])
       g.report.part5_dictionary(metadatadir = metadatadir, sep_reports = params_output[["sep_reports"]])
-    } else {
-      if (verbose == TRUE) cat("\nSkipped because no milestone data available")
+    }
+  }
+  if (length(which(do.report == 6)) > 0) {
+    N.files.ms6.out = length(dir(paste0(metadatadir, "/meta/ms6.out")))
+    if (N.files.ms6.out > 0) {
+      if (verbose == TRUE) print_console_header("Report part 6")
+      if (N.files.ms6.out < f0) f0 = 1
+      if (N.files.ms6.out < f1) f1 = N.files.ms6.out
+      if (f1 == 0) f1 = N.files.ms6.out
+      g.report.part6(metadatadir = metadatadir, f0 = f0, f1 = f1,
+                     params_cleaning = params_cleaning,
+                     verbose = verbose, sep_reports = params_output[["sep_reports"]])
     }
   }
   if (params_output[["visualreport"]] == TRUE) {
@@ -420,8 +439,9 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
     files.ms4.out = dir(paste0(metadatadir, "/meta/ms4.out"))
     # at least one metafile from the same recording in each folder
     files.available = Reduce(intersect, list(files.basic, files.ms3.out, files.ms4.out))
-    if (verbose == TRUE) print_console_header("Generate visual reports")
+
     if (length(files.available) > 0) {
+      if (verbose == TRUE) print_console_header("Generate visual reports")
       g.plot5(metadatadir = metadatadir,
               dofirstpage = params_output[["dofirstpage"]],
               viewingwindow = params_output[["viewingwindow"]],
@@ -436,8 +456,6 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
               includedaycrit = params_cleaning[["includedaycrit"]],
               includenightcrit = params_cleaning[["includenightcrit"]],
               verbose = TRUE)
-    } else {
-      if (verbose == TRUE) cat("\nSkipped because no milestone data available")
     }
   }
 }