From d8008d779a20c6e4b9c73befc32461268feb8ac2 Mon Sep 17 00:00:00 2001
From: aaroncherian <aaron.cherian2@gmail.com>
Date: Tue, 6 Feb 2024 10:51:05 -0500
Subject: [PATCH 01/11] openpose things

---
 skellytracker/process_folder_of_videos.py     |   6 +
 .../openpose_tracker/openpose_model_info.py   |   0
 .../openpose_tracker/openpose_parser.py       |  68 +++++++++++
 .../openpose_tracker/openpose_recorder.py     |  78 +++++++++++++
 .../openpose_tracker/openpose_tracker.py      | 106 ++++++++++++++++++
 5 files changed, 258 insertions(+)
 create mode 100644 skellytracker/trackers/openpose_tracker/openpose_model_info.py
 create mode 100644 skellytracker/trackers/openpose_tracker/openpose_parser.py
 create mode 100644 skellytracker/trackers/openpose_tracker/openpose_recorder.py
 create mode 100644 skellytracker/trackers/openpose_tracker/openpose_tracker.py

diff --git a/skellytracker/process_folder_of_videos.py b/skellytracker/process_folder_of_videos.py
index c32a2b6..15dbceb 100644
--- a/skellytracker/process_folder_of_videos.py
+++ b/skellytracker/process_folder_of_videos.py
@@ -21,6 +21,10 @@
 from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (
     MediapipeTrackingParams,
 )
+
+
+from skellytracker.trackers.openpose_tracker.openpose_tracker import OpenPoseTracker
+
 from skellytracker.utilities.get_video_paths import get_video_paths
 
 logger = logging.getLogger(__name__)
@@ -162,6 +166,8 @@ def get_tracker(tracker_name: str, tracking_params: BaseModel) -> BaseTracker:
     elif tracker_name == "BrightestPointTracker":
         tracker = BrightestPointTracker()
 
+    elif tracker_name = 'OpenPoseTracker':
+        
     else:
         raise ValueError("Invalid tracker type")
 
diff --git a/skellytracker/trackers/openpose_tracker/openpose_model_info.py b/skellytracker/trackers/openpose_tracker/openpose_model_info.py
new file mode 100644
index 0000000..e69de29
diff --git a/skellytracker/trackers/openpose_tracker/openpose_parser.py b/skellytracker/trackers/openpose_tracker/openpose_parser.py
new file mode 100644
index 0000000..4067f8e
--- /dev/null
+++ b/skellytracker/trackers/openpose_tracker/openpose_parser.py
@@ -0,0 +1,68 @@
+import json
+import numpy as np
+from pathlib import Path
+from tqdm import tqdm
+import re
+
+def extract_frame_index(filename):
+    """Extract the numeric part indicating the frame index from the filename."""
+    match = re.search(r"_(\d{12})_keypoints", filename)
+    return int(match.group(1)) if match else None
+
+def parse_openpose_jsons(main_directory):
+    """Parse OpenPose JSON files from subdirectories within the main directory."""
+    main_directory = Path(main_directory)
+    subdirectories = [d for d in main_directory.iterdir() if d.is_dir()]
+    num_cams = len(subdirectories)
+    
+    # Check the first subdirectory to determine the number of frames
+    sample_files = list(subdirectories[0].glob("*.json"))
+    num_frames = len(sample_files)
+    frame_indices = [extract_frame_index(f.name) for f in sample_files]
+    frame_indices.sort()
+    
+    # Assuming standard OpenPose output
+    body_markers = 25
+    hand_markers = 21  # Per hand
+    face_markers = 70
+    num_markers = body_markers + 2 * hand_markers + face_markers
+
+    data_array = np.full((num_cams, num_frames, num_markers, 3), np.nan)
+    
+    for cam_index, subdir in enumerate(subdirectories):
+        json_files = sorted(subdir.glob("*.json"), key=lambda x: extract_frame_index(x.stem))
+        
+        for file_index, json_file in tqdm(enumerate(json_files), desc = f'Processing {subdir.name} JSONS'):
+            with open(json_file) as f:
+                data = json.load(f)
+            
+            if data["people"]:
+                keypoints = extract_keypoints(data["people"][0], body_markers, hand_markers, face_markers)
+                data_array[cam_index, frame_indices[file_index], :, :] = keypoints
+
+    return data_array
+
+def extract_keypoints(person_data, body_markers, hand_markers, face_markers):
+    """Extract and organize keypoints from person data."""
+    # Initialize a full array of NaNs for keypoints
+    keypoints_array = np.full((body_markers + 2 * hand_markers + face_markers, 3), np.nan)
+    
+    # Populate the array with available data
+    if "pose_keypoints_2d" in person_data:
+        keypoints_array[:body_markers, :] = np.reshape(person_data["pose_keypoints_2d"], (-1, 3))[:body_markers, :]
+    if "hand_left_keypoints_2d" in person_data and "hand_right_keypoints_2d" in person_data:
+        keypoints_array[body_markers:body_markers + hand_markers, :] = np.reshape(person_data["hand_left_keypoints_2d"], (-1, 3))[:hand_markers, :]
+        keypoints_array[body_markers + hand_markers:body_markers + 2*hand_markers, :] = np.reshape(person_data["hand_right_keypoints_2d"], (-1, 3))[:hand_markers, :]
+    if "face_keypoints_2d" in person_data:
+        keypoints_array[body_markers + 2*hand_markers:, :] = np.reshape(person_data["face_keypoints_2d"], (-1, 3))[:face_markers, :]
+
+    return keypoints_array
+
+
+path_to_recording_folder = Path(r'D:\steen_pantsOn_gait_3_cameras')
+path_to_json_folder = path_to_recording_folder/'output_data'/'raw_data'/'openpose_json'
+path_to_save_raw_data = path_to_recording_folder/'output_data'/'raw_data'/'openpose2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy'
+
+data = parse_openpose_jsons(path_to_json_folder)
+np.save(path_to_save_raw_data, data)
+f = 2
\ No newline at end of file
diff --git a/skellytracker/trackers/openpose_tracker/openpose_recorder.py b/skellytracker/trackers/openpose_tracker/openpose_recorder.py
new file mode 100644
index 0000000..b7c3cc4
--- /dev/null
+++ b/skellytracker/trackers/openpose_tracker/openpose_recorder.py
@@ -0,0 +1,78 @@
+import json
+import numpy as np
+from pathlib import Path
+from skellytracker.trackers.base_tracker.base_recorder import BaseRecorder
+import re
+from tqdm import tqdm
+
+class OpenPoseRecorder(BaseRecorder):
+    def __init__(self, json_directory_path):
+        super().__init__()
+        self.json_directory_path = Path(json_directory_path)
+
+    def extract_frame_index(self, filename):
+        """Extract the numeric part indicating the frame index from the filename."""
+        match = re.search(r"_(\d{12})_keypoints", filename)
+        return int(match.group(1)) if match else None
+
+    def record(self, tracked_objects=None, annotated_image=None) -> None:
+        """
+        Override the record method to read from JSON files instead of receiving data directly.
+        """
+        # This method is adapted to fit the requirement of reading from JSON,
+        # hence tracked_objects and annotated_image are not used.
+        self.recorded_objects = self.parse_openpose_jsons(self.json_directory_path)
+
+    def parse_openpose_jsons(self, main_directory):
+        subdirectories = [d for d in main_directory.iterdir() if d.is_dir()]
+        num_cams = len(subdirectories)
+
+        # Assuming the first subdirectory to determine the number of frames
+        sample_files = list(subdirectories[0].glob("*.json"))
+        num_frames = len(sample_files)
+        frame_indices = [self.extract_frame_index(f.name) for f in sample_files]
+        frame_indices.sort()
+
+        # Assuming standard OpenPose output
+        body_markers, hand_markers, face_markers = 25, 21, 70
+        num_markers = body_markers + 2 * hand_markers + face_markers
+
+        data_array = np.full((num_cams, num_frames, num_markers, 3), np.nan)
+
+        for cam_index, subdir in enumerate(subdirectories):
+            json_files = sorted(subdir.glob("*.json"), key=lambda x: self.extract_frame_index(x.stem))
+
+            for file_index, json_file in enumerate(tqdm(json_files, desc=f'Processing {subdir.name} JSONs')):
+                with open(json_file) as f:
+                    data = json.load(f)
+
+                if data["people"]:
+                    keypoints = self.extract_keypoints(data["people"][0], body_markers, hand_markers, face_markers)
+                    data_array[cam_index, frame_indices[file_index], :, :] = keypoints
+
+        return data_array
+
+    def extract_keypoints(self,person_data, body_markers, hand_markers, face_markers):
+        """Extract and organize keypoints from person data."""
+        # Initialize a full array of NaNs for keypoints
+        keypoints_array = np.full((body_markers + 2 * hand_markers + face_markers, 3), np.nan)
+        
+        # Populate the array with available data
+        if "pose_keypoints_2d" in person_data:
+            keypoints_array[:body_markers, :] = np.reshape(person_data["pose_keypoints_2d"], (-1, 3))[:body_markers, :]
+        if "hand_left_keypoints_2d" in person_data and "hand_right_keypoints_2d" in person_data:
+            keypoints_array[body_markers:body_markers + hand_markers, :] = np.reshape(person_data["hand_left_keypoints_2d"], (-1, 3))[:hand_markers, :]
+            keypoints_array[body_markers + hand_markers:body_markers + 2*hand_markers, :] = np.reshape(person_data["hand_right_keypoints_2d"], (-1, 3))[:hand_markers, :]
+        if "face_keypoints_2d" in person_data:
+            keypoints_array[body_markers + 2*hand_markers:, :] = np.reshape(person_data["face_keypoints_2d"], (-1, 3))[:face_markers, :]
+
+        return keypoints_array
+
+    def process_tracked_objects(self, **kwargs) -> np.ndarray:
+        """
+        Convert the recorded JSON data into the structured numpy array format.
+        """
+        # In this case, the recorded_objects are already in the desired format,
+        # so we simply return them.
+        self.recorded_objects_array = self.recorded_objects
+        return self.recorded_objects_array
diff --git a/skellytracker/trackers/openpose_tracker/openpose_tracker.py b/skellytracker/trackers/openpose_tracker/openpose_tracker.py
new file mode 100644
index 0000000..c1f9194
--- /dev/null
+++ b/skellytracker/trackers/openpose_tracker/openpose_tracker.py
@@ -0,0 +1,106 @@
+import subprocess
+from pathlib import Path
+from skellytracker.trackers.base_tracker.base_tracker import BaseTracker
+from skellytracker.trackers.openpose_tracker.openpose_recorder import OpenPoseRecorder
+from typing import Dict, Any
+import numpy as np
+
+import os
+class OpenPoseTracker(BaseTracker):
+    def __init__(self, openpose_exe_path, output_json_path, net_resolution="-1x320", number_people_max=1):
+        """
+        Initialize the OpenPoseTracker.
+
+        :param recorder: An instance of OpenPoseRecorder for handling the output.
+        :param openpose_exe_path: Path to the OpenPose executable.
+        :param output_json_path: Directory where JSON files will be saved.
+        :param net_resolution: Network resolution for OpenPose processing.
+        :param number_people_max: Maximum number of people to detect.
+        """
+        # super().__init__(recorder=recorder)
+        self.openpose_exe_path = Path(openpose_exe_path)
+        self.output_json_path = Path(output_json_path)
+        self.net_resolution = net_resolution
+        self.number_people_max = number_people_max
+
+        super().__init__(
+            tracked_object_names=[],
+            recorder=OpenPoseRecorder(json_directory_path=output_json_path),
+        )
+
+
+        os.chdir(self.openpose_exe_path)
+
+    def run_openpose(self, input_video_filepath, output_video_folder):
+        """
+        Run the OpenPose demo on a video file to generate JSON outputs
+        in a unique directory for each video.
+        """
+
+        # Extract video name without extension to use as a unique folder name
+        video_name = Path(input_video_filepath).stem
+        unique_json_output_path = self.output_json_path / video_name
+        unique_json_output_path.mkdir(parents=True, exist_ok=True)  # Create the directory if it doesn't exist
+
+        video_save_path = output_video_folder / f"{video_name}_openpose.avi"
+
+        # Update the subprocess command to use the unique output directory
+        subprocess.run(
+            [
+                "bin\OpenPoseDemo.exe",
+                "--video", str(input_video_filepath),
+                "--write_json", str(unique_json_output_path),
+                "--net_resolution", self.net_resolution,
+                "--hand",
+                "--face",
+                "--number_people_max", str(self.number_people_max),
+                '--write_video', 
+                str(video_save_path),
+                "--output_resolution",
+                "-1x-1",
+
+            ],
+            shell=True,
+        )
+
+    def process_video(self, input_video_filepath, output_video_folder, save_data_bool=False, use_tqdm=True):
+        # Run OpenPose on the input video
+        self.run_openpose(input_video_filepath, output_video_folder)
+
+        if self.recorder is not None:
+            self.recorder.record(self.tracked_objects)
+
+
+    def process_image(self, image: np.ndarray, **kwargs) -> Dict[str, Any]:
+        """Process an individual frame/image."""
+
+        raise NotImplementedError("OpenPoseTracker processes video files directly and does not process individual images.")
+
+    def annotate_image(self, image: np.ndarray, tracked_objects: Dict[str, Any], **kwargs) -> np.ndarray:
+        """Annotate an image with tracking data."""
+
+        raise NotImplementedError("OpenPoseTracker does not annotate images directly. Annotations are handled by OpenPose's output.")
+
+
+if __name__ == '__main__':
+    # Example usage
+    from pathlib import Path
+
+    input_video_folder = Path(r'C:\Users\aaron\FreeMocap_Data\recording_sessions\freemocap_sample_data')
+    input_video_filepath = input_video_folder/'synchronized_videos'/'sesh_2022-09-19_16_16_50_in_class_jsm_synced_Cam1.mp4'
+    
+    output_video_folder = input_video_folder/'openpose_annotated_videos'
+    output_video_folder.mkdir(parents=True, exist_ok=True)
+
+    output_json_path = input_video_folder/'output_data'/'raw_data'/'openpose_jsons'
+    output_json_path.mkdir(parents=True, exist_ok=True)
+
+    openpose_exe_path = r'C:\openpose'
+    # output_json_path = r'C:\openpose\output_json'
+    # input_video_filepath = r'C:\path\to\input\video.mp4'
+    # output_video_filepath = r'C:\path\to\output\video.mp4'
+    tracker = OpenPoseTracker(
+        openpose_exe_path=str(openpose_exe_path),
+        output_json_path=str(output_json_path),
+    )
+    tracker.process_video(input_video_filepath, output_video_folder)
\ No newline at end of file

From 14ce558c25e548cd64954ff6c93acd476d42df57 Mon Sep 17 00:00:00 2001
From: aaroncherian <aaron.cherian2@gmail.com>
Date: Tue, 6 Feb 2024 12:49:46 -0500
Subject: [PATCH 02/11] more updates

---
 skellytracker/process_folder_of_videos.py     | 43 ++++++++++++++++---
 .../openpose_tracker/openpose_model_info.py   | 10 +++++
 .../openpose_tracker/openpose_tracker.py      |  2 +-
 3 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/skellytracker/process_folder_of_videos.py b/skellytracker/process_folder_of_videos.py
index 15dbceb..d816060 100644
--- a/skellytracker/process_folder_of_videos.py
+++ b/skellytracker/process_folder_of_videos.py
@@ -24,6 +24,7 @@
 
 
 from skellytracker.trackers.openpose_tracker.openpose_tracker import OpenPoseTracker
+from skellytracker.trackers.openpose_tracker.openpose_model_info import OpenPoseTrackingParams
 
 from skellytracker.utilities.get_video_paths import get_video_paths
 
@@ -34,6 +35,7 @@
     "YOLOMediapipeComboTracker": "mediapipe2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy",
     "YOLOPoseTracker": "yolo2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy",
     "BrightestPointTracker": "brightestPoint2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy",
+    "OpenPoseTracker": "openpose2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy",
 }
 
 
@@ -166,7 +168,13 @@ def get_tracker(tracker_name: str, tracking_params: BaseModel) -> BaseTracker:
     elif tracker_name == "BrightestPointTracker":
         tracker = BrightestPointTracker()
 
-    elif tracker_name = 'OpenPoseTracker':
+    elif tracker_name == 'OpenPoseTracker':
+        tracker = OpenPoseTracker(
+            openpose_exe_path=tracking_params.openpose_exe_path,
+            output_json_path=tracking_params.output_json_path,
+            net_resolution=tracking_params.net_resolution,
+            number_people_max=tracking_params.number_people_max,
+        )
         
     else:
         raise ValueError("Invalid tracker type")
@@ -175,15 +183,40 @@ def get_tracker(tracker_name: str, tracking_params: BaseModel) -> BaseTracker:
 
 
 if __name__ == "__main__":
+
     synchronized_video_path = Path(
         "/Users/philipqueen/freemocap_data/recording_sessions/freemocap_sample_data/synchronized_videos"
     )
-    tracker_name = "YOLOMediapipeComboTracker"
-    num_processes = None
+    # tracker_name = "YOLOMediapipeComboTracker"
+    # num_processes = None
+
+    # process_folder_of_videos(
+    #     tracker_name=tracker_name,
+    #     tracking_params=MediapipeTrackingParams(),
+    #     synchronized_video_path=synchronized_video_path,
+    #     num_processes=num_processes,
+    # )
+
+    tracker_name = "OpenPoseTracker"
+    num_processes = 1
+    
+    input_video_folder = Path(r'C:\Users\aaron\FreeMocap_Data\recording_sessions\freemocap_sample_data')
+    input_video_filepath = input_video_folder/'synchronized_videos'
+    
+    output_video_folder = input_video_folder/'openpose_annotated_videos'
+    output_video_folder.mkdir(parents=True, exist_ok=True)
+
+    output_json_path = input_video_folder/'output_data'/'raw_data'/'openpose_jsons'
+    output_json_path.mkdir(parents=True, exist_ok=True)
+
+    openpose_exe_path = r'C:\openpose'
 
     process_folder_of_videos(
         tracker_name=tracker_name,
-        tracking_params=MediapipeTrackingParams(),
-        synchronized_video_path=synchronized_video_path,
+        tracking_params=OpenPoseTrackingParams(
+            openpose_exe_path=str(openpose_exe_path),
+            output_json_path=str(output_json_path),
+        ),
+        synchronized_video_path=input_video_filepath,
         num_processes=num_processes,
     )
diff --git a/skellytracker/trackers/openpose_tracker/openpose_model_info.py b/skellytracker/trackers/openpose_tracker/openpose_model_info.py
index e69de29..09cbdba 100644
--- a/skellytracker/trackers/openpose_tracker/openpose_model_info.py
+++ b/skellytracker/trackers/openpose_tracker/openpose_model_info.py
@@ -0,0 +1,10 @@
+from skellytracker.trackers.base_tracker.base_tracking_params import BaseTrackingParams
+
+
+class OpenPoseTrackingParams(BaseTrackingParams):
+    openpose_exe_path: str
+    output_json_path: str
+    net_resolution: str = "-1x320"
+    number_people_max: int = 1
+    write_video: bool = True
+    openpose_output_resolution: str = "-1x-1"
diff --git a/skellytracker/trackers/openpose_tracker/openpose_tracker.py b/skellytracker/trackers/openpose_tracker/openpose_tracker.py
index c1f9194..ae03a7f 100644
--- a/skellytracker/trackers/openpose_tracker/openpose_tracker.py
+++ b/skellytracker/trackers/openpose_tracker/openpose_tracker.py
@@ -7,7 +7,7 @@
 
 import os
 class OpenPoseTracker(BaseTracker):
-    def __init__(self, openpose_exe_path, output_json_path, net_resolution="-1x320", number_people_max=1):
+    def __init__(self, openpose_exe_path, output_json_path, net_resolution="-1x640", number_people_max=1):
         """
         Initialize the OpenPoseTracker.
 

From ce0f4249c5cd9512b7706653898c23bbd84f301d Mon Sep 17 00:00:00 2001
From: philipqueen <queen.philip@gmail.com>
Date: Tue, 6 Feb 2024 18:10:47 -0700
Subject: [PATCH 03/11] remove annotated image as argument to base recorder

---
 skellytracker/trackers/base_tracker/base_recorder.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/skellytracker/trackers/base_tracker/base_recorder.py b/skellytracker/trackers/base_tracker/base_recorder.py
index f81f0b3..4aac353 100644
--- a/skellytracker/trackers/base_tracker/base_recorder.py
+++ b/skellytracker/trackers/base_tracker/base_recorder.py
@@ -20,13 +20,12 @@ def __init__(self):
 
     @abstractmethod
     def record(
-        self, tracked_objects: Dict[str, TrackedObject], annotated_image: np.ndarray
+        self, tracked_objects: Dict[str, TrackedObject],
     ) -> None:
         """
         Record the tracked objects as they are created by the tracker.
 
         :param tracked_object: A tracked objects dictionary.
-        :param annotated_image: Image array with tracking results annotated.
         :return: None
         """
         pass

From fb42f663712122b8f312f84a529ebd34e7f423ee Mon Sep 17 00:00:00 2001
From: philipqueen <queen.philip@gmail.com>
Date: Tue, 6 Feb 2024 18:20:53 -0700
Subject: [PATCH 04/11] fixing the linting errors

---
 skellytracker/trackers/base_tracker/base_recorder.py | 5 +++--
 skellytracker/trackers/base_tracker/video_handler.py | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/skellytracker/trackers/base_tracker/base_recorder.py b/skellytracker/trackers/base_tracker/base_recorder.py
index 4aac353..35fb524 100644
--- a/skellytracker/trackers/base_tracker/base_recorder.py
+++ b/skellytracker/trackers/base_tracker/base_recorder.py
@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 import logging
-from typing import Dict
+from pathlib import Path
+from typing import Dict, Union
 
 import numpy as np
 
@@ -44,7 +45,7 @@ def clear_recorded_objects(self):
         self.recorded_objects = []
         self.recorded_objects_array = None
 
-    def save(self, file_path: str) -> None:
+    def save(self, file_path: Union[str, Path]) -> None:
         """
         Save the recorded objects to a file.
 
diff --git a/skellytracker/trackers/base_tracker/video_handler.py b/skellytracker/trackers/base_tracker/video_handler.py
index 13cc064..13d7249 100644
--- a/skellytracker/trackers/base_tracker/video_handler.py
+++ b/skellytracker/trackers/base_tracker/video_handler.py
@@ -24,7 +24,7 @@ def __init__(
         :param codec: The codec to use for the output video.
         """
         self.output_path = output_path
-        fourcc = cv2.VideoWriter_fourcc(*codec)
+        fourcc = cv2.VideoWriter.fourcc(*codec)
         self.video_writer = cv2.VideoWriter(
             str(output_path), fourcc, fps, frame_size
         )

From ba54193ac3c6a0dbd8af582a417b435065ffa04a Mon Sep 17 00:00:00 2001
From: philipqueen <queen.philip@gmail.com>
Date: Tue, 6 Feb 2024 18:27:36 -0700
Subject: [PATCH 05/11] create abstract cumulative tracker, does not support
 by-image methods

---
 .../trackers/base_tracker/base_tracker.py     | 56 ++++++++++++++++++-
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/skellytracker/trackers/base_tracker/base_tracker.py b/skellytracker/trackers/base_tracker/base_tracker.py
index 4e19280..c136db9 100644
--- a/skellytracker/trackers/base_tracker/base_tracker.py
+++ b/skellytracker/trackers/base_tracker/base_tracker.py
@@ -66,7 +66,7 @@ def process_video(
         output_video_filepath: Optional[Union[str, Path]] = None,
         save_data_bool: bool = False,
         use_tqdm: bool = True,
-    ) -> np.ndarray:
+    ) -> Union[np.ndarray, None]:
         """
         Run the tracker on a video.
 
@@ -99,7 +99,7 @@ def process_video(
         if use_tqdm:
             iterator = tqdm(
                 range(number_of_frames),
-                desc=f"processing video: {input_video_filepath.name}",
+                desc=f"processing video: {Path(input_video_filepath).name}",
                 total=number_of_frames,
                 colour="magenta",
                 unit="frames",
@@ -159,3 +159,55 @@ def image_demo(self, image_path: Path) -> None:
 
         image_viewer = ImageDemoViewer(self, self.__class__.__name__)
         image_viewer.run(image_path=image_path)
+
+
+class BaseCumulativeTracker(BaseTracker):
+    """
+    A base class for tracking algorithms that run cumulatively, i.e are not able to process videos frame by frame.
+    Throws a descriptive error for the abstract methods of BaseTracker that do not apply to this type of tracker.
+    Trackers inheriting from this will need to overwrite the `process_video` method.
+    """
+
+    def __init__(
+        self,
+        tracked_object_names: List[str] = None,
+        recorder: BaseRecorder = None,
+        **data: Any,
+    ):
+        super().__init__(
+            tracked_object_names=tracked_object_names, recorder=recorder, **data
+        )
+
+    def process_image(self, **kwargs) -> None:
+        raise NotImplementedError(
+            "This tracker does not support processing individual images, please use process_video instead."
+        )
+
+    def annotate_image(self, **kwargs) -> None:
+        raise NotImplementedError(
+            "This tracker does not support processing individual images, please use process_video instead."
+        )
+
+    @abstractmethod
+    def process_video(
+        self,
+        input_video_filepath: Union[str, Path],
+        output_video_filepath: Optional[Union[str, Path]] = None,
+        save_data_bool: bool = False,
+        use_tqdm: bool = True,
+    ) -> Union[np.ndarray, None]:
+        """
+        Run the tracker on a video.
+
+        :param input_video_filepath: Path to video file.
+        :param output_video_filepath: Path to save annotated video to, does not save video if None.
+        :param save_data_bool: Whether to save the data to a file.
+        :param use_tqdm: Whether to use tqdm to show a progress bar
+        :return: Array of tracked keypoint data
+        """
+        pass
+
+    def image_demo(self, image_path: Path) -> None:
+        raise NotImplementedError(
+            "This tracker does not support processing individual images, please use process_video instead."
+        )

From db894bcbdf05112802f0d1236d2f63aca1a42204 Mon Sep 17 00:00:00 2001
From: philipqueen <queen.philip@gmail.com>
Date: Tue, 6 Feb 2024 18:44:42 -0700
Subject: [PATCH 06/11] make a cumulative recorder for openpose recorder to
 inherit from

---
 .../trackers/base_tracker/base_recorder.py     | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/skellytracker/trackers/base_tracker/base_recorder.py b/skellytracker/trackers/base_tracker/base_recorder.py
index 35fb524..cb104b7 100644
--- a/skellytracker/trackers/base_tracker/base_recorder.py
+++ b/skellytracker/trackers/base_tracker/base_recorder.py
@@ -21,7 +21,8 @@ def __init__(self):
 
     @abstractmethod
     def record(
-        self, tracked_objects: Dict[str, TrackedObject],
+        self,
+        tracked_objects: Dict[str, TrackedObject],
     ) -> None:
         """
         Record the tracked objects as they are created by the tracker.
@@ -56,3 +57,18 @@ def save(self, file_path: Union[str, Path]) -> None:
             self.process_tracked_objects()
         logger.info(f"Saving recorded objects to {file_path}")
         np.save(file_path, self.recorded_objects_array)
+
+
+class BaseCumulativeRecorder(BaseRecorder):
+    """
+    A base class for recording data from cumulative trackers.
+    Throws a descriptive error for methods that do not apply to recording data from this type of tracker.
+    Trackers implementing this will only use the process_tracked_objects method to get data in the proper format.
+    """
+    def __init__(self):
+        super().__init__()
+
+    def record(self, tracked_objects: Dict[str, TrackedObject]) -> None:
+        raise NotImplementedError(
+            "This tracker does not support by frame recording, please use process_tracked_objects instead"
+        )

From b14cffebf364d38bc27015c025051a0de0b5246e Mon Sep 17 00:00:00 2001
From: philipqueen <queen.philip@gmail.com>
Date: Tue, 6 Feb 2024 18:54:15 -0700
Subject: [PATCH 07/11] type hints and todos in openpose_recorder

---
 .../openpose_tracker/openpose_recorder.py     | 75 ++++++++++++-------
 1 file changed, 49 insertions(+), 26 deletions(-)

diff --git a/skellytracker/trackers/openpose_tracker/openpose_recorder.py b/skellytracker/trackers/openpose_tracker/openpose_recorder.py
index b7c3cc4..16103d0 100644
--- a/skellytracker/trackers/openpose_tracker/openpose_recorder.py
+++ b/skellytracker/trackers/openpose_tracker/openpose_recorder.py
@@ -1,30 +1,24 @@
 import json
+from typing import Union
 import numpy as np
 from pathlib import Path
-from skellytracker.trackers.base_tracker.base_recorder import BaseRecorder
+from skellytracker.trackers.base_tracker.base_recorder import BaseCumulativeRecorder
 import re
 from tqdm import tqdm
 
-class OpenPoseRecorder(BaseRecorder):
-    def __init__(self, json_directory_path):
+
+class OpenPoseRecorder(BaseCumulativeRecorder):
+    def __init__(self, json_directory_path: Union[Path, str]):
         super().__init__()
         self.json_directory_path = Path(json_directory_path)
 
-    def extract_frame_index(self, filename):
+    def extract_frame_index(self, filename: str) -> Union[int, None]:
         """Extract the numeric part indicating the frame index from the filename."""
         match = re.search(r"_(\d{12})_keypoints", filename)
         return int(match.group(1)) if match else None
 
-    def record(self, tracked_objects=None, annotated_image=None) -> None:
-        """
-        Override the record method to read from JSON files instead of receiving data directly.
-        """
-        # This method is adapted to fit the requirement of reading from JSON,
-        # hence tracked_objects and annotated_image are not used.
-        self.recorded_objects = self.parse_openpose_jsons(self.json_directory_path)
-
-    def parse_openpose_jsons(self, main_directory):
-        subdirectories = [d for d in main_directory.iterdir() if d.is_dir()]
+    def parse_openpose_jsons(self, main_directory: Union[Path, str]) -> np.ndarray:
+        subdirectories = [d for d in Path(main_directory).iterdir() if d.is_dir()]
         num_cams = len(subdirectories)
 
         # Assuming the first subdirectory to determine the number of frames
@@ -34,37 +28,64 @@ def parse_openpose_jsons(self, main_directory):
         frame_indices.sort()
 
         # Assuming standard OpenPose output
+        # TODO: move these definitions to openpose model info
         body_markers, hand_markers, face_markers = 25, 21, 70
         num_markers = body_markers + 2 * hand_markers + face_markers
 
         data_array = np.full((num_cams, num_frames, num_markers, 3), np.nan)
 
         for cam_index, subdir in enumerate(subdirectories):
-            json_files = sorted(subdir.glob("*.json"), key=lambda x: self.extract_frame_index(x.stem))
+            json_files = sorted(
+                subdir.glob("*.json"), key=lambda x: self.extract_frame_index(x.stem)
+            )
 
-            for file_index, json_file in enumerate(tqdm(json_files, desc=f'Processing {subdir.name} JSONs')):
+            for file_index, json_file in enumerate(
+                tqdm(json_files, desc=f"Processing {subdir.name} JSONs")
+            ):
                 with open(json_file) as f:
                     data = json.load(f)
 
                 if data["people"]:
-                    keypoints = self.extract_keypoints(data["people"][0], body_markers, hand_markers, face_markers)
+                    keypoints = self.extract_keypoints(
+                        data["people"][0], body_markers, hand_markers, face_markers
+                    )
                     data_array[cam_index, frame_indices[file_index], :, :] = keypoints
 
         return data_array
 
-    def extract_keypoints(self,person_data, body_markers, hand_markers, face_markers):
+    def extract_keypoints(
+        self, person_data, body_markers, hand_markers, face_markers
+    ) -> (
+        np.ndarray
+    ):  # TODO: type hint person_data - is this an ndarray yet or something else?
+        # TODO: marker numbers don't need to be passed into this function, they can just be referred to as OpenPoseModelInfo.xyz
         """Extract and organize keypoints from person data."""
         # Initialize a full array of NaNs for keypoints
-        keypoints_array = np.full((body_markers + 2 * hand_markers + face_markers, 3), np.nan)
-        
+        keypoints_array = np.full(
+            (body_markers + 2 * hand_markers + face_markers, 3), np.nan
+        )
+
         # Populate the array with available data
         if "pose_keypoints_2d" in person_data:
-            keypoints_array[:body_markers, :] = np.reshape(person_data["pose_keypoints_2d"], (-1, 3))[:body_markers, :]
-        if "hand_left_keypoints_2d" in person_data and "hand_right_keypoints_2d" in person_data:
-            keypoints_array[body_markers:body_markers + hand_markers, :] = np.reshape(person_data["hand_left_keypoints_2d"], (-1, 3))[:hand_markers, :]
-            keypoints_array[body_markers + hand_markers:body_markers + 2*hand_markers, :] = np.reshape(person_data["hand_right_keypoints_2d"], (-1, 3))[:hand_markers, :]
+            keypoints_array[:body_markers, :] = np.reshape(
+                person_data["pose_keypoints_2d"], (-1, 3)
+            )[:body_markers, :]
+        if (
+            "hand_left_keypoints_2d" in person_data
+            and "hand_right_keypoints_2d" in person_data
+        ):
+            keypoints_array[body_markers : body_markers + hand_markers, :] = np.reshape(
+                person_data["hand_left_keypoints_2d"], (-1, 3)
+            )[:hand_markers, :]
+            keypoints_array[
+                body_markers + hand_markers : body_markers + 2 * hand_markers, :
+            ] = np.reshape(person_data["hand_right_keypoints_2d"], (-1, 3))[
+                :hand_markers, :
+            ]
         if "face_keypoints_2d" in person_data:
-            keypoints_array[body_markers + 2*hand_markers:, :] = np.reshape(person_data["face_keypoints_2d"], (-1, 3))[:face_markers, :]
+            keypoints_array[body_markers + 2 * hand_markers :, :] = np.reshape(
+                person_data["face_keypoints_2d"], (-1, 3)
+            )[:face_markers, :]
 
         return keypoints_array
 
@@ -74,5 +95,7 @@ def process_tracked_objects(self, **kwargs) -> np.ndarray:
         """
         # In this case, the recorded_objects are already in the desired format,
         # so we simply return them.
-        self.recorded_objects_array = self.recorded_objects
+        self.recorded_objects_array = self.parse_openpose_jsons(
+            self.json_directory_path
+        )
         return self.recorded_objects_array

From a1a4d527d656733ecd44a32818acb54255e03871 Mon Sep 17 00:00:00 2001
From: philipqueen <queen.philip@gmail.com>
Date: Tue, 6 Feb 2024 19:05:41 -0700
Subject: [PATCH 08/11] move some functions, add type hints and TODOs

---
 .../openpose_tracker/openpose_tracker.py      | 97 ++++++++++---------
 1 file changed, 52 insertions(+), 45 deletions(-)

diff --git a/skellytracker/trackers/openpose_tracker/openpose_tracker.py b/skellytracker/trackers/openpose_tracker/openpose_tracker.py
index ae03a7f..f854bce 100644
--- a/skellytracker/trackers/openpose_tracker/openpose_tracker.py
+++ b/skellytracker/trackers/openpose_tracker/openpose_tracker.py
@@ -1,13 +1,20 @@
 import subprocess
 from pathlib import Path
-from skellytracker.trackers.base_tracker.base_tracker import BaseTracker
+from typing import Union
+from skellytracker.trackers.base_tracker.base_tracker import BaseCumulativeTracker
 from skellytracker.trackers.openpose_tracker.openpose_recorder import OpenPoseRecorder
-from typing import Dict, Any
-import numpy as np
 
 import os
-class OpenPoseTracker(BaseTracker):
-    def __init__(self, openpose_exe_path, output_json_path, net_resolution="-1x640", number_people_max=1):
+
+
+class OpenPoseTracker(BaseCumulativeTracker):
+    def __init__(
+        self,
+        openpose_exe_path: Union[str, Path],
+        output_json_path: Union[str, Path],
+        net_resolution: str = "-1x640",
+        number_people_max: int = 1,
+    ):
         """
         Initialize the OpenPoseTracker.
 
@@ -28,10 +35,16 @@ def __init__(self, openpose_exe_path, output_json_path, net_resolution="-1x640",
             recorder=OpenPoseRecorder(json_directory_path=output_json_path),
         )
 
+        os.chdir(self.openpose_exe_path) # TODO: this can mess with things downstream if we're not sure to change the path back, we should just be explicit about the path throughout
 
-        os.chdir(self.openpose_exe_path)
-
-    def run_openpose(self, input_video_filepath, output_video_folder):
+    def process_video(
+        self,
+        input_video_filepath: Union[str, Path],
+        output_video_folder: Union[str, Path],
+        save_data_bool: bool = False,
+        use_tqdm: bool = True,
+        **kwargs,
+    ):
         """
         Run the OpenPose demo on a video file to generate JSON outputs
         in a unique directory for each video.
@@ -40,62 +53,56 @@ def run_openpose(self, input_video_filepath, output_video_folder):
         # Extract video name without extension to use as a unique folder name
         video_name = Path(input_video_filepath).stem
         unique_json_output_path = self.output_json_path / video_name
-        unique_json_output_path.mkdir(parents=True, exist_ok=True)  # Create the directory if it doesn't exist
+        unique_json_output_path.mkdir(
+            parents=True, exist_ok=True
+        )  # Create the directory if it doesn't exist
 
-        video_save_path = output_video_folder / f"{video_name}_openpose.avi"
+        video_save_path = Path(output_video_folder) / f"{video_name}_openpose.avi"
 
         # Update the subprocess command to use the unique output directory
         subprocess.run(
             [
-                "bin\OpenPoseDemo.exe",
-                "--video", str(input_video_filepath),
-                "--write_json", str(unique_json_output_path),
-                "--net_resolution", self.net_resolution,
+                "bin\OpenPoseDemo.exe",  # TODO: this is throwing the error `S607 Starting a process with a partial executable path`, needs to be something like usr/bin/OpenPoseDemo.exe, but I can't verify where it is (also does it need two backslashes to escape properly?)
+                "--video",
+                str(input_video_filepath),
+                "--write_json",
+                str(unique_json_output_path),
+                "--net_resolution",
+                self.net_resolution,
                 "--hand",
                 "--face",
-                "--number_people_max", str(self.number_people_max),
-                '--write_video', 
+                "--number_people_max",
+                str(self.number_people_max),
+                "--write_video",
                 str(video_save_path),
                 "--output_resolution",
                 "-1x-1",
-
             ],
-            shell=True,
+            shell=True,  # TODO: This is generally frowned upon if there's not a reason for it, see `S602 subprocess call with shell=True identified, security issue.`
         )
 
-    def process_video(self, input_video_filepath, output_video_folder, save_data_bool=False, use_tqdm=True):
-        # Run OpenPose on the input video
-        self.run_openpose(input_video_filepath, output_video_folder)
-
-        if self.recorder is not None:
-            self.recorder.record(self.tracked_objects)
-
-
-    def process_image(self, image: np.ndarray, **kwargs) -> Dict[str, Any]:
-        """Process an individual frame/image."""
-
-        raise NotImplementedError("OpenPoseTracker processes video files directly and does not process individual images.")
 
-    def annotate_image(self, image: np.ndarray, tracked_objects: Dict[str, Any], **kwargs) -> np.ndarray:
-        """Annotate an image with tracking data."""
-
-        raise NotImplementedError("OpenPoseTracker does not annotate images directly. Annotations are handled by OpenPose's output.")
-
-
-if __name__ == '__main__':
+if __name__ == "__main__":
     # Example usage
-    from pathlib import Path
 
-    input_video_folder = Path(r'C:\Users\aaron\FreeMocap_Data\recording_sessions\freemocap_sample_data')
-    input_video_filepath = input_video_folder/'synchronized_videos'/'sesh_2022-09-19_16_16_50_in_class_jsm_synced_Cam1.mp4'
-    
-    output_video_folder = input_video_folder/'openpose_annotated_videos'
+    input_video_folder = Path(
+        r"C:\Users\aaron\FreeMocap_Data\recording_sessions\freemocap_sample_data"
+    )
+    input_video_filepath = (
+        input_video_folder
+        / "synchronized_videos"
+        / "sesh_2022-09-19_16_16_50_in_class_jsm_synced_Cam1.mp4"
+    )
+
+    output_video_folder = input_video_folder / "openpose_annotated_videos"
     output_video_folder.mkdir(parents=True, exist_ok=True)
 
-    output_json_path = input_video_folder/'output_data'/'raw_data'/'openpose_jsons'
+    output_json_path = (
+        input_video_folder / "output_data" / "raw_data" / "openpose_jsons"
+    )
     output_json_path.mkdir(parents=True, exist_ok=True)
 
-    openpose_exe_path = r'C:\openpose'
+    openpose_exe_path = r"C:\openpose"
     # output_json_path = r'C:\openpose\output_json'
     # input_video_filepath = r'C:\path\to\input\video.mp4'
     # output_video_filepath = r'C:\path\to\output\video.mp4'
@@ -103,4 +110,4 @@ def annotate_image(self, image: np.ndarray, tracked_objects: Dict[str, Any], **k
         openpose_exe_path=str(openpose_exe_path),
         output_json_path=str(output_json_path),
     )
-    tracker.process_video(input_video_filepath, output_video_folder)
\ No newline at end of file
+    tracker.process_video(input_video_filepath, output_video_folder)

From fec1c3891cfb97a6add21e26e4c2ca00c3ef220f Mon Sep 17 00:00:00 2001
From: philipqueen <queen.philip@gmail.com>
Date: Tue, 6 Feb 2024 19:06:16 -0700
Subject: [PATCH 09/11] add kwargs to process_video function for cumulative
 tracker

---
 skellytracker/trackers/base_tracker/base_recorder.py | 1 +
 skellytracker/trackers/base_tracker/base_tracker.py  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/skellytracker/trackers/base_tracker/base_recorder.py b/skellytracker/trackers/base_tracker/base_recorder.py
index cb104b7..c3767cd 100644
--- a/skellytracker/trackers/base_tracker/base_recorder.py
+++ b/skellytracker/trackers/base_tracker/base_recorder.py
@@ -65,6 +65,7 @@ class BaseCumulativeRecorder(BaseRecorder):
     Throws a descriptive error for methods that do not apply to recording data from this type of tracker.
     Trackers implementing this will only use the process_tracked_objects method to get data in the proper format.
     """
+
     def __init__(self):
         super().__init__()
 
diff --git a/skellytracker/trackers/base_tracker/base_tracker.py b/skellytracker/trackers/base_tracker/base_tracker.py
index c136db9..2509d03 100644
--- a/skellytracker/trackers/base_tracker/base_tracker.py
+++ b/skellytracker/trackers/base_tracker/base_tracker.py
@@ -195,6 +195,7 @@ def process_video(
         output_video_filepath: Optional[Union[str, Path]] = None,
         save_data_bool: bool = False,
         use_tqdm: bool = True,
+        **kwargs,
     ) -> Union[np.ndarray, None]:
         """
         Run the tracker on a video.

From 3b56761e844fc1497612cedb11114e1605805a3a Mon Sep 17 00:00:00 2001
From: philipqueen <queen.philip@gmail.com>
Date: Tue, 6 Feb 2024 19:07:50 -0700
Subject: [PATCH 10/11] suggest parameter change

---
 skellytracker/trackers/openpose_tracker/openpose_tracker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/skellytracker/trackers/openpose_tracker/openpose_tracker.py b/skellytracker/trackers/openpose_tracker/openpose_tracker.py
index f854bce..bf402e2 100644
--- a/skellytracker/trackers/openpose_tracker/openpose_tracker.py
+++ b/skellytracker/trackers/openpose_tracker/openpose_tracker.py
@@ -27,7 +27,7 @@ def __init__(
         # super().__init__(recorder=recorder)
         self.openpose_exe_path = Path(openpose_exe_path)
         self.output_json_path = Path(output_json_path)
-        self.net_resolution = net_resolution
+        self.net_resolution = net_resolution # TODO: this and num_people should be parameters for process_video, since we could use this one tracker to process videos with different parameters
         self.number_people_max = number_people_max
 
         super().__init__(

From 547038c267bfc18f5d762b2bf24a7241e3ec0206 Mon Sep 17 00:00:00 2001
From: philipqueen <queen.philip@gmail.com>
Date: Tue, 6 Feb 2024 19:40:15 -0700
Subject: [PATCH 11/11] add some notes on error handling

---
 skellytracker/trackers/openpose_tracker/openpose_tracker.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/skellytracker/trackers/openpose_tracker/openpose_tracker.py b/skellytracker/trackers/openpose_tracker/openpose_tracker.py
index bf402e2..bee0031 100644
--- a/skellytracker/trackers/openpose_tracker/openpose_tracker.py
+++ b/skellytracker/trackers/openpose_tracker/openpose_tracker.py
@@ -50,6 +50,8 @@ def process_video(
         in a unique directory for each video.
         """
 
+        # TODO: Add some check that the openpose executable exists (and maybe do a cross platform search like we do for blender in fmc?)
+
         # Extract video name without extension to use as a unique folder name
         video_name = Path(input_video_filepath).stem
         unique_json_output_path = self.output_json_path / video_name
@@ -60,6 +62,7 @@ def process_video(
         video_save_path = Path(output_video_folder) / f"{video_name}_openpose.avi"
 
         # Update the subprocess command to use the unique output directory
+        # TODO: subprocess call should probably be in a try/except with some form of error handling
         subprocess.run(
             [
                 "bin\OpenPoseDemo.exe",  # TODO: this is throwing the error `S607 Starting a process with a partial executable path`, needs to be something like usr/bin/OpenPoseDemo.exe, but I can't verify where it is (also does it need two backslashes to escape properly?)