From f789dd26a1f9ab40313f70b817a087d69573872a Mon Sep 17 00:00:00 2001 From: Chadwick Boulay Date: Fri, 20 Dec 2019 14:40:30 -0500 Subject: [PATCH] Added more stream info and added events stream. --- CMakeLists.txt | 1 + README.md | 13 ++- cmake/FindTobii.cmake | 3 + mainwindow.cpp | 222 ++++++++++++++++++++++++++++++++---------- mainwindow.h | 1 + mainwindow.ui | 4 +- 6 files changed, 184 insertions(+), 60 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c13b6d6..9732308 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,7 @@ cmake_minimum_required(VERSION 3.5) project(TobiiStreamEngine LANGUAGES CXX VERSION 0.1) +cmake_policy(SET CMP0074 NEW) #: [project](https://cmake.org/cmake/help/latest/command/project.html) sets the #: name of the app, the languages used and the version. The version is later on diff --git a/README.md b/README.md index 9ef4e7b..3dce839 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,15 @@ # Application Description Stream data from Tobii consumer devices over LSL. +So for only a single stream with 2-D gaze points on screen in normalized coordinates is provided. -## Dependencies +## Download + +Find the latest release on [the release page](https://github.com/labstreaminglayer/App-TobiiStreamEngine/releases). +Note that you will need to provide your own copy of tobii_stream_engine.dll and put it in the same folder. +Find it [here](https://developer.tobii.com/consumer-eye-trackers/stream-engine/getting-started/) + +## Build Dependencies Download and unzip the Stream Engine for Windows x64 from the bottom of [this page](https://developer.tobii.com/consumer-eye-trackers/stream-engine/getting-started/). @@ -13,10 +20,6 @@ Download and unzip the latest liblsl binaries for Win64 from [the liblsl release I downloaded liblsl-1.13.0-Win64.7z. I unzipped into C:\Users\chboulay\Tools\liblsl\liblsl . This folder name will be provided to cmake as `LSL_INSTALL_ROOT`. -## Download - -When the app is done, downloads will be available on the releases page. - # Build Follow the generic LSL-App build instructions for building apps using diff --git a/cmake/FindTobii.cmake b/cmake/FindTobii.cmake index 6b91965..7b29915 100644 --- a/cmake/FindTobii.cmake +++ b/cmake/FindTobii.cmake @@ -37,6 +37,9 @@ # 2018 Ryan Pavlik for Sensics, Inc. # # Copyright Sensics, Inc. 2018. +# +# Modified by Chadwick Boulay, 2019-12-20 +# set(Tobii_ROOT diff --git a/mainwindow.cpp b/mainwindow.cpp index e889df4..bf5b753 100644 --- a/mainwindow.cpp +++ b/mainwindow.cpp @@ -1,5 +1,5 @@ #pragma warning(push) -#pragma warning(disable : 26812) +#pragma warning(disable : 26812 26439 26451 26495 26498) #include "mainwindow.h" #include "ui_mainwindow.h" @@ -13,28 +13,31 @@ #include //: standard C++ headers #include -#include #include #include -#pragma warning( pop ) +#include + +#include //: Device headers #include #include +#pragma warning( pop ) + -static auto list_devices(tobii_api_t* api) + + +static auto list_devices(tobii_api_t* api, tobii_error_t& err) { std::vector result; - auto error = tobii_enumerate_local_device_urls(api, + err = tobii_enumerate_local_device_urls(api, [](char const* url, void* user_data) // Use a lambda for url receiver function { // Add url string to the supplied result vector auto list = (std::vector*) user_data; list->push_back(url); }, &result); - // TODO: Handle error. - //if (error != TOBII_ERROR_NO_ERROR) std::cerr << "Failed to enumerate devices." << std::endl; return result; } @@ -42,7 +45,7 @@ static auto list_devices(tobii_api_t* api) static auto device_connect(tobii_api_t* api, std::string url, const unsigned int retries, const unsigned int interval, tobii_device_t** device) { - // std::cout << "Connecting to " << url << " (trying " << retries << " times with " << interval << " milliseconds intervals)." << std::endl; + std::cout << "Connecting to " << url << " (trying " << retries << " times with " << interval << " milliseconds intervals)." << std::endl; unsigned int retry = 0; auto error = TOBII_ERROR_NO_ERROR; @@ -59,9 +62,9 @@ static auto device_connect(tobii_api_t* api, std::string url, const unsigned int // std::cerr << "Failed connecting to " << url; if ((error == TOBII_ERROR_CONNECTION_FAILED) || (error == TOBII_ERROR_FIRMWARE_UPGRADE_IN_PROGRESS)) { - // std::cerr << " (tried " << retries << " times with " << interval << " milliseconds intervals)"; + std::cerr << " (tried " << retries << " times with " << interval << " milliseconds intervals)"; } - // std::cerr << "." << std::endl; + std::cerr << "." << std::endl; } return error; @@ -70,7 +73,7 @@ static auto device_connect(tobii_api_t* api, std::string url, const unsigned int static auto device_reconnect(tobii_device_t* device, const unsigned int retries, const unsigned int interval) { - // std::cout << "Reconnecting (trying " << retries << " times with " << interval << " milliseconds intervals)." << std::endl; + std::cout << "Reconnecting (trying " << retries << " times with " << interval << " milliseconds intervals)." << std::endl; unsigned int retry = 0; auto error = TOBII_ERROR_NO_ERROR; @@ -84,12 +87,12 @@ static auto device_reconnect(tobii_device_t* device, const unsigned int retries, if (error != TOBII_ERROR_NO_ERROR) { - // std::cerr << "Failed reconnecting"; + std::cerr << "Failed reconnecting"; if ((error == TOBII_ERROR_CONNECTION_FAILED) || (error == TOBII_ERROR_FIRMWARE_UPGRADE_IN_PROGRESS)) { - // std::cerr << " (tried " << retry << " times with " << interval << " milliseconds intervals)"; + std::cerr << " (tried " << retry << " times with " << interval << " milliseconds intervals)"; } - // std::cerr << "." << std::endl; + std::cerr << "." << std::endl; } return error; @@ -128,25 +131,36 @@ MainWindow::MainWindow(QWidget *parent, const char *config_file) void MainWindow::refreshDevices() { tobii_api_t* api; - auto error = tobii_api_create(&api, nullptr, nullptr); + tobii_error_t error = tobii_api_create(&api, nullptr, nullptr); if (error != TOBII_ERROR_NO_ERROR) { - //std::cerr << "Failed to initialize the Tobii Stream Engine API." << std::endl; + updateStatus("Failed to initialize the Tobii Stream Engine API."); } - std::vector devices = list_devices(api); + std::vector devices = list_devices(api, error); + if (error != TOBII_ERROR_NO_ERROR) { + updateStatus("Failed to enumerate devices."); + } - // TOOD: populate a dropdown list. + // Populate a dropdown list. ui->comboBox_device->clear(); - for each (auto dev_url in devices) + for each (std::string dev_url in devices) { ui->comboBox_device->addItem(QString::fromStdString(dev_url)); } - tobii_api_destroy(api); + error = tobii_api_destroy(api); + if (error != TOBII_ERROR_NO_ERROR) { + updateStatus("Failed to destroy API after enumerating devices."); + } } +void MainWindow::updateStatus(const char* message) +{ + this->statusBar()->showMessage(QString(message)); +} + void MainWindow::load_config(const QString &filename) { QSettings settings(filename, QSettings::Format::IniFormat); @@ -190,25 +204,19 @@ void recording_thread_function( const unsigned int retries = 300; const unsigned int interval = 100; tobii_device_t* device = nullptr; + int64_t last_sync_time; - //: create an outlet and a send buffer - lsl::stream_info info(name, "gaze", 2); - lsl::stream_outlet outlet(info); - // Create interface to Tobii API - auto error = tobii_api_create(&api, nullptr, nullptr); - if (error != TOBII_ERROR_NO_ERROR) - { - //std::cerr << "Failed to initialize the Tobii Stream Engine API." << std::endl; - goto cleanup_nodevice; - } + tobii_error_t error = tobii_api_create(&api, nullptr, nullptr); + assert(error == TOBII_ERROR_NO_ERROR); + error = tobii_system_clock(api, &last_sync_time); + assert(error == TOBII_ERROR_NO_ERROR); // Get list of devices. - devices = list_devices(api); + devices = list_devices(api, error); if (devices.size() == 0) { - //std::cerr << "No stream engine compatible device(s) found." << std::endl; - goto cleanup; + std::cerr << "No stream engine compatible device(s) found." << std::endl; } // Find device matching device_url @@ -217,39 +225,148 @@ void recording_thread_function( selected_device = device_url; } else { - goto cleanup; } error = device_connect(api, selected_device, retries, interval, &device); - if (error != TOBII_ERROR_NO_ERROR) - { - goto cleanup; - } + assert(error == TOBII_ERROR_NO_ERROR); + + tobii_device_info_t device_info; + error = tobii_get_device_info(device, &device_info); + assert(error == TOBII_ERROR_NO_ERROR); + + /* + tobii_supported_t supported; + error = tobii_stream_supported(device, TOBII_STREAM_GAZE_POINT, &supported); + assert(error == TOBII_ERROR_NO_ERROR); + if (supported == TOBII_SUPPORTED) + std::cout << "Device supports gaze point stream." << std::endl; + else + std::cout << "Device does not support gaze point stream." << std::endl; + */ + + // create an outlet for gaze_point data + lsl::stream_info info(name + "_gaze", "gaze", 2, lsl::IRREGULAR_RATE, lsl::cf_float32, std::string(device_info.serial_number) + "_gaze"); + info.desc().append_child_value("manufacturer", "Tobii"); + info.desc().append_child_value("model", device_info.model); + info.desc().append_child_value("serial", device_info.serial_number); + info.desc().append_child_value("generation", device_info.generation); + lsl::xml_element chns = info.desc().append_child("channels"); + for each (std::string chan_name in std::vector{ "x", "y" }) + chns.append_child("channel") + .append_child_value("label", chan_name) + .append_child_value("unit", "normalized") + .append_child_value("type", "screen"); + lsl::stream_outlet gaze_outlet(info); + + /* + error = tobii_stream_supported(device, TOBII_STREAM_WEARABLE_CONSUMER, &supported); + assert(error == TOBII_ERROR_NO_ERROR); + if (supported == TOBII_SUPPORTED) + std::cout << "Device supports wearable stream." << std::endl; + else + std::cout << "Device does not support wearable stream." << std::endl; + */ // Start subscribing to gaze and supply lambda callback function to handle the gaze point data error = tobii_gaze_point_subscribe(device, [](tobii_gaze_point_t const* gaze_point, void* user_data) { auto p_outlet = (lsl::stream_outlet*)user_data; // Cast user_data back to outlet pointer + double timestamp_s = (double)gaze_point->timestamp_us / 1000000; if (gaze_point->validity == TOBII_VALIDITY_VALID) { - double timestamp_s = (double)gaze_point->timestamp_us / 1000000; - std::vector sample = {gaze_point->position_xy[0], gaze_point->position_xy[1] }; + std::vector sample = { gaze_point->position_xy[0], gaze_point->position_xy[1] }; p_outlet->push_sample(sample, timestamp_s); } else { - //std::cout << "Gaze point: " << gaze_point->timestamp_us << " INVALID" << std::endl; + std::vector sample = { NAN, NAN }; + p_outlet->push_sample(sample, timestamp_s); } - }, &outlet); + }, &gaze_outlet); + assert(error == TOBII_ERROR_NO_ERROR); + + // create an outlet for Tobii events + lsl::stream_info event_info(name + "_events", "gaze_events", 1, lsl::IRREGULAR_RATE, lsl::cf_float32, std::string(device_info.serial_number) + "_events"); + info.desc().append_child_value("manufacturer", "Tobii"); + info.desc().append_child_value("model", device_info.model); + info.desc().append_child_value("serial", device_info.serial_number); + info.desc().append_child_value("generation", device_info.generation); + lsl::stream_outlet event_outlet(event_info); + error = tobii_user_presence_subscribe(device, + [](tobii_user_presence_status_t status, int64_t timestamp_us, void* user_data) { + auto p_outlet = (lsl::stream_outlet*)user_data; // Cast user_data back to outlet pointer + double timestamp_s = (double)timestamp_us / 1000000; + switch (status) + { + case TOBII_USER_PRESENCE_STATUS_UNKNOWN: + p_outlet->push_sample("USER_PRESENCE_STATUS_UNKNOWN", timestamp_s); + break; + case TOBII_USER_PRESENCE_STATUS_AWAY: + p_outlet->push_sample("USER_PRESENCE_STATUS_AWAY", timestamp_s); + break; + case TOBII_USER_PRESENCE_STATUS_PRESENT: + p_outlet->push_sample("USER_PRESENCE_STATUS_PRESENT", timestamp_s); + break; + default: + break; + } + }, &event_outlet); + assert(error == TOBII_ERROR_NO_ERROR); + error = tobii_notifications_subscribe(device, + [](tobii_notification_t const* notification, void* user_data) { + auto p_outlet = (lsl::stream_outlet*)user_data; // Cast user_data back to outlet pointer + std::string ev_str; + switch (notification->type) + { + case TOBII_NOTIFICATION_TYPE_CALIBRATION_STATE_CHANGED: + if (notification->value.state == TOBII_STATE_BOOL_TRUE) + ev_str = "Calibration started"; + else + ev_str = "Calibration stopped"; + break; + case TOBII_NOTIFICATION_TYPE_FRAMERATE_CHANGED: + ev_str = "FRAMERATE_CHANGED"; + break; + case TOBII_NOTIFICATION_TYPE_DEVICE_PAUSED_STATE_CHANGED: + if (notification->value.state == TOBII_STATE_BOOL_TRUE) + ev_str = "Paused"; + else + ev_str = "Unpaused"; + break; + case TOBII_NOTIFICATION_TYPE_CALIBRATION_ENABLED_EYE_CHANGED: + ev_str = "CALIBRATION_ENABLED_EYE_CHANGED"; + break; + case TOBII_NOTIFICATION_TYPE_COMBINED_GAZE_EYE_SELECTION_CHANGED: + ev_str = "COMBINED_GAZE_EYE_SELECTION_CHANGED"; + break; + case TOBII_NOTIFICATION_TYPE_CALIBRATION_ID_CHANGED: + ev_str = "Calibration ID changed to " + std::to_string(notification->value.uint_); + break; + default: + break; + } + if (ev_str != "") + p_outlet->push_sample(ev_str.c_str()); + }, &event_outlet); + assert(error == TOBII_ERROR_NO_ERROR); while (!shutdown) { + // Every ~30 seconds, update the timesync + int64_t current_time; + error = tobii_system_clock(api, ¤t_time); + if ((current_time - last_sync_time) > 30000000) + { + tobii_update_timesync(device); + last_sync_time = current_time; + } + auto error = tobii_wait_for_callbacks(1, &device); if (error == TOBII_ERROR_TIMED_OUT) continue; // If timed out, redo the wait for callbacks call else if (error != TOBII_ERROR_NO_ERROR) { - //std::cerr << "tobii_wait_for_callbacks failed: " << tobii_error_message(error) << "." << std::endl; + std::cerr << "tobii_wait_for_callbacks failed: " << tobii_error_message(error) << "." << std::endl; break; } // Calling this function will execute the subscription callback functions @@ -260,30 +377,29 @@ void recording_thread_function( error = device_reconnect(device, interval, retries); if (error != TOBII_ERROR_NO_ERROR) { - //std::cerr << "Connection was lost and reconnection failed." << std::endl; + std::cerr << "Connection was lost and reconnection failed." << std::endl; break; } continue; } else if (error != TOBII_ERROR_NO_ERROR) { - //std::cerr << "tobii_device_process_callbacks failed: " << tobii_error_message(error) << "." << std::endl; + std::cerr << "tobii_device_process_callbacks failed: " << tobii_error_message(error) << "." << std::endl; break; } } -cleanup: + error = tobii_notifications_unsubscribe(device); + assert(error == TOBII_ERROR_NO_ERROR); + error = tobii_user_presence_unsubscribe(device); + assert(error == TOBII_ERROR_NO_ERROR); error = tobii_gaze_point_unsubscribe(device); -cleanup_nodevice: - if (error != TOBII_ERROR_NO_ERROR) {} - //std::cerr << "Failed to unsubscribe from gaze stream." << std::endl; + assert(error == TOBII_ERROR_NO_ERROR); error = tobii_device_destroy(device); - if (error != TOBII_ERROR_NO_ERROR) {} - //std::cerr << "Failed to destroy device." << std::endl; + assert(error == TOBII_ERROR_NO_ERROR); error = tobii_api_destroy(api); - if (error != TOBII_ERROR_NO_ERROR) {} - //std::cerr << "Failed to destroy API." << std::endl; + assert(error == TOBII_ERROR_NO_ERROR); } diff --git a/mainwindow.h b/mainwindow.h index 0443d47..5047071 100644 --- a/mainwindow.h +++ b/mainwindow.h @@ -24,6 +24,7 @@ private slots: void closeEvent(QCloseEvent *ev) override; void toggleRecording(); void refreshDevices(); + void updateStatus(const char* message); private: // function for loading / saving the config file diff --git a/mainwindow.ui b/mainwindow.ui index c92fa4a..3f2543a 100644 --- a/mainwindow.ui +++ b/mainwindow.ui @@ -7,11 +7,11 @@ 0 0 281 - 166 + 189 - BestPracticesGUI Connector + Tobii (Stream Engine) LSL