Initial script engine audio listener

overte-org · Dec 24, 2024 · 803be05 · 803be05
1 parent 2986352
commit 803be05
Show file tree

Hide file tree

Showing 5 changed files with 107 additions and 0 deletions.
diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
@@ -448,6 +448,34 @@ void AudioClient::setAudioPaused(bool pause) {
     }
 }
 
+QUuid AudioClient::registerScriptListener() {
+    std::lock_guard<std::mutex> lock(_scriptListenersMutex);
+    QUuid listenerID = QUuid::createUuid();
+    while (_scriptListeners.contains(listenerID)) {
+        listenerID = QUuid::createUuid();
+    }
+    _scriptListeners.insert(listenerID, std::make_shared<ScriptAudioListener>());
+    return listenerID;
+}
+
+void AudioClient::unregisterScriptListener(const QUuid& listener) {
+    std::lock_guard<std::mutex> lock(_scriptListenersMutex);
+}
+
+QByteArray AudioClient::getPCMData(const QUuid& listenerID) {
+    std::shared_ptr<ScriptAudioListener> listener;
+    {
+        std::lock_guard<std::mutex> lock(_scriptListenersMutex);
+        if (_scriptListeners.contains(listenerID)) {
+            listener = _scriptListeners[listenerID];
+        } else {
+            qDebug() << "AudioClient::getPCMData: Script listener doesn't exist: " << listenerID;
+            return QByteArray();
+        }
+    }
+    return listener->getData();
+}
+
 HifiAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName, const QString& hmdName, bool isHmd=false) {
     HifiAudioDeviceInfo result;
     foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode,hmdName)) {
@@ -2450,6 +2478,18 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
         _audio->_audioFileWav.addRawAudioChunk(data, bytesWritten);
     }
 
+    // Send audio data to script engines.
+    // Hash table needs to be copied to avoid deadlocks and improve performance
+    QHash<QUuid, std::shared_ptr<ScriptAudioListener>> scriptListeners;
+    {
+        std::lock_guard<std::mutex> lock(_audio->_scriptListenersMutex);
+        scriptListeners = _audio->_scriptListeners;
+    }
+    QByteArray dataArray(data, bytesWritten);
+    for (auto listener : _audio->_scriptListeners) {
+        listener->putData(dataArray);
+    }
+
     int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree();
     float msecsAudioOutputUnplayed = bytesAudioOutputUnplayed / (float)_audio->_outputFormat.bytesForDuration(USECS_PER_MSEC);
     _audio->_stats.updateOutputMsUnplayed(msecsAudioOutputUnplayed);

diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h
@@ -54,6 +54,7 @@
 
 #include <plugins/CodecPlugin.h>
 
+#include "AudioScriptingInterface.h"
 #include "AudioIOStats.h"
 #include "AudioFileWav.h"
 #include "HifiAudioDeviceInfo.h"
@@ -187,6 +188,10 @@ class AudioClient : public AbstractAudioInterface, public Dependency {
 
     AudioSolo& getAudioSolo() override { return _solo; }
 
+    QUuid registerScriptListener() override;
+    void unregisterScriptListener(const QUuid& listenerID) override;
+    QByteArray getPCMData(const QUuid& listener) override;
+
 #ifdef Q_OS_WIN
     static QString getWinDeviceName(wchar_t* guid);
 #endif
@@ -535,6 +540,9 @@ public slots:
     Mutex _checkPeakValuesMutex;
     QTimer* _checkPeakValuesTimer { nullptr };
 
+    Mutex _scriptListenersMutex;
+    QHash<QUuid, std::shared_ptr<ScriptAudioListener>> _scriptListeners;
+
     bool _isRecording { false };
 };
 

diff --git a/libraries/audio/src/AbstractAudioInterface.h b/libraries/audio/src/AbstractAudioInterface.h
@@ -41,6 +41,10 @@ class AbstractAudioInterface : public QObject {
 
     virtual AudioSolo& getAudioSolo() = 0;
 
+    virtual QUuid registerScriptListener() = 0;
+    virtual void unregisterScriptListener(const QUuid& listener) = 0;
+    virtual QByteArray getPCMData(const QUuid& listener) = 0;
+
 public slots:
     virtual bool shouldLoopbackInjectors() { return false; }
 

diff --git a/libraries/audio/src/AudioScriptingInterface.cpp b/libraries/audio/src/AudioScriptingInterface.cpp
@@ -40,6 +40,22 @@ void registerAudioMetaTypes(ScriptEngine* engine) {
     scriptRegisterMetaType<SharedSoundPointer, soundSharedPointerToScriptValue, soundSharedPointerFromScriptValue>(engine);
 }
 
+QByteArray ScriptAudioListener::getData() {
+    QByteArray result;
+    {
+        std::lock_guard<std::mutex> lock(_dataMutex);
+        while (!_data.empty()) {
+            result.append(_data.front());
+            _data.pop();
+        }
+    }
+    return result;
+}
+
+void ScriptAudioListener::putData(const QByteArray &data) {
+    std::lock_guard<std::mutex> lock(_dataMutex);
+    _data.push(data);
+}
 
 void AudioScriptingInterface::setLocalAudioInterface(AbstractAudioInterface* audioInterface) {
     if (_localAudioInterface) {
@@ -96,6 +112,28 @@ bool AudioScriptingInterface::isStereoInput() {
     return stereoEnabled;
 }
 
+QUuid AudioScriptingInterface::registerScriptListener() {
+    if (_localAudioInterface) {
+        return _localAudioInterface->registerScriptListener();
+    } else {
+        return QUuid();
+    }
+}
+
+void AudioScriptingInterface::unregisterScriptListener(const QUuid& listener) {
+    if (_localAudioInterface) {
+        _localAudioInterface->unregisterScriptListener(listener);
+    }
+}
+
+QByteArray AudioScriptingInterface::getPCMData(const QUuid& listener) {
+    if (_localAudioInterface) {
+        return _localAudioInterface->getPCMData(listener);
+    } else {
+        return QByteArray();
+    }
+}
+
 bool AudioScriptingInterface::getServerEcho() {
     bool serverEchoEnabled = false;
     if (_localAudioInterface) {

diff --git a/libraries/audio/src/AudioScriptingInterface.h b/libraries/audio/src/AudioScriptingInterface.h
@@ -17,6 +17,8 @@
 #ifndef hifi_AudioScriptingInterface_h
 #define hifi_AudioScriptingInterface_h
 
+#include <queue>
+
 #include "AbstractAudioInterface.h"
 #include "AudioInjector.h"
 #include <DependencyManager.h>
@@ -25,6 +27,17 @@
 class ScriptAudioInjector;
 class ScriptEngine;
 
+class ScriptAudioListener {
+    //TODO: add total data size limit
+    //TODO: unregister on script engine destruction
+public:
+    QByteArray getData();
+    void putData(const QByteArray &data);
+private:
+    std::mutex _dataMutex;
+    std::queue<QByteArray> _data;
+};
+
 /// Provides the <code><a href="https://apidocs.overte.org/Audio.html">Audio</a></code> scripting API
 class AudioScriptingInterface : public QObject, public Dependency {
     Q_OBJECT
@@ -224,6 +237,10 @@ class AudioScriptingInterface : public QObject, public Dependency {
      */
     Q_INVOKABLE bool isStereoInput();
 
+    Q_INVOKABLE QUuid registerScriptListener();
+    Q_INVOKABLE void unregisterScriptListener(const QUuid& listener);
+    Q_INVOKABLE QByteArray getPCMData(const QUuid& listener);
+
 signals:
 
     /*@jsdoc