1603 speech to text hotkey (#1771)

* Added ctrl + enter hotkeys to init speach to text * Ran linter * Fixed speech transcript from being submitted twice when the user clicks the send button. Updated speech hotkeys. * Added pulse animation to mic * Fixed prompt double-send when clicking the send button or ending the TTS session. * Fixed comment grammar * Update mic hotkeys --------- Co-authored-by: Timothy Carambat <[email protected]>
Mintplex-Labs · Aug 7, 2024 · 6a0f068 · 6a0f068
1 parent d93c43f
commit 6a0f068
Show file tree

Hide file tree

Showing 3 changed files with 79 additions and 5 deletions.
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx
@@ -1,10 +1,11 @@
-import { useEffect } from "react";
+import { useEffect, useCallback } from "react";
 import { Microphone } from "@phosphor-icons/react";
 import { Tooltip } from "react-tooltip";
 import _regeneratorRuntime from "regenerator-runtime";
 import SpeechRecognition, {
   useSpeechRecognition,
 } from "react-speech-recognition";
+import { PROMPT_INPUT_EVENT } from "../../PromptInput";
 
 let timeout;
 const SILENCE_INTERVAL = 3_200; // wait in seconds of silence before closing.
@@ -45,15 +46,49 @@ export default function SpeechToText({ sendCommand }) {
     clearTimeout(timeout);
   }
 
+  const handleKeyPress = useCallback(
+    (event) => {
+      if (event.ctrlKey && event.keyCode === 77) {
+        if (listening) {
+          endTTSSession();
+        } else {
+          startSTTSession();
+        }
+      }
+    },
+    [listening, endTTSSession, startSTTSession]
+  );
+
+  function handlePromptUpdate(e) {
+    if (!e?.detail && timeout) {
+      endTTSSession();
+      clearTimeout(timeout);
+    }
+  }
+
+  useEffect(() => {
+    document.addEventListener("keydown", handleKeyPress);
+    return () => {
+      document.removeEventListener("keydown", handleKeyPress);
+    };
+  }, [handleKeyPress]);
+
+  useEffect(() => {
+    if (!!window)
+      window.addEventListener(PROMPT_INPUT_EVENT, handlePromptUpdate);
+    return () =>
+      window?.removeEventListener(PROMPT_INPUT_EVENT, handlePromptUpdate);
+  }, []);
+
   useEffect(() => {
-    if (transcript?.length > 0) {
+    if (transcript?.length > 0 && listening) {
       sendCommand(transcript, false);
       clearTimeout(timeout);
       timeout = setTimeout(() => {
         endTTSSession();
       }, SILENCE_INTERVAL);
     }
-  }, [transcript]);
+  }, [transcript, listening]);
 
   if (!browserSupportsSpeechRecognition) return null;
   return (
@@ -69,7 +104,9 @@ export default function SpeechToText({ sendCommand }) {
     >
       <Microphone
         weight="fill"
-        className="w-6 h-6 pointer-events-none text-white"
+        className={`w-6 h-6 pointer-events-none text-white overflow-hidden rounded-full ${
+          listening ? "animate-pulse" : ""
+        }`}
       />
       <Tooltip
         id="tooltip-text-size-btn"

diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx
@@ -14,6 +14,9 @@ import handleSocketResponse, {
   AGENT_SESSION_START,
 } from "@/utils/chat/agent";
 import DnDFileUploaderWrapper from "./DnDWrapper";
+import SpeechRecognition, {
+  useSpeechRecognition,
+} from "react-speech-recognition";
 
 export default function ChatContainer({ workspace, knownHistory = [] }) {
   const { threadSlug = null } = useParams();
@@ -29,6 +32,10 @@ export default function ChatContainer({ workspace, knownHistory = [] }) {
     setMessage(event.target.value);
   };
 
+  const { listening, resetTranscript } = useSpeechRecognition({
+    clearTranscriptOnListen: true,
+  });
+
   // Emit an update to the state of the prompt input without directly
   // passing a prop in so that it does not re-render constantly.
   function setMessageEmit(messageContent = "") {
@@ -57,11 +64,20 @@ export default function ChatContainer({ workspace, knownHistory = [] }) {
       },
     ];
 
+    if (listening) {
+      // Stop the mic if the send button is clicked
+      endTTSSession();
+    }
     setChatHistory(prevChatHistory);
     setMessageEmit("");
     setLoadingResponse(true);
   };
 
+  function endTTSSession() {
+    SpeechRecognition.stopListening();
+    resetTranscript();
+  }
+
   const regenerateAssistantMessage = (chatId) => {
     const updatedHistory = chatHistory.slice(0, -1);
     const lastUserMessage = updatedHistory.slice(-1)[0];

diff --git a/frontend/tailwind.config.js b/frontend/tailwind.config.js
@@ -86,7 +86,8 @@ export default {
         ]
       },
       animation: {
-        sweep: "sweep 0.5s ease-in-out"
+        sweep: "sweep 0.5s ease-in-out",
+        pulse: "pulse 1.5s infinite"
       },
       keyframes: {
         sweep: {
@@ -100,6 +101,26 @@ export default {
         fadeOut: {
           "0%": { opacity: 1 },
           "100%": { opacity: 0 }
+        },
+        pulse: {
+          "0%": {
+            opacity: 1,
+            transform: "scale(1)",
+            boxShadow: "0 0 0 rgba(255, 255, 255, 0.0)",
+            backgroundColor: "rgba(255, 255, 255, 0.0)"
+          },
+          "50%": {
+            opacity: 1,
+            transform: "scale(1.1)",
+            boxShadow: "0 0 15px rgba(255, 255, 255, 0.2)",
+            backgroundColor: "rgba(255, 255, 255, 0.1)"
+          },
+          "100%": {
+            opacity: 1,
+            transform: "scale(1)",
+            boxShadow: "0 0 0 rgba(255, 255, 255, 0.0)",
+            backgroundColor: "rgba(255, 255, 255, 0.0)"
+          }
         }
       }
     }