New feature added: Integrated WebView with Native Speech and PiperTTS

.vscode/launch.json

@@ -14,7 +14,8 @@
         // Pass a directory to manually test in
         "${workspaceFolder}/manual-testing-sandbox",
         "${workspaceFolder}/manual-testing-sandbox/test.js",
-        "--extensionDevelopmentPath=${workspaceFolder}/extensions/vscode"
+        "--extensionDevelopmentPath=${workspaceFolder}/extensions/vscode",
+        "--disable-extensions"
       ],
       "pauseForSourceMap": false,
       "outFiles": ["${workspaceFolder}/extensions/vscode/out/extension.js"],

core/config/types.ts

@@ -1049,7 +1049,7 @@ declare global {
     /**
      * Automatically read LLM chat responses aloud using system TTS models
      */
-    readResponseTTS?: boolean;
+    readResponseTTS?: {type:string,lang?:string,flush?:boolean};
 
     /**
      * If set to true, we will attempt to pull down and install an instance of Chromium

core/core.ts

@@ -37,7 +37,7 @@ import {
 } from "./util/paths";
 import { Telemetry } from "./util/posthog";
 import { getSymbolsForManyFiles } from "./util/treeSitter";
-import { TTS } from "./util/tts";
+import { sanitizeMessageForTTS, TTS } from "./util/tts";
 
 import { type ContextItemId, type IDE, type IndexingProgressUpdate } from ".";
 import type { FromCoreProtocol, ToCoreProtocol } from "./protocol";
@@ -392,7 +392,8 @@ export class Core {
       }
 
       // Stop TTS on new StreamChat
-      if (config.experimental?.readResponseTTS) {
+
+      if (config.experimental?.readResponseTTS?.type==="native") {
         void TTS.kill();
       }
 
@@ -426,10 +427,19 @@ export class Core {
         next = await gen.next();
       }
 
-      if (config.experimental?.readResponseTTS && "completion" in next.value) {
-        void TTS.read(next.value?.completion);
+      if("completion" in next.value){
+        let message=next.value?.completion;
+        const lang=config.experimental?.readResponseTTS?.lang??"en-US";
+        const flush=config.experimental?.readResponseTTS?.flush??false;
+        if (config.experimental?.readResponseTTS?.type==="native" ) {
+          void TTS.read(message,lang);
+        }else if (config.experimental?.readResponseTTS?.type==="piperTTS" ) {
+          message = sanitizeMessageForTTS(message);
+          void TTS.messenger.request("setPiperTTS",{lang,message,flush});
+        }
       }
 
+
       void Telemetry.capture(
         "chat",
         {

core/index.d.ts

@@ -1057,7 +1057,7 @@ export interface ExperimentalConfig {
   /**
    * Automatically read LLM chat responses aloud using system TTS models
    */
-  readResponseTTS?: boolean;
+  readResponseTTS?: {type:string,lang?:string,flush?:boolean};
 
   /**
    * If set to true, we will attempt to pull down and install an instance of Chromium

core/protocol/passThrough.ts

@@ -72,6 +72,8 @@ export const CORE_TO_WEBVIEW_PASS_THROUGH: (keyof ToWebviewFromCoreProtocol)[] =
     "isContinueInputFocused",
     "didChangeAvailableProfiles",
     "setTTSActive",
+    "setTTSNative",
+    "setPiperTTS",
     "getWebviewHistoryLength",
     "getCurrentSessionId",
     "signInToControlPlane",

core/protocol/webview.ts

@@ -31,6 +31,8 @@ export type ToWebviewFromIdeOrCoreProtocol = {
     void,
   ];
   setTTSActive: [boolean, void];
+  setTTSNative:[{lang:string,message:string}, boolean];
+  setPiperTTS:[{lang:string,message:string,flush:boolean}, void];
   getWebviewHistoryLength: [undefined, number];
   getCurrentSessionId: [undefined, string];
   signInToControlPlane: [undefined, void];

core/util/tts.ts

@@ -36,45 +36,47 @@ export class TTS {
   static handle: ChildProcess | undefined = undefined;
   static messenger: IMessenger<ToCoreProtocol, FromCoreProtocol>;
 
-  static async read(message: string) {
+  static async read(message: string,lang: string) {
     message = sanitizeMessageForTTS(message);
-
-    try {
-      // Kill any active TTS processes
-      await TTS.kill();
-    } catch (e) {
-      console.warn("Error killing TTS process: ", e);
-      return;
-    }
-
-    switch (TTS.os) {
-      case "darwin":
-        TTS.handle = exec(`say "${message}"`);
-        break;
-      case "win32":
-        // Replace single quotes on windows
-        TTS.handle = exec(
-          `powershell -Command "Add-Type -AssemblyName System.Speech; (New-Object System.Speech.Synthesis.SpeechSynthesizer).Speak('${message.replace(
-            /'/g,
-            "''",
-          )}')"`,
-        );
-        break;
-      case "linux":
-        TTS.handle = exec(`espeak "${message}"`);
-        break;
-      default:
-        console.log(
-          "Text-to-speech is not supported on this operating system.",
-        );
+    const native = await TTS.messenger.request("setTTSNative",{lang,message});
+    if(!native){
+      try {
+        // Kill any active TTS processes
+        await TTS.kill();
+      } catch (e) {
+        console.warn("Error killing TTS process: ", e);
         return;
-    }
+      }
+
+      switch (TTS.os) {
+        case "darwin":
+          TTS.handle = exec(`say "${message}"`);
+          break;
+        case "win32":
+          // Replace single quotes on windows
+          TTS.handle = exec(
+            `powershell -Command "Add-Type -AssemblyName System.Speech; (New-Object System.Speech.Synthesis.SpeechSynthesizer).Speak('${message.replace(
+              /'/g,
+              "''",
+            )}')"`,
+          );
+          break;
+        case "linux":
+          TTS.handle = exec(`espeak "${message}"`);
+          break;
+        default:
+          console.log(
+            "Text-to-speech is not supported on this operating system.",
+          );
+          return;
+      }
 
-    void TTS.messenger.request("setTTSActive", true);
+      void TTS.messenger.request("setTTSActive", true);
 
-    TTS.handle?.once("exit", () => {
-      void TTS.messenger.request("setTTSActive", false);
-    });
+      TTS.handle?.once("exit", () => {
+        void TTS.messenger.request("setTTSActive", false);
+      });
+  }
   }
 
   static async kill(): Promise<void> {

extensions/vscode/config_schema.json

@@ -3064,9 +3064,22 @@
               }
             },
             "readResponseTTS": {
-              "type": "boolean",
-              "default": true,
-              "description": "Automatically read LLM chat responses aloud using system TTS models"
+              "type": "object",
+              "description": "Automatically read LLM chat responses aloud using system TTS models",
+              "properties": {
+                "type": {
+                  "type": "string",
+                  "default": "native"
+                },
+                "lang": {
+                  "type": "string",
+                  "default": "en-US"
+                },
+                "flush": {
+                  "type": "boolean",
+                  "default": false
+                }
+              }
             },
             "promptPath": {
               "type": "string"