Merge pull request 'STT' (#6) from STT into main

Reviewed-on: Spilum.Net/Many-Tailed-Fox#6

.gitignore

@@ -1,2 +1,4 @@
 .env
 node_modules/
+.venv/
+src/models/vosk*

package-lock.json

@@ -9,6 +9,7 @@
       "version": "1.0.0",
       "license": "ISC",
       "dependencies": {
+        "mic": "^2.1.2",
         "redis": "^6.1.0"
       },
       "devDependencies": {
@@ -320,6 +321,12 @@
         "node": ">=0.12.0"
       }
     },
+    "node_modules/mic": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/mic/-/mic-2.1.2.tgz",
+      "integrity": "sha512-rpl4tgdXX24sAzYwjRc5OZfGNAuhUIIjdd0cw8+Ubq7rp3iGhi40AdqcwurDWhEZADk60tPOxb3E2MpoeLeyxw==",
+      "license": "MIT"
+    },
     "node_modules/minimatch": {
       "version": "10.2.5",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz",

package.json

@@ -12,6 +12,7 @@
   "license": "ISC",
   "type": "commonjs",
   "dependencies": {
+    "mic": "^2.1.2",
     "redis": "^6.1.0"
   },
   "devDependencies": {

src/config/redis.config.js

@@ -1,4 +1,4 @@
 module.exports = {
-	REDIS_HOST: process.env.REDIS_HOST,
+	HOST: process.env.REDIS_HOST,
-	REDIS_PORT: process.env.REDIS_PORT
+	PORT: process.env.REDIS_PORT
 }

src/config/transcribe.config.js

@@ -0,0 +1,11 @@
+module.exports = {
+  wakeword: process.env.TRANS_WAKEWORD,
+  wakewordThreshold: Number(process.env.TRANS_WAKEWORD_THRESHOLD),
+  cooldownSeconds: Number(process.env.TRANS_COOLDOWN_SECONDS),
+  confirmFrames: Number(process.env.TRANS_CONFIRM_FRAMES),
+  listenSeconds: Number(process.env.TRANS_LISTEN_SECONDS),
+  silenceThreshold: Number(process.env.TRANS_SILENCE_THRESHOLD),
+  silenceFrames: Number(process.env.TRANS_SIELNCE_FRAMES),
+  voskModelPath: process.env.TRANS_VOSK_MODEL_PATH,
+  sampleRate: Number(process.env.TRANS_SAMPLE_RATE),
+};

src/models/index.js

@@ -6,12 +6,10 @@ const Redis = require('redis');
 const redis = {};
 
 redis.client = Redis.createClient({
-    url: `redis://default@${redisConfig.REDIS_HOST}:${redisConfig.REDIS_PORT}`
+    url: `redis://default@${redisConfig.HOST}:${redisConfig.PORT}`
 });
 
 redis.ensureGroup = async (stream, group) => {
-    await redis.client.connect().catch(console.error);
-
     try {
         await redis.client.xGroupCreate(stream, group, "0", {MKSTREAM: true});
 
@@ -23,7 +21,19 @@ redis.ensureGroup = async (stream, group) => {
             console.error(err);
         }
     }
-    await redis.client.quit();
+}
+
+redis.sendTranscriptEvent = async (streamKey, device_id, text) => {
+    try {
+        await redis.client.xAdd(streamKey, "*", {
+            text,
+            device_id: device_id,
+        });
+
+        console.log("Transcript added to stream");
+    } catch (err) {
+        console.error(err);
+    }
 }
 
 module.exports = {

src/scripts/py/config.json

@@ -0,0 +1,11 @@
+{
+  "wakeword": "alexa",
+  "wakewordThreshold": 0.80,
+  "cooldownSeconds": 2.5,
+  "confirmFrames": 3,
+  "listenSeconds": 12,
+  "silenceThreshold": 0.015,
+  "silenceFrames": 10,
+  "voskModelPath": "./vosk-model-small-en-us-0.15",
+  "sampleRate": 16000
+}

src/scripts/py/info.txt

@@ -0,0 +1 @@
+PyAudio requires Python version <=3.13

src/scripts/py/pythonOutput.js

@@ -0,0 +1,41 @@
+// pythonOutput.js
+let buffer = "";
+
+function handlePythonOutput(chunk, onMessage) {
+  buffer += chunk;
+  const lines = buffer.split("\n");
+  buffer = lines.pop() || "";
+
+  for (const line of lines) {
+    if (!line.trim()) continue;
+
+    try {
+      const msg = JSON.parse(line);
+      switch (msg.event) {
+        case "ready":
+          console.log("Python ready");
+          break;
+        case "wakeword":
+          console.log(`Wakeword detected: ${msg.model} (${msg.score})`);
+          break;
+        case "partial":
+          console.log("Partial:", msg.text);
+          break;
+        case "text":
+        case "final":
+          console.log("Transcript:", msg.text);
+          onMessage?.(msg).catch(console.error);
+          break;
+        case "done":
+          console.log("Back to wakeword mode");
+          break;
+        default:
+          console.log("[PY]", line);
+      }
+    } catch {
+      console.log("[PY]", line);
+    }
+  }
+}
+
+module.exports = { handlePythonOutput };

src/scripts/py/requirements.txt

@@ -0,0 +1,4 @@
+openwakeword
+onnxruntime
+numpy
+pyaudio

src/scripts/py/transcribe.py

@@ -0,0 +1,112 @@
+import json
+import sys
+import time
+import numpy as np
+import pyaudio
+import vosk
+from openwakeword.model import Model
+
+CONFIG = json.loads(sys.argv[1])
+
+RATE = CONFIG.get("sampleRate", 16000)
+CHUNK = 1280
+WAKEWORD = CONFIG.get("wakeword", "alexa")
+THRESHOLD = CONFIG.get("wakewordThreshold", 0.75)
+COOLDOWN = CONFIG.get("cooldownSeconds", 2.5)
+CONFIRM_FRAMES = CONFIG.get("confirmFrames", 3)
+LISTEN_SECONDS = CONFIG.get("listenSeconds", 12)
+SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.02)
+SILENCE_FRAMES = CONFIG.get("silenceFrames", 12)
+VOSK_MODEL_PATH = CONFIG["voskModelPath"]
+
+vosk.SetLogLevel(-1)
+vosk_model = vosk.Model(VOSK_MODEL_PATH)
+recognizer = vosk.KaldiRecognizer(vosk_model, RATE)
+recognizer.SetWords(True)
+
+oww = Model(wakeword_models=[WAKEWORD], inference_framework="onnx")
+
+audio = pyaudio.PyAudio()
+stream = audio.open(
+    format=pyaudio.paInt16,
+    channels=1,
+    rate=RATE,
+    input=True,
+    frames_per_buffer=CHUNK,
+)
+
+print(json.dumps({"event": "ready"}), flush=True)
+
+mode = "wakeword"
+high_count = 0
+last_trigger = 0
+silence_count = 0
+listen_deadline = None
+
+def rms_level(samples):
+    x = samples.astype(np.float32) / 32768.0
+    return np.sqrt(np.mean(x * x))
+
+try:
+    while True:
+        data = stream.read(CHUNK, exception_on_overflow=False)
+        samples = np.frombuffer(data, dtype=np.int16)
+        now = time.time()
+
+        if mode == "wakeword":
+            preds = oww.predict(samples)
+            score = preds.get(WAKEWORD, 0.0)
+
+            if score >= THRESHOLD:
+                high_count += 1
+            else:
+                high_count = 0
+
+            if high_count >= CONFIRM_FRAMES and (now - last_trigger) > COOLDOWN:
+                last_trigger = now
+                high_count = 0
+                print(json.dumps({
+                    "event": "wakeword",
+                    "model": WAKEWORD,
+                    "score": float(score)
+                }), flush=True)
+                mode = "transcribe"
+                recognizer.Reset()
+                silence_count = 0
+                listen_deadline = now + LISTEN_SECONDS
+
+        else:
+            if recognizer.AcceptWaveform(data):
+                result = json.loads(recognizer.Result())
+                if result.get("text"):
+                    print(json.dumps({"event": "final", "text": result["text"]}), flush=True)
+                    listen_deadline = now + LISTEN_SECONDS
+                    silence_count = 0
+            else:
+                partial = json.loads(recognizer.PartialResult()).get("partial", "")
+                if partial:
+                    print(json.dumps({"event": "partial", "text": partial}), flush=True)
+
+            level = rms_level(samples)
+            if level < SILENCE_THRESHOLD:
+                silence_count += 1
+            else:
+                silence_count = 0
+
+            if silence_count >= SILENCE_FRAMES or now > listen_deadline:
+                final = json.loads(recognizer.FinalResult())
+                if final.get("text"):
+                    print(json.dumps({"event": "final", "text": final["text"]}), flush=True)
+
+                print(json.dumps({"event": "done"}), flush=True)
+                mode = "wakeword"
+                silence_count = 0
+                high_count = 0
+                listen_deadline = None
+
+except KeyboardInterrupt:
+    pass
+finally:
+    stream.stop_stream()
+    stream.close()
+    audio.terminate()

src/server.js

@@ -3,31 +3,21 @@ if (process.env.NODE_ENV !== "production") {
 	require("dotenv").config();
 }
 
+const { spawn } = require('child_process');
+
 const models = require("./models");
 const redis = models.redis;
 
 const config = require("./config");
+const transConfig = require("./config/transcribe.config")
 
-// Send Speech to Text to Redis Stream
+const py = spawn('python', ['./src/scripts/py/transcribe.py', JSON.stringify(transConfig)], {
-const sendTranscriptEvent = async (streamKey, device_id, text) => {
+    stdio: ['ignore', 'pipe', 'pipe']
-    await redis.client.xAdd(
+});
-        streamKey,
+const { handlePythonOutput } = require("./scripts/py/pythonOutput");
-        '*',
-        {
-            text: text,
-            device_id: device_id
-        }
-    )
-
-    console.log("Transcript added to stream");
-}
 
 // Read Redis Stream group
-const processor = async (group, consumer, streamKey) => {
+async function processor (group, consumer, streamKey) {
-    await redis.client.connect().catch(console.error);
-
-    sendTranscriptEvent("transcripts", config.DEVICE_ID, "TEST");
-
     while (true) {
         const res = await redis.client.xReadGroup(
             group,
@@ -46,14 +36,29 @@ const processor = async (group, consumer, streamKey) => {
             }
         }
     }
-
-    await redis.client.quit();
 }
 
-const main = async () => {
+async function main() {
-    await redis.ensureGroup("commands", 'assistant');
+    await redis.client.connect().catch(console.error);
+    await redis.ensureGroup("commands", "assistant");
 
-    processor('assistant', `${config.DEVICE_ID}`, "commands");
+    py.stdout.on("data", (data) => {
+        handlePythonOutput(data.toString(), async (msg) => {
+            if (msg.event === "text" || msg.event === "final") {
+                await redis.sendTranscriptEvent("transcripts", config.DEVICE_ID, msg.text);
+            }
+        });
+    });
+
+    py.stderr.on("data", (data) => {
+        console.error("[PY ERR]", data.toString());
+    });
+
+    py.on("close", (code) => {
+        console.log("Python exited with code", code);
+    });
+
+    processor("assistant", `${config.DEVICE_ID}`, "commands").catch(console.error);
 }
 
 main().catch(console.error);