diff --git a/.gitignore b/.gitignore index 2d7ec5c..4f02d84 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .env node_modules/ +.venv/ +src/models/vosk* \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 4b13c72..a4c8ad9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "mic": "^2.1.2", "redis": "^6.1.0" }, "devDependencies": { @@ -320,6 +321,12 @@ "node": ">=0.12.0" } }, + "node_modules/mic": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mic/-/mic-2.1.2.tgz", + "integrity": "sha512-rpl4tgdXX24sAzYwjRc5OZfGNAuhUIIjdd0cw8+Ubq7rp3iGhi40AdqcwurDWhEZADk60tPOxb3E2MpoeLeyxw==", + "license": "MIT" + }, "node_modules/minimatch": { "version": "10.2.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", diff --git a/package.json b/package.json index a69a95b..23ccdde 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "license": "ISC", "type": "commonjs", "dependencies": { + "mic": "^2.1.2", "redis": "^6.1.0" }, "devDependencies": { diff --git a/src/config/redis.config.js b/src/config/redis.config.js index 627c270..fc0ff3e 100644 --- a/src/config/redis.config.js +++ b/src/config/redis.config.js @@ -1,4 +1,4 @@ module.exports = { - REDIS_HOST: process.env.REDIS_HOST, - REDIS_PORT: process.env.REDIS_PORT + HOST: process.env.REDIS_HOST, + PORT: process.env.REDIS_PORT } diff --git a/src/config/transcribe.config.js b/src/config/transcribe.config.js new file mode 100644 index 0000000..d7799c5 --- /dev/null +++ b/src/config/transcribe.config.js @@ -0,0 +1,11 @@ +module.exports = { + wakeword: process.env.TRANS_WAKEWORD, + wakewordThreshold: Number(process.env.TRANS_WAKEWORD_THRESHOLD), + cooldownSeconds: Number(process.env.TRANS_COOLDOWN_SECONDS), + confirmFrames: Number(process.env.TRANS_CONFIRM_FRAMES), + listenSeconds: Number(process.env.TRANS_LISTEN_SECONDS), + silenceThreshold: Number(process.env.TRANS_SILENCE_THRESHOLD), + silenceFrames: Number(process.env.TRANS_SIELNCE_FRAMES), + voskModelPath: process.env.TRANS_VOSK_MODEL_PATH, + sampleRate: Number(process.env.TRANS_SAMPLE_RATE), +}; diff --git a/src/models/index.js b/src/models/index.js index c02adc1..605d2f2 100644 --- a/src/models/index.js +++ b/src/models/index.js @@ -6,12 +6,10 @@ const Redis = require('redis'); const redis = {}; redis.client = Redis.createClient({ - url: `redis://default@${redisConfig.REDIS_HOST}:${redisConfig.REDIS_PORT}` + url: `redis://default@${redisConfig.HOST}:${redisConfig.PORT}` }); redis.ensureGroup = async (stream, group) => { - await redis.client.connect().catch(console.error); - try { await redis.client.xGroupCreate(stream, group, "0", {MKSTREAM: true}); @@ -23,7 +21,19 @@ redis.ensureGroup = async (stream, group) => { console.error(err); } } - await redis.client.quit(); +} + +redis.sendTranscriptEvent = async (streamKey, device_id, text) => { + try { + await redis.client.xAdd(streamKey, "*", { + text, + device_id: device_id, + }); + + console.log("Transcript added to stream"); + } catch (err) { + console.error(err); + } } module.exports = { diff --git a/src/scripts/py/config.json b/src/scripts/py/config.json new file mode 100644 index 0000000..8f1cd46 --- /dev/null +++ b/src/scripts/py/config.json @@ -0,0 +1,11 @@ +{ + "wakeword": "alexa", + "wakewordThreshold": 0.80, + "cooldownSeconds": 2.5, + "confirmFrames": 3, + "listenSeconds": 12, + "silenceThreshold": 0.015, + "silenceFrames": 10, + "voskModelPath": "./vosk-model-small-en-us-0.15", + "sampleRate": 16000 +} diff --git a/src/scripts/py/info.txt b/src/scripts/py/info.txt new file mode 100644 index 0000000..c30b258 --- /dev/null +++ b/src/scripts/py/info.txt @@ -0,0 +1 @@ +PyAudio requires Python version <=3.13 \ No newline at end of file diff --git a/src/scripts/py/pythonOutput.js b/src/scripts/py/pythonOutput.js new file mode 100644 index 0000000..e09e33b --- /dev/null +++ b/src/scripts/py/pythonOutput.js @@ -0,0 +1,41 @@ +// pythonOutput.js +let buffer = ""; + +function handlePythonOutput(chunk, onMessage) { + buffer += chunk; + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + + for (const line of lines) { + if (!line.trim()) continue; + + try { + const msg = JSON.parse(line); + switch (msg.event) { + case "ready": + console.log("Python ready"); + break; + case "wakeword": + console.log(`Wakeword detected: ${msg.model} (${msg.score})`); + break; + case "partial": + console.log("Partial:", msg.text); + break; + case "text": + case "final": + console.log("Transcript:", msg.text); + onMessage?.(msg).catch(console.error); + break; + case "done": + console.log("Back to wakeword mode"); + break; + default: + console.log("[PY]", line); + } + } catch { + console.log("[PY]", line); + } + } +} + +module.exports = { handlePythonOutput }; diff --git a/src/scripts/py/requirements.txt b/src/scripts/py/requirements.txt new file mode 100644 index 0000000..b787d28 --- /dev/null +++ b/src/scripts/py/requirements.txt @@ -0,0 +1,4 @@ +openwakeword +onnxruntime +numpy +pyaudio \ No newline at end of file diff --git a/src/scripts/py/transcribe.py b/src/scripts/py/transcribe.py new file mode 100644 index 0000000..947b956 --- /dev/null +++ b/src/scripts/py/transcribe.py @@ -0,0 +1,112 @@ +import json +import sys +import time +import numpy as np +import pyaudio +import vosk +from openwakeword.model import Model + +CONFIG = json.loads(sys.argv[1]) + +RATE = CONFIG.get("sampleRate", 16000) +CHUNK = 1280 +WAKEWORD = CONFIG.get("wakeword", "alexa") +THRESHOLD = CONFIG.get("wakewordThreshold", 0.75) +COOLDOWN = CONFIG.get("cooldownSeconds", 2.5) +CONFIRM_FRAMES = CONFIG.get("confirmFrames", 3) +LISTEN_SECONDS = CONFIG.get("listenSeconds", 12) +SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.02) +SILENCE_FRAMES = CONFIG.get("silenceFrames", 12) +VOSK_MODEL_PATH = CONFIG["voskModelPath"] + +vosk.SetLogLevel(-1) +vosk_model = vosk.Model(VOSK_MODEL_PATH) +recognizer = vosk.KaldiRecognizer(vosk_model, RATE) +recognizer.SetWords(True) + +oww = Model(wakeword_models=[WAKEWORD], inference_framework="onnx") + +audio = pyaudio.PyAudio() +stream = audio.open( + format=pyaudio.paInt16, + channels=1, + rate=RATE, + input=True, + frames_per_buffer=CHUNK, +) + +print(json.dumps({"event": "ready"}), flush=True) + +mode = "wakeword" +high_count = 0 +last_trigger = 0 +silence_count = 0 +listen_deadline = None + +def rms_level(samples): + x = samples.astype(np.float32) / 32768.0 + return np.sqrt(np.mean(x * x)) + +try: + while True: + data = stream.read(CHUNK, exception_on_overflow=False) + samples = np.frombuffer(data, dtype=np.int16) + now = time.time() + + if mode == "wakeword": + preds = oww.predict(samples) + score = preds.get(WAKEWORD, 0.0) + + if score >= THRESHOLD: + high_count += 1 + else: + high_count = 0 + + if high_count >= CONFIRM_FRAMES and (now - last_trigger) > COOLDOWN: + last_trigger = now + high_count = 0 + print(json.dumps({ + "event": "wakeword", + "model": WAKEWORD, + "score": float(score) + }), flush=True) + mode = "transcribe" + recognizer.Reset() + silence_count = 0 + listen_deadline = now + LISTEN_SECONDS + + else: + if recognizer.AcceptWaveform(data): + result = json.loads(recognizer.Result()) + if result.get("text"): + print(json.dumps({"event": "final", "text": result["text"]}), flush=True) + listen_deadline = now + LISTEN_SECONDS + silence_count = 0 + else: + partial = json.loads(recognizer.PartialResult()).get("partial", "") + if partial: + print(json.dumps({"event": "partial", "text": partial}), flush=True) + + level = rms_level(samples) + if level < SILENCE_THRESHOLD: + silence_count += 1 + else: + silence_count = 0 + + if silence_count >= SILENCE_FRAMES or now > listen_deadline: + final = json.loads(recognizer.FinalResult()) + if final.get("text"): + print(json.dumps({"event": "final", "text": final["text"]}), flush=True) + + print(json.dumps({"event": "done"}), flush=True) + mode = "wakeword" + silence_count = 0 + high_count = 0 + listen_deadline = None + +except KeyboardInterrupt: + pass +finally: + stream.stop_stream() + stream.close() + audio.terminate() diff --git a/src/server.js b/src/server.js index 1536db7..bdb4b2e 100644 --- a/src/server.js +++ b/src/server.js @@ -3,31 +3,21 @@ if (process.env.NODE_ENV !== "production") { require("dotenv").config(); } +const { spawn } = require('child_process'); + const models = require("./models"); const redis = models.redis; const config = require("./config"); +const transConfig = require("./config/transcribe.config") -// Send Speech to Text to Redis Stream -const sendTranscriptEvent = async (streamKey, device_id, text) => { - await redis.client.xAdd( - streamKey, - '*', - { - text: text, - device_id: device_id - } - ) - - console.log("Transcript added to stream"); -} +const py = spawn('python', ['./src/scripts/py/transcribe.py', JSON.stringify(transConfig)], { + stdio: ['ignore', 'pipe', 'pipe'] +}); +const { handlePythonOutput } = require("./scripts/py/pythonOutput"); // Read Redis Stream group -const processor = async (group, consumer, streamKey) => { - await redis.client.connect().catch(console.error); - - sendTranscriptEvent("transcripts", config.DEVICE_ID, "TEST"); - +async function processor (group, consumer, streamKey) { while (true) { const res = await redis.client.xReadGroup( group, @@ -46,14 +36,29 @@ const processor = async (group, consumer, streamKey) => { } } } - - await redis.client.quit(); } -const main = async () => { - await redis.ensureGroup("commands", 'assistant'); +async function main() { + await redis.client.connect().catch(console.error); + await redis.ensureGroup("commands", "assistant"); - processor('assistant', `${config.DEVICE_ID}`, "commands"); + py.stdout.on("data", (data) => { + handlePythonOutput(data.toString(), async (msg) => { + if (msg.event === "text" || msg.event === "final") { + await redis.sendTranscriptEvent("transcripts", config.DEVICE_ID, msg.text); + } + }); + }); + + py.stderr.on("data", (data) => { + console.error("[PY ERR]", data.toString()); + }); + + py.on("close", (code) => { + console.log("Python exited with code", code); + }); + + processor("assistant", `${config.DEVICE_ID}`, "commands").catch(console.error); } main().catch(console.error); \ No newline at end of file