diff --git a/.gitignore b/.gitignore index 49f898c..2a5b053 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .env node_modules/ -.venv/ \ No newline at end of file +.venv/ +src/scripts/py/vosk-model*/ \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 4b13c72..a4c8ad9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "mic": "^2.1.2", "redis": "^6.1.0" }, "devDependencies": { @@ -320,6 +321,12 @@ "node": ">=0.12.0" } }, + "node_modules/mic": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mic/-/mic-2.1.2.tgz", + "integrity": "sha512-rpl4tgdXX24sAzYwjRc5OZfGNAuhUIIjdd0cw8+Ubq7rp3iGhi40AdqcwurDWhEZADk60tPOxb3E2MpoeLeyxw==", + "license": "MIT" + }, "node_modules/minimatch": { "version": "10.2.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", diff --git a/package.json b/package.json index a69a95b..23ccdde 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "license": "ISC", "type": "commonjs", "dependencies": { + "mic": "^2.1.2", "redis": "^6.1.0" }, "devDependencies": { diff --git a/src/scripts/py/config.json b/src/scripts/py/config.json new file mode 100644 index 0000000..8f1cd46 --- /dev/null +++ b/src/scripts/py/config.json @@ -0,0 +1,11 @@ +{ + "wakeword": "alexa", + "wakewordThreshold": 0.80, + "cooldownSeconds": 2.5, + "confirmFrames": 3, + "listenSeconds": 12, + "silenceThreshold": 0.015, + "silenceFrames": 10, + "voskModelPath": "./vosk-model-small-en-us-0.15", + "sampleRate": 16000 +} diff --git a/src/scripts/py/info.txt b/src/scripts/py/info.txt new file mode 100644 index 0000000..c30b258 --- /dev/null +++ b/src/scripts/py/info.txt @@ -0,0 +1 @@ +PyAudio requires Python version <=3.13 \ No newline at end of file diff --git a/src/scripts/py/requirements.txt b/src/scripts/py/requirements.txt new file mode 100644 index 0000000..b787d28 --- /dev/null +++ b/src/scripts/py/requirements.txt @@ -0,0 +1,4 @@ +openwakeword +onnxruntime +numpy +pyaudio \ No newline at end of file diff --git a/src/scripts/py/transcribe.js b/src/scripts/py/transcribe.js new file mode 100644 index 0000000..50137ca --- /dev/null +++ b/src/scripts/py/transcribe.js @@ -0,0 +1,46 @@ +const { spawn } = require('child_process'); +const fs = require('fs'); + +const config = JSON.parse(fs.readFileSync('./config.json', 'utf8')); + +const py = spawn('python', ['wakeword.py', JSON.stringify(config)], { + stdio: ['ignore', 'pipe', 'pipe'] +}); + +let buffer = ''; + +py.stdout.on('data', (data) => { + buffer += data.toString(); + const lines = buffer.split('\n'); + buffer = lines.pop(); + + for (const line of lines) { + if (!line.trim()) continue; + + try { + const msg = JSON.parse(line); + + if (msg.event === 'ready') { + console.log('Python ready'); + } else if (msg.event === 'wakeword') { + console.log(`Wakeword detected: ${msg.model} (${msg.score})`); + } else if (msg.event === 'partial') { + console.log('Partial:', msg.text); + } else if (msg.event === 'text' || msg.event === 'final') { + console.log('Transcript:', msg.text); + } else if (msg.event === 'done') { + console.log('Back to wakeword mode'); + } + } catch { + console.log('[PY]', line); + } + } +}); + +py.stderr.on('data', (data) => { + console.error('[PY ERR]', data.toString()); +}); + +py.on('close', (code) => { + console.log('Python exited with code', code); +}); diff --git a/src/scripts/py/wakeword.py b/src/scripts/py/wakeword.py new file mode 100644 index 0000000..947b956 --- /dev/null +++ b/src/scripts/py/wakeword.py @@ -0,0 +1,112 @@ +import json +import sys +import time +import numpy as np +import pyaudio +import vosk +from openwakeword.model import Model + +CONFIG = json.loads(sys.argv[1]) + +RATE = CONFIG.get("sampleRate", 16000) +CHUNK = 1280 +WAKEWORD = CONFIG.get("wakeword", "alexa") +THRESHOLD = CONFIG.get("wakewordThreshold", 0.75) +COOLDOWN = CONFIG.get("cooldownSeconds", 2.5) +CONFIRM_FRAMES = CONFIG.get("confirmFrames", 3) +LISTEN_SECONDS = CONFIG.get("listenSeconds", 12) +SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.02) +SILENCE_FRAMES = CONFIG.get("silenceFrames", 12) +VOSK_MODEL_PATH = CONFIG["voskModelPath"] + +vosk.SetLogLevel(-1) +vosk_model = vosk.Model(VOSK_MODEL_PATH) +recognizer = vosk.KaldiRecognizer(vosk_model, RATE) +recognizer.SetWords(True) + +oww = Model(wakeword_models=[WAKEWORD], inference_framework="onnx") + +audio = pyaudio.PyAudio() +stream = audio.open( + format=pyaudio.paInt16, + channels=1, + rate=RATE, + input=True, + frames_per_buffer=CHUNK, +) + +print(json.dumps({"event": "ready"}), flush=True) + +mode = "wakeword" +high_count = 0 +last_trigger = 0 +silence_count = 0 +listen_deadline = None + +def rms_level(samples): + x = samples.astype(np.float32) / 32768.0 + return np.sqrt(np.mean(x * x)) + +try: + while True: + data = stream.read(CHUNK, exception_on_overflow=False) + samples = np.frombuffer(data, dtype=np.int16) + now = time.time() + + if mode == "wakeword": + preds = oww.predict(samples) + score = preds.get(WAKEWORD, 0.0) + + if score >= THRESHOLD: + high_count += 1 + else: + high_count = 0 + + if high_count >= CONFIRM_FRAMES and (now - last_trigger) > COOLDOWN: + last_trigger = now + high_count = 0 + print(json.dumps({ + "event": "wakeword", + "model": WAKEWORD, + "score": float(score) + }), flush=True) + mode = "transcribe" + recognizer.Reset() + silence_count = 0 + listen_deadline = now + LISTEN_SECONDS + + else: + if recognizer.AcceptWaveform(data): + result = json.loads(recognizer.Result()) + if result.get("text"): + print(json.dumps({"event": "final", "text": result["text"]}), flush=True) + listen_deadline = now + LISTEN_SECONDS + silence_count = 0 + else: + partial = json.loads(recognizer.PartialResult()).get("partial", "") + if partial: + print(json.dumps({"event": "partial", "text": partial}), flush=True) + + level = rms_level(samples) + if level < SILENCE_THRESHOLD: + silence_count += 1 + else: + silence_count = 0 + + if silence_count >= SILENCE_FRAMES or now > listen_deadline: + final = json.loads(recognizer.FinalResult()) + if final.get("text"): + print(json.dumps({"event": "final", "text": final["text"]}), flush=True) + + print(json.dumps({"event": "done"}), flush=True) + mode = "wakeword" + silence_count = 0 + high_count = 0 + listen_deadline = None + +except KeyboardInterrupt: + pass +finally: + stream.stop_stream() + stream.close() + audio.terminate()