Spilum.Net/mtf-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: Spilum.Net:4a2c6aa76118607bd3377e0e36f7cc14426ea2fb
Branches Tags
Spilum.Net:main
..
compare: Spilum.Net:6e8c689639434baa15644878fecb039abc2281fc
Branches Tags
Spilum.Net:main

No commits in common. "4a2c6aa76118607bd3377e0e36f7cc14426ea2fb" and "6e8c689639434baa15644878fecb039abc2281fc" have entirely different histories.
4a2c6aa761 ... 6e8c689639

12 changed files with 96 additions and 546 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,5 +1,5 @@
.env
node_modules/
.venv/
src/scripts/py/zipformer*
src/models/vosk*
output.wav

279
package-lock.json generated
View File

@ -9,7 +9,6 @@
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"axios": "^1.18.1",
"mic": "^2.1.2",
"redis": "^6.1.0",
"speaker": "^0.5.5",
@ -101,18 +100,6 @@
"node": ">=12.0"
}
},
"node_modules/agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"license": "MIT",
"dependencies": {
"debug": "4"
},
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/anymatch": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
@ -127,24 +114,6 @@
"node": ">= 8"
}
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
"license": "MIT"
},
"node_modules/axios": {
"version": "1.18.1",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.18.1.tgz",
"integrity": "sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==",
"license": "MIT",
"dependencies": {
"follow-redirects": "^1.16.0",
"form-data": "^4.0.5",
"https-proxy-agent": "^5.0.1",
"proxy-from-env": "^2.1.0"
}
},
"node_modules/balanced-match": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
@ -221,19 +190,6 @@
"resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
"integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ=="
},
"node_modules/call-bind-apply-helpers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/chokidar": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz",
@ -268,18 +224,6 @@
"node": ">=0.10.0"
}
},
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"license": "MIT",
"dependencies": {
"delayed-stream": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/commander": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
@ -337,15 +281,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"license": "MIT",
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/dotenv": {
"version": "17.4.2",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz",
@ -359,20 +294,6 @@
"url": "https://dotenvx.com"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.1",
"es-errors": "^1.3.0",
"gopd": "^1.2.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-define-property": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
@ -389,33 +310,6 @@
"node": ">= 0.4"
}
},
"node_modules/es-object-atoms": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz",
"integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-set-tostringtag": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.6",
"has-tostringtag": "^1.0.2",
"hasown": "^2.0.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/escape-string-regexp": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
@ -445,42 +339,6 @@
"node": ">=8"
}
},
"node_modules/follow-redirects": {
"version": "1.16.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
"integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
}
],
"license": "MIT",
"engines": {
"node": ">=4.0"
},
"peerDependenciesMeta": {
"debug": {
"optional": true
}
}
},
"node_modules/form-data": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz",
"integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==",
"license": "MIT",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"es-set-tostringtag": "^2.1.0",
"hasown": "^2.0.4",
"mime-types": "^2.1.35"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
@ -496,52 +354,6 @@
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/function-bind": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
"integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-intrinsic": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"es-define-property": "^1.0.1",
"es-errors": "^1.3.0",
"es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
"get-proto": "^1.0.1",
"gopd": "^1.2.0",
"has-symbols": "^1.1.0",
"hasown": "^2.0.2",
"math-intrinsics": "^1.1.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
"license": "MIT",
"dependencies": {
"dunder-proto": "^1.0.1",
"es-object-atoms": "^1.0.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/glob-parent": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
@ -616,58 +428,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has-symbols": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has-tostringtag": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
"license": "MIT",
"dependencies": {
"has-symbols": "^1.0.3"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/hasown": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz",
"integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==",
"license": "MIT",
"dependencies": {
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/https-proxy-agent": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
"integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
"license": "MIT",
"dependencies": {
"agent-base": "6",
"debug": "4"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/ignore-by-default": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz",
@ -735,42 +495,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/mic": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/mic/-/mic-2.1.2.tgz",
"integrity": "sha512-rpl4tgdXX24sAzYwjRc5OZfGNAuhUIIjdd0cw8+Ubq7rp3iGhi40AdqcwurDWhEZADk60tPOxb3E2MpoeLeyxw==",
"license": "MIT"
},
"node_modules/mime-db": {
"version": "1.52.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "2.1.35",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"license": "MIT",
"dependencies": {
"mime-db": "1.52.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/minimatch": {
"version": "10.2.5",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz",
@ -874,15 +604,6 @@
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/proxy-from-env": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz",
"integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==",
"license": "MIT",
"engines": {
"node": ">=10"
}
},
"node_modules/pstree.remy": {
"version": "1.1.8",
"resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz",

1
package.json
View File

@ -12,7 +12,6 @@
"license": "ISC",
"type": "commonjs",
"dependencies": {
"axios": "^1.18.1",
"mic": "^2.1.2",
"redis": "^6.1.0",
"speaker": "^0.5.5",

4
src/config/index.js
View File

@ -2,7 +2,5 @@ module.exports = {
FORCE_DB_SYNC: process.env.FORCE_DB_SYNC,
NODE_ENV: process.env.NODE_ENV,
PORT: process.env.PORT,
DEVICE_ID: process.env.DEVICE_ID,
HA_URL: process.env.HA_URL,
HA_TOKEN: process.env.HA_TOKEN
DEVICE_ID: process.env.DEVICE_ID
};

4
src/config/redis.config.js Normal file
View File

@ -0,0 +1,4 @@
module.exports = {
HOST: process.env.REDIS_HOST,
PORT: process.env.REDIS_PORT
}

7
src/config/transcribe.config.js
View File

@ -6,11 +6,6 @@ module.exports = {
listenSeconds: Number(process.env.TRANS_LISTEN_SECONDS),
silenceThreshold: Number(process.env.TRANS_SILENCE_THRESHOLD),
silenceFrames: Number(process.env.TRANS_SILENCE_FRAMES),
voskModelPath: process.env.TRANS_VOSK_MODEL_PATH,
sampleRate: Number(process.env.TRANS_SAMPLE_RATE),
sherpaModelPath: {
encoder: process.env.TRANS_SHERPA_ENCODER,
decoder: process.env.TRANS_SHERPA_DECODER,
joiner: process.env.TRANS_SHERPA_JOINER,
tokens: process.env.TRANS_SHERPA_TOKENS,
},
};

2
src/modules/redis

@ -1 +1 @@
Subproject commit b335675847af7b92b3464d807939d1dbe558968d
Subproject commit e41a75e7436a059ab963c5db7b62ca524cacc4e5

9
src/scripts/py/config.json
View File

@ -6,11 +6,6 @@
"listenSeconds": 12,
"silenceThreshold": 0.015,
"silenceFrames": 10,
"sampleRate": 16000,
"sherpaModelPath": {
"encoder": "./zipformer-small/encoder.onnx",
"decoder": "./zipformer-small/decoder.onnx",
"joiner": "./zipformer-small/joiner.onnx",
"tokens": "./zipformer-small/data/tokens.txt"
}
"voskModelPath": "./vosk-model-small-en-us-0.15",
"sampleRate": 16000
}

1
src/scripts/py/pythonOutput.js
View File

@ -1,3 +1,4 @@
// pythonOutput.js
let buffer = "";
function handlePythonOutput(chunk, onMessage) {

1
src/scripts/py/requirements.txt
View File

@ -3,4 +3,3 @@ onnxruntime
numpy
pyaudio
vosk
sherpa_onnx

269
src/scripts/py/transcribe.py
View File

@ -1,229 +1,116 @@
import collections, json, os, sys
from dataclasses import dataclass
from time import monotonic
import json
import sys
import time
import numpy as np
import openwakeword
import pyaudio
import sherpa_onnx
import vosk
import openwakeword
from openwakeword.model import Model
CONFIG = json.loads(sys.argv[1])
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
RATE = CONFIG.get("sampleRate", 16000)
CHUNK = 640
# Wakeword
WAKEWORD = CONFIG.get("wakeword", "hey jarvis")
THRESHOLD = CONFIG.get("wakewordThreshold", 0.80)
CHUNK = 1280
WAKEWORD = CONFIG.get("wakeword", "alexa")
THRESHOLD = CONFIG.get("wakewordThreshold", 0.75)
COOLDOWN = CONFIG.get("cooldownSeconds", 2.5)
CONFIRM_FRAMES = CONFIG.get("confirmFrames", 3)
LISTEN_SECONDS = CONFIG.get("listenSeconds", 12)
SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.02)
SILENCE_FRAMES = CONFIG.get("silenceFrames", 12)
VOSK_MODEL_PATH = CONFIG["voskModelPath"]
# Speech/silence gating (prevents wakeword firing right after transcript)
# Values assume threshold is on a normalized amplitude scale (0..1).
SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.03)
SILENCE_FRAMES = CONFIG.get("silenceFrames", 6)
vosk.SetLogLevel(-1)
vosk_model = vosk.Model(VOSK_MODEL_PATH)
recognizer = vosk.KaldiRecognizer(vosk_model, RATE)
recognizer.SetWords(True)
# After transcribe finishes, ignore wakeword for a short period
POST_TRANSCRIPT_IGNORE_SECONDS = CONFIG.get("postTranscriptIgnoreSeconds", 1.5)
# Download all models (TODO: Import single model)
openwakeword.utils.download_models()
# ---- Transcription endpointing ----
ENDPOINT_RULE1 = CONFIG.get("endpoint_rule1_min_trailing_silence", 2.4)
ENDPOINT_RULE2 = CONFIG.get("endpoint_rule2_min_trailing_silence", 1.2)
ENDPOINT_RULE3 = CONFIG.get("endpoint_rule3_min_utterance_length", 30)
MIN_TRANSCRIBE_SECONDS_BEFORE_ENDPOINT = CONFIG.get("min_transcribe_before_endpoint", 1.0)
ENABLE_ENDPOINT_DETECTION = CONFIG.get("enable_endpoint_detection", True)
MODE_WAKE = "wakeword"
MODE_TRANSCRIBE = "transcribe"
# ---- Sherpa model paths ----
mc = CONFIG["sherpaModelPath"]
paths = {k: os.path.join(SCRIPT_DIR, v.lstrip("./")) for k, v in mc.items()}
recognizer = sherpa_onnx.OnlineRecognizer.from_transducer(
encoder=paths["encoder"],
decoder=paths["decoder"],
joiner=paths["joiner"],
tokens=paths["tokens"],
sample_rate=RATE,
feature_dim=80,
decoding_method="greedy_search",
num_threads=max(1, (os.cpu_count() or 1) // 2),
enable_endpoint_detection=ENABLE_ENDPOINT_DETECTION,
rule1_min_trailing_silence=ENDPOINT_RULE1,
rule2_min_trailing_silence=ENDPOINT_RULE2,
rule3_min_utterance_length=ENDPOINT_RULE3,
)
# ---- openwakeword model ----
oww = Model(wakeword_models=[WAKEWORD], inference_framework="onnx")
# ---- Audio ----
pa = pyaudio.PyAudio()
mic = pa.open(
audio = pyaudio.PyAudio()
stream = audio.open(
format=pyaudio.paInt16,
channels=1,
rate=RATE,
input=True,
frames_per_buffer=CHUNK
frames_per_buffer=CHUNK,
)
@dataclass
class State:
mode: str = MODE_WAKE
high_count: int = 0
last_trigger: float = 0.0
print(json.dumps({"event": "ready"}), flush=True)
# Transcription timing
listen_deadline: float | None = None
transcribe_start: float | None = None
mode = "wakeword"
high_count = 0
last_trigger = 0
silence_count = 0
listen_deadline = None
# Wakeword anti-double-trigger
post_transcript_ignore_deadline: float | None = None
# Silence gate
silence_count: int = 0
sherpa_stream: object = None
last_partial: str = ""
state = State()
float_buf = np.empty(CHUNK, np.float32)
def emit(event, **kw):
print(json.dumps({"event": event, **kw}, ensure_ascii=False), flush=True)
def decode_pending(st):
while recognizer.is_ready(st):
recognizer.decode_stream(st)
def reset_oww():
if hasattr(oww, "accumulated_predictions") and WAKEWORD in oww.accumulated_predictions:
oww.accumulated_predictions[WAKEWORD].clear()
def flush():
# Drain mic so next phase starts with fresh audio
while True:
try:
available = mic.get_read_available()
except Exception:
break
if available <= 0:
break
mic.read(min(available, CHUNK), exception_on_overflow=False)
def start(now):
state.mode = MODE_TRANSCRIBE
state.high_count = 0
state.last_trigger = now
state.listen_deadline = now + LISTEN_SECONDS
state.transcribe_start = now
state.last_partial = ""
state.sherpa_stream = recognizer.create_stream()
def stop(now):
# Reset back to wake mode
reset_oww()
flush()
state.mode = MODE_WAKE
state.high_count = 0
state.listen_deadline = None
state.transcribe_start = None
state.last_partial = ""
state.sherpa_stream = None
# Cooldown baseline should start AFTER the transcript ends
state.last_trigger = now
# And ignore wakeword for a short grace period
state.post_transcript_ignore_deadline = now + POST_TRANSCRIPT_IGNORE_SECONDS
emit("ready")
def rms_level(samples):
x = samples.astype(np.float32) / 32768.0
return np.sqrt(np.mean(x * x))
try:
while True:
pcm_i16 = mic.read(CHUNK, exception_on_overflow=False)
samples_i16 = np.frombuffer(pcm_i16, dtype=np.int16)
data = stream.read(CHUNK, exception_on_overflow=False)
samples = np.frombuffer(data, dtype=np.int16)
now = time.time()
now = monotonic()
if mode == "wakeword":
preds = oww.predict(samples)
score = preds.get(WAKEWORD, 0.0)
if state.mode == MODE_WAKE:
# -------- Silence gate (prevents wake right after transcript) --------
# normalized mean absolute amplitude
amp = float(np.mean(np.abs(samples_i16)) / 32768.0)
if amp <= SILENCE_THRESHOLD:
state.silence_count += 1
if score >= THRESHOLD:
high_count += 1
else:
state.silence_count = 0
high_count = 0
silence_ok = state.silence_count >= SILENCE_FRAMES
if high_count >= CONFIRM_FRAMES and (now - last_trigger) > COOLDOWN:
last_trigger = now
high_count = 0
print(json.dumps({
"event": "wakeword",
"model": WAKEWORD,
"score": float(score)
}), flush=True)
mode = "transcribe"
recognizer.Reset()
silence_count = 0
listen_deadline = now + LISTEN_SECONDS
# Ignore wakeword right after transcript ends
ignore_ok = not (
state.post_transcript_ignore_deadline is not None
and now < state.post_transcript_ignore_deadline
)
else:
if recognizer.AcceptWaveform(data):
result = json.loads(recognizer.Result())
if result.get("text"):
print(json.dumps({"event": "final", "text": result["text"]}), flush=True)
listen_deadline = now + LISTEN_SECONDS
silence_count = 0
else:
partial = json.loads(recognizer.PartialResult()).get("partial", "")
if partial:
print(json.dumps({"event": "partial", "text": partial}), flush=True)
if not (silence_ok and ignore_ok):
continue
level = rms_level(samples)
if level < SILENCE_THRESHOLD:
silence_count += 1
else:
silence_count = 0
# -------- Wakeword scoring --------
score = oww.predict(samples_i16).get(WAKEWORD, 0.0)
if silence_count >= SILENCE_FRAMES or now > listen_deadline:
final = json.loads(recognizer.FinalResult())
if final.get("text"):
print(json.dumps({"event": "final", "text": final["text"]}), flush=True)
state.high_count = (state.high_count + 1) if score >= THRESHOLD else 0
if state.high_count >= CONFIRM_FRAMES and (now - state.last_trigger) > COOLDOWN:
start(now)
emit("wakeword", model=WAKEWORD, score=float(score))
continue
# ---- Transcribe mode ----
np.multiply(samples_i16, 1 / 32768.0, out=float_buf, casting="unsafe")
st = state.sherpa_stream
st.accept_waveform(RATE, float_buf)
decode_pending(st)
txt = recognizer.get_result(st)
if txt and txt != state.last_partial:
state.last_partial = txt
emit("partial", text=txt)
endpoint_ok = False
if state.listen_deadline is not None and now >= state.listen_deadline:
endpoint_ok = True
elif ENABLE_ENDPOINT_DETECTION and recognizer.is_endpoint(st):
# Don’t allow endpointing immediately after switching to transcribe
if state.transcribe_start is not None and now >= (state.transcribe_start + MIN_TRANSCRIBE_SECONDS_BEFORE_ENDPOINT):
endpoint_ok = True
if not endpoint_ok:
continue
st.input_finished()
decode_pending(st)
final = recognizer.get_result(st)
if final:
emit("final", text=final)
emit("done")
stop(now)
print(json.dumps({"event": "done"}), flush=True)
mode = "wakeword"
silence_count = 0
high_count = 0
listen_deadline = None
except KeyboardInterrupt:
pass
finally:
mic.stop_stream()
mic.close()
pa.terminate()
stream.stop_stream()
stream.close()
audio.terminate()

59
src/server.js
View File

@ -3,14 +3,9 @@ if (process.env.NODE_ENV !== "production") {
require("dotenv").config();
}
// Axios
const axios = require("axios");
// TinyTTS
const TinyTTS = require("tiny-tts");
const tts = new TinyTTS();
// General imports
const fs = require("fs");
const os = require("os");
const path = require("path");
@ -18,26 +13,12 @@ const wav = require("wav-decoder");
const Speaker = require("speaker");
const { spawn } = require("child_process");
// Modules
const {redis} = require('./modules/redis');
// Configs
const config = require("./config");
const transConfig = require("./config/transcribe.config");
const { handlePythonOutput } = require("./scripts/py/pythonOutput");
// === HomeAssistant ===
const ha = {}
ha.client = axios.create({
baseURL: config.HA_URL,
headers: {
Authorization: `Bearer ${config.HA_TOKEN}`,
"Content-Type": "application/json",
},
});
// === Speaker ===
let speaker = null;
let speakerChannels = null;
@ -88,7 +69,6 @@ function floatsToPCM16LE(channelData) {
return out;
}
// === TTS ===
async function playAudio(text) {
const outFile = path.join(
os.tmpdir(),
@ -124,7 +104,6 @@ async function playAudio(text) {
}
}
// === Transcriber ===
const py = spawn("python", ["./src/scripts/py/transcribe.py", JSON.stringify(transConfig)], {
stdio: ["ignore", "pipe", "pipe"],
});
@ -157,26 +136,6 @@ async function processor(group, consumer, streamKey) {
await playAudio("Reminder, " + String(msg.message.value));
break;
case "light_on":
await playAudio("Turning on light.");
await ha.client.post("/api/services/homeassistant/turn_on", {
entity_id: "light.living_room_living_room"
}).catch(async (err) => {
await playAudio("I have failed you father.");
console.error(err);
});
break;
case "light_off":
await playAudio("Turning off light.");
await ha.client.post("/api/services/homeassistant/turn_off", {
entity_id: "light.living_room_living_room"
}).catch(async (err) => {
await playAudio("I have failed you father.");
console.error(err);
});
break;
default:
await playAudio("Command, " + String(msg.message.command) + " is undefined.")
}
@ -187,9 +146,12 @@ async function processor(group, consumer, streamKey) {
}
}
// === Main ===
async function main() {
// Attach python listeners FIRST to avoid missing logs
await redis.connect();
await redis.ensureGroup("commands", "assistant");
await playAudio("Assistant, online.");
py.stdout.on("data", (data) => {
handlePythonOutput(data.toString(), async (msg) => {
if (msg.event === "text" || msg.event === "final") {
@ -206,18 +168,7 @@ async function main() {
console.log("Python exited with code", code);
});
try {
await redis.connect();
await redis.ensureGroup("commands", "assistant");
await playAudio("Assistant, online.");
// Start the command processor
processor("assistant", `${config.DEVICE_ID}`, "commands").catch(console.error);
console.log("System fully initialized.");
} catch (err) {
console.error("Initialization failed:", err);
}
}
main().catch(console.error);