From 22bcfa31f3e1dbfe883195f06085d54ed7e00654 Mon Sep 17 00:00:00 2001 From: Apher Date: Fri, 17 Jul 2026 03:40:02 +0000 Subject: [PATCH] Use sherpa instead of vosk --- src/config/transcribe.config.js | 7 +- src/scripts/py/config.json | 9 +- src/scripts/py/transcribe.py | 205 +++++++++++++++++--------------- 3 files changed, 119 insertions(+), 102 deletions(-) diff --git a/src/config/transcribe.config.js b/src/config/transcribe.config.js index e18145c..40f3071 100644 --- a/src/config/transcribe.config.js +++ b/src/config/transcribe.config.js @@ -6,6 +6,11 @@ module.exports = { listenSeconds: Number(process.env.TRANS_LISTEN_SECONDS), silenceThreshold: Number(process.env.TRANS_SILENCE_THRESHOLD), silenceFrames: Number(process.env.TRANS_SILENCE_FRAMES), - voskModelPath: process.env.TRANS_VOSK_MODEL_PATH, sampleRate: Number(process.env.TRANS_SAMPLE_RATE), + sherpaModelPath: { + encoder: process.env.TRANS_SHERPA_ENCODER, + decoder: process.env.TRANS_SHERPA_DECODER, + joiner: process.env.TRANS_SHERPA_JOINER, + tokens: process.env.TRANS_SHERPA_TOKENS, + }, }; diff --git a/src/scripts/py/config.json b/src/scripts/py/config.json index 8f1cd46..06ed71c 100644 --- a/src/scripts/py/config.json +++ b/src/scripts/py/config.json @@ -6,6 +6,11 @@ "listenSeconds": 12, "silenceThreshold": 0.015, "silenceFrames": 10, - "voskModelPath": "./vosk-model-small-en-us-0.15", - "sampleRate": 16000 + "sampleRate": 16000, + "sherpaModelPath": { + "encoder": "./zipformer-small/encoder.onnx", + "decoder": "./zipformer-small/decoder.onnx", + "joiner": "./zipformer-small/joiner.onnx", + "tokens": "./zipformer-small/data/tokens.txt" + } } diff --git a/src/scripts/py/transcribe.py b/src/scripts/py/transcribe.py index 9fa7cd1..7a972e6 100644 --- a/src/scripts/py/transcribe.py +++ b/src/scripts/py/transcribe.py @@ -1,116 +1,123 @@ -import json -import sys -import time + +# Refactored template based on your original. +# NOTE: Replace MODEL_CONFIG loading as needed. + +import collections, json, os, sys +from dataclasses import dataclass +from time import monotonic + import numpy as np -import pyaudio -import vosk import openwakeword +import pyaudio +import sherpa_onnx from openwakeword.model import Model -CONFIG = json.loads(sys.argv[1]) +CONFIG=json.loads(sys.argv[1]) +SCRIPT_DIR=os.path.dirname(os.path.abspath(__file__)) +RATE=CONFIG.get("sampleRate",16000) +CHUNK=640 +WAKEWORD=CONFIG.get("wakeword","alexa") +THRESHOLD=CONFIG.get("wakewordThreshold",0.75) +COOLDOWN=CONFIG.get("cooldownSeconds",2.5) +CONFIRM_FRAMES=CONFIG.get("confirmFrames",3) +LISTEN_SECONDS=CONFIG.get("listenSeconds",8) +MODE_WAKE="wakeword" +MODE_TRANSCRIBE="transcribe" -RATE = CONFIG.get("sampleRate", 16000) -CHUNK = 1280 -WAKEWORD = CONFIG.get("wakeword", "alexa") -THRESHOLD = CONFIG.get("wakewordThreshold", 0.75) -COOLDOWN = CONFIG.get("cooldownSeconds", 2.5) -CONFIRM_FRAMES = CONFIG.get("confirmFrames", 3) -LISTEN_SECONDS = CONFIG.get("listenSeconds", 12) -SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.02) -SILENCE_FRAMES = CONFIG.get("silenceFrames", 12) -VOSK_MODEL_PATH = CONFIG["voskModelPath"] +mc=CONFIG["sherpaModelPath"] +paths={k:os.path.join(SCRIPT_DIR,v.lstrip("./")) for k,v in mc.items()} -vosk.SetLogLevel(-1) -vosk_model = vosk.Model(VOSK_MODEL_PATH) -recognizer = vosk.KaldiRecognizer(vosk_model, RATE) -recognizer.SetWords(True) - -# Download all models (TODO: Import single model) -openwakeword.utils.download_models() - -oww = Model(wakeword_models=[WAKEWORD], inference_framework="onnx") - -audio = pyaudio.PyAudio() -stream = audio.open( - format=pyaudio.paInt16, - channels=1, - rate=RATE, - input=True, - frames_per_buffer=CHUNK, +recognizer=sherpa_onnx.OnlineRecognizer.from_transducer( + encoder=paths["encoder"],decoder=paths["decoder"],joiner=paths["joiner"], + tokens=paths["tokens"],sample_rate=RATE,feature_dim=80, + decoding_method="greedy_search",num_threads=max(1,(os.cpu_count() or 1)//2), + enable_endpoint_detection=True, + rule1_min_trailing_silence=1.5, + rule2_min_trailing_silence=0.3, + rule3_min_utterance_length=10, ) -print(json.dumps({"event": "ready"}), flush=True) +oww=Model(wakeword_models=[WAKEWORD], inference_framework="onnx") +pa=pyaudio.PyAudio() +mic=pa.open(format=pyaudio.paInt16,channels=1,rate=RATE,input=True,frames_per_buffer=CHUNK) -mode = "wakeword" -high_count = 0 -last_trigger = 0 -silence_count = 0 -listen_deadline = None +@dataclass +class State: + mode:str=MODE_WAKE + high_count:int=0 + last_trigger:float=0.0 + listen_deadline:float|None=None + sherpa_stream=None + last_partial:str="" -def rms_level(samples): - x = samples.astype(np.float32) / 32768.0 - return np.sqrt(np.mean(x * x)) +state=State() +float_buf=np.empty(CHUNK,np.float32) +def emit(event,**kw): + print(json.dumps({"event":event,**kw},ensure_ascii=False),flush=True) + +def decode_pending(st): + while recognizer.is_ready(st): + recognizer.decode_stream(st) + +def reset_oww(): + if hasattr(oww,"accumulated_predictions") and WAKEWORD in oww.accumulated_predictions: + oww.accumulated_predictions[WAKEWORD].clear() + +def flush(): + while True: + try:a=mic.get_read_available() + except:break + if a<=0:break + mic.read(min(a,CHUNK),exception_on_overflow=False) + +def start(now): + state.mode=MODE_TRANSCRIBE + state.high_count=0 + state.last_trigger=now + state.listen_deadline=now+LISTEN_SECONDS + state.last_partial="" + state.sherpa_stream=recognizer.create_stream() + +def stop(): + reset_oww(); flush() + state.mode=MODE_WAKE + state.high_count=0 + state.listen_deadline=None + state.last_partial="" + state.sherpa_stream=None + +emit("ready") try: while True: - data = stream.read(CHUNK, exception_on_overflow=False) - samples = np.frombuffer(data, dtype=np.int16) - now = time.time() - - if mode == "wakeword": - preds = oww.predict(samples) - score = preds.get(WAKEWORD, 0.0) - - if score >= THRESHOLD: - high_count += 1 - else: - high_count = 0 - - if high_count >= CONFIRM_FRAMES and (now - last_trigger) > COOLDOWN: - last_trigger = now - high_count = 0 - print(json.dumps({ - "event": "wakeword", - "model": WAKEWORD, - "score": float(score) - }), flush=True) - mode = "transcribe" - recognizer.Reset() - silence_count = 0 - listen_deadline = now + LISTEN_SECONDS - - else: - if recognizer.AcceptWaveform(data): - result = json.loads(recognizer.Result()) - if result.get("text"): - print(json.dumps({"event": "final", "text": result["text"]}), flush=True) - listen_deadline = now + LISTEN_SECONDS - silence_count = 0 - else: - partial = json.loads(recognizer.PartialResult()).get("partial", "") - if partial: - print(json.dumps({"event": "partial", "text": partial}), flush=True) - - level = rms_level(samples) - if level < SILENCE_THRESHOLD: - silence_count += 1 - else: - silence_count = 0 - - if silence_count >= SILENCE_FRAMES or now > listen_deadline: - final = json.loads(recognizer.FinalResult()) - if final.get("text"): - print(json.dumps({"event": "final", "text": final["text"]}), flush=True) - - print(json.dumps({"event": "done"}), flush=True) - mode = "wakeword" - silence_count = 0 - high_count = 0 - listen_deadline = None + samples=np.frombuffer(mic.read(CHUNK,exception_on_overflow=False),dtype=np.int16) + now=monotonic() + if state.mode==MODE_WAKE: + score=oww.predict(samples).get(WAKEWORD,0.0) + state.high_count=state.high_count+1 if score>=THRESHOLD else 0 + if state.high_count>=CONFIRM_FRAMES and now-state.last_trigger>COOLDOWN: + start(now) + emit("wakeword",model=WAKEWORD,score=float(score)) + continue + np.multiply(samples,1/32768.0,out=float_buf,casting="unsafe") + st=state.sherpa_stream + st.accept_waveform(RATE,float_buf) + decode_pending(st) + txt=recognizer.get_result(st) + if txt and txt!=state.last_partial: + state.last_partial=txt + emit("partial",text=txt) + if not(recognizer.is_endpoint(st) or now>=state.listen_deadline): + continue + st.input_finished() + decode_pending(st) + final=recognizer.get_result(st) + if final: + emit("final",text=final) + emit("done") + stop() except KeyboardInterrupt: pass finally: - stream.stop_stream() - stream.close() - audio.terminate() + mic.stop_stream(); mic.close(); pa.terminate()