Use sherpa instead of vosk

src/config/transcribe.config.js

@@ -6,6 +6,11 @@ module.exports = {
   listenSeconds: Number(process.env.TRANS_LISTEN_SECONDS),
   silenceThreshold: Number(process.env.TRANS_SILENCE_THRESHOLD),
   silenceFrames: Number(process.env.TRANS_SILENCE_FRAMES),
-  voskModelPath: process.env.TRANS_VOSK_MODEL_PATH,
   sampleRate: Number(process.env.TRANS_SAMPLE_RATE),
+  sherpaModelPath: {
+    encoder: process.env.TRANS_SHERPA_ENCODER,
+    decoder: process.env.TRANS_SHERPA_DECODER,
+    joiner: process.env.TRANS_SHERPA_JOINER,
+    tokens: process.env.TRANS_SHERPA_TOKENS,
+  },
 };

src/scripts/py/config.json

@@ -6,6 +6,11 @@
   "listenSeconds": 12,
   "silenceThreshold": 0.015,
   "silenceFrames": 10,
-  "voskModelPath": "./vosk-model-small-en-us-0.15",
+  "sampleRate": 16000,
-  "sampleRate": 16000
+  "sherpaModelPath": {
+    "encoder": "./zipformer-small/encoder.onnx",
+    "decoder": "./zipformer-small/decoder.onnx",
+    "joiner": "./zipformer-small/joiner.onnx",
+    "tokens": "./zipformer-small/data/tokens.txt"
+  }
 }

src/scripts/py/transcribe.py

@@ -1,116 +1,123 @@
-import json
+
-import sys
+# Refactored template based on your original.
-import time
+# NOTE: Replace MODEL_CONFIG loading as needed.
+
+import collections, json, os, sys
+from dataclasses import dataclass
+from time import monotonic
+
 import numpy as np
-import pyaudio
-import vosk
 import openwakeword
+import pyaudio
+import sherpa_onnx
 from openwakeword.model import Model
 
-CONFIG = json.loads(sys.argv[1])
+CONFIG=json.loads(sys.argv[1])
+SCRIPT_DIR=os.path.dirname(os.path.abspath(__file__))
+RATE=CONFIG.get("sampleRate",16000)
+CHUNK=640
+WAKEWORD=CONFIG.get("wakeword","alexa")
+THRESHOLD=CONFIG.get("wakewordThreshold",0.75)
+COOLDOWN=CONFIG.get("cooldownSeconds",2.5)
+CONFIRM_FRAMES=CONFIG.get("confirmFrames",3)
+LISTEN_SECONDS=CONFIG.get("listenSeconds",8)
+MODE_WAKE="wakeword"
+MODE_TRANSCRIBE="transcribe"
 
-RATE = CONFIG.get("sampleRate", 16000)
+mc=CONFIG["sherpaModelPath"]
-CHUNK = 1280
+paths={k:os.path.join(SCRIPT_DIR,v.lstrip("./")) for k,v in mc.items()}
-WAKEWORD = CONFIG.get("wakeword", "alexa")
-THRESHOLD = CONFIG.get("wakewordThreshold", 0.75)
-COOLDOWN = CONFIG.get("cooldownSeconds", 2.5)
-CONFIRM_FRAMES = CONFIG.get("confirmFrames", 3)
-LISTEN_SECONDS = CONFIG.get("listenSeconds", 12)
-SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.02)
-SILENCE_FRAMES = CONFIG.get("silenceFrames", 12)
-VOSK_MODEL_PATH = CONFIG["voskModelPath"]
 
-vosk.SetLogLevel(-1)
+recognizer=sherpa_onnx.OnlineRecognizer.from_transducer(
-vosk_model = vosk.Model(VOSK_MODEL_PATH)
+    encoder=paths["encoder"],decoder=paths["decoder"],joiner=paths["joiner"],
-recognizer = vosk.KaldiRecognizer(vosk_model, RATE)
+    tokens=paths["tokens"],sample_rate=RATE,feature_dim=80,
-recognizer.SetWords(True)
+    decoding_method="greedy_search",num_threads=max(1,(os.cpu_count() or 1)//2),
-
+    enable_endpoint_detection=True,
-# Download all models (TODO: Import single model)
+    rule1_min_trailing_silence=1.5,
-openwakeword.utils.download_models()
+    rule2_min_trailing_silence=0.3,
-
+    rule3_min_utterance_length=10,
-oww = Model(wakeword_models=[WAKEWORD], inference_framework="onnx")
-
-audio = pyaudio.PyAudio()
-stream = audio.open(
-    format=pyaudio.paInt16,
-    channels=1,
-    rate=RATE,
-    input=True,
-    frames_per_buffer=CHUNK,
 )
 
-print(json.dumps({"event": "ready"}), flush=True)
+oww=Model(wakeword_models=[WAKEWORD], inference_framework="onnx")
+pa=pyaudio.PyAudio()
+mic=pa.open(format=pyaudio.paInt16,channels=1,rate=RATE,input=True,frames_per_buffer=CHUNK)
 
-mode = "wakeword"
+@dataclass
-high_count = 0
+class State:
-last_trigger = 0
+    mode:str=MODE_WAKE
-silence_count = 0
+    high_count:int=0
-listen_deadline = None
+    last_trigger:float=0.0
+    listen_deadline:float|None=None
+    sherpa_stream=None
+    last_partial:str=""
 
-def rms_level(samples):
+state=State()
-    x = samples.astype(np.float32) / 32768.0
+float_buf=np.empty(CHUNK,np.float32)
-    return np.sqrt(np.mean(x * x))
 
+def emit(event,**kw):
+    print(json.dumps({"event":event,**kw},ensure_ascii=False),flush=True)
+
+def decode_pending(st):
+    while recognizer.is_ready(st):
+        recognizer.decode_stream(st)
+
+def reset_oww():
+    if hasattr(oww,"accumulated_predictions") and WAKEWORD in oww.accumulated_predictions:
+        oww.accumulated_predictions[WAKEWORD].clear()
+
+def flush():
+    while True:
+        try:a=mic.get_read_available()
+        except:break
+        if a<=0:break
+        mic.read(min(a,CHUNK),exception_on_overflow=False)
+
+def start(now):
+    state.mode=MODE_TRANSCRIBE
+    state.high_count=0
+    state.last_trigger=now
+    state.listen_deadline=now+LISTEN_SECONDS
+    state.last_partial=""
+    state.sherpa_stream=recognizer.create_stream()
+
+def stop():
+    reset_oww(); flush()
+    state.mode=MODE_WAKE
+    state.high_count=0
+    state.listen_deadline=None
+    state.last_partial=""
+    state.sherpa_stream=None
+
+emit("ready")
 try:
     while True:
-        data = stream.read(CHUNK, exception_on_overflow=False)
+        samples=np.frombuffer(mic.read(CHUNK,exception_on_overflow=False),dtype=np.int16)
-        samples = np.frombuffer(data, dtype=np.int16)
+        now=monotonic()
-        now = time.time()
+        if state.mode==MODE_WAKE:
-
+            score=oww.predict(samples).get(WAKEWORD,0.0)
-        if mode == "wakeword":
+            state.high_count=state.high_count+1 if score>=THRESHOLD else 0
-            preds = oww.predict(samples)
+            if state.high_count>=CONFIRM_FRAMES and now-state.last_trigger>COOLDOWN:
-            score = preds.get(WAKEWORD, 0.0)
+                start(now)
-
+                emit("wakeword",model=WAKEWORD,score=float(score))
-            if score >= THRESHOLD:
+            continue
-                high_count += 1
-            else:
-                high_count = 0
-
-            if high_count >= CONFIRM_FRAMES and (now - last_trigger) > COOLDOWN:
-                last_trigger = now
-                high_count = 0
-                print(json.dumps({
-                    "event": "wakeword",
-                    "model": WAKEWORD,
-                    "score": float(score)
-                }), flush=True)
-                mode = "transcribe"
-                recognizer.Reset()
-                silence_count = 0
-                listen_deadline = now + LISTEN_SECONDS
-
-        else:
-            if recognizer.AcceptWaveform(data):
-                result = json.loads(recognizer.Result())
-                if result.get("text"):
-                    print(json.dumps({"event": "final", "text": result["text"]}), flush=True)
-                    listen_deadline = now + LISTEN_SECONDS
-                    silence_count = 0
-            else:
-                partial = json.loads(recognizer.PartialResult()).get("partial", "")
-                if partial:
-                    print(json.dumps({"event": "partial", "text": partial}), flush=True)
-
-            level = rms_level(samples)
-            if level < SILENCE_THRESHOLD:
-                silence_count += 1
-            else:
-                silence_count = 0
-
-            if silence_count >= SILENCE_FRAMES or now > listen_deadline:
-                final = json.loads(recognizer.FinalResult())
-                if final.get("text"):
-                    print(json.dumps({"event": "final", "text": final["text"]}), flush=True)
-
-                print(json.dumps({"event": "done"}), flush=True)
-                mode = "wakeword"
-                silence_count = 0
-                high_count = 0
-                listen_deadline = None
 
+        np.multiply(samples,1/32768.0,out=float_buf,casting="unsafe")
+        st=state.sherpa_stream
+        st.accept_waveform(RATE,float_buf)
+        decode_pending(st)
+        txt=recognizer.get_result(st)
+        if txt and txt!=state.last_partial:
+            state.last_partial=txt
+            emit("partial",text=txt)
+        if not(recognizer.is_endpoint(st) or now>=state.listen_deadline):
+            continue
+        st.input_finished()
+        decode_pending(st)
+        final=recognizer.get_result(st)
+        if final:
+            emit("final",text=final)
+        emit("done")
+        stop()
 except KeyboardInterrupt:
     pass
 finally:
-    stream.stop_stream()
+    mic.stop_stream(); mic.close(); pa.terminate()
-    stream.close()
-    audio.terminate()