Use sherpa instead of vosk
This commit is contained in:
parent
b92ffb7a73
commit
22bcfa31f3
@ -6,6 +6,11 @@ module.exports = {
|
||||
listenSeconds: Number(process.env.TRANS_LISTEN_SECONDS),
|
||||
silenceThreshold: Number(process.env.TRANS_SILENCE_THRESHOLD),
|
||||
silenceFrames: Number(process.env.TRANS_SILENCE_FRAMES),
|
||||
voskModelPath: process.env.TRANS_VOSK_MODEL_PATH,
|
||||
sampleRate: Number(process.env.TRANS_SAMPLE_RATE),
|
||||
sherpaModelPath: {
|
||||
encoder: process.env.TRANS_SHERPA_ENCODER,
|
||||
decoder: process.env.TRANS_SHERPA_DECODER,
|
||||
joiner: process.env.TRANS_SHERPA_JOINER,
|
||||
tokens: process.env.TRANS_SHERPA_TOKENS,
|
||||
},
|
||||
};
|
||||
|
||||
@ -6,6 +6,11 @@
|
||||
"listenSeconds": 12,
|
||||
"silenceThreshold": 0.015,
|
||||
"silenceFrames": 10,
|
||||
"voskModelPath": "./vosk-model-small-en-us-0.15",
|
||||
"sampleRate": 16000
|
||||
"sampleRate": 16000,
|
||||
"sherpaModelPath": {
|
||||
"encoder": "./zipformer-small/encoder.onnx",
|
||||
"decoder": "./zipformer-small/decoder.onnx",
|
||||
"joiner": "./zipformer-small/joiner.onnx",
|
||||
"tokens": "./zipformer-small/data/tokens.txt"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,116 +1,123 @@
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Refactored template based on your original.
|
||||
# NOTE: Replace MODEL_CONFIG loading as needed.
|
||||
|
||||
import collections, json, os, sys
|
||||
from dataclasses import dataclass
|
||||
from time import monotonic
|
||||
|
||||
import numpy as np
|
||||
import pyaudio
|
||||
import vosk
|
||||
import openwakeword
|
||||
import pyaudio
|
||||
import sherpa_onnx
|
||||
from openwakeword.model import Model
|
||||
|
||||
CONFIG = json.loads(sys.argv[1])
|
||||
CONFIG=json.loads(sys.argv[1])
|
||||
SCRIPT_DIR=os.path.dirname(os.path.abspath(__file__))
|
||||
RATE=CONFIG.get("sampleRate",16000)
|
||||
CHUNK=640
|
||||
WAKEWORD=CONFIG.get("wakeword","alexa")
|
||||
THRESHOLD=CONFIG.get("wakewordThreshold",0.75)
|
||||
COOLDOWN=CONFIG.get("cooldownSeconds",2.5)
|
||||
CONFIRM_FRAMES=CONFIG.get("confirmFrames",3)
|
||||
LISTEN_SECONDS=CONFIG.get("listenSeconds",8)
|
||||
MODE_WAKE="wakeword"
|
||||
MODE_TRANSCRIBE="transcribe"
|
||||
|
||||
RATE = CONFIG.get("sampleRate", 16000)
|
||||
CHUNK = 1280
|
||||
WAKEWORD = CONFIG.get("wakeword", "alexa")
|
||||
THRESHOLD = CONFIG.get("wakewordThreshold", 0.75)
|
||||
COOLDOWN = CONFIG.get("cooldownSeconds", 2.5)
|
||||
CONFIRM_FRAMES = CONFIG.get("confirmFrames", 3)
|
||||
LISTEN_SECONDS = CONFIG.get("listenSeconds", 12)
|
||||
SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.02)
|
||||
SILENCE_FRAMES = CONFIG.get("silenceFrames", 12)
|
||||
VOSK_MODEL_PATH = CONFIG["voskModelPath"]
|
||||
mc=CONFIG["sherpaModelPath"]
|
||||
paths={k:os.path.join(SCRIPT_DIR,v.lstrip("./")) for k,v in mc.items()}
|
||||
|
||||
vosk.SetLogLevel(-1)
|
||||
vosk_model = vosk.Model(VOSK_MODEL_PATH)
|
||||
recognizer = vosk.KaldiRecognizer(vosk_model, RATE)
|
||||
recognizer.SetWords(True)
|
||||
|
||||
# Download all models (TODO: Import single model)
|
||||
openwakeword.utils.download_models()
|
||||
|
||||
oww = Model(wakeword_models=[WAKEWORD], inference_framework="onnx")
|
||||
|
||||
audio = pyaudio.PyAudio()
|
||||
stream = audio.open(
|
||||
format=pyaudio.paInt16,
|
||||
channels=1,
|
||||
rate=RATE,
|
||||
input=True,
|
||||
frames_per_buffer=CHUNK,
|
||||
recognizer=sherpa_onnx.OnlineRecognizer.from_transducer(
|
||||
encoder=paths["encoder"],decoder=paths["decoder"],joiner=paths["joiner"],
|
||||
tokens=paths["tokens"],sample_rate=RATE,feature_dim=80,
|
||||
decoding_method="greedy_search",num_threads=max(1,(os.cpu_count() or 1)//2),
|
||||
enable_endpoint_detection=True,
|
||||
rule1_min_trailing_silence=1.5,
|
||||
rule2_min_trailing_silence=0.3,
|
||||
rule3_min_utterance_length=10,
|
||||
)
|
||||
|
||||
print(json.dumps({"event": "ready"}), flush=True)
|
||||
oww=Model(wakeword_models=[WAKEWORD], inference_framework="onnx")
|
||||
pa=pyaudio.PyAudio()
|
||||
mic=pa.open(format=pyaudio.paInt16,channels=1,rate=RATE,input=True,frames_per_buffer=CHUNK)
|
||||
|
||||
mode = "wakeword"
|
||||
high_count = 0
|
||||
last_trigger = 0
|
||||
silence_count = 0
|
||||
listen_deadline = None
|
||||
@dataclass
|
||||
class State:
|
||||
mode:str=MODE_WAKE
|
||||
high_count:int=0
|
||||
last_trigger:float=0.0
|
||||
listen_deadline:float|None=None
|
||||
sherpa_stream=None
|
||||
last_partial:str=""
|
||||
|
||||
def rms_level(samples):
|
||||
x = samples.astype(np.float32) / 32768.0
|
||||
return np.sqrt(np.mean(x * x))
|
||||
state=State()
|
||||
float_buf=np.empty(CHUNK,np.float32)
|
||||
|
||||
def emit(event,**kw):
|
||||
print(json.dumps({"event":event,**kw},ensure_ascii=False),flush=True)
|
||||
|
||||
def decode_pending(st):
|
||||
while recognizer.is_ready(st):
|
||||
recognizer.decode_stream(st)
|
||||
|
||||
def reset_oww():
|
||||
if hasattr(oww,"accumulated_predictions") and WAKEWORD in oww.accumulated_predictions:
|
||||
oww.accumulated_predictions[WAKEWORD].clear()
|
||||
|
||||
def flush():
|
||||
while True:
|
||||
try:a=mic.get_read_available()
|
||||
except:break
|
||||
if a<=0:break
|
||||
mic.read(min(a,CHUNK),exception_on_overflow=False)
|
||||
|
||||
def start(now):
|
||||
state.mode=MODE_TRANSCRIBE
|
||||
state.high_count=0
|
||||
state.last_trigger=now
|
||||
state.listen_deadline=now+LISTEN_SECONDS
|
||||
state.last_partial=""
|
||||
state.sherpa_stream=recognizer.create_stream()
|
||||
|
||||
def stop():
|
||||
reset_oww(); flush()
|
||||
state.mode=MODE_WAKE
|
||||
state.high_count=0
|
||||
state.listen_deadline=None
|
||||
state.last_partial=""
|
||||
state.sherpa_stream=None
|
||||
|
||||
emit("ready")
|
||||
try:
|
||||
while True:
|
||||
data = stream.read(CHUNK, exception_on_overflow=False)
|
||||
samples = np.frombuffer(data, dtype=np.int16)
|
||||
now = time.time()
|
||||
|
||||
if mode == "wakeword":
|
||||
preds = oww.predict(samples)
|
||||
score = preds.get(WAKEWORD, 0.0)
|
||||
|
||||
if score >= THRESHOLD:
|
||||
high_count += 1
|
||||
else:
|
||||
high_count = 0
|
||||
|
||||
if high_count >= CONFIRM_FRAMES and (now - last_trigger) > COOLDOWN:
|
||||
last_trigger = now
|
||||
high_count = 0
|
||||
print(json.dumps({
|
||||
"event": "wakeword",
|
||||
"model": WAKEWORD,
|
||||
"score": float(score)
|
||||
}), flush=True)
|
||||
mode = "transcribe"
|
||||
recognizer.Reset()
|
||||
silence_count = 0
|
||||
listen_deadline = now + LISTEN_SECONDS
|
||||
|
||||
else:
|
||||
if recognizer.AcceptWaveform(data):
|
||||
result = json.loads(recognizer.Result())
|
||||
if result.get("text"):
|
||||
print(json.dumps({"event": "final", "text": result["text"]}), flush=True)
|
||||
listen_deadline = now + LISTEN_SECONDS
|
||||
silence_count = 0
|
||||
else:
|
||||
partial = json.loads(recognizer.PartialResult()).get("partial", "")
|
||||
if partial:
|
||||
print(json.dumps({"event": "partial", "text": partial}), flush=True)
|
||||
|
||||
level = rms_level(samples)
|
||||
if level < SILENCE_THRESHOLD:
|
||||
silence_count += 1
|
||||
else:
|
||||
silence_count = 0
|
||||
|
||||
if silence_count >= SILENCE_FRAMES or now > listen_deadline:
|
||||
final = json.loads(recognizer.FinalResult())
|
||||
if final.get("text"):
|
||||
print(json.dumps({"event": "final", "text": final["text"]}), flush=True)
|
||||
|
||||
print(json.dumps({"event": "done"}), flush=True)
|
||||
mode = "wakeword"
|
||||
silence_count = 0
|
||||
high_count = 0
|
||||
listen_deadline = None
|
||||
samples=np.frombuffer(mic.read(CHUNK,exception_on_overflow=False),dtype=np.int16)
|
||||
now=monotonic()
|
||||
if state.mode==MODE_WAKE:
|
||||
score=oww.predict(samples).get(WAKEWORD,0.0)
|
||||
state.high_count=state.high_count+1 if score>=THRESHOLD else 0
|
||||
if state.high_count>=CONFIRM_FRAMES and now-state.last_trigger>COOLDOWN:
|
||||
start(now)
|
||||
emit("wakeword",model=WAKEWORD,score=float(score))
|
||||
continue
|
||||
|
||||
np.multiply(samples,1/32768.0,out=float_buf,casting="unsafe")
|
||||
st=state.sherpa_stream
|
||||
st.accept_waveform(RATE,float_buf)
|
||||
decode_pending(st)
|
||||
txt=recognizer.get_result(st)
|
||||
if txt and txt!=state.last_partial:
|
||||
state.last_partial=txt
|
||||
emit("partial",text=txt)
|
||||
if not(recognizer.is_endpoint(st) or now>=state.listen_deadline):
|
||||
continue
|
||||
st.input_finished()
|
||||
decode_pending(st)
|
||||
final=recognizer.get_result(st)
|
||||
if final:
|
||||
emit("final",text=final)
|
||||
emit("done")
|
||||
stop()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
stream.stop_stream()
|
||||
stream.close()
|
||||
audio.terminate()
|
||||
mic.stop_stream(); mic.close(); pa.terminate()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user