diff --git a/src/scripts/py/transcribe.py b/src/scripts/py/transcribe.py index 7a972e6..977a653 100644 --- a/src/scripts/py/transcribe.py +++ b/src/scripts/py/transcribe.py @@ -1,7 +1,3 @@ - -# Refactored template based on your original. -# NOTE: Replace MODEL_CONFIG loading as needed. - import collections, json, os, sys from dataclasses import dataclass from time import monotonic @@ -12,112 +8,222 @@ import pyaudio import sherpa_onnx from openwakeword.model import Model -CONFIG=json.loads(sys.argv[1]) -SCRIPT_DIR=os.path.dirname(os.path.abspath(__file__)) -RATE=CONFIG.get("sampleRate",16000) -CHUNK=640 -WAKEWORD=CONFIG.get("wakeword","alexa") -THRESHOLD=CONFIG.get("wakewordThreshold",0.75) -COOLDOWN=CONFIG.get("cooldownSeconds",2.5) -CONFIRM_FRAMES=CONFIG.get("confirmFrames",3) -LISTEN_SECONDS=CONFIG.get("listenSeconds",8) -MODE_WAKE="wakeword" -MODE_TRANSCRIBE="transcribe" +CONFIG = json.loads(sys.argv[1]) +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -mc=CONFIG["sherpaModelPath"] -paths={k:os.path.join(SCRIPT_DIR,v.lstrip("./")) for k,v in mc.items()} +RATE = CONFIG.get("sampleRate", 16000) +CHUNK = 640 -recognizer=sherpa_onnx.OnlineRecognizer.from_transducer( - encoder=paths["encoder"],decoder=paths["decoder"],joiner=paths["joiner"], - tokens=paths["tokens"],sample_rate=RATE,feature_dim=80, - decoding_method="greedy_search",num_threads=max(1,(os.cpu_count() or 1)//2), - enable_endpoint_detection=True, - rule1_min_trailing_silence=1.5, - rule2_min_trailing_silence=0.3, - rule3_min_utterance_length=10, +# Wakeword +WAKEWORD = CONFIG.get("wakeword", "hey jarvis") +THRESHOLD = CONFIG.get("wakewordThreshold", 0.80) +COOLDOWN = CONFIG.get("cooldownSeconds", 2.5) +CONFIRM_FRAMES = CONFIG.get("confirmFrames", 3) +LISTEN_SECONDS = CONFIG.get("listenSeconds", 12) + +# Speech/silence gating (prevents wakeword firing right after transcript) +# Values assume threshold is on a normalized amplitude scale (0..1). +SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.03) +SILENCE_FRAMES = CONFIG.get("silenceFrames", 6) + +# After transcribe finishes, ignore wakeword for a short period +POST_TRANSCRIPT_IGNORE_SECONDS = CONFIG.get("postTranscriptIgnoreSeconds", 1.5) + +# ---- Transcription endpointing ---- +ENDPOINT_RULE1 = CONFIG.get("endpoint_rule1_min_trailing_silence", 2.4) +ENDPOINT_RULE2 = CONFIG.get("endpoint_rule2_min_trailing_silence", 1.2) +ENDPOINT_RULE3 = CONFIG.get("endpoint_rule3_min_utterance_length", 30) +MIN_TRANSCRIBE_SECONDS_BEFORE_ENDPOINT = CONFIG.get("min_transcribe_before_endpoint", 1.0) +ENABLE_ENDPOINT_DETECTION = CONFIG.get("enable_endpoint_detection", True) + +MODE_WAKE = "wakeword" +MODE_TRANSCRIBE = "transcribe" + +# ---- Sherpa model paths ---- +mc = CONFIG["sherpaModelPath"] +paths = {k: os.path.join(SCRIPT_DIR, v.lstrip("./")) for k, v in mc.items()} + +recognizer = sherpa_onnx.OnlineRecognizer.from_transducer( + encoder=paths["encoder"], + decoder=paths["decoder"], + joiner=paths["joiner"], + tokens=paths["tokens"], + sample_rate=RATE, + feature_dim=80, + decoding_method="greedy_search", + num_threads=max(1, (os.cpu_count() or 1) // 2), + + enable_endpoint_detection=ENABLE_ENDPOINT_DETECTION, + rule1_min_trailing_silence=ENDPOINT_RULE1, + rule2_min_trailing_silence=ENDPOINT_RULE2, + rule3_min_utterance_length=ENDPOINT_RULE3, ) -oww=Model(wakeword_models=[WAKEWORD], inference_framework="onnx") -pa=pyaudio.PyAudio() -mic=pa.open(format=pyaudio.paInt16,channels=1,rate=RATE,input=True,frames_per_buffer=CHUNK) +# ---- openwakeword model ---- +oww = Model(wakeword_models=[WAKEWORD], inference_framework="onnx") + +# ---- Audio ---- +pa = pyaudio.PyAudio() +mic = pa.open( + format=pyaudio.paInt16, + channels=1, + rate=RATE, + input=True, + frames_per_buffer=CHUNK +) @dataclass class State: - mode:str=MODE_WAKE - high_count:int=0 - last_trigger:float=0.0 - listen_deadline:float|None=None - sherpa_stream=None - last_partial:str="" + mode: str = MODE_WAKE + high_count: int = 0 + last_trigger: float = 0.0 -state=State() -float_buf=np.empty(CHUNK,np.float32) + # Transcription timing + listen_deadline: float | None = None + transcribe_start: float | None = None -def emit(event,**kw): - print(json.dumps({"event":event,**kw},ensure_ascii=False),flush=True) + # Wakeword anti-double-trigger + post_transcript_ignore_deadline: float | None = None + + # Silence gate + silence_count: int = 0 + + sherpa_stream: object = None + last_partial: str = "" + +state = State() + +float_buf = np.empty(CHUNK, np.float32) + +def emit(event, **kw): + print(json.dumps({"event": event, **kw}, ensure_ascii=False), flush=True) def decode_pending(st): while recognizer.is_ready(st): recognizer.decode_stream(st) def reset_oww(): - if hasattr(oww,"accumulated_predictions") and WAKEWORD in oww.accumulated_predictions: + if hasattr(oww, "accumulated_predictions") and WAKEWORD in oww.accumulated_predictions: oww.accumulated_predictions[WAKEWORD].clear() def flush(): + # Drain mic so next phase starts with fresh audio while True: - try:a=mic.get_read_available() - except:break - if a<=0:break - mic.read(min(a,CHUNK),exception_on_overflow=False) + try: + available = mic.get_read_available() + except Exception: + break + if available <= 0: + break + mic.read(min(available, CHUNK), exception_on_overflow=False) def start(now): - state.mode=MODE_TRANSCRIBE - state.high_count=0 - state.last_trigger=now - state.listen_deadline=now+LISTEN_SECONDS - state.last_partial="" - state.sherpa_stream=recognizer.create_stream() + state.mode = MODE_TRANSCRIBE + state.high_count = 0 + state.last_trigger = now -def stop(): - reset_oww(); flush() - state.mode=MODE_WAKE - state.high_count=0 - state.listen_deadline=None - state.last_partial="" - state.sherpa_stream=None + state.listen_deadline = now + LISTEN_SECONDS + state.transcribe_start = now + + state.last_partial = "" + state.sherpa_stream = recognizer.create_stream() + +def stop(now): + # Reset back to wake mode + reset_oww() + flush() + + state.mode = MODE_WAKE + state.high_count = 0 + + state.listen_deadline = None + state.transcribe_start = None + state.last_partial = "" + state.sherpa_stream = None + + # Cooldown baseline should start AFTER the transcript ends + state.last_trigger = now + + # And ignore wakeword for a short grace period + state.post_transcript_ignore_deadline = now + POST_TRANSCRIPT_IGNORE_SECONDS emit("ready") + try: while True: - samples=np.frombuffer(mic.read(CHUNK,exception_on_overflow=False),dtype=np.int16) - now=monotonic() - if state.mode==MODE_WAKE: - score=oww.predict(samples).get(WAKEWORD,0.0) - state.high_count=state.high_count+1 if score>=THRESHOLD else 0 - if state.high_count>=CONFIRM_FRAMES and now-state.last_trigger>COOLDOWN: + pcm_i16 = mic.read(CHUNK, exception_on_overflow=False) + samples_i16 = np.frombuffer(pcm_i16, dtype=np.int16) + + now = monotonic() + + if state.mode == MODE_WAKE: + # -------- Silence gate (prevents wake right after transcript) -------- + # normalized mean absolute amplitude + amp = float(np.mean(np.abs(samples_i16)) / 32768.0) + + if amp <= SILENCE_THRESHOLD: + state.silence_count += 1 + else: + state.silence_count = 0 + + silence_ok = state.silence_count >= SILENCE_FRAMES + + # Ignore wakeword right after transcript ends + ignore_ok = not ( + state.post_transcript_ignore_deadline is not None + and now < state.post_transcript_ignore_deadline + ) + + if not (silence_ok and ignore_ok): + continue + + # -------- Wakeword scoring -------- + score = oww.predict(samples_i16).get(WAKEWORD, 0.0) + + state.high_count = (state.high_count + 1) if score >= THRESHOLD else 0 + + if state.high_count >= CONFIRM_FRAMES and (now - state.last_trigger) > COOLDOWN: start(now) - emit("wakeword",model=WAKEWORD,score=float(score)) + emit("wakeword", model=WAKEWORD, score=float(score)) + continue - np.multiply(samples,1/32768.0,out=float_buf,casting="unsafe") - st=state.sherpa_stream - st.accept_waveform(RATE,float_buf) + # ---- Transcribe mode ---- + np.multiply(samples_i16, 1 / 32768.0, out=float_buf, casting="unsafe") + + st = state.sherpa_stream + st.accept_waveform(RATE, float_buf) decode_pending(st) - txt=recognizer.get_result(st) - if txt and txt!=state.last_partial: - state.last_partial=txt - emit("partial",text=txt) - if not(recognizer.is_endpoint(st) or now>=state.listen_deadline): + + txt = recognizer.get_result(st) + if txt and txt != state.last_partial: + state.last_partial = txt + emit("partial", text=txt) + + endpoint_ok = False + if state.listen_deadline is not None and now >= state.listen_deadline: + endpoint_ok = True + elif ENABLE_ENDPOINT_DETECTION and recognizer.is_endpoint(st): + # Don’t allow endpointing immediately after switching to transcribe + if state.transcribe_start is not None and now >= (state.transcribe_start + MIN_TRANSCRIBE_SECONDS_BEFORE_ENDPOINT): + endpoint_ok = True + + if not endpoint_ok: continue + st.input_finished() decode_pending(st) - final=recognizer.get_result(st) + + final = recognizer.get_result(st) if final: - emit("final",text=final) + emit("final", text=final) + emit("done") - stop() + stop(now) + except KeyboardInterrupt: pass finally: - mic.stop_stream(); mic.close(); pa.terminate() + mic.stop_stream() + mic.close() + pa.terminate()