Spilum.Net/mtf-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: Spilum.Net:4a2c6aa76118607bd3377e0e36f7cc14426ea2fb
Branches Tags
Spilum.Net:main
..
compare: Spilum.Net:6e8c689639434baa15644878fecb039abc2281fc
Branches Tags
Spilum.Net:main

No commits in common. "4a2c6aa76118607bd3377e0e36f7cc14426ea2fb" and "6e8c689639434baa15644878fecb039abc2281fc" have entirely different histories.
4a2c6aa761 ... 6e8c689639

12 changed files with 96 additions and 546 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,5 +1,5 @@
.env .env
node_modules/ node_modules/
.venv/ .venv/
src/scripts/py/zipformer* src/models/vosk*
output.wav output.wav

279
package-lock.json generated
View File

@ -9,7 +9,6 @@
"version": "1.0.0", "version": "1.0.0",
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"axios": "^1.18.1",
"mic": "^2.1.2", "mic": "^2.1.2",
"redis": "^6.1.0", "redis": "^6.1.0",
"speaker": "^0.5.5", "speaker": "^0.5.5",
@ -101,18 +100,6 @@
"node": ">=12.0" "node": ">=12.0"
} }
}, },
"node_modules/agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"license": "MIT",
"dependencies": {
"debug": "4"
},
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/anymatch": { "node_modules/anymatch": {
"version": "3.1.3", "version": "3.1.3",
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
@ -127,24 +114,6 @@
"node": ">= 8" "node": ">= 8"
} }
}, },
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
"license": "MIT"
},
"node_modules/axios": {
"version": "1.18.1",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.18.1.tgz",
"integrity": "sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==",
"license": "MIT",
"dependencies": {
"follow-redirects": "^1.16.0",
"form-data": "^4.0.5",
"https-proxy-agent": "^5.0.1",
"proxy-from-env": "^2.1.0"
}
},
"node_modules/balanced-match": { "node_modules/balanced-match": {
"version": "4.0.4", "version": "4.0.4",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
@ -221,19 +190,6 @@
"resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
"integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==" "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ=="
}, },
"node_modules/call-bind-apply-helpers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/chokidar": { "node_modules/chokidar": {
"version": "3.6.0", "version": "3.6.0",
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz",
@ -268,18 +224,6 @@
"node": ">=0.10.0" "node": ">=0.10.0"
} }
}, },
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"license": "MIT",
"dependencies": {
"delayed-stream": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/commander": { "node_modules/commander": {
"version": "12.1.0", "version": "12.1.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
@ -337,15 +281,6 @@
"url": "https://github.com/sponsors/ljharb" "url": "https://github.com/sponsors/ljharb"
} }
}, },
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"license": "MIT",
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/dotenv": { "node_modules/dotenv": {
"version": "17.4.2", "version": "17.4.2",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz",
@ -359,20 +294,6 @@
"url": "https://dotenvx.com" "url": "https://dotenvx.com"
} }
}, },
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.1",
"es-errors": "^1.3.0",
"gopd": "^1.2.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-define-property": { "node_modules/es-define-property": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
@ -389,33 +310,6 @@
"node": ">= 0.4" "node": ">= 0.4"
} }
}, },
"node_modules/es-object-atoms": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz",
"integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-set-tostringtag": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.6",
"has-tostringtag": "^1.0.2",
"hasown": "^2.0.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/escape-string-regexp": { "node_modules/escape-string-regexp": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
@ -445,42 +339,6 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/follow-redirects": {
"version": "1.16.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
"integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
}
],
"license": "MIT",
"engines": {
"node": ">=4.0"
},
"peerDependenciesMeta": {
"debug": {
"optional": true
}
}
},
"node_modules/form-data": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz",
"integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==",
"license": "MIT",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"es-set-tostringtag": "^2.1.0",
"hasown": "^2.0.4",
"mime-types": "^2.1.35"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/fsevents": { "node_modules/fsevents": {
"version": "2.3.3", "version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
@ -496,52 +354,6 @@
"node": "^8.16.0 || ^10.6.0 || >=11.0.0" "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
} }
}, },
"node_modules/function-bind": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
"integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-intrinsic": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"es-define-property": "^1.0.1",
"es-errors": "^1.3.0",
"es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
"get-proto": "^1.0.1",
"gopd": "^1.2.0",
"has-symbols": "^1.1.0",
"hasown": "^2.0.2",
"math-intrinsics": "^1.1.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
"license": "MIT",
"dependencies": {
"dunder-proto": "^1.0.1",
"es-object-atoms": "^1.0.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/glob-parent": { "node_modules/glob-parent": {
"version": "5.1.2", "version": "5.1.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
@ -616,58 +428,6 @@
"url": "https://github.com/sponsors/ljharb" "url": "https://github.com/sponsors/ljharb"
} }
}, },
"node_modules/has-symbols": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has-tostringtag": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
"license": "MIT",
"dependencies": {
"has-symbols": "^1.0.3"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/hasown": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz",
"integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==",
"license": "MIT",
"dependencies": {
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/https-proxy-agent": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
"integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
"license": "MIT",
"dependencies": {
"agent-base": "6",
"debug": "4"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/ignore-by-default": { "node_modules/ignore-by-default": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz",
@ -735,42 +495,12 @@
"url": "https://github.com/sponsors/sindresorhus" "url": "https://github.com/sponsors/sindresorhus"
} }
}, },
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/mic": { "node_modules/mic": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/mic/-/mic-2.1.2.tgz", "resolved": "https://registry.npmjs.org/mic/-/mic-2.1.2.tgz",
"integrity": "sha512-rpl4tgdXX24sAzYwjRc5OZfGNAuhUIIjdd0cw8+Ubq7rp3iGhi40AdqcwurDWhEZADk60tPOxb3E2MpoeLeyxw==", "integrity": "sha512-rpl4tgdXX24sAzYwjRc5OZfGNAuhUIIjdd0cw8+Ubq7rp3iGhi40AdqcwurDWhEZADk60tPOxb3E2MpoeLeyxw==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/mime-db": {
"version": "1.52.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "2.1.35",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"license": "MIT",
"dependencies": {
"mime-db": "1.52.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/minimatch": { "node_modules/minimatch": {
"version": "10.2.5", "version": "10.2.5",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz",
@ -874,15 +604,6 @@
"url": "https://github.com/sponsors/jonschlinkert" "url": "https://github.com/sponsors/jonschlinkert"
} }
}, },
"node_modules/proxy-from-env": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz",
"integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==",
"license": "MIT",
"engines": {
"node": ">=10"
}
},
"node_modules/pstree.remy": { "node_modules/pstree.remy": {
"version": "1.1.8", "version": "1.1.8",
"resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz",

1
package.json
View File

@ -12,7 +12,6 @@
"license": "ISC", "license": "ISC",
"type": "commonjs", "type": "commonjs",
"dependencies": { "dependencies": {
"axios": "^1.18.1",
"mic": "^2.1.2", "mic": "^2.1.2",
"redis": "^6.1.0", "redis": "^6.1.0",
"speaker": "^0.5.5", "speaker": "^0.5.5",

4
src/config/index.js
View File

@ -2,7 +2,5 @@ module.exports = {
FORCE_DB_SYNC: process.env.FORCE_DB_SYNC, FORCE_DB_SYNC: process.env.FORCE_DB_SYNC,
NODE_ENV: process.env.NODE_ENV, NODE_ENV: process.env.NODE_ENV,
PORT: process.env.PORT, PORT: process.env.PORT,
DEVICE_ID: process.env.DEVICE_ID, DEVICE_ID: process.env.DEVICE_ID
HA_URL: process.env.HA_URL,
HA_TOKEN: process.env.HA_TOKEN
}; };

4
src/config/redis.config.js Normal file
View File

@ -0,0 +1,4 @@
module.exports = {
HOST: process.env.REDIS_HOST,
PORT: process.env.REDIS_PORT
}

7
src/config/transcribe.config.js
View File

@ -6,11 +6,6 @@ module.exports = {
listenSeconds: Number(process.env.TRANS_LISTEN_SECONDS), listenSeconds: Number(process.env.TRANS_LISTEN_SECONDS),
silenceThreshold: Number(process.env.TRANS_SILENCE_THRESHOLD), silenceThreshold: Number(process.env.TRANS_SILENCE_THRESHOLD),
silenceFrames: Number(process.env.TRANS_SILENCE_FRAMES), silenceFrames: Number(process.env.TRANS_SILENCE_FRAMES),
voskModelPath: process.env.TRANS_VOSK_MODEL_PATH,
sampleRate: Number(process.env.TRANS_SAMPLE_RATE), sampleRate: Number(process.env.TRANS_SAMPLE_RATE),
sherpaModelPath: {
encoder: process.env.TRANS_SHERPA_ENCODER,
decoder: process.env.TRANS_SHERPA_DECODER,
joiner: process.env.TRANS_SHERPA_JOINER,
tokens: process.env.TRANS_SHERPA_TOKENS,
},
}; };

2
src/modules/redis

@ -1 +1 @@
Subproject commit b335675847af7b92b3464d807939d1dbe558968d Subproject commit e41a75e7436a059ab963c5db7b62ca524cacc4e5

9
src/scripts/py/config.json
View File

@ -6,11 +6,6 @@
"listenSeconds": 12, "listenSeconds": 12,
"silenceThreshold": 0.015, "silenceThreshold": 0.015,
"silenceFrames": 10, "silenceFrames": 10,
"sampleRate": 16000, "voskModelPath": "./vosk-model-small-en-us-0.15",
"sherpaModelPath": { "sampleRate": 16000
"encoder": "./zipformer-small/encoder.onnx",
"decoder": "./zipformer-small/decoder.onnx",
"joiner": "./zipformer-small/joiner.onnx",
"tokens": "./zipformer-small/data/tokens.txt"
}
} }

1
src/scripts/py/pythonOutput.js
View File

@ -1,3 +1,4 @@
// pythonOutput.js
let buffer = ""; let buffer = "";
function handlePythonOutput(chunk, onMessage) { function handlePythonOutput(chunk, onMessage) {

3
src/scripts/py/requirements.txt
View File

@ -2,5 +2,4 @@ openwakeword
onnxruntime onnxruntime
numpy numpy
pyaudio pyaudio
vosk vosk
sherpa_onnx

269
src/scripts/py/transcribe.py
View File

@ -1,229 +1,116 @@
import collections, json, os, sys import json
from dataclasses import dataclass import sys
from time import monotonic import time
import numpy as np import numpy as np
import openwakeword
import pyaudio import pyaudio
import sherpa_onnx import vosk
import openwakeword
from openwakeword.model import Model from openwakeword.model import Model
CONFIG = json.loads(sys.argv[1]) CONFIG = json.loads(sys.argv[1])
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
RATE = CONFIG.get("sampleRate", 16000) RATE = CONFIG.get("sampleRate", 16000)
CHUNK = 640 CHUNK = 1280
WAKEWORD = CONFIG.get("wakeword", "alexa")
# Wakeword THRESHOLD = CONFIG.get("wakewordThreshold", 0.75)
WAKEWORD = CONFIG.get("wakeword", "hey jarvis")
THRESHOLD = CONFIG.get("wakewordThreshold", 0.80)
COOLDOWN = CONFIG.get("cooldownSeconds", 2.5) COOLDOWN = CONFIG.get("cooldownSeconds", 2.5)
CONFIRM_FRAMES = CONFIG.get("confirmFrames", 3) CONFIRM_FRAMES = CONFIG.get("confirmFrames", 3)
LISTEN_SECONDS = CONFIG.get("listenSeconds", 12) LISTEN_SECONDS = CONFIG.get("listenSeconds", 12)
SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.02)
SILENCE_FRAMES = CONFIG.get("silenceFrames", 12)
VOSK_MODEL_PATH = CONFIG["voskModelPath"]
# Speech/silence gating (prevents wakeword firing right after transcript) vosk.SetLogLevel(-1)
# Values assume threshold is on a normalized amplitude scale (0..1). vosk_model = vosk.Model(VOSK_MODEL_PATH)
SILENCE_THRESHOLD = CONFIG.get("silenceThreshold", 0.03) recognizer = vosk.KaldiRecognizer(vosk_model, RATE)
SILENCE_FRAMES = CONFIG.get("silenceFrames", 6) recognizer.SetWords(True)
# After transcribe finishes, ignore wakeword for a short period # Download all models (TODO: Import single model)
POST_TRANSCRIPT_IGNORE_SECONDS = CONFIG.get("postTranscriptIgnoreSeconds", 1.5) openwakeword.utils.download_models()
# ---- Transcription endpointing ----
ENDPOINT_RULE1 = CONFIG.get("endpoint_rule1_min_trailing_silence", 2.4)
ENDPOINT_RULE2 = CONFIG.get("endpoint_rule2_min_trailing_silence", 1.2)
ENDPOINT_RULE3 = CONFIG.get("endpoint_rule3_min_utterance_length", 30)
MIN_TRANSCRIBE_SECONDS_BEFORE_ENDPOINT = CONFIG.get("min_transcribe_before_endpoint", 1.0)
ENABLE_ENDPOINT_DETECTION = CONFIG.get("enable_endpoint_detection", True)
MODE_WAKE = "wakeword"
MODE_TRANSCRIBE = "transcribe"
# ---- Sherpa model paths ----
mc = CONFIG["sherpaModelPath"]
paths = {k: os.path.join(SCRIPT_DIR, v.lstrip("./")) for k, v in mc.items()}
recognizer = sherpa_onnx.OnlineRecognizer.from_transducer(
encoder=paths["encoder"],
decoder=paths["decoder"],
joiner=paths["joiner"],
tokens=paths["tokens"],
sample_rate=RATE,
feature_dim=80,
decoding_method="greedy_search",
num_threads=max(1, (os.cpu_count() or 1) // 2),
enable_endpoint_detection=ENABLE_ENDPOINT_DETECTION,
rule1_min_trailing_silence=ENDPOINT_RULE1,
rule2_min_trailing_silence=ENDPOINT_RULE2,
rule3_min_utterance_length=ENDPOINT_RULE3,
)
# ---- openwakeword model ----
oww = Model(wakeword_models=[WAKEWORD], inference_framework="onnx") oww = Model(wakeword_models=[WAKEWORD], inference_framework="onnx")
# ---- Audio ---- audio = pyaudio.PyAudio()
pa = pyaudio.PyAudio() stream = audio.open(
mic = pa.open(
format=pyaudio.paInt16, format=pyaudio.paInt16,
channels=1, channels=1,
rate=RATE, rate=RATE,
input=True, input=True,
frames_per_buffer=CHUNK frames_per_buffer=CHUNK,
) )
@dataclass print(json.dumps({"event": "ready"}), flush=True)
class State:
mode: str = MODE_WAKE
high_count: int = 0
last_trigger: float = 0.0
# Transcription timing mode = "wakeword"
listen_deadline: float | None = None high_count = 0
transcribe_start: float | None = None last_trigger = 0
silence_count = 0
listen_deadline = None
# Wakeword anti-double-trigger def rms_level(samples):
post_transcript_ignore_deadline: float | None = None x = samples.astype(np.float32) / 32768.0
return np.sqrt(np.mean(x * x))
# Silence gate
silence_count: int = 0
sherpa_stream: object = None
last_partial: str = ""
state = State()
float_buf = np.empty(CHUNK, np.float32)
def emit(event, **kw):
print(json.dumps({"event": event, **kw}, ensure_ascii=False), flush=True)
def decode_pending(st):
while recognizer.is_ready(st):
recognizer.decode_stream(st)
def reset_oww():
if hasattr(oww, "accumulated_predictions") and WAKEWORD in oww.accumulated_predictions:
oww.accumulated_predictions[WAKEWORD].clear()
def flush():
# Drain mic so next phase starts with fresh audio
while True:
try:
available = mic.get_read_available()
except Exception:
break
if available <= 0:
break
mic.read(min(available, CHUNK), exception_on_overflow=False)
def start(now):
state.mode = MODE_TRANSCRIBE
state.high_count = 0
state.last_trigger = now
state.listen_deadline = now + LISTEN_SECONDS
state.transcribe_start = now
state.last_partial = ""
state.sherpa_stream = recognizer.create_stream()
def stop(now):
# Reset back to wake mode
reset_oww()
flush()
state.mode = MODE_WAKE
state.high_count = 0
state.listen_deadline = None
state.transcribe_start = None
state.last_partial = ""
state.sherpa_stream = None
# Cooldown baseline should start AFTER the transcript ends
state.last_trigger = now
# And ignore wakeword for a short grace period
state.post_transcript_ignore_deadline = now + POST_TRANSCRIPT_IGNORE_SECONDS
emit("ready")
try: try:
while True: while True:
pcm_i16 = mic.read(CHUNK, exception_on_overflow=False) data = stream.read(CHUNK, exception_on_overflow=False)
samples_i16 = np.frombuffer(pcm_i16, dtype=np.int16) samples = np.frombuffer(data, dtype=np.int16)
now = time.time()
now = monotonic() if mode == "wakeword":
preds = oww.predict(samples)
score = preds.get(WAKEWORD, 0.0)
if state.mode == MODE_WAKE: if score >= THRESHOLD:
# -------- Silence gate (prevents wake right after transcript) -------- high_count += 1
# normalized mean absolute amplitude
amp = float(np.mean(np.abs(samples_i16)) / 32768.0)
if amp <= SILENCE_THRESHOLD:
state.silence_count += 1
else: else:
state.silence_count = 0 high_count = 0
silence_ok = state.silence_count >= SILENCE_FRAMES if high_count >= CONFIRM_FRAMES and (now - last_trigger) > COOLDOWN:
last_trigger = now
high_count = 0
print(json.dumps({
"event": "wakeword",
"model": WAKEWORD,
"score": float(score)
}), flush=True)
mode = "transcribe"
recognizer.Reset()
silence_count = 0
listen_deadline = now + LISTEN_SECONDS
# Ignore wakeword right after transcript ends else:
ignore_ok = not ( if recognizer.AcceptWaveform(data):
state.post_transcript_ignore_deadline is not None result = json.loads(recognizer.Result())
and now < state.post_transcript_ignore_deadline if result.get("text"):
) print(json.dumps({"event": "final", "text": result["text"]}), flush=True)
listen_deadline = now + LISTEN_SECONDS
silence_count = 0
else:
partial = json.loads(recognizer.PartialResult()).get("partial", "")
if partial:
print(json.dumps({"event": "partial", "text": partial}), flush=True)
if not (silence_ok and ignore_ok): level = rms_level(samples)
continue if level < SILENCE_THRESHOLD:
silence_count += 1
else:
silence_count = 0
# -------- Wakeword scoring -------- if silence_count >= SILENCE_FRAMES or now > listen_deadline:
score = oww.predict(samples_i16).get(WAKEWORD, 0.0) final = json.loads(recognizer.FinalResult())
if final.get("text"):
print(json.dumps({"event": "final", "text": final["text"]}), flush=True)
state.high_count = (state.high_count + 1) if score >= THRESHOLD else 0 print(json.dumps({"event": "done"}), flush=True)
mode = "wakeword"
if state.high_count >= CONFIRM_FRAMES and (now - state.last_trigger) > COOLDOWN: silence_count = 0
start(now) high_count = 0
emit("wakeword", model=WAKEWORD, score=float(score)) listen_deadline = None
continue
# ---- Transcribe mode ----
np.multiply(samples_i16, 1 / 32768.0, out=float_buf, casting="unsafe")
st = state.sherpa_stream
st.accept_waveform(RATE, float_buf)
decode_pending(st)
txt = recognizer.get_result(st)
if txt and txt != state.last_partial:
state.last_partial = txt
emit("partial", text=txt)
endpoint_ok = False
if state.listen_deadline is not None and now >= state.listen_deadline:
endpoint_ok = True
elif ENABLE_ENDPOINT_DETECTION and recognizer.is_endpoint(st):
# Don’t allow endpointing immediately after switching to transcribe
if state.transcribe_start is not None and now >= (state.transcribe_start + MIN_TRANSCRIBE_SECONDS_BEFORE_ENDPOINT):
endpoint_ok = True
if not endpoint_ok:
continue
st.input_finished()
decode_pending(st)
final = recognizer.get_result(st)
if final:
emit("final", text=final)
emit("done")
stop(now)
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass
finally: finally:
mic.stop_stream() stream.stop_stream()
mic.close() stream.close()
pa.terminate() audio.terminate()

61
src/server.js
View File

@ -3,14 +3,9 @@ if (process.env.NODE_ENV !== "production") {
require("dotenv").config(); require("dotenv").config();
} }
// Axios
const axios = require("axios");
// TinyTTS
const TinyTTS = require("tiny-tts"); const TinyTTS = require("tiny-tts");
const tts = new TinyTTS(); const tts = new TinyTTS();
// General imports
const fs = require("fs"); const fs = require("fs");
const os = require("os"); const os = require("os");
const path = require("path"); const path = require("path");
@ -18,26 +13,12 @@ const wav = require("wav-decoder");
const Speaker = require("speaker"); const Speaker = require("speaker");
const { spawn } = require("child_process"); const { spawn } = require("child_process");
// Modules
const {redis} = require('./modules/redis'); const {redis} = require('./modules/redis');
// Configs
const config = require("./config"); const config = require("./config");
const transConfig = require("./config/transcribe.config"); const transConfig = require("./config/transcribe.config");
const { handlePythonOutput } = require("./scripts/py/pythonOutput"); const { handlePythonOutput } = require("./scripts/py/pythonOutput");
// === HomeAssistant ===
const ha = {}
ha.client = axios.create({
baseURL: config.HA_URL,
headers: {
Authorization: `Bearer ${config.HA_TOKEN}`,
"Content-Type": "application/json",
},
});
// === Speaker ===
let speaker = null; let speaker = null;
let speakerChannels = null; let speakerChannels = null;
@ -88,7 +69,6 @@ function floatsToPCM16LE(channelData) {
return out; return out;
} }
// === TTS ===
async function playAudio(text) { async function playAudio(text) {
const outFile = path.join( const outFile = path.join(
os.tmpdir(), os.tmpdir(),
@ -124,7 +104,6 @@ async function playAudio(text) {
} }
} }
// === Transcriber ===
const py = spawn("python", ["./src/scripts/py/transcribe.py", JSON.stringify(transConfig)], { const py = spawn("python", ["./src/scripts/py/transcribe.py", JSON.stringify(transConfig)], {
stdio: ["ignore", "pipe", "pipe"], stdio: ["ignore", "pipe", "pipe"],
}); });
@ -156,26 +135,6 @@ async function processor(group, consumer, streamKey) {
case "remind": case "remind":
await playAudio("Reminder, " + String(msg.message.value)); await playAudio("Reminder, " + String(msg.message.value));
break; break;
case "light_on":
await playAudio("Turning on light.");
await ha.client.post("/api/services/homeassistant/turn_on", {
entity_id: "light.living_room_living_room"
}).catch(async (err) => {
await playAudio("I have failed you father.");
console.error(err);
});
break;
case "light_off":
await playAudio("Turning off light.");
await ha.client.post("/api/services/homeassistant/turn_off", {
entity_id: "light.living_room_living_room"
}).catch(async (err) => {
await playAudio("I have failed you father.");
console.error(err);
});
break;
default: default:
await playAudio("Command, " + String(msg.message.command) + " is undefined.") await playAudio("Command, " + String(msg.message.command) + " is undefined.")
@ -187,9 +146,12 @@ async function processor(group, consumer, streamKey) {
} }
} }
// === Main ===
async function main() { async function main() {
// Attach python listeners FIRST to avoid missing logs await redis.connect();
await redis.ensureGroup("commands", "assistant");
await playAudio("Assistant, online.");
py.stdout.on("data", (data) => { py.stdout.on("data", (data) => {
handlePythonOutput(data.toString(), async (msg) => { handlePythonOutput(data.toString(), async (msg) => {
if (msg.event === "text" || msg.event === "final") { if (msg.event === "text" || msg.event === "final") {
@ -206,18 +168,7 @@ async function main() {
console.log("Python exited with code", code); console.log("Python exited with code", code);
}); });
try { processor("assistant", `${config.DEVICE_ID}`, "commands").catch(console.error);
await redis.connect();
await redis.ensureGroup("commands", "assistant");
await playAudio("Assistant, online.");
// Start the command processor
processor("assistant", `${config.DEVICE_ID}`, "commands").catch(console.error);
console.log("System fully initialized.");
} catch (err) {
console.error("Initialization failed:", err);
}
} }
main().catch(console.error); main().catch(console.error);