diff --git a/.cargo/config.toml b/.cargo/config.toml index 2dc4a5047..6d9b1f650 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,7 +2,7 @@ rustflags = ["-C", "target-feature=+crt-static"] [target.aarch64-pc-windows-msvc] -rustflags = ["-C", "target-feature=+crt-static"] +rustflags = [] [target.x86_64-apple-darwin] rustflags = ["-C", "link-args=-ObjC"] diff --git a/.changeset/add_uniffi_interface_for_livekit_wakeword.md b/.changeset/add_uniffi_interface_for_livekit_wakeword.md new file mode 100644 index 000000000..3b2e20ed8 --- /dev/null +++ b/.changeset/add_uniffi_interface_for_livekit_wakeword.md @@ -0,0 +1,10 @@ +--- +livekit-uniffi: minor +--- + +# Add UniFFI interface for livekit-wakeword + +## Summary +- Expose `WakeWordDetector` as a UniFFI Object wrapping `WakeWordModel` with `Mutex` for interior mutability +- Export `new`, `load_model`, and `predict` methods across FFI +- Use `#[uniffi::remote(Error)]` with `#[uniffi(flat_error)]` for `WakeWordError`, matching the existing `AccessTokenError` pattern diff --git a/Cargo.lock b/Cargo.lock index 0f005f9d4..f6368c094 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2972,6 +2972,12 @@ dependencies = [ "digest", ] +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + [[package]] name = "home" version = "0.5.12" @@ -3949,6 +3955,7 @@ version = "0.1.0" dependencies = [ "livekit-api", "livekit-protocol", + "livekit-wakeword", "log", "once_cell", "tokio", @@ -4043,6 +4050,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-rust2" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69" + [[package]] name = "mach2" version = "0.4.3" @@ -5103,6 +5116,7 @@ dependencies = [ "ort-sys", "smallvec", "tracing", + "ureq", ] [[package]] @@ -5110,6 +5124,11 @@ name = "ort-sys" version = "2.0.0-rc.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06503bb33f294c5f1ba484011e053bfa6ae227074bdb841e9863492dc5960d4b" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] [[package]] name = "ort-tract" @@ -6821,6 +6840,17 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "soxr-sys" version = "0.1.2" @@ -7905,6 +7935,36 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc" +dependencies = [ + "base64 0.22.1", + "der", + "log", + "native-tls", + "percent-encoding", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf-8", + "webpki-root-certs", +] + +[[package]] +name = "ureq-proto" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f" +dependencies = [ + "base64 0.22.1", + "http 1.4.0", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -8275,6 +8335,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.25.4" diff --git a/Cargo.toml b/Cargo.toml index ecf593d5d..936bd7f0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ livekit-api = { version = "0.4.14", path = "livekit-api" } livekit-ffi = { version = "0.12.48", path = "livekit-ffi" } livekit-protocol = { version = "0.7.1", path = "livekit-protocol" } livekit-runtime = { version = "0.4.0", path = "livekit-runtime" } +livekit-wakeword = { version = "0.1.0", path = "livekit-wakeword" } soxr-sys = { version = "0.1.2", path = "soxr-sys" } webrtc-sys = { version = "0.3.23", path = "webrtc-sys" } webrtc-sys-build = { version = "0.3.13", path = "webrtc-sys/build" } diff --git a/livekit-uniffi/Cargo.toml b/livekit-uniffi/Cargo.toml index 8293512e9..899ce19d0 100644 --- a/livekit-uniffi/Cargo.toml +++ b/livekit-uniffi/Cargo.toml @@ -11,9 +11,14 @@ repository.workspace = true readme = "README.md" publish = false +[features] +default = ["wakeword"] +wakeword = ["livekit-wakeword"] + [dependencies] livekit-protocol = { workspace = true } livekit-api = { workspace = true } +livekit-wakeword = { workspace = true, optional = true } uniffi = { version = "0.30.0", features = ["cli", "scaffolding-ffi-buffer-fns"] } log = { workspace = true } tokio = { workspace = true, features = ["sync"] } diff --git a/livekit-uniffi/src/lib.rs b/livekit-uniffi/src/lib.rs index 698bd4d0b..eddf55631 100644 --- a/livekit-uniffi/src/lib.rs +++ b/livekit-uniffi/src/lib.rs @@ -21,4 +21,8 @@ pub mod log_forward; /// Information about the build such as version. pub mod build_info; +/// Wake word detection from [`livekit-wakeword`]. +#[cfg(feature = "wakeword")] +pub mod wakeword; + uniffi::setup_scaffolding!(); diff --git a/livekit-uniffi/src/wakeword.rs b/livekit-uniffi/src/wakeword.rs new file mode 100644 index 000000000..05e5a9d75 --- /dev/null +++ b/livekit-uniffi/src/wakeword.rs @@ -0,0 +1,74 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use livekit_wakeword::{WakeWordError, WakeWordModel}; +use std::collections::HashMap; +use std::sync::Mutex; + +#[uniffi::remote(Error)] +#[uniffi(flat_error)] +pub enum WakeWordError { + Ort, + Shape, + Io, + ModelNotFound, + UnsupportedSampleRate, + Resample, +} + +/// Wake word detector backed by ONNX classifier models. +/// +/// Wraps [`livekit_wakeword::WakeWordModel`] for use across FFI boundaries. +/// Uses interior mutability since the underlying model requires `&mut self`. +#[derive(uniffi::Object)] +pub struct WakeWordDetector { + inner: Mutex, +} + +#[uniffi::export] +impl WakeWordDetector { + /// Create a new wake word detector. + /// + /// `model_paths` are filesystem paths to ONNX classifier models. + /// `sample_rate` is the sample rate of audio that will be passed to + /// [`predict`](Self::predict). Supported rates: 16000 (recommended), + /// 22050, 32000, 44100, 48000, 88200, 96000, 176400, 192000, 384000 Hz. + #[uniffi::constructor] + pub fn new(model_paths: Vec, sample_rate: u32) -> Result { + let model = WakeWordModel::new(&model_paths, sample_rate)?; + Ok(Self { inner: Mutex::new(model) }) + } + + /// Load an additional wake word classifier ONNX model from disk. + /// + /// If `model_name` is `None`, the file stem is used as the classifier name. + pub fn load_model( + &self, + model_path: String, + model_name: Option, + ) -> Result<(), WakeWordError> { + let mut inner = self.inner.lock().unwrap(); + inner.load_model(&model_path, model_name.as_deref())?; + Ok(()) + } + + /// Get wake word predictions for an audio chunk. + /// + /// Pass ~2 seconds of i16 PCM audio at the sample rate configured in + /// [`new`](Self::new). Returns a map of classifier name to confidence score. + pub fn predict(&self, audio_chunk: Vec) -> Result, WakeWordError> { + let mut inner = self.inner.lock().unwrap(); + Ok(inner.predict(&audio_chunk)?) + } +} diff --git a/livekit-wakeword/Cargo.toml b/livekit-wakeword/Cargo.toml index a716e4765..4bd6b9ea4 100644 --- a/livekit-wakeword/Cargo.toml +++ b/livekit-wakeword/Cargo.toml @@ -7,7 +7,7 @@ license.workspace = true [dependencies] ndarray = "0.17.2" -ort = { version = "2.0.0-rc.11", default-features = false, features = ["ndarray", "std"] } +ort = { version = "2.0.0-rc.11", default-features = false, features = ["ndarray", "std", "download-binaries", "tls-native"] } resampler = "0.4" thiserror = "2"