From 9132a98717e409bdc5a64abf9edb18e2ecec4abb Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Mon, 8 Jun 2026 13:58:53 +0200 Subject: [PATCH 1/4] All platform audio changes before vacation --- Runtime/Plugins/iOS.meta | 8 + Runtime/Plugins/iOS/LiveKitAudioSession.mm | 70 +++ .../Plugins/iOS/LiveKitAudioSession.mm.meta | 42 ++ Runtime/Plugins/iOS/liblivekit_ffi.a | 3 + Runtime/Plugins/iOS/liblivekit_ffi.a.meta | 27 ++ Runtime/Scripts/Internal/FFIClient.cs | 70 +++ .../FFIClients/FfiRequestExtensions.cs | 19 + Runtime/Scripts/Internal/NativeMethods.cs | 14 + Runtime/Scripts/PlatformAudio.cs | 408 ++++++++++++++++++ Runtime/Scripts/PlatformAudio.cs.meta | 11 + Runtime/Scripts/PlatformAudioSource.cs | 161 +++++++ Runtime/Scripts/PlatformAudioSource.cs.meta | 11 + Runtime/Scripts/Track.cs | 20 +- .../Meet/Assets/Editor/MeetManagerEditor.cs | 74 ++++ Samples~/Meet/Assets/Runtime/MeetManager.cs | 181 +++++++- Samples~/Meet/Assets/Scenes/MeetApp.unity | 6 +- 16 files changed, 1110 insertions(+), 15 deletions(-) create mode 100644 Runtime/Plugins/iOS.meta create mode 100644 Runtime/Plugins/iOS/LiveKitAudioSession.mm create mode 100644 Runtime/Plugins/iOS/LiveKitAudioSession.mm.meta create mode 100644 Runtime/Plugins/iOS/liblivekit_ffi.a create mode 100644 Runtime/Plugins/iOS/liblivekit_ffi.a.meta create mode 100644 Runtime/Scripts/PlatformAudio.cs create mode 100644 Runtime/Scripts/PlatformAudio.cs.meta create mode 100644 Runtime/Scripts/PlatformAudioSource.cs create mode 100644 Runtime/Scripts/PlatformAudioSource.cs.meta create mode 100644 Samples~/Meet/Assets/Editor/MeetManagerEditor.cs diff --git a/Runtime/Plugins/iOS.meta b/Runtime/Plugins/iOS.meta new file mode 100644 index 00000000..bff0c975 --- /dev/null +++ b/Runtime/Plugins/iOS.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: f9dc9cc15b6914ff0bc3ccd2a0096200 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Plugins/iOS/LiveKitAudioSession.mm b/Runtime/Plugins/iOS/LiveKitAudioSession.mm new file mode 100644 index 00000000..e6c2b7b2 --- /dev/null +++ b/Runtime/Plugins/iOS/LiveKitAudioSession.mm @@ -0,0 +1,70 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +extern "C" { + +/// Configures the iOS audio session for VoIP/WebRTC use. +/// This sets AVAudioSessionCategoryPlayAndRecord with VoiceChat mode, +/// which enables the VPIO (Voice Processing IO) AudioUnit for: +/// - Hardware echo cancellation (AEC) +/// - Automatic gain control (AGC) +/// - Noise suppression (NS) +/// +/// Call this before creating PlatformAudio to ensure WebRTC can +/// properly initialize the microphone and speaker. +void LiveKit_ConfigureAudioSessionForVoIP() { + AVAudioSession* session = [AVAudioSession sharedInstance]; + NSError* error = nil; + + // Configure for VoIP with echo cancellation + BOOL success = [session setCategory:AVAudioSessionCategoryPlayAndRecord + mode:AVAudioSessionModeVoiceChat + options:AVAudioSessionCategoryOptionDefaultToSpeaker | + AVAudioSessionCategoryOptionAllowBluetooth | + AVAudioSessionCategoryOptionAllowBluetoothA2DP + error:&error]; + + if (!success || error) { + NSLog(@"LiveKit: Failed to configure VoIP audio session: %@", error.localizedDescription); + return; + } + + // Activate the audio session + success = [session setActive:YES error:&error]; + if (!success || error) { + NSLog(@"LiveKit: Failed to activate audio session: %@", error.localizedDescription); + return; + } + + NSLog(@"LiveKit: Audio session configured for VoIP (PlayAndRecord + VoiceChat mode)"); +} + +/// Restores the audio session to the default ambient category. +/// Call this when PlatformAudio is disposed if you want to restore +/// the original audio behavior. +void LiveKit_RestoreDefaultAudioSession() { + AVAudioSession* session = [AVAudioSession sharedInstance]; + NSError* error = nil; + + [session setCategory:AVAudioSessionCategoryAmbient error:&error]; + if (error) { + NSLog(@"LiveKit: Failed to restore default audio session: %@", error.localizedDescription); + } +} + +} diff --git a/Runtime/Plugins/iOS/LiveKitAudioSession.mm.meta b/Runtime/Plugins/iOS/LiveKitAudioSession.mm.meta new file mode 100644 index 00000000..8a957390 --- /dev/null +++ b/Runtime/Plugins/iOS/LiveKitAudioSession.mm.meta @@ -0,0 +1,42 @@ +fileFormatVersion: 2 +guid: 27d51b1df7f4b469a8a9c97f2349face +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 1 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + VisionOS: VisionOS + second: + enabled: 1 + settings: {} + - first: + iPhone: iOS + second: + enabled: 1 + settings: {} + - first: + tvOS: tvOS + second: + enabled: 1 + settings: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Plugins/iOS/liblivekit_ffi.a b/Runtime/Plugins/iOS/liblivekit_ffi.a new file mode 100644 index 00000000..f6e59d9e --- /dev/null +++ b/Runtime/Plugins/iOS/liblivekit_ffi.a @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07f507aab5a0a241d674b81460dee4a2d99b0996debb3d07a92bf41f63be783d +size 564011360 diff --git a/Runtime/Plugins/iOS/liblivekit_ffi.a.meta b/Runtime/Plugins/iOS/liblivekit_ffi.a.meta new file mode 100644 index 00000000..3534c748 --- /dev/null +++ b/Runtime/Plugins/iOS/liblivekit_ffi.a.meta @@ -0,0 +1,27 @@ +fileFormatVersion: 2 +guid: 96afae52e55d443c796a803ca8331e93 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 1 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Internal/FFIClient.cs b/Runtime/Scripts/Internal/FFIClient.cs index 55c140c0..487fecaa 100644 --- a/Runtime/Scripts/Internal/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFIClient.cs @@ -133,6 +133,48 @@ static void GetMainContext() Utils.Debug("Main Context created"); } +#if UNITY_ANDROID && !UNITY_EDITOR + /// + /// Get the Android application context as a raw jobject pointer. + /// This is passed to the native library for WebRTC audio initialization. + /// + /// IntPtr to the application context jobject, or IntPtr.Zero on failure + private static IntPtr GetAndroidApplicationContext() + { + try + { + // Get the Unity activity + using var unityPlayer = new AndroidJavaClass("com.unity3d.player.UnityPlayer"); + using var currentActivity = unityPlayer.GetStatic("currentActivity"); + + if (currentActivity == null) + { + Utils.Error("FFIServer - Failed to get Unity currentActivity"); + return IntPtr.Zero; + } + + // Get the application context from the activity + var applicationContext = currentActivity.Call("getApplicationContext"); + + if (applicationContext == null) + { + Utils.Error("FFIServer - Failed to get Android applicationContext"); + return IntPtr.Zero; + } + + // Get the raw jobject pointer + // Note: We don't dispose the applicationContext here because we're passing + // the raw pointer to native code. The native code will create its own global ref. + return applicationContext.GetRawObject(); + } + catch (System.Exception e) + { + Utils.Error($"FFIServer - Failed to get Android application context: {e.Message}"); + return IntPtr.Zero; + } + } +#endif + private static void InitializeSdk() { #if NO_LIVEKIT_MODE @@ -145,6 +187,34 @@ private static void InitializeSdk() const bool captureLogs = false; #endif +#if UNITY_ANDROID && !UNITY_EDITOR + // Initialize Android WebRTC before the main FFI initialization. + // This initializes the JVM and ContextUtils (required for PlatformAudio). + try + { + IntPtr javaVmPtr = AndroidJNI.GetJavaVM(); + IntPtr contextPtr = GetAndroidApplicationContext(); + + if (javaVmPtr != IntPtr.Zero && contextPtr != IntPtr.Zero) + { + bool contextInitialized = NativeMethods.LiveKitInitializeAndroidContext(javaVmPtr, contextPtr); + if (!contextInitialized) + { + // JVM init still succeeded; only PlatformAudio won't work + Utils.Error("FFIServer - Android context init failed; PlatformAudio will not work"); + } + } + else + { + Utils.Error("FFIServer - Failed to get JavaVM or context for Android init"); + } + } + catch (System.Exception e) + { + Utils.Error($"FFIServer - Android initialization failed: {e.Message}"); + } +#endif + var sdkVersion = PackageVersion.Get(); NativeMethods.LiveKitInitialize(FFICallback, captureLogs, "unity", sdkVersion); diff --git a/Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs b/Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs index adcfe263..a39e9fb9 100644 --- a/Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs +++ b/Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs @@ -152,6 +152,25 @@ public static void Inject(this FfiRequest ffiRequest, T request) case RemixAndResampleRequest remixAndResampleRequest: ffiRequest.RemixAndResample = remixAndResampleRequest; break; + // PlatformAudio + case NewPlatformAudioRequest newPlatformAudioRequest: + ffiRequest.NewPlatformAudio = newPlatformAudioRequest; + break; + case GetAudioDevicesRequest getAudioDevicesRequest: + ffiRequest.GetAudioDevices = getAudioDevicesRequest; + break; + case SetRecordingDeviceRequest setRecordingDeviceRequest: + ffiRequest.SetRecordingDevice = setRecordingDeviceRequest; + break; + case SetPlayoutDeviceRequest setPlayoutDeviceRequest: + ffiRequest.SetPlayoutDevice = setPlayoutDeviceRequest; + break; + case StartRecordingRequest startRecordingRequest: + ffiRequest.StartRecording = startRecordingRequest; + break; + case StopRecordingRequest stopRecordingRequest: + ffiRequest.StopRecording = stopRecordingRequest; + break; case LocalTrackMuteRequest localTrackMuteRequest: ffiRequest.LocalTrackMute = localTrackMuteRequest; break; diff --git a/Runtime/Scripts/Internal/NativeMethods.cs b/Runtime/Scripts/Internal/NativeMethods.cs index 96a70fbd..4587b345 100644 --- a/Runtime/Scripts/Internal/NativeMethods.cs +++ b/Runtime/Scripts/Internal/NativeMethods.cs @@ -25,5 +25,19 @@ internal static class NativeMethods [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_initialize")] internal extern static FfiHandleId LiveKitInitialize(FFICallbackDelegate cb, bool captureLogs, string sdk, string sdkVersion); + +#if UNITY_ANDROID && !UNITY_EDITOR + /// + /// Initialize Android WebRTC with the application context. + /// This initializes both the JVM and ContextUtils, which is required for + /// Android audio (microphone/speaker) to work via PlatformAudio. + /// + /// Pointer to the JavaVM + /// The Android application context (jobject) + /// true if context initialization succeeded, false otherwise. + /// Note: JVM initialization happens regardless of return value. + [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_initialize_android_context")] + internal extern static bool LiveKitInitializeAndroidContext(IntPtr javaVmPtr, IntPtr contextPtr); +#endif } } \ No newline at end of file diff --git a/Runtime/Scripts/PlatformAudio.cs b/Runtime/Scripts/PlatformAudio.cs new file mode 100644 index 00000000..0dfe2215 --- /dev/null +++ b/Runtime/Scripts/PlatformAudio.cs @@ -0,0 +1,408 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using LiveKit.Proto; +using LiveKit.Internal; +using LiveKit.Internal.FFIClients.Requests; + +#if PLATFORM_ANDROID +using UnityEngine.Android; +#endif + +namespace LiveKit +{ +#if UNITY_IOS && !UNITY_EDITOR + internal static class IOSAudioSessionHelper + { + /// + /// Configures the iOS audio session for VoIP/WebRTC. + /// Must be called before creating PlatformAudio. + /// + [DllImport("__Internal")] + internal static extern void LiveKit_ConfigureAudioSessionForVoIP(); + + /// + /// Restores the iOS audio session to ambient mode. + /// + [DllImport("__Internal")] + internal static extern void LiveKit_RestoreDefaultAudioSession(); + } +#endif + + /// + /// Information about an audio device (microphone or speaker). + /// + public struct AudioDevice + { + /// Device index (0-based). Note: indices can change when devices are added/removed. + public uint Index; + /// Device name as reported by the operating system. + public string Name; + /// + /// Platform-specific unique device identifier (GUID). + /// This is stable across device additions/removals and should be preferred + /// over index for device selection. + /// + public string Guid; + } + + /// + /// Platform audio device management using WebRTC's Audio Device Module (ADM). + /// + /// PlatformAudio provides access to the platform's audio devices (microphones and + /// speakers) and enables automatic audio capture and playback through WebRTC's ADM. + /// + /// Key features: + /// - Echo cancellation (AEC) + /// - Automatic gain control (AGC) + /// - Noise suppression (NS) + /// - Automatic speaker playout for remote audio + /// + /// Usage: + /// 1. Create a PlatformAudio instance (enables ADM) + /// 2. Optionally enumerate and select devices + /// 3. Create audio tracks using PlatformAudioSource + /// 4. Remote audio automatically plays through speakers + /// + /// + /// + /// // Create PlatformAudio (enables ADM) + /// var platformAudio = new PlatformAudio(); + /// + /// // Enumerate devices + /// var (recording, playout) = platformAudio.GetDevices(); + /// foreach (var device in recording) + /// Debug.Log($"Mic {device.Index}: {device.Name}"); + /// + /// // Select devices + /// platformAudio.SetRecordingDevice(0); + /// platformAudio.SetPlayoutDevice(0); + /// + /// // Create audio source and track + /// var source = new PlatformAudioSource(platformAudio); + /// var track = LocalAudioTrack.CreateAudioTrack("microphone", source, room); + /// + /// // Publish track + /// await room.LocalParticipant.PublishTrack(track, options); + /// + /// // Dispose when done + /// platformAudio.Dispose(); + /// + /// + public sealed class PlatformAudio : IDisposable + { + internal readonly FfiHandle Handle; + private readonly PlatformAudioInfo _info; + private bool _disposed = false; + + /// + /// Number of available recording (microphone) devices. + /// + public int RecordingDeviceCount => _info.RecordingDeviceCount; + + /// + /// Number of available playout (speaker) devices. + /// + public int PlayoutDeviceCount => _info.PlayoutDeviceCount; + + /// + /// Creates a new PlatformAudio instance, enabling the platform ADM. + /// + /// This must be called before creating any PlatformAudioSource or connecting + /// to a room if you want automatic speaker playout for remote audio. + /// + /// On iOS, this automatically configures the audio session for VoIP mode + /// (PlayAndRecord category with VoiceChat mode) to enable hardware echo + /// cancellation and microphone input. + /// + /// + /// Thrown if the platform ADM could not be initialized (e.g., no audio devices, + /// missing permissions). + /// + public PlatformAudio() + { +#if UNITY_IOS && !UNITY_EDITOR + // Configure iOS audio session for VoIP before initializing WebRTC ADM. + // This sets PlayAndRecord category with VoiceChat mode for hardware AEC. + IOSAudioSessionHelper.LiveKit_ConfigureAudioSessionForVoIP(); +#endif + + using var request = FFIBridge.Instance.NewRequest(); + using var response = request.Send(); + FfiResponse res = response; + + if (res.NewPlatformAudio.MessageCase == NewPlatformAudioResponse.MessageOneofCase.Error) + throw new InvalidOperationException($"Failed to create PlatformAudio: {res.NewPlatformAudio.Error}"); + + var platformAudio = res.NewPlatformAudio.PlatformAudio; + Handle = FfiHandle.FromOwnedHandle(platformAudio.Handle); + _info = platformAudio.Info; + + Utils.Debug($"PlatformAudio created: {RecordingDeviceCount} recording devices, {PlayoutDeviceCount} playout devices"); + } + + /// + /// Gets the lists of available recording and playout devices. + /// + /// + /// A tuple containing: + /// - Recording: List of available microphones + /// - Playout: List of available speakers/headphones + /// + /// + /// Thrown if device enumeration failed. + /// + public (List Recording, List Playout) GetDevices() + { + using var request = FFIBridge.Instance.NewRequest(); + request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); + + using var response = request.Send(); + FfiResponse res = response; + + if (res.GetAudioDevices.HasError && !string.IsNullOrEmpty(res.GetAudioDevices.Error)) + throw new InvalidOperationException($"Failed to get audio devices: {res.GetAudioDevices.Error}"); + + var recording = new List(); + foreach (var device in res.GetAudioDevices.RecordingDevices) + { + recording.Add(new AudioDevice { + Index = device.Index, + Name = device.Name, + Guid = device.HasGuid ? device.Guid : null + }); + } + + var playout = new List(); + foreach (var device in res.GetAudioDevices.PlayoutDevices) + { + playout.Add(new AudioDevice { + Index = device.Index, + Name = device.Name, + Guid = device.HasGuid ? device.Guid : null + }); + } + + return (recording, playout); + } + + /// + /// Sets the recording device (microphone) by index. + /// + /// Call this before creating audio tracks to select which microphone to use. + /// Device indices are 0-based and must be less than RecordingDeviceCount. + /// + /// Note: Prefer SetRecordingDevice(string deviceId) for robust device selection across hot-plug events. + /// + /// Device index from GetDevices().Recording + /// + /// Thrown if the device index is invalid or the operation failed. + /// + public void SetRecordingDevice(uint index) + { + // Look up the device GUID by index + var (recording, _) = GetDevices(); + if (index >= recording.Count) + throw new InvalidOperationException($"Recording device index {index} out of range (max: {recording.Count - 1})"); + + var deviceId = recording[(int)index].Guid; + + // Note: On Android, devices don't have GUIDs - they're identified by index only. + // Android also only reports a single "default" microphone because the system + // automatically selects the best input source based on the audio mode. + // If GUID is empty, we pass an empty string which triggers index-0 fallback in native code. + SetRecordingDevice(deviceId ?? ""); + Utils.Debug($"PlatformAudio: set recording device to index {index} (GUID: {(string.IsNullOrEmpty(deviceId) ? "" : deviceId)})"); + } + + /// + /// Sets the recording device (microphone) by device ID (GUID). + /// + /// This is the preferred method for device selection as device IDs are stable + /// across device hot-plug events, unlike indices which can change. + /// + /// Device ID/GUID from GetDevices().Recording[i].Guid + /// + /// Thrown if the device is not found or the operation failed. + /// + public void SetRecordingDevice(string deviceId) + { +#if UNITY_IOS && !UNITY_EDITOR + // iOS exposes only one logical WebRTC recording device, and AudioDeviceIOS::RecordingDeviceName + // returns -1, so set_recording_device_by_guid in native code can never match and always + // returns "Device not found". Mic input routing on iOS is governed by AVAudioSession + // (configured via IOSAudioSessionHelper.LiveKit_ConfigureAudioSessionForVoIP), not by + // WebRTC device selection, so skipping the FFI call here is the correct behavior. + Utils.Debug($"PlatformAudio: skipping SetRecordingDevice on iOS (deviceId '{deviceId}'); input is governed by AVAudioSession"); + return; +#else + using var request = FFIBridge.Instance.NewRequest(); + request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); + request.request.DeviceId = deviceId; + + using var response = request.Send(); + FfiResponse res = response; + + if (res.SetRecordingDevice.HasError && !string.IsNullOrEmpty(res.SetRecordingDevice.Error)) + throw new InvalidOperationException($"Failed to set recording device: {res.SetRecordingDevice.Error}"); + + Utils.Debug($"PlatformAudio: set recording device to {deviceId}"); +#endif + } + + /// + /// Sets the playout device (speaker/headphones) by index. + /// + /// Call this before connecting to select which speaker to use for remote audio. + /// Device indices are 0-based and must be less than PlayoutDeviceCount. + /// + /// Note: Prefer SetPlayoutDevice(string deviceId) for robust device selection across hot-plug events. + /// + /// Device index from GetDevices().Playout + /// + /// Thrown if the device index is invalid or the operation failed. + /// + public void SetPlayoutDevice(uint index) + { + // Look up the device GUID by index + var (_, playout) = GetDevices(); + if (index >= playout.Count) + throw new InvalidOperationException($"Playout device index {index} out of range (max: {playout.Count - 1})"); + + var deviceId = playout[(int)index].Guid; + + // Note: On Android, devices don't have GUIDs - they're identified by index only. + // Android also only reports a single "default" device because audio routing + // (speaker vs earpiece vs Bluetooth) is handled by the system via AudioManager, + // not through WebRTC device selection. Use Android's AudioManager API to switch outputs. + // If GUID is empty, we pass an empty string which triggers index-0 fallback in native code. + SetPlayoutDevice(deviceId ?? ""); + Utils.Debug($"PlatformAudio: set playout device to index {index} (GUID: {(string.IsNullOrEmpty(deviceId) ? "" : deviceId)})"); + } + + /// + /// Sets the playout device (speaker/headphones) by device ID (GUID). + /// + /// This is the preferred method for device selection as device IDs are stable + /// across device hot-plug events, unlike indices which can change. + /// + /// Device ID/GUID from GetDevices().Playout[i].Guid + /// + /// Thrown if the device is not found or the operation failed. + /// + public void SetPlayoutDevice(string deviceId) + { +#if UNITY_IOS && !UNITY_EDITOR + // Same iOS limitation as SetRecordingDevice: AudioDeviceIOS::PlayoutDeviceName returns -1, + // so set_playout_device_by_guid never matches. Speaker vs earpiece vs Bluetooth routing on + // iOS is governed by AVAudioSession, not by WebRTC device selection. + Utils.Debug($"PlatformAudio: skipping SetPlayoutDevice on iOS (deviceId '{deviceId}'); output is governed by AVAudioSession"); + return; +#else + using var request = FFIBridge.Instance.NewRequest(); + request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); + request.request.DeviceId = deviceId; + + using var response = request.Send(); + FfiResponse res = response; + + if (res.SetPlayoutDevice.HasError && !string.IsNullOrEmpty(res.SetPlayoutDevice.Error)) + throw new InvalidOperationException($"Failed to set playout device: {res.SetPlayoutDevice.Error}"); + + Utils.Debug($"PlatformAudio: set playout device to {deviceId}"); +#endif + } + + /// + /// Starts recording from the microphone. + /// + /// Recording is started automatically when PlatformAudio is created. + /// Use this to resume recording after calling StopRecording. + /// This turns on the system's recording privacy indicator (e.g., on macOS/iOS). + /// + /// + /// Thrown if the operation failed. + /// + public IEnumerator StartRecording() + { +#if PLATFORM_ANDROID + if (!Permission.HasUserAuthorizedPermission(Permission.Microphone)) + { + // Fire the system permission dialog and yield until the user resolves it. + // PermissionCallbacks delivers the result asynchronously from the Android OS; + // we poll the captured flag from this coroutine until one of the callbacks + // sets it. Without this gate, the WebRTC Android ADM would crash the process + // when AudioRecord fails to open due to the missing permission. + bool? granted = null; + var callbacks = new PermissionCallbacks(); + callbacks.PermissionGranted += _ => granted = true; + callbacks.PermissionDenied += _ => granted = false; + callbacks.PermissionDeniedAndDontAskAgain += _ => granted = false; + Permission.RequestUserPermission(Permission.Microphone, callbacks); + + while (granted == null) + yield return null; + + if (granted == false) + throw new InvalidOperationException( + "Microphone permission denied by user; cannot start recording."); + } +#endif + + using var request = FFIBridge.Instance.NewRequest(); + request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); + + using var response = request.Send(); + FfiResponse res = response; + + if (res.StartRecording.HasError && !string.IsNullOrEmpty(res.StartRecording.Error)) + throw new InvalidOperationException($"Failed to start recording: {res.StartRecording.Error}"); + + Utils.Debug("PlatformAudio: started recording"); + + // Ensures this method is always a valid iterator even when the PLATFORM_ANDROID + // branch is compiled out (no `yield return` would otherwise be reachable on + // non-Android builds, which is a compile error for IEnumerator-returning methods). + yield break; + } + + /// + /// Stops recording from the microphone. + /// + /// Use this to temporarily stop recording without disposing PlatformAudio. + /// This turns off the system's recording privacy indicator (e.g., on macOS/iOS). + /// Call StartRecording to resume recording. + /// + /// + /// Thrown if the operation failed. + /// + public void StopRecording() + { + using var request = FFIBridge.Instance.NewRequest(); + request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); + + using var response = request.Send(); + FfiResponse res = response; + + if (res.StopRecording.HasError && !string.IsNullOrEmpty(res.StopRecording.Error)) + throw new InvalidOperationException($"Failed to stop recording: {res.StopRecording.Error}"); + + Utils.Debug("PlatformAudio: stopped recording"); + } + + /// + /// Releases the PlatformAudio resources. + /// + /// When disposed, the platform ADM may be disabled if this was the last + /// PlatformAudio instance. + /// + public void Dispose() + { + if (_disposed) return; + Handle.Dispose(); + _disposed = true; + Utils.Debug("PlatformAudio disposed"); + } + } +} diff --git a/Runtime/Scripts/PlatformAudio.cs.meta b/Runtime/Scripts/PlatformAudio.cs.meta new file mode 100644 index 00000000..6fadaf57 --- /dev/null +++ b/Runtime/Scripts/PlatformAudio.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 84e509f50b0674233812310ab67ccd9c +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/PlatformAudioSource.cs b/Runtime/Scripts/PlatformAudioSource.cs new file mode 100644 index 00000000..f0be42b5 --- /dev/null +++ b/Runtime/Scripts/PlatformAudioSource.cs @@ -0,0 +1,161 @@ +using System; +using LiveKit.Proto; +using LiveKit.Internal; +using LiveKit.Internal.FFIClients.Requests; + +namespace LiveKit +{ + /// + /// Options for audio processing when creating a PlatformAudioSource. + /// + public struct AudioProcessingOptions + { + /// Enable echo cancellation (AEC). Default: true. + public bool EchoCancellation; + /// Enable noise suppression (NS). Default: true. + public bool NoiseSuppression; + /// Enable automatic gain control (AGC). Default: true. + public bool AutoGainControl; + /// Prefer hardware audio processing (e.g., iOS VPIO). Lower latency. Default: true. + public bool PreferHardware; + + /// + /// Default audio processing options with all processing enabled and hardware preferred. + /// + public static AudioProcessingOptions Default => new AudioProcessingOptions + { + EchoCancellation = true, + NoiseSuppression = true, + AutoGainControl = true, + PreferHardware = true + }; + } + + /// + /// Audio source that captures from the platform microphone via WebRTC's ADM. + /// + /// Unlike MicrophoneSource which uses Unity's Microphone API and manually pushes + /// audio frames, PlatformAudioSource lets WebRTC's ADM handle capture directly. + /// This provides: + /// - Echo cancellation (AEC) that works with speaker playout + /// - Lower latency (single audio path, no Unity intermediate) + /// - Automatic gain control and noise suppression + /// + /// Requires PlatformAudio to be created first to enable the ADM. + /// + /// + /// + /// // Create PlatformAudio first (enables ADM) + /// var platformAudio = new PlatformAudio(); + /// + /// // Create audio source with default options + /// var source = new PlatformAudioSource(platformAudio); + /// + /// // Or with custom options + /// var options = new AudioProcessingOptions { + /// EchoCancellation = true, + /// NoiseSuppression = true, + /// AutoGainControl = true, + /// PreferHardware = true + /// }; + /// var source = new PlatformAudioSource(platformAudio, options); + /// + /// // Create and publish track + /// var track = LocalAudioTrack.CreateAudioTrack("microphone", source, room); + /// await room.LocalParticipant.PublishTrack(track, options); + /// + /// + public sealed class PlatformAudioSource : IRtcSource, IDisposable + { + internal readonly FfiHandle Handle; + private readonly PlatformAudio _platformAudio; + private bool _disposed = false; + private bool _muted = false; + + /// + /// Whether the audio source is muted. + /// + public override bool Muted => _muted; + + /// + /// Creates a new platform audio source with default audio processing options. + /// + /// The source will capture audio from the microphone selected via + /// PlatformAudio.SetRecordingDevice(), or the default device if none selected. + /// + /// + /// The PlatformAudio instance. Must be kept alive while this source is in use. + /// + /// + /// Thrown if platformAudio is null. + /// + public PlatformAudioSource(PlatformAudio platformAudio) + : this(platformAudio, AudioProcessingOptions.Default) + { + } + + /// + /// Creates a new platform audio source with custom audio processing options. + /// + /// The source will capture audio from the microphone selected via + /// PlatformAudio.SetRecordingDevice(), or the default device if none selected. + /// + /// + /// The PlatformAudio instance. Must be kept alive while this source is in use. + /// + /// Audio processing options to configure on the ADM. + /// + /// Thrown if platformAudio is null. + /// + public PlatformAudioSource(PlatformAudio platformAudio, AudioProcessingOptions options) + { + if (platformAudio == null) + throw new ArgumentNullException(nameof(platformAudio)); + + _platformAudio = platformAudio; + + using var request = FFIBridge.Instance.NewRequest(); + var newAudioSource = request.request; + newAudioSource.Type = AudioSourceType.AudioSourcePlatform; + newAudioSource.NumChannels = 2; + newAudioSource.SampleRate = 48000; + + // Pass the platform audio handle so the Rust side can configure audio processing + newAudioSource.PlatformAudioHandle = (ulong)platformAudio.Handle.DangerousGetHandle(); + + // Configure audio processing options + newAudioSource.Options = request.TempResource(); + newAudioSource.Options.EchoCancellation = options.EchoCancellation; + newAudioSource.Options.AutoGainControl = options.AutoGainControl; + newAudioSource.Options.NoiseSuppression = options.NoiseSuppression; + newAudioSource.Options.PreferHardware = options.PreferHardware; + + using var response = request.Send(); + FfiResponse res = response; + + Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle); + Utils.Debug($"PlatformAudioSource created: handle={Handle.DangerousGetHandle()}, AEC={options.EchoCancellation}, NS={options.NoiseSuppression}, AGC={options.AutoGainControl}, HW={options.PreferHardware}"); + } + + /// + /// Mutes or unmutes the audio source. + /// + /// True to mute, false to unmute. + public override void SetMute(bool muted) + { + _muted = muted; + Utils.Debug($"PlatformAudioSource: muted={muted}"); + } + + /// + /// Releases the audio source resources. + /// + public void Dispose() + { + if (_disposed) return; + Handle.Dispose(); + _disposed = true; + Utils.Debug("PlatformAudioSource disposed"); + } + } +} diff --git a/Runtime/Scripts/PlatformAudioSource.cs.meta b/Runtime/Scripts/PlatformAudioSource.cs.meta new file mode 100644 index 00000000..c05fe981 --- /dev/null +++ b/Runtime/Scripts/PlatformAudioSource.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: f15bdf32fdae64d69ba90ab6cabc47d5 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Track.cs b/Runtime/Scripts/Track.cs index 419c25df..2fa873b6 100644 --- a/Runtime/Scripts/Track.cs +++ b/Runtime/Scripts/Track.cs @@ -117,7 +117,7 @@ internal void DisposeHandles() public sealed class LocalAudioTrack : Track, ILocalTrack, IAudioTrack { - RtcAudioSource _source; + IRtcSource _source; IRtcSource ILocalTrack.source { get => _source; } @@ -125,6 +125,10 @@ internal LocalAudioTrack(OwnedTrack track, Room room, RtcAudioSource source) : b _source = source; } + internal LocalAudioTrack(OwnedTrack track, Room room, PlatformAudioSource source) : base(track, room, room?.LocalParticipant) { + _source = source; + } + public static LocalAudioTrack CreateAudioTrack(string name, RtcAudioSource source, Room room) { using var request = FFIBridge.Instance.NewRequest(); @@ -138,6 +142,20 @@ public static LocalAudioTrack CreateAudioTrack(string name, RtcAudioSource sourc var track = new LocalAudioTrack(trackInfo, room, source); return track; } + + public static LocalAudioTrack CreateAudioTrack(string name, PlatformAudioSource source, Room room) + { + using var request = FFIBridge.Instance.NewRequest(); + var createTrack = request.request; + createTrack.Name = name; + createTrack.SourceHandle = (ulong)source.Handle.DangerousGetHandle(); + + using var resp = request.Send(); + FfiResponse res = resp; + var trackInfo = res.CreateAudioTrack.Track; + var track = new LocalAudioTrack(trackInfo, room, source); + return track; + } } public sealed class LocalVideoTrack : Track, ILocalTrack, IVideoTrack diff --git a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs new file mode 100644 index 00000000..1ca6c412 --- /dev/null +++ b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs @@ -0,0 +1,74 @@ +using UnityEngine; +using UnityEditor; + +[CustomEditor(typeof(MeetManager))] +public class MeetManagerEditor : Editor +{ + private SerializedProperty buttonBar; + private SerializedProperty videoTrackParent; + private SerializedProperty participantTilePrefab; + private SerializedProperty frameRate; + private SerializedProperty usePlatformAudio; + private SerializedProperty echoCancellation; + private SerializedProperty noiseSuppression; + private SerializedProperty autoGainControl; + private SerializedProperty preferHardwareProcessing; + + private void OnEnable() + { + buttonBar = serializedObject.FindProperty("buttonBar"); + videoTrackParent = serializedObject.FindProperty("videoTrackParent"); + participantTilePrefab = serializedObject.FindProperty("participantTilePrefab"); + frameRate = serializedObject.FindProperty("frameRate"); + usePlatformAudio = serializedObject.FindProperty("usePlatformAudio"); + echoCancellation = serializedObject.FindProperty("echoCancellation"); + noiseSuppression = serializedObject.FindProperty("noiseSuppression"); + autoGainControl = serializedObject.FindProperty("autoGainControl"); + preferHardwareProcessing = serializedObject.FindProperty("preferHardwareProcessing"); + } + + public override void OnInspectorGUI() + { + serializedObject.Update(); + + EditorGUILayout.LabelField("UI", EditorStyles.boldLabel); + EditorGUILayout.PropertyField(buttonBar); + + EditorGUILayout.Space(); + EditorGUILayout.LabelField("Video Layout", EditorStyles.boldLabel); + EditorGUILayout.PropertyField(videoTrackParent); + EditorGUILayout.PropertyField(participantTilePrefab); + EditorGUILayout.PropertyField(frameRate); + + EditorGUILayout.Space(); + EditorGUILayout.LabelField("Audio Mode", EditorStyles.boldLabel); + EditorGUILayout.PropertyField(usePlatformAudio, new GUIContent("Use Platform Audio", + "Use PlatformAudio (WebRTC ADM) for microphone capture and automatic speaker playout. " + + "Provides AEC, AGC, and NS. Disable to use Unity's Microphone API instead.")); + + EditorGUILayout.Space(); + EditorGUILayout.LabelField("Audio Processing (PlatformAudio only)", EditorStyles.boldLabel); + + // Gray out audio processing options when PlatformAudio is disabled + bool platformAudioEnabled = usePlatformAudio.boolValue; + + using (new EditorGUI.DisabledGroupScope(!platformAudioEnabled)) + { + if (!platformAudioEnabled) + { + EditorGUILayout.HelpBox("Audio processing options are only available when 'Use Platform Audio' is enabled.", MessageType.Info); + } + + EditorGUILayout.PropertyField(echoCancellation, new GUIContent("Echo Cancellation", + "Enable echo cancellation to remove echo from speaker playback.")); + EditorGUILayout.PropertyField(noiseSuppression, new GUIContent("Noise Suppression", + "Enable noise suppression to remove background noise.")); + EditorGUILayout.PropertyField(autoGainControl, new GUIContent("Auto Gain Control", + "Enable auto gain control to normalize audio levels.")); + EditorGUILayout.PropertyField(preferHardwareProcessing, new GUIContent("Prefer Hardware Processing", + "Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")); + } + + serializedObject.ApplyModifiedProperties(); + } +} diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index 225c7a0c..b7afbfbf 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -8,6 +8,12 @@ /// /// Manages a LiveKit room connection with local/remote audio and video tracks. +/// +/// Supports two audio modes: +/// - PlatformAudio (default): Uses WebRTC's ADM for microphone capture and automatic +/// speaker playout. Provides echo cancellation (AEC), AGC, and noise suppression. +/// - Unity Audio: Uses Unity's Microphone API and AudioStream for manual audio handling. +/// No AEC support but gives more control over audio processing. /// [RequireComponent(typeof(TokenSourceComponent))] public class MeetManager : MonoBehaviour @@ -23,6 +29,21 @@ public class MeetManager : MonoBehaviour [SerializeField] private ParticipantTile participantTilePrefab; [SerializeField] private int frameRate = 30; + [Header("Audio Mode")] + [Tooltip("Use PlatformAudio (WebRTC ADM) for microphone capture and automatic speaker playout. " + + "Provides AEC, AGC, and NS. Disable to use Unity's Microphone API instead.")] + [SerializeField] private bool usePlatformAudio = true; + + [Header("Audio Processing (PlatformAudio only)")] + [Tooltip("Enable echo cancellation to remove echo from speaker playback.")] + [SerializeField] private bool echoCancellation = true; + [Tooltip("Enable noise suppression to remove background noise.")] + [SerializeField] private bool noiseSuppression = true; + [Tooltip("Enable auto gain control to normalize audio levels.")] + [SerializeField] private bool autoGainControl = true; + [Tooltip("Prefer hardware audio processing (e.g., iOS VPIO). Lower latency but may have different quality characteristics.")] + [SerializeField] private bool preferHardwareProcessing = true; + private const string PlaceholderTextureResourceName = "PlaceholderTileSquare"; private Texture _placeholderTexture; @@ -42,11 +63,14 @@ public class MeetManager : MonoBehaviour private RtcVideoSource _localRtcVideoSource; private RtcAudioSource _localRtcAudioSource; + private PlatformAudioSource _platformAudioSource; private LocalVideoTrack _localVideoTrack; private LocalAudioTrack _localAudioTrack; private bool _cameraActive; private bool _microphoneActive; + private PlatformAudio _platformAudio; + #region Lifecycle private void Start() @@ -62,6 +86,41 @@ private void Start() _audioTrackParent = new GameObject("AudioTrackParent").transform; _placeholderTexture = Resources.Load(PlaceholderTextureResourceName); + + if (usePlatformAudio) + InitializePlatformAudio(); + } + + private void InitializePlatformAudio() + { + try + { + _platformAudio = new PlatformAudio(); + Debug.Log($"PlatformAudio initialized: {_platformAudio.RecordingDeviceCount} mics, " + + $"{_platformAudio.PlayoutDeviceCount} speakers"); + + var (recording, playout) = _platformAudio.GetDevices(); + Debug.Log("Recording devices:"); + foreach (var device in recording) + Debug.Log($" [{device.Index}] {device.Name}"); + + Debug.Log("Playout devices:"); + foreach (var device in playout) + Debug.Log($" [{device.Index}] {device.Name}"); + + if (_platformAudio.RecordingDeviceCount > 0) + _platformAudio.SetRecordingDevice(0); + if (_platformAudio.PlayoutDeviceCount > 0) + _platformAudio.SetPlayoutDevice(0); + + Debug.Log($"PlatformAudio ready. AEC={echoCancellation}, NS={noiseSuppression}, AGC={autoGainControl}, HW={preferHardwareProcessing}"); + } + catch (System.Exception e) + { + Debug.LogError($"Failed to initialize PlatformAudio, falling back to Unity audio: {e.Message}"); + usePlatformAudio = false; + _platformAudio = null; + } } private void OnApplicationPause(bool pause) @@ -90,6 +149,8 @@ private void OnDestroy() } CleanUpAllTracks(); _webCamTexture?.Stop(); + _platformAudioSource?.Dispose(); + _platformAudio?.Dispose(); _room?.Disconnect(); } @@ -181,7 +242,7 @@ private IEnumerator ConnectToRoom() yield break; } - Debug.Log($"Connected to {_room.Name}"); + Debug.Log($"Connected to {_room.Name} (PlatformAudio: {usePlatformAudio})"); _localId = _room.LocalParticipant.Identity; buttonBar.SetConnected(true); @@ -295,6 +356,15 @@ private void RemoveExtraVideoTile(string sid) private void AddRemoteAudioTrack(RemoteAudioTrack audioTrack) { var sid = audioTrack.Sid; + + if (usePlatformAudio && _platformAudio != null) + { + // PlatformAudio mode: ADM handles speaker playback automatically. + // No AudioStream / GameObject needed. + Debug.Log($"Remote audio track {sid} will play via PlatformAudio (automatic)"); + return; + } + var audioObject = new GameObject($"AudioTrack: {sid}"); audioObject.transform.SetParent(_audioTrackParent); @@ -309,8 +379,8 @@ private void RemoveRemoteAudioTrack(string sid) { if (_audioObjects.TryGetValue(sid, out var obj)) { - obj.GetComponent()?.Stop(); - Destroy(obj); + if (obj != null) obj.GetComponent()?.Stop(); + if (obj != null) Destroy(obj); _audioObjects.Remove(sid); } @@ -451,7 +521,65 @@ private void UnpublishLocalCamera() private IEnumerator PublishLocalMicrophone() { - if (_audioObjects.ContainsKey(LocalAudioTrackName)) yield break; + if (_microphoneActive) yield break; + + if (usePlatformAudio && _platformAudio != null) + yield return PublishLocalMicrophonePlatform(); + else + yield return PublishLocalMicrophoneUnity(); + + if (_microphoneActive && _participantTiles.TryGetValue(_localId, out var tile)) + tile.SetMicMuted(false); + } + + private IEnumerator PublishLocalMicrophonePlatform() + { + Debug.Log("Publishing microphone using PlatformAudio (ADM)"); + + // Start recording (in case it was stopped by a previous mute). + // This turns on the privacy indicator on macOS/iOS. On Android this also + // awaits the RECORD_AUDIO runtime permission dialog if not yet granted. + if (_platformAudio != null) + { + yield return _platformAudio.StartRecording(); + } + + var audioOptions = new AudioProcessingOptions + { + EchoCancellation = echoCancellation, + NoiseSuppression = noiseSuppression, + AutoGainControl = autoGainControl, + PreferHardware = preferHardwareProcessing + }; + + _platformAudioSource = new PlatformAudioSource(_platformAudio, audioOptions); + _localAudioTrack = LocalAudioTrack.CreateAudioTrack(LocalAudioTrackName, _platformAudioSource, _room); + + var options = new TrackPublishOptions + { + AudioEncoding = new AudioEncoding { MaxBitrate = 64000 }, + Source = TrackSource.SourceMicrophone + }; + + var publish = _room.LocalParticipant.PublishTrack(_localAudioTrack, options); + yield return publish; + + if (publish.IsError) + { + Debug.LogError("Failed to publish microphone track"); + _platformAudioSource?.Dispose(); + _platformAudioSource = null; + _localAudioTrack = null; + yield break; + } + + _microphoneActive = true; + Debug.Log("Microphone published via PlatformAudio (AEC enabled)"); + } + + private IEnumerator PublishLocalMicrophoneUnity() + { + Debug.Log("Publishing microphone using Unity Microphone API"); Microphone.Start(null, true, 10, 44100); @@ -471,26 +599,50 @@ private IEnumerator PublishLocalMicrophone() var publish = _room.LocalParticipant.PublishTrack(_localAudioTrack, options); yield return publish; - if (publish.IsError) yield break; + if (publish.IsError) + { + Destroy(audioObject); + _localAudioTrack = null; + yield break; + } _microphoneActive = true; _audioObjects[LocalAudioTrackName] = audioObject; _localRtcAudioSource = rtcSource; rtcSource.Start(); - if (_participantTiles.TryGetValue(_localId, out var tile)) - tile.SetMicMuted(false); + Debug.Log("Microphone published via Unity Microphone API (no AEC)"); } private void UnpublishLocalMicrophone() { - DisposeSource(ref _localRtcAudioSource); + if (usePlatformAudio && _platformAudioSource != null) + { + try + { + _platformAudio?.StopRecording(); + } + catch (System.Exception e) + { + Debug.LogWarning($"Failed to stop recording: {e.Message}"); + } - if (_audioObjects.TryGetValue(LocalAudioTrackName, out var obj)) + _platformAudioSource.Dispose(); + _platformAudioSource = null; + } + else { - obj.GetComponent()?.Stop(); - Destroy(obj); - _audioObjects.Remove(LocalAudioTrackName); + DisposeSource(ref _localRtcAudioSource); + + if (_audioObjects.TryGetValue(LocalAudioTrackName, out var obj)) + { + if (obj != null) + { + obj.GetComponent()?.Stop(); + Destroy(obj); + } + _audioObjects.Remove(LocalAudioTrackName); + } } _room.LocalParticipant.UnpublishTrack(_localAudioTrack, false); @@ -564,6 +716,9 @@ private void CleanUpAllTracks() DisposeSource(ref _localRtcAudioSource); DisposeSource(ref _localRtcVideoSource); + _platformAudioSource?.Dispose(); + _platformAudioSource = null; + foreach (var obj in _audioObjects.Values) { if (obj == null) continue; @@ -607,4 +762,4 @@ private void CleanUpAllTracks() } #endregion -} \ No newline at end of file +} diff --git a/Samples~/Meet/Assets/Scenes/MeetApp.unity b/Samples~/Meet/Assets/Scenes/MeetApp.unity index 2205fdcf..91fc1e07 100644 --- a/Samples~/Meet/Assets/Scenes/MeetApp.unity +++ b/Samples~/Meet/Assets/Scenes/MeetApp.unity @@ -38,7 +38,6 @@ RenderSettings: m_ReflectionIntensity: 1 m_CustomReflection: {fileID: 0} m_Sun: {fileID: 0} - m_IndirectSpecularColor: {r: 0, g: 0, b: 0, a: 1} m_UseRadianceAmbientProbe: 0 --- !u!157 &3 LightmapSettings: @@ -903,6 +902,11 @@ MonoBehaviour: videoTrackParent: {fileID: 2128321498} participantTilePrefab: {fileID: 4315784896331113596, guid: bec493bbc3d574c07b5bbf8dd2be26b3, type: 3} frameRate: 30 + usePlatformAudio: 1 + echoCancellation: 1 + noiseSuppression: 1 + autoGainControl: 1 + preferHardwareProcessing: 1 --- !u!114 &1478206705 MonoBehaviour: m_ObjectHideFlags: 0 From 14fbb935aafddad0b6dedbe2a7babc3a6373bc7a Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Mon, 8 Jun 2026 15:45:21 +0200 Subject: [PATCH 2/4] Cleaned up no nop for mobile platforms --- Runtime/Scripts/PlatformAudio.cs | 78 ++++++------------- .../Assets/Editor/MeetManagerEditor.cs.meta | 11 +++ .../ProjectSettings/ProjectSettings.asset | 2 +- 3 files changed, 37 insertions(+), 54 deletions(-) create mode 100644 Samples~/Meet/Assets/Editor/MeetManagerEditor.cs.meta diff --git a/Runtime/Scripts/PlatformAudio.cs b/Runtime/Scripts/PlatformAudio.cs index 0dfe2215..1525eb42 100644 --- a/Runtime/Scripts/PlatformAudio.cs +++ b/Runtime/Scripts/PlatformAudio.cs @@ -1,11 +1,14 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Runtime.InteropServices; using LiveKit.Proto; using LiveKit.Internal; using LiveKit.Internal.FFIClients.Requests; +#if UNITY_IOS && !UNITY_EDITOR +using System.Runtime.InteropServices; +#endif + #if PLATFORM_ANDROID using UnityEngine.Android; #endif @@ -75,7 +78,9 @@ public struct AudioDevice /// foreach (var device in recording) /// Debug.Log($"Mic {device.Index}: {device.Name}"); /// - /// // Select devices + /// // Select devices (no-op on Android/iOS; routing there is governed by the OS). + /// // Use the uint overload for quick index-based selection, or the string overload + /// // with a GUID from GetDevices() to persist a stable selection across hot-plug. /// platformAudio.SetRecordingDevice(0); /// platformAudio.SetPlayoutDevice(0); /// @@ -190,37 +195,30 @@ public PlatformAudio() /// /// Sets the recording device (microphone) by index. /// - /// Call this before creating audio tracks to select which microphone to use. - /// Device indices are 0-based and must be less than RecordingDeviceCount. - /// - /// Note: Prefer SetRecordingDevice(string deviceId) for robust device selection across hot-plug events. + /// Convenience wrapper around that looks + /// up the GUID from . Prefer the GUID overload for code + /// that persists a selection — indices can shift when devices are added/removed. /// /// Device index from GetDevices().Recording /// - /// Thrown if the device index is invalid or the operation failed. + /// Thrown if the device index is out of range or the operation failed. /// public void SetRecordingDevice(uint index) { - // Look up the device GUID by index var (recording, _) = GetDevices(); if (index >= recording.Count) throw new InvalidOperationException($"Recording device index {index} out of range (max: {recording.Count - 1})"); - var deviceId = recording[(int)index].Guid; - - // Note: On Android, devices don't have GUIDs - they're identified by index only. - // Android also only reports a single "default" microphone because the system - // automatically selects the best input source based on the audio mode. - // If GUID is empty, we pass an empty string which triggers index-0 fallback in native code. - SetRecordingDevice(deviceId ?? ""); - Utils.Debug($"PlatformAudio: set recording device to index {index} (GUID: {(string.IsNullOrEmpty(deviceId) ? "" : deviceId)})"); + SetRecordingDevice(recording[(int)index].Guid ?? ""); } /// /// Sets the recording device (microphone) by device ID (GUID). /// - /// This is the preferred method for device selection as device IDs are stable - /// across device hot-plug events, unlike indices which can change. + /// On Android and iOS this is a no-op in the native ADM: input routing is + /// governed by the OS (AVAudioSession on iOS, AudioManager on Android) and + /// the call is acknowledged but ignored. The method is still safe to call, + /// and the response carries no error. /// /// Device ID/GUID from GetDevices().Recording[i].Guid /// @@ -228,15 +226,6 @@ public void SetRecordingDevice(uint index) /// public void SetRecordingDevice(string deviceId) { -#if UNITY_IOS && !UNITY_EDITOR - // iOS exposes only one logical WebRTC recording device, and AudioDeviceIOS::RecordingDeviceName - // returns -1, so set_recording_device_by_guid in native code can never match and always - // returns "Device not found". Mic input routing on iOS is governed by AVAudioSession - // (configured via IOSAudioSessionHelper.LiveKit_ConfigureAudioSessionForVoIP), not by - // WebRTC device selection, so skipping the FFI call here is the correct behavior. - Utils.Debug($"PlatformAudio: skipping SetRecordingDevice on iOS (deviceId '{deviceId}'); input is governed by AVAudioSession"); - return; -#else using var request = FFIBridge.Instance.NewRequest(); request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); request.request.DeviceId = deviceId; @@ -248,44 +237,35 @@ public void SetRecordingDevice(string deviceId) throw new InvalidOperationException($"Failed to set recording device: {res.SetRecordingDevice.Error}"); Utils.Debug($"PlatformAudio: set recording device to {deviceId}"); -#endif } /// /// Sets the playout device (speaker/headphones) by index. /// - /// Call this before connecting to select which speaker to use for remote audio. - /// Device indices are 0-based and must be less than PlayoutDeviceCount. - /// - /// Note: Prefer SetPlayoutDevice(string deviceId) for robust device selection across hot-plug events. + /// Convenience wrapper around that looks + /// up the GUID from . Prefer the GUID overload for code + /// that persists a selection — indices can shift when devices are added/removed. /// /// Device index from GetDevices().Playout /// - /// Thrown if the device index is invalid or the operation failed. + /// Thrown if the device index is out of range or the operation failed. /// public void SetPlayoutDevice(uint index) { - // Look up the device GUID by index var (_, playout) = GetDevices(); if (index >= playout.Count) throw new InvalidOperationException($"Playout device index {index} out of range (max: {playout.Count - 1})"); - var deviceId = playout[(int)index].Guid; - - // Note: On Android, devices don't have GUIDs - they're identified by index only. - // Android also only reports a single "default" device because audio routing - // (speaker vs earpiece vs Bluetooth) is handled by the system via AudioManager, - // not through WebRTC device selection. Use Android's AudioManager API to switch outputs. - // If GUID is empty, we pass an empty string which triggers index-0 fallback in native code. - SetPlayoutDevice(deviceId ?? ""); - Utils.Debug($"PlatformAudio: set playout device to index {index} (GUID: {(string.IsNullOrEmpty(deviceId) ? "" : deviceId)})"); + SetPlayoutDevice(playout[(int)index].Guid ?? ""); } /// /// Sets the playout device (speaker/headphones) by device ID (GUID). /// - /// This is the preferred method for device selection as device IDs are stable - /// across device hot-plug events, unlike indices which can change. + /// On Android and iOS this is a no-op in the native ADM: output routing is + /// governed by the OS (AVAudioSession on iOS, AudioManager on Android) and + /// the call is acknowledged but ignored. The method is still safe to call, + /// and the response carries no error. /// /// Device ID/GUID from GetDevices().Playout[i].Guid /// @@ -293,13 +273,6 @@ public void SetPlayoutDevice(uint index) /// public void SetPlayoutDevice(string deviceId) { -#if UNITY_IOS && !UNITY_EDITOR - // Same iOS limitation as SetRecordingDevice: AudioDeviceIOS::PlayoutDeviceName returns -1, - // so set_playout_device_by_guid never matches. Speaker vs earpiece vs Bluetooth routing on - // iOS is governed by AVAudioSession, not by WebRTC device selection. - Utils.Debug($"PlatformAudio: skipping SetPlayoutDevice on iOS (deviceId '{deviceId}'); output is governed by AVAudioSession"); - return; -#else using var request = FFIBridge.Instance.NewRequest(); request.request.PlatformAudioHandle = (ulong)Handle.DangerousGetHandle(); request.request.DeviceId = deviceId; @@ -311,7 +284,6 @@ public void SetPlayoutDevice(string deviceId) throw new InvalidOperationException($"Failed to set playout device: {res.SetPlayoutDevice.Error}"); Utils.Debug($"PlatformAudio: set playout device to {deviceId}"); -#endif } /// diff --git a/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs.meta b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs.meta new file mode 100644 index 00000000..31bc7e05 --- /dev/null +++ b/Samples~/Meet/Assets/Editor/MeetManagerEditor.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: d2c79b6f60f764430a2420ca32c2cd06 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Samples~/Meet/ProjectSettings/ProjectSettings.asset b/Samples~/Meet/ProjectSettings/ProjectSettings.asset index 0848f64e..ce10bd1a 100644 --- a/Samples~/Meet/ProjectSettings/ProjectSettings.asset +++ b/Samples~/Meet/ProjectSettings/ProjectSettings.asset @@ -171,7 +171,7 @@ PlayerSettings: tvOS: 0 overrideDefaultApplicationIdentifier: 1 AndroidBundleVersionCode: 1 - AndroidMinSdkVersion: 22 + AndroidMinSdkVersion: 25 AndroidTargetSdkVersion: 0 AndroidPreferredInstallLocation: 1 aotOptions: nimt-trampolines=1024 From 0f2ee148711a608a7c9668179698abb707571a0f Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:50:16 +0200 Subject: [PATCH 3/4] Set liblivekit_ffi.a for iOS to link only into iOS --- Runtime/Plugins/iOS/liblivekit_ffi.a.meta | 57 ++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/Runtime/Plugins/iOS/liblivekit_ffi.a.meta b/Runtime/Plugins/iOS/liblivekit_ffi.a.meta index 3534c748..c2afeeb9 100644 --- a/Runtime/Plugins/iOS/liblivekit_ffi.a.meta +++ b/Runtime/Plugins/iOS/liblivekit_ffi.a.meta @@ -11,17 +11,72 @@ PluginImporter: isExplicitlyReferenced: 0 validateReferences: 1 platformData: + - first: + : Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 1 + Exclude Linux64: 1 + Exclude OSXUniversal: 1 + Exclude WebGL: 1 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 0 + - first: + Android: Android + second: + enabled: 0 + settings: + AndroidSharedLibraryType: Executable + CPU: ARMv7 - first: Any: second: - enabled: 1 + enabled: 0 settings: {} - first: Editor: Editor second: enabled: 0 settings: + CPU: AnyCPU DefaultValueInitialized: true + OS: AnyOS + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: None + - first: + iPhone: iOS + second: + enabled: 1 + settings: + AddToEmbeddedBinaries: false + CPU: AnyCPU + CompileFlags: + FrameworkDependencies: userData: assetBundleName: assetBundleVariant: From 3856dce7fc06407e2066e41d4b8fb721d884d89d Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Wed, 10 Jun 2026 11:07:22 +0200 Subject: [PATCH 4/4] Add PlatformAudio unit and integration tests Mirror the C++ PlatformAudio test suites in the Unity Test Framework: - EditMode (pure-managed, always run): AudioProcessingOptions defaults, AudioDevice struct, PlatformAudioSource null-arg guard. - PlayMode unit (ADM-backed, no server): create source/track, custom options, enumerate + select-by-GUID, out-of-range index, start/stop recording. Skips via Assert.Ignore when no platform ADM is available. - PlayMode E2E (Category=E2E): publish/unpublish round-trip, multiple sources from one manager, and media flow verified via an InboundRtp audio stat (the batch-mode-friendly substitute for the C++ frame callback). Shared PlatformAudioTestHelper.TryCreateOrIgnore mirrors GTEST_SKIP for environments without an ADM. Co-Authored-By: Claude Opus 4.8 (1M context) --- Tests/EditMode/PlatformAudioTests.cs | 48 ++++ Tests/EditMode/PlatformAudioTests.cs.meta | 11 + .../PlayMode/PlatformAudioIntegrationTests.cs | 213 ++++++++++++++++++ .../PlatformAudioIntegrationTests.cs.meta | 11 + Tests/PlayMode/PlatformAudioTests.cs | 135 +++++++++++ Tests/PlayMode/PlatformAudioTests.cs.meta | 11 + .../PlayMode/Utils/PlatformAudioTestHelper.cs | 30 +++ .../Utils/PlatformAudioTestHelper.cs.meta | 11 + 8 files changed, 470 insertions(+) create mode 100644 Tests/EditMode/PlatformAudioTests.cs create mode 100644 Tests/EditMode/PlatformAudioTests.cs.meta create mode 100644 Tests/PlayMode/PlatformAudioIntegrationTests.cs create mode 100644 Tests/PlayMode/PlatformAudioIntegrationTests.cs.meta create mode 100644 Tests/PlayMode/PlatformAudioTests.cs create mode 100644 Tests/PlayMode/PlatformAudioTests.cs.meta create mode 100644 Tests/PlayMode/Utils/PlatformAudioTestHelper.cs create mode 100644 Tests/PlayMode/Utils/PlatformAudioTestHelper.cs.meta diff --git a/Tests/EditMode/PlatformAudioTests.cs b/Tests/EditMode/PlatformAudioTests.cs new file mode 100644 index 00000000..b7608ac7 --- /dev/null +++ b/Tests/EditMode/PlatformAudioTests.cs @@ -0,0 +1,48 @@ +using System; +using NUnit.Framework; + +namespace LiveKit.EditModeTests +{ + /// + /// Pure-managed unit tests for the PlatformAudio public surface. These never touch + /// the FFI, the platform Audio Device Module, or a server, so they always run + /// (including on headless CI runners with no audio subsystem). + /// + public class PlatformAudioTests + { + [Test] + public void AudioProcessingOptions_Default_EnablesProcessingAndHardware() + { + var options = AudioProcessingOptions.Default; + + Assert.IsTrue(options.EchoCancellation, "AEC should be enabled by default"); + Assert.IsTrue(options.NoiseSuppression, "NS should be enabled by default"); + Assert.IsTrue(options.AutoGainControl, "AGC should be enabled by default"); + // Unlike the C++ defaults (prefer_hardware == false), the Unity default prefers + // hardware processing (e.g. iOS VPIO) for lower latency. + Assert.IsTrue(options.PreferHardware, "Unity default prefers hardware processing"); + } + + [Test] + public void AudioDevice_StoresIndexNameGuid() + { + var device = new AudioDevice + { + Index = 1, + Name = "Microphone", + Guid = "device-guid" + }; + + Assert.AreEqual(1u, device.Index); + Assert.AreEqual("Microphone", device.Name); + Assert.AreEqual("device-guid", device.Guid); + } + + [Test] + public void PlatformAudioSource_NullPlatformAudio_Throws() + { + // The null guard runs before any FFI call, so this is safe without an ADM. + Assert.Throws(() => new PlatformAudioSource(null)); + } + } +} diff --git a/Tests/EditMode/PlatformAudioTests.cs.meta b/Tests/EditMode/PlatformAudioTests.cs.meta new file mode 100644 index 00000000..23f54f6f --- /dev/null +++ b/Tests/EditMode/PlatformAudioTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 741f9c7e3754f484188e93e27abe80c7 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/PlatformAudioIntegrationTests.cs b/Tests/PlayMode/PlatformAudioIntegrationTests.cs new file mode 100644 index 00000000..d521f3c7 --- /dev/null +++ b/Tests/PlayMode/PlatformAudioIntegrationTests.cs @@ -0,0 +1,213 @@ +using System.Collections; +using LiveKit.PlayModeTests.Utils; +using LiveKit.Proto; +using NUnit.Framework; +using UnityEngine; +using UnityEngine.TestTools; + +namespace LiveKit.PlayModeTests +{ + /// + /// End-to-end tests that publish a platform-ADM-backed audio track to a second + /// participant via a local LiveKit server. Skipped (Assert.Ignore) when no platform + /// ADM is available. Mirrors the C++ PlatformAudioIntegrationTest suite. + /// + public class PlatformAudioIntegrationTests + { + static TrackPublishOptions MicOptions() => + new TrackPublishOptions { Source = TrackSource.SourceMicrophone }; + + static (TestRoomContext.ConnectionOptions publisher, TestRoomContext.ConnectionOptions subscriber) TwoPeers() + { + var publisher = TestRoomContext.ConnectionOptions.Default; + publisher.Identity = "platform-audio-publisher"; + var subscriber = TestRoomContext.ConnectionOptions.Default; + subscriber.Identity = "platform-audio-subscriber"; + return (publisher, subscriber); + } + + [UnityTest, Category("E2E")] + public IEnumerator PublishPlatformAudioTrack_EndToEnd() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + var (publisher, subscriber) = TwoPeers(); + using var context = new TestRoomContext(new[] { publisher, subscriber }); + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + + var publisherRoom = context.Rooms[0]; + var subscriberRoom = context.Rooms[1]; + + using var source = new PlatformAudioSource(platformAudio); + const string trackName = "platform-mic"; + var track = LocalAudioTrack.CreateAudioTrack(trackName, source, publisherRoom); + + var subscribedExp = new Expectation(timeoutSeconds: 20f); + subscriberRoom.TrackSubscribed += (remoteTrack, publication, participant) => + { + if (remoteTrack.Kind == TrackKind.KindAudio && publication.Name == trackName) + subscribedExp.Fulfill(); + }; + + var pub = publisherRoom.LocalParticipant.PublishTrack(track, MicOptions()); + yield return pub; + Assert.IsFalse(pub.IsError, "publish failed"); + + yield return subscribedExp.Wait(); + Assert.IsNull(subscribedExp.Error, "receiver never subscribed to the platform audio track"); + } + + [UnityTest, Category("E2E")] + public IEnumerator UnpublishPlatformAudioTrack_Propagates() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + var (publisher, subscriber) = TwoPeers(); + using var context = new TestRoomContext(new[] { publisher, subscriber }); + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + + var publisherRoom = context.Rooms[0]; + var subscriberRoom = context.Rooms[1]; + + using var source = new PlatformAudioSource(platformAudio); + const string trackName = "platform-mic-unpublish"; + var track = LocalAudioTrack.CreateAudioTrack(trackName, source, publisherRoom); + + var subscribedExp = new Expectation(timeoutSeconds: 20f); + subscriberRoom.TrackSubscribed += (remoteTrack, publication, participant) => + { + if (remoteTrack.Kind == TrackKind.KindAudio && publication.Name == trackName) + subscribedExp.Fulfill(); + }; + + var removedExp = new Expectation(timeoutSeconds: 20f); + subscriberRoom.TrackUnsubscribed += (_, _, _) => removedExp.Fulfill(); + subscriberRoom.TrackUnpublished += (_, _) => removedExp.Fulfill(); + + var pub = publisherRoom.LocalParticipant.PublishTrack(track, MicOptions()); + yield return pub; + Assert.IsFalse(pub.IsError, "publish failed"); + + yield return subscribedExp.Wait(); + Assert.IsNull(subscribedExp.Error, "receiver never subscribed to the platform audio track"); + + var unpub = publisherRoom.LocalParticipant.UnpublishTrack(track, stopOnUnpublish: false); + yield return unpub; + Assert.IsFalse(unpub.IsError, "unpublish failed"); + + yield return removedExp.Wait(); + Assert.IsNull(removedExp.Error, "receiver never saw the platform audio track removed"); + } + + [UnityTest, Category("E2E")] + public IEnumerator MultipleSourcesFromOneManager_Publish() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + var (publisher, subscriber) = TwoPeers(); + using var context = new TestRoomContext(new[] { publisher, subscriber }); + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + + var publisherRoom = context.Rooms[0]; + var subscriberRoom = context.Rooms[1]; + + using var sourceA = new PlatformAudioSource(platformAudio); + using var sourceB = new PlatformAudioSource(platformAudio); + var handleA = (long)sourceA.Handle.DangerousGetHandle(); + var handleB = (long)sourceB.Handle.DangerousGetHandle(); + Assert.AreNotEqual(0, handleA, "source A handle should be non-zero"); + Assert.AreNotEqual(0, handleB, "source B handle should be non-zero"); + Assert.AreNotEqual(handleA, handleB, "sources should have distinct FFI handles"); + + const string nameA = "platform-mic-a"; + const string nameB = "platform-mic-b"; + var trackA = LocalAudioTrack.CreateAudioTrack(nameA, sourceA, publisherRoom); + var trackB = LocalAudioTrack.CreateAudioTrack(nameB, sourceB, publisherRoom); + + var subscribedA = new Expectation(timeoutSeconds: 20f); + var subscribedB = new Expectation(timeoutSeconds: 20f); + subscriberRoom.TrackSubscribed += (remoteTrack, publication, participant) => + { + if (publication.Name == nameA) subscribedA.Fulfill(); + else if (publication.Name == nameB) subscribedB.Fulfill(); + }; + + var pubA = publisherRoom.LocalParticipant.PublishTrack(trackA, MicOptions()); + yield return pubA; + Assert.IsFalse(pubA.IsError, "publish A failed"); + + var pubB = publisherRoom.LocalParticipant.PublishTrack(trackB, MicOptions()); + yield return pubB; + Assert.IsFalse(pubB.IsError, "publish B failed"); + + yield return subscribedA.Wait(); + Assert.IsNull(subscribedA.Error, "receiver did not subscribe to track A"); + yield return subscribedB.Wait(); + Assert.IsNull(subscribedB.Error, "receiver did not subscribe to track B"); + } + + [UnityTest, Category("E2E")] + public IEnumerator PlatformAudioFramesReachRemote_ViaStats() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + var (publisher, subscriber) = TwoPeers(); + using var context = new TestRoomContext(new[] { publisher, subscriber }); + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + + var publisherRoom = context.Rooms[0]; + var subscriberRoom = context.Rooms[1]; + + using var source = new PlatformAudioSource(platformAudio); + const string trackName = "platform-mic-frames"; + var track = LocalAudioTrack.CreateAudioTrack(trackName, source, publisherRoom); + + var subscribedExp = new Expectation(timeoutSeconds: 20f); + IRemoteTrack receivedRemoteTrack = null; + subscriberRoom.TrackSubscribed += (remoteTrack, publication, participant) => + { + if (remoteTrack.Kind == TrackKind.KindAudio && publication.Name == trackName) + { + receivedRemoteTrack = remoteTrack; + subscribedExp.Fulfill(); + } + }; + + var pub = publisherRoom.LocalParticipant.PublishTrack(track, MicOptions()); + yield return pub; + Assert.IsFalse(pub.IsError, "publish failed"); + + yield return subscribedExp.Wait(); + Assert.IsNull(subscribedExp.Error, "receiver never subscribed to the platform audio track"); + Assert.IsNotNull(receivedRemoteTrack); + + // Give the RTP pipeline a moment to collect inbound measurements. Unity's audio + // subsystem does not deliver decoded frames in -batchmode, so we assert media flow + // via an InboundRtp audio stat (the headless-friendly equivalent of the C++ frame + // callback test). + yield return new WaitForSeconds(1.5f); + + var statsInstruction = receivedRemoteTrack.GetStats(); + yield return statsInstruction; + Assert.IsFalse(statsInstruction.IsError, statsInstruction.Error); + Assert.IsNotNull(statsInstruction.Stats); + + RtcStats.Types.InboundRtp inboundRtp = null; + foreach (var stat in statsInstruction.Stats) + { + if (stat.StatsCase == RtcStats.StatsOneofCase.InboundRtp) + { + inboundRtp = stat.InboundRtp; + break; + } + } + + Assert.IsNotNull(inboundRtp, "expected an InboundRtp stat for the platform audio track"); + Assert.AreEqual("audio", inboundRtp.Stream.Kind); + } + } +} diff --git a/Tests/PlayMode/PlatformAudioIntegrationTests.cs.meta b/Tests/PlayMode/PlatformAudioIntegrationTests.cs.meta new file mode 100644 index 00000000..d1783643 --- /dev/null +++ b/Tests/PlayMode/PlatformAudioIntegrationTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 593a3c8811bd44133a623d717b634662 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/PlatformAudioTests.cs b/Tests/PlayMode/PlatformAudioTests.cs new file mode 100644 index 00000000..28b0ed7b --- /dev/null +++ b/Tests/PlayMode/PlatformAudioTests.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using LiveKit.PlayModeTests.Utils; +using LiveKit.Proto; +using NUnit.Framework; +using UnityEngine.TestTools; + +namespace LiveKit.PlayModeTests +{ + /// + /// ADM-backed unit tests for PlatformAudio / PlatformAudioSource. These exercise the + /// FFI but do NOT need a LiveKit server, so they are not tagged Category("E2E"). + /// Each test skips cleanly (Assert.Ignore) when no platform ADM is available. + /// + public class PlatformAudioTests + { + [UnityTest] + public IEnumerator CreateSourceAndTrack_WhenAvailable() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + using var source = new PlatformAudioSource(platformAudio); + Assert.AreNotEqual(0, (long)source.Handle.DangerousGetHandle(), "source handle should be non-zero"); + + // CreateAudioTrack only uses room?.LocalParticipant, so a null room is fine for + // constructing the track from the source handle. + var track = LocalAudioTrack.CreateAudioTrack("platform-mic", source, null); + Assert.IsNotNull(track); + Assert.AreEqual("platform-mic", track.Name); + Assert.AreEqual(TrackKind.KindAudio, track.Kind); + + yield break; + } + + [UnityTest] + public IEnumerator CreateSource_WithCustomOptions() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + var options = new AudioProcessingOptions + { + EchoCancellation = false, + NoiseSuppression = false, + AutoGainControl = false, + PreferHardware = true + }; + + using var source = new PlatformAudioSource(platformAudio, options); + Assert.AreNotEqual(0, (long)source.Handle.DangerousGetHandle(), "source handle should be non-zero"); + + yield break; + } + + [UnityTest] + public IEnumerator EnumerateDevices_AndSelectByGuid_DoesNotThrow() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + // Enumeration must succeed even on headless runners (it may return empty lists). + List recording = null; + List playout = null; + Assert.DoesNotThrow(() => (recording, playout) = platformAudio.GetDevices()); + + // Selecting a real device by its stable GUID must not throw. Headless runners + // usually report no devices, so guard the assertion behind availability. + var selectedAny = false; + foreach (var device in recording) + { + if (!string.IsNullOrEmpty(device.Guid)) + { + Assert.DoesNotThrow(() => platformAudio.SetRecordingDevice(device.Guid)); + selectedAny = true; + break; + } + } + foreach (var device in playout) + { + if (!string.IsNullOrEmpty(device.Guid)) + { + Assert.DoesNotThrow(() => platformAudio.SetPlayoutDevice(device.Guid)); + selectedAny = true; + break; + } + } + + if (!selectedAny) + Assert.Ignore("No audio devices with stable GUIDs available to select"); + + yield break; + } + + [UnityTest] + public IEnumerator SetRecordingDeviceByIndex_OutOfRange_Throws() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + // The uint convenience overload validates the index against the enumerated list + // before calling the GUID overload. 9999 is out of range regardless of device count. + Assert.Throws(() => platformAudio.SetRecordingDevice((uint)9999)); + + yield break; + } + + [UnityTest] + public IEnumerator StartThenStopRecording_DoesNotThrow() + { + using var platformAudio = PlatformAudioTestHelper.TryCreateOrIgnore(); + + // StartRecording is a coroutine (it awaits the Android permission dialog on-device). + // In the editor there is no PLATFORM_ANDROID branch, so it sends the FFI request + // synchronously. A headless ADM may legitimately fail to start recording; treat that + // as "ADM can't record here" and skip rather than fail. + var start = platformAudio.StartRecording(); + while (true) + { + bool moved; + try + { + moved = start.MoveNext(); + } + catch (InvalidOperationException e) + { + Assert.Ignore($"Recording unavailable in this environment: {e.Message}"); + yield break; + } + + if (!moved) break; + yield return start.Current; + } + + Assert.DoesNotThrow(() => platformAudio.StopRecording()); + } + } +} diff --git a/Tests/PlayMode/PlatformAudioTests.cs.meta b/Tests/PlayMode/PlatformAudioTests.cs.meta new file mode 100644 index 00000000..2c7b35d5 --- /dev/null +++ b/Tests/PlayMode/PlatformAudioTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: aaf36fb61b8234f4a8a97f9f0defc8df +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/Utils/PlatformAudioTestHelper.cs b/Tests/PlayMode/Utils/PlatformAudioTestHelper.cs new file mode 100644 index 00000000..fd12ad9e --- /dev/null +++ b/Tests/PlayMode/Utils/PlatformAudioTestHelper.cs @@ -0,0 +1,30 @@ +using System; +using NUnit.Framework; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// Shared helpers for PlatformAudio tests. PlatformAudio requires a working platform + /// Audio Device Module; constructing it throws on environments without one (e.g. a + /// headless CI runner with no audio subsystem). Mirrors the C++ suite's GTEST_SKIP. + /// + internal static class PlatformAudioTestHelper + { + /// + /// Creates a PlatformAudio, or skips the calling test (Assert.Ignore) when the + /// platform ADM is unavailable. Never returns null on the happy path. + /// + internal static PlatformAudio TryCreateOrIgnore() + { + try + { + return new PlatformAudio(); + } + catch (InvalidOperationException e) + { + Assert.Ignore($"PlatformAudio unavailable: {e.Message}"); + return null; // unreachable: Assert.Ignore throws + } + } + } +} diff --git a/Tests/PlayMode/Utils/PlatformAudioTestHelper.cs.meta b/Tests/PlayMode/Utils/PlatformAudioTestHelper.cs.meta new file mode 100644 index 00000000..60bb5254 --- /dev/null +++ b/Tests/PlayMode/Utils/PlatformAudioTestHelper.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6779fae67af4248ce86821f5d88cd4f8 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: