Platform audio for Unity

Runtime/Plugins/iOS/LiveKitAudioSession.mm

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <AVFoundation/AVFoundation.h>
+
+extern "C" {
+
+/// Configures the iOS audio session for VoIP/WebRTC use.
+/// This sets AVAudioSessionCategoryPlayAndRecord with VoiceChat mode,
+/// which enables the VPIO (Voice Processing IO) AudioUnit for:
+/// - Hardware echo cancellation (AEC)
+/// - Automatic gain control (AGC)
+/// - Noise suppression (NS)
+///
+/// Call this before creating PlatformAudio to ensure WebRTC can
+/// properly initialize the microphone and speaker.
+void LiveKit_ConfigureAudioSessionForVoIP() {
+    AVAudioSession* session = [AVAudioSession sharedInstance];
+    NSError* error = nil;
+
+    // Configure for VoIP with echo cancellation
+    BOOL success = [session setCategory:AVAudioSessionCategoryPlayAndRecord
+                                   mode:AVAudioSessionModeVoiceChat
+                                options:AVAudioSessionCategoryOptionDefaultToSpeaker |
+                                        AVAudioSessionCategoryOptionAllowBluetooth |
+                                        AVAudioSessionCategoryOptionAllowBluetoothA2DP
+                                  error:&error];
+
+    if (!success || error) {
+        NSLog(@"LiveKit: Failed to configure VoIP audio session: %@", error.localizedDescription);
+        return;
+    }
+
+    // Activate the audio session
+    success = [session setActive:YES error:&error];
+    if (!success || error) {
+        NSLog(@"LiveKit: Failed to activate audio session: %@", error.localizedDescription);
+        return;
+    }
+
+    NSLog(@"LiveKit: Audio session configured for VoIP (PlayAndRecord + VoiceChat mode)");
+}
+
+/// Restores the audio session to the default ambient category.
+/// Call this when PlatformAudio is disposed if you want to restore
+/// the original audio behavior.
+void LiveKit_RestoreDefaultAudioSession() {
+    AVAudioSession* session = [AVAudioSession sharedInstance];
+    NSError* error = nil;
+
+    [session setCategory:AVAudioSessionCategoryAmbient error:&error];
+    if (error) {
+        NSLog(@"LiveKit: Failed to restore default audio session: %@", error.localizedDescription);
+    }
+}
+
+}

Runtime/Scripts/Internal/FFIClient.cs

@@ -133,6 +133,48 @@ static void GetMainContext()
             Utils.Debug("Main Context created");
         }
 
+#if UNITY_ANDROID && !UNITY_EDITOR
+        /// <summary>
+        /// Get the Android application context as a raw jobject pointer.
+        /// This is passed to the native library for WebRTC audio initialization.
+        /// </summary>
+        /// <returns>IntPtr to the application context jobject, or IntPtr.Zero on failure</returns>
+        private static IntPtr GetAndroidApplicationContext()
+        {
+            try
+            {
+                // Get the Unity activity
+                using var unityPlayer = new AndroidJavaClass("com.unity3d.player.UnityPlayer");
+                using var currentActivity = unityPlayer.GetStatic<AndroidJavaObject>("currentActivity");
+
+                if (currentActivity == null)
+                {
+                    Utils.Error("FFIServer - Failed to get Unity currentActivity");
+                    return IntPtr.Zero;
+                }
+
+                // Get the application context from the activity
+                var applicationContext = currentActivity.Call<AndroidJavaObject>("getApplicationContext");
+
+                if (applicationContext == null)
+                {
+                    Utils.Error("FFIServer - Failed to get Android applicationContext");
+                    return IntPtr.Zero;
+                }
+
+                // Get the raw jobject pointer
+                // Note: We don't dispose the applicationContext here because we're passing
+                // the raw pointer to native code. The native code will create its own global ref.
+                return applicationContext.GetRawObject();
+            }
+            catch (System.Exception e)
+            {
+                Utils.Error($"FFIServer - Failed to get Android application context: {e.Message}");
+                return IntPtr.Zero;
+            }
+        }
+#endif
+
         private static void InitializeSdk()
         {
 #if NO_LIVEKIT_MODE
@@ -145,6 +187,34 @@ private static void InitializeSdk()
             const bool captureLogs = false;
 #endif
 
+#if UNITY_ANDROID && !UNITY_EDITOR
+            // Initialize Android WebRTC before the main FFI initialization.
+            // This initializes the JVM and ContextUtils (required for PlatformAudio).
+            try
+            {
+                IntPtr javaVmPtr = AndroidJNI.GetJavaVM();
+                IntPtr contextPtr = GetAndroidApplicationContext();
+
+                if (javaVmPtr != IntPtr.Zero && contextPtr != IntPtr.Zero)
+                {
+                    bool contextInitialized = NativeMethods.LiveKitInitializeAndroidContext(javaVmPtr, contextPtr);
+                    if (!contextInitialized)
+                    {
+                        // JVM init still succeeded; only PlatformAudio won't work
+                        Utils.Error("FFIServer - Android context init failed; PlatformAudio will not work");
+                    }
+                }
+                else
+                {
+                    Utils.Error("FFIServer - Failed to get JavaVM or context for Android init");
+                }
+            }
+            catch (System.Exception e)
+            {
+                Utils.Error($"FFIServer - Android initialization failed: {e.Message}");
+            }
+#endif
+
             var sdkVersion = PackageVersion.Get();
             NativeMethods.LiveKitInitialize(FFICallback, captureLogs, "unity", sdkVersion);
 

Runtime/Scripts/Internal/FFIClients/FfiRequestExtensions.cs

@@ -152,6 +152,25 @@ public static void Inject<T>(this FfiRequest ffiRequest, T request)
                 case RemixAndResampleRequest remixAndResampleRequest:
                     ffiRequest.RemixAndResample = remixAndResampleRequest;
                     break;
+                // PlatformAudio
+                case NewPlatformAudioRequest newPlatformAudioRequest:
+                    ffiRequest.NewPlatformAudio = newPlatformAudioRequest;
+                    break;
+                case GetAudioDevicesRequest getAudioDevicesRequest:
+                    ffiRequest.GetAudioDevices = getAudioDevicesRequest;
+                    break;
+                case SetRecordingDeviceRequest setRecordingDeviceRequest:
+                    ffiRequest.SetRecordingDevice = setRecordingDeviceRequest;
+                    break;
+                case SetPlayoutDeviceRequest setPlayoutDeviceRequest:
+                    ffiRequest.SetPlayoutDevice = setPlayoutDeviceRequest;
+                    break;
+                case StartRecordingRequest startRecordingRequest:
+                    ffiRequest.StartRecording = startRecordingRequest;
+                    break;
+                case StopRecordingRequest stopRecordingRequest:
+                    ffiRequest.StopRecording = stopRecordingRequest;
+                    break;
                 case LocalTrackMuteRequest localTrackMuteRequest:
                     ffiRequest.LocalTrackMute = localTrackMuteRequest;
                     break;

Runtime/Scripts/Internal/NativeMethods.cs

@@ -25,5 +25,19 @@ internal static class NativeMethods
 
         [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_initialize")]
         internal extern static FfiHandleId LiveKitInitialize(FFICallbackDelegate cb, bool captureLogs, string sdk, string sdkVersion);
+
+#if UNITY_ANDROID && !UNITY_EDITOR
+        /// <summary>
+        /// Initialize Android WebRTC with the application context.
+        /// This initializes both the JVM and ContextUtils, which is required for
+        /// Android audio (microphone/speaker) to work via PlatformAudio.
+        /// </summary>
+        /// <param name="javaVmPtr">Pointer to the JavaVM</param>
+        /// <param name="contextPtr">The Android application context (jobject)</param>
+        /// <returns>true if context initialization succeeded, false otherwise.
+        /// Note: JVM initialization happens regardless of return value.</returns>
+        [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_initialize_android_context")]
+        internal extern static bool LiveKitInitializeAndroidContext(IntPtr javaVmPtr, IntPtr contextPtr);
+#endif
     }
 }