diff --git a/.changes/adaptive-stream-pixel-density b/.changes/adaptive-stream-pixel-density new file mode 100644 index 000000000..94de7146f --- /dev/null +++ b/.changes/adaptive-stream-pixel-density @@ -0,0 +1 @@ +patch type="fixed" "Fix adaptive stream dimensions on high-density displays" diff --git a/lib/src/publication/remote.dart b/lib/src/publication/remote.dart index 3d2ceb46d..09c2a64e3 100644 --- a/lib/src/publication/remote.dart +++ b/lib/src/publication/remote.dart @@ -158,19 +158,30 @@ class RemoteTrackPublication extends TrackPublication final videoTrack = track as VideoTrack; - // filter visible build contexts - final viewSizes = videoTrack.viewKeys - .map((e) => e.currentContext) + // Filter visible build contexts and scale each view's logical size by its + // own pixel density, so the server is asked for physical-pixel dimensions + // (retina-aware). Each view's density is configured on its VideoTrackRenderer + // and resolved per-view; with AdaptiveStreamPixelDensity.auto the actual + // device pixel ratio is read from that view via MediaQuery. The largest + // resulting size across all of the track's views is requested. + final viewSizes = videoTrack.viewRegistrations + .map((registration) { + final context = registration.key.currentContext; + if (context == null) return null; + final renderBox = context.findRenderObject() as RenderBox?; + if (renderBox == null || !renderBox.hasSize) return null; + final density = registration.pixelDensity.resolve( + MediaQuery.maybeDevicePixelRatioOf(context) ?? 1.0, + ); + return renderBox.size * density; + }) .nonNulls - .map((e) => e.findRenderObject() as RenderBox?) - .nonNulls - .where((e) => e.hasSize) - .map((e) => e.size); + .toList(); logger.finer('[Visibility] ${track?.sid} watching ${viewSizes.length} views...'); if (viewSizes.isNotEmpty) { - final largestSize = viewSizes.reduce((value, element) => maxOfSizes(value, element)); + final largestSize = viewSizes.reduce(maxOfSizes); _adaptiveStreamDimensions = VideoDimensions( largestSize.width.ceil(), largestSize.height.ceil(), @@ -226,7 +237,7 @@ class RemoteTrackPublication extends TrackPublication (_) => _computeVideoViewVisibility(), ); - newValue.onVideoViewBuild = (_) { + newValue.onVideoViewBuild = () { logger.finer('[Visibility] VideoView did build'); if (_lastSentTrackSettings?.disabled == true) { // quick enable diff --git a/lib/src/track/local/local.dart b/lib/src/track/local/local.dart index e520b8712..6fb854a4b 100644 --- a/lib/src/track/local/local.dart +++ b/lib/src/track/local/local.dart @@ -38,24 +38,42 @@ import '../track.dart'; import 'audio.dart'; import 'video.dart'; +@internal +class VideoTrackViewRegistration { + /// The widget key used by adaptive stream to find this view's render context. + final GlobalKey key = GlobalKey(); + + /// The pixel density used to convert this view's logical size to physical + /// pixels when computing adaptive-stream dimensions. + AdaptiveStreamPixelDensity pixelDensity; + + VideoTrackViewRegistration({ + this.pixelDensity = AdaptiveStreamPixelDensity.auto, + }); +} + /// Used to group [LocalVideoTrack] and [RemoteVideoTrack]. mixin VideoTrack on Track { + /// The views attached to this track. Set by [VideoTrackRenderer] and read by + /// the visibility observer to compute adaptive-stream dimensions. @internal - final List viewKeys = []; + final List viewRegistrations = []; @internal - Function(Key)? onVideoViewBuild; + VoidCallback? onVideoViewBuild; @internal - GlobalKey addViewKey() { - final key = GlobalKey(); - viewKeys.add(key); - return key; + VideoTrackViewRegistration addViewRegistration({ + AdaptiveStreamPixelDensity pixelDensity = AdaptiveStreamPixelDensity.auto, + }) { + final registration = VideoTrackViewRegistration(pixelDensity: pixelDensity); + viewRegistrations.add(registration); + return registration; } @internal - void removeViewKey(GlobalKey key) { - viewKeys.remove(key); + void removeViewRegistration(VideoTrackViewRegistration registration) { + viewRegistrations.remove(registration); } } diff --git a/lib/src/types/other.dart b/lib/src/types/other.dart index d6d99d77a..08d2494ff 100644 --- a/lib/src/types/other.dart +++ b/lib/src/types/other.dart @@ -233,3 +233,50 @@ class ParticipantTrackPermission { this.allowedTrackSids, ); } + +/// Controls how a video view's logical size is scaled to physical pixels when +/// computing adaptive-stream dimensions. Mirrors the JS SDK's `pixelDensity` +/// option (`number | 'screen'`). +/// +/// Server layers are sized in physical pixels, so on high-density (retina) +/// displays the logical view size under-represents the pixels needed. Set on a +/// view via [VideoTrackRenderer]; the largest result is requested across all +/// views attached to the track. +class AdaptiveStreamPixelDensity { + /// Upper bound applied to the resolved density to keep bandwidth in check. + static const maxDensity = 3.0; + + /// Fixed multiplier, or `null` to use the view's device pixel ratio ([auto]). + final double? value; + + const AdaptiveStreamPixelDensity._(this.value); + + /// Use the view's actual device pixel ratio, read via `MediaQuery`. + /// Equivalent to the JS SDK's `'screen'` setting. Capped at [maxDensity]. + static const auto = AdaptiveStreamPixelDensity._(null); + + /// A positive fixed pixel-density multiplier (fractional allowed, e.g. `1.5`, + /// `2.0`, `2.75`). The effective value is capped at [maxDensity] (3x) when + /// resolved. + const AdaptiveStreamPixelDensity.fixed(double density) + : assert(density > 0, 'density must be positive'), + value = density; + + /// Resolves the effective multiplier, capped at [maxDensity]. For [auto], + /// falls back to the supplied [devicePixelRatio]. + double resolve(double devicePixelRatio) { + final density = value ?? devicePixelRatio; + if (density.isNaN || density <= 0) return 1.0; + return density > maxDensity ? maxDensity : density; + } + + @override + bool operator ==(Object other) => + identical(this, other) || (other is AdaptiveStreamPixelDensity && other.value == value); + + @override + int get hashCode => value.hashCode; + + @override + String toString() => value == null ? 'AdaptiveStreamPixelDensity.auto' : 'AdaptiveStreamPixelDensity.fixed($value)'; +} diff --git a/lib/src/widgets/video_track_renderer.dart b/lib/src/widgets/video_track_renderer.dart index 95b683bce..47c2cc20c 100644 --- a/lib/src/widgets/video_track_renderer.dart +++ b/lib/src/widgets/video_track_renderer.dart @@ -69,6 +69,12 @@ class VideoTrackRenderer extends StatefulWidget { /// wrap the video view in a Center widget (if [fit] is [VideoViewFit.contain]) final bool autoCenter; + /// Controls how this view's logical size is converted to the physical-pixel + /// dimensions requested from the server when adaptive stream is enabled. + /// Defaults to [AdaptiveStreamPixelDensity.auto] (the view's own device pixel + /// ratio), avoiding an under-sized layer on retina / high-density displays. + final AdaptiveStreamPixelDensity adaptiveStreamPixelDensity; + const VideoTrackRenderer( this.track, { this.fit = VideoViewFit.contain, @@ -77,6 +83,7 @@ class VideoTrackRenderer extends StatefulWidget { this.autoDisposeRenderer = true, this.cachedRenderer, this.autoCenter = true, + this.adaptiveStreamPixelDensity = AdaptiveStreamPixelDensity.auto, Key? key, }) : super(key: key); @@ -91,7 +98,7 @@ class _VideoTrackRendererState extends State { double? _aspectRatio; EventsListener? _listener; // Used to compute visibility information - late GlobalKey _internalKey; + late VideoTrackViewRegistration _viewRegistration; Future _initializeRenderer() async { if (lkPlatformIs(PlatformType.iOS) && widget.renderMode == VideoRenderMode.platformView) { @@ -142,7 +149,7 @@ class _VideoTrackRendererState extends State { if (widget.cachedRenderer != null) { _renderer = widget.cachedRenderer; } - _internalKey = widget.track.addViewKey(); + _viewRegistration = widget.track.addViewRegistration(pixelDensity: widget.adaptiveStreamPixelDensity); if (kIsWeb) { unawaited(() async { await _initializeRenderer(); @@ -154,7 +161,7 @@ class _VideoTrackRendererState extends State { @override void dispose() { - widget.track.removeViewKey(_internalKey); + widget.track.removeViewRegistration(_viewRegistration); unawaited(_listener?.dispose()); if (widget.autoDisposeRenderer) { disposeRenderer(); @@ -188,11 +195,13 @@ class _VideoTrackRendererState extends State { void didUpdateWidget(covariant VideoTrackRenderer oldWidget) { super.didUpdateWidget(oldWidget); if (widget.track != oldWidget.track) { - oldWidget.track.removeViewKey(_internalKey); - _internalKey = widget.track.addViewKey(); + oldWidget.track.removeViewRegistration(_viewRegistration); + _viewRegistration = widget.track.addViewRegistration(pixelDensity: widget.adaptiveStreamPixelDensity); unawaited(() async { await _attach(); }()); + } else if (widget.adaptiveStreamPixelDensity != oldWidget.adaptiveStreamPixelDensity) { + _viewRegistration.pixelDensity = widget.adaptiveStreamPixelDensity; } if ([BrowserType.safari, BrowserType.firefox].contains(lkBrowser()) && oldWidget.key != widget.key) { @@ -203,11 +212,11 @@ class _VideoTrackRendererState extends State { Widget _videoViewForWeb() => !_rendererReadyForWeb ? Container() : Builder( - key: _internalKey, + key: _viewRegistration.key, builder: (ctx) { // let it render before notifying build WidgetsBindingCompatible.instance?.addPostFrameCallback((timeStamp) { - widget.track.onVideoViewBuild?.call(_internalKey); + widget.track.onVideoViewBuild?.call(); }); return rtc.RTCVideoView( _renderer! as rtc.RTCVideoRenderer, @@ -244,11 +253,11 @@ class _VideoTrackRendererState extends State { if ((snapshot.hasData && _renderer != null) || (lkPlatformIs(PlatformType.iOS) && widget.renderMode == VideoRenderMode.platformView)) { return Builder( - key: _internalKey, + key: _viewRegistration.key, builder: (ctx) { // let it render before notifying build WidgetsBindingCompatible.instance?.addPostFrameCallback((timeStamp) { - widget.track.onVideoViewBuild?.call(_internalKey); + widget.track.onVideoViewBuild?.call(); }); if (!lkPlatformIsMobile() || widget.track is! LocalVideoTrack) { diff --git a/test/options/adaptive_stream_pixel_density_test.dart b/test/options/adaptive_stream_pixel_density_test.dart new file mode 100644 index 000000000..d2c248795 --- /dev/null +++ b/test/options/adaptive_stream_pixel_density_test.dart @@ -0,0 +1,76 @@ +// Copyright 2025 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/src/types/other.dart'; + +void main() { + group('AdaptiveStreamPixelDensity.resolve', () { + test('fixed densities ignore the device pixel ratio', () { + expect(const AdaptiveStreamPixelDensity.fixed(1.0).resolve(3.0), 1.0); + expect(const AdaptiveStreamPixelDensity.fixed(2.0).resolve(1.0), 2.0); + }); + + test('fractional fixed densities are supported', () { + expect(const AdaptiveStreamPixelDensity.fixed(1.5).resolve(3.0), 1.5); + expect(const AdaptiveStreamPixelDensity.fixed(2.75).resolve(1.0), 2.75); + }); + + test('auto falls back to the supplied device pixel ratio', () { + expect(AdaptiveStreamPixelDensity.auto.resolve(1.0), 1.0); + expect(AdaptiveStreamPixelDensity.auto.resolve(2.0), 2.0); + expect(AdaptiveStreamPixelDensity.auto.resolve(2.625), 2.625); + }); + + test('caps at 3x for both fixed and auto', () { + expect(const AdaptiveStreamPixelDensity.fixed(4.0).resolve(1.0), 3.0); + expect(AdaptiveStreamPixelDensity.auto.resolve(4.0), 3.0); + expect(AdaptiveStreamPixelDensity.maxDensity, 3.0); + }); + + test('falls back for invalid auto device pixel ratios', () { + expect(AdaptiveStreamPixelDensity.auto.resolve(0), 1.0); + expect(AdaptiveStreamPixelDensity.auto.resolve(-2.0), 1.0); + expect(AdaptiveStreamPixelDensity.auto.resolve(double.nan), 1.0); + }); + + test('fixed densities must be positive', () { + expect( + () => AdaptiveStreamPixelDensity.fixed(0), + throwsA(isA()), + ); + expect( + () => AdaptiveStreamPixelDensity.fixed(-1.0), + throwsA(isA()), + ); + }); + + test('value is null only for auto', () { + expect(AdaptiveStreamPixelDensity.auto.value, isNull); + expect(const AdaptiveStreamPixelDensity.fixed(1.5).value, 1.5); + }); + + test('equality is by value', () { + expect( + const AdaptiveStreamPixelDensity.fixed(2.0), + const AdaptiveStreamPixelDensity.fixed(2.0), + ); + expect( + const AdaptiveStreamPixelDensity.fixed(2.0) == AdaptiveStreamPixelDensity.auto, + isFalse, + ); + }); + }); +}