diff --git a/.changes/adaptive-stream-manual-quality-merge b/.changes/adaptive-stream-manual-quality-merge new file mode 100644 index 000000000..083a3def6 --- /dev/null +++ b/.changes/adaptive-stream-manual-quality-merge @@ -0,0 +1 @@ +patch type="improved" "Allow manual video quality selection with adaptive stream enabled" diff --git a/lib/src/publication/remote.dart b/lib/src/publication/remote.dart index f9e30bb5e..3d2ceb46d 100644 --- a/lib/src/publication/remote.dart +++ b/lib/src/publication/remote.dart @@ -33,6 +33,7 @@ import '../types/other.dart'; import '../types/video_dimensions.dart'; import '../utils.dart'; import 'track_publication.dart'; +import 'track_settings.dart'; /// Represents a track publication from a RemoteParticipant. Provides methods to /// control if we should subscribe to the track, and its quality (for video). @@ -41,18 +42,31 @@ class RemoteTrackPublication extends TrackPublication @override final RemoteParticipant participant; - bool get enabled => _enabled; - bool _enabled = true; + bool get enabled => _enabledPreference != TrackEnabledPreference.disabled; + + /// The user's explicit enable/disable request via [enable] / [disable]. + /// [TrackEnabledPreference.unset] means no explicit request, in which case + /// adaptive-stream visibility decides. An explicit request takes precedence + /// over visibility. + TrackEnabledPreference _enabledPreference = TrackEnabledPreference.unset; /// The current desired FPS of the track. This is only available for video tracks that support SVC. int? _fps; int get fps => _fps ?? 0; - VideoQuality? _videoQuality = VideoQuality.HIGH; - VideoQuality get videoQuality => _videoQuality ?? VideoQuality.HIGH; + // Manual settings (set by user via setVideoQuality / setVideoDimensions) + VideoSettings? _userPreference; + + // Adaptive stream state (set automatically by visibility observer) + VideoDimensions? _adaptiveStreamDimensions; + // Whether adaptive stream is active for this publication (room option on + + // remote video track). When false, view visibility never gates `disabled`. + bool _adaptiveStreamActive = false; + // Whether at least one view of this track is currently visible/sized. + bool _adaptiveStreamVisible = true; - VideoDimensions? _videoDimensions; - VideoDimensions? get videoDimensions => _videoDimensions; + VideoQuality get videoQuality => _userPreference?.quality ?? VideoQuality.HIGH; + VideoDimensions? get videoDimensions => _userPreference?.dimensions; /// The server may pause the track when they are bandwidth limitations and resume /// when there is more capacity. This property will be updated when the track is @@ -144,11 +158,6 @@ class RemoteTrackPublication extends TrackPublication final videoTrack = track as VideoTrack; - final settings = lk_rtc.UpdateTrackSettings( - trackSids: [sid], - disabled: true, - ); - // filter visible build contexts final viewSizes = videoTrack.viewKeys .map((e) => e.currentContext) @@ -161,15 +170,19 @@ class RemoteTrackPublication extends TrackPublication logger.finer('[Visibility] ${track?.sid} watching ${viewSizes.length} views...'); if (viewSizes.isNotEmpty) { - // compute largest size final largestSize = viewSizes.reduce((value, element) => maxOfSizes(value, element)); - - settings - ..disabled = false - ..width = largestSize.width.ceil() - ..height = largestSize.height.ceil(); + _adaptiveStreamDimensions = VideoDimensions( + largestSize.width.ceil(), + largestSize.height.ceil(), + ); + _adaptiveStreamVisible = true; + } else { + _adaptiveStreamDimensions = null; + _adaptiveStreamVisible = false; } + final settings = _buildTrackSettings(); + // Only send new settings to server if it changed if (settings != _lastSentTrackSettings) { _lastSentTrackSettings = settings; @@ -182,7 +195,13 @@ class RemoteTrackPublication extends TrackPublication } } - void _sendPendingTrackSettingsUpdateRequest(lk_rtc.UpdateTrackSettings settings) { + void _sendPendingTrackSettingsUpdateRequest(lk_rtc.UpdateTrackSettings _) { + // Re-build from the current state at fire time instead of replaying the + // snapshot captured when the debounce was scheduled. Otherwise a stale + // snapshot could be sent after newer state (e.g. a manual setVideoQuality) + // has already been applied, clobbering it. + final settings = _buildTrackSettings(); + _lastSentTrackSettings = settings; logger.fine('[Visibility] Sending... ${settings.toProto3Json()}'); participant.room.engine.signalClient.sendUpdateTrackSettings(settings); } @@ -200,6 +219,7 @@ class RemoteTrackPublication extends TrackPublication final roomOptions = participant.room.roomOptions; if (roomOptions.adaptiveStream && newValue is RemoteVideoTrack) { + _adaptiveStreamActive = true; // Start monitoring visibility _visibilityTimer = Timer.periodic( const Duration(milliseconds: 300), @@ -214,6 +234,8 @@ class RemoteTrackPublication extends TrackPublication _computeVideoViewVisibility(quick: true); } }; + } else { + _adaptiveStreamActive = false; } if (newValue != null) { @@ -229,7 +251,7 @@ class RemoteTrackPublication extends TrackPublication return didUpdate; } - bool _canUpdateManualVideoSettings() { + bool _isManualOperationAllowed() { if (kind != TrackType.VIDEO) { logger.warning('Manual video setting updates are only supported for video tracks'); return false; @@ -240,55 +262,59 @@ class RemoteTrackPublication extends TrackPublication return false; } - if (participant.room.roomOptions.adaptiveStream) { - logger.warning('Manual video setting update ignored because adaptive stream is enabled'); - return false; - } - return true; } + /// For tracks that support simulcasting, adjust subscribed quality. + /// + /// This indicates the highest quality the client can accept. If network + /// bandwidth does not allow, the server will automatically reduce quality to + /// optimize for uninterrupted video. + /// + /// When adaptive stream is active, this preference is merged client-side with + /// the dimensions computed from the visible views, and the smaller (more + /// conservative) of the two is sent to the server. Future setVideoQuality(VideoQuality newValue) async { - if (newValue == _videoQuality) return; - if (!_canUpdateManualVideoSettings()) return; - _videoQuality = newValue; - _videoDimensions = null; - sendUpdateTrackSettings(); + if (newValue == _userPreference?.quality) return; + if (!_isManualOperationAllowed()) return; + _userPreference = VideoSettings.quality(newValue); + _emitTrackUpdate(); } /// Set preferred video dimensions for this track. /// /// Server will choose the appropriate layer based on these dimensions. /// Will override previous calls to [setVideoQuality]. + /// + /// When adaptive stream is active, this preference is merged client-side with + /// the dimensions computed from the visible views, and the smaller (more + /// conservative) of the two is sent to the server. Future setVideoDimensions(VideoDimensions newValue) async { - if (newValue.width == _videoDimensions?.width && newValue.height == _videoDimensions?.height) { - return; - } - if (!_canUpdateManualVideoSettings()) return; - _videoDimensions = newValue; - _videoQuality = null; - sendUpdateTrackSettings(); + if (newValue == _userPreference?.dimensions) return; + if (!_isManualOperationAllowed()) return; + _userPreference = VideoSettings.dimensions(newValue); + _emitTrackUpdate(); } /// Set desired FPS, server will do its best to return FPS close to this. /// It's only supported for video codecs that support SVC currently. Future setVideoFPS(int newValue) async { if (newValue == _fps) return; - if (!_canUpdateManualVideoSettings()) return; + if (!_isManualOperationAllowed()) return; _fps = newValue; - sendUpdateTrackSettings(); + _emitTrackUpdate(); } Future enable() async { - if (_enabled) return; - _enabled = true; - sendUpdateTrackSettings(); + if (_enabledPreference == TrackEnabledPreference.enabled) return; + _enabledPreference = TrackEnabledPreference.enabled; + _emitTrackUpdate(); } Future disable() async { - if (!_enabled) return; - _enabled = false; - sendUpdateTrackSettings(); + if (_enabledPreference == TrackEnabledPreference.disabled) return; + _enabledPreference = TrackEnabledPreference.disabled; + _emitTrackUpdate(); } Future subscribe() async { @@ -333,26 +359,49 @@ class RemoteTrackPublication extends TrackPublication participant.room.engine.signalClient.sendUpdateSubscription(subscription); } - @internal - void sendUpdateTrackSettings() { - final settings = lk_rtc.UpdateTrackSettings( - trackSids: [sid], - disabled: !_enabled, + lk_rtc.UpdateTrackSettings _buildTrackSettings() { + final isDisabled = resolveDisabled( + enabledPreference: _enabledPreference, + adaptiveStreamActive: _adaptiveStreamActive, + adaptiveStreamVisible: _adaptiveStreamVisible, ); - if (kind == TrackType.VIDEO) { - if (_videoDimensions != null) { - settings.width = _videoDimensions!.width; - settings.height = _videoDimensions!.height; - } else if (_videoQuality != null) { - settings.quality = _videoQuality!.toPBType(); - } else { - settings.quality = VideoQuality.HIGH.toPBType(); - } - if (_fps != null) settings.fps = _fps!; + + if (kind != TrackType.VIDEO) { + return buildUpdateTrackSettings(sid: sid, disabled: isDisabled); } + + final resolved = resolveVideoSettings( + adaptiveStreamDimensions: _adaptiveStreamDimensions, + userPreference: _userPreference, + layerDimensionsForQuality: (quality) { + final pbQuality = quality.toPBType(); + final layer = latestInfo?.layers.where((l) => l.quality == pbQuality).firstOrNull; + if (layer == null) return null; + return VideoDimensions(layer.width, layer.height); + }, + ); + + return buildUpdateTrackSettings( + sid: sid, + disabled: isDisabled, + dimensions: resolved.dimensions, + quality: resolved.quality?.toPBType(), + fps: _fps, + ); + } + + void _emitTrackUpdate() { + // Cancel any pending debounced visibility update so its (now potentially + // stale) snapshot cannot fire after — and clobber — this immediate update. + _cancelPendingTrackSettingsUpdateRequest?.call(); + final settings = _buildTrackSettings(); + _lastSentTrackSettings = settings; participant.room.engine.signalClient.sendUpdateTrackSettings(settings); } + @internal + void sendUpdateTrackSettings() => _emitTrackUpdate(); + @internal // Update internal var and return true if changed Future updateSubscriptionAllowed(bool allowed) async { diff --git a/lib/src/publication/track_settings.dart b/lib/src/publication/track_settings.dart new file mode 100644 index 000000000..08a054cdc --- /dev/null +++ b/lib/src/publication/track_settings.dart @@ -0,0 +1,136 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:meta/meta.dart' show immutable, internal; + +import '../proto/livekit_models.pb.dart' as lk_models; +import '../proto/livekit_rtc.pb.dart' as lk_rtc; +import '../types/other.dart'; +import '../types/video_dimensions.dart'; + +/// Represents a video quality setting — either explicit dimensions or a +/// quality level (LOW/MEDIUM/HIGH), never both. +/// +/// Used for both user-requested settings and the resolved merge result. +@internal +@immutable +class VideoSettings { + final VideoDimensions? dimensions; + final VideoQuality? quality; + + const VideoSettings.dimensions(VideoDimensions this.dimensions) : quality = null; + + const VideoSettings.quality(VideoQuality this.quality) : dimensions = null; + + static const high = VideoSettings.quality(VideoQuality.HIGH); +} + +/// Merges adaptive stream dimensions with manual [VideoSettings], +/// always picking the more conservative (smaller) of the two. +/// +/// This matches the JS SDK's merge behavior in `emitTrackUpdate()`. +@internal +VideoSettings resolveVideoSettings({ + VideoDimensions? adaptiveStreamDimensions, + VideoSettings? userPreference, + VideoDimensions? Function(VideoQuality quality)? layerDimensionsForQuality, +}) { + VideoDimensions? minDimensions = userPreference?.dimensions; + + if (adaptiveStreamDimensions != null) { + if (minDimensions != null) { + // Use the smaller of adaptive vs manually requested dimensions + if (adaptiveStreamDimensions.area() < minDimensions.area()) { + minDimensions = adaptiveStreamDimensions; + } + } else if (userPreference?.quality != null) { + // Compare adaptive dimensions with the max quality layer dimensions + final maxQualityLayer = layerDimensionsForQuality?.call(userPreference!.quality!); + if (maxQualityLayer != null && adaptiveStreamDimensions.area() < maxQualityLayer.area()) { + minDimensions = adaptiveStreamDimensions; + } + } else { + minDimensions = adaptiveStreamDimensions; + } + } + + if (minDimensions != null) { + return VideoSettings.dimensions(minDimensions); + } else if (userPreference?.quality != null) { + return VideoSettings.quality(userPreference!.quality!); + } + return VideoSettings.high; +} + +/// The user's explicit enable/disable request for a track, used to decide +/// whether visibility may gate the track. Equivalent to the JS SDK's +/// `requestedDisabled` tri-state (`undefined` / `false` / `true`). +@internal +enum TrackEnabledPreference { + /// No explicit request; adaptive-stream visibility decides. + unset, + + /// User explicitly enabled; overrides visibility (track keeps streaming). + enabled, + + /// User explicitly disabled; overrides visibility (track stays off). + disabled, +} + +/// Resolves whether a subscribed track should be sent as `disabled`. +/// +/// Mirrors the JS SDK's `isEnabled` precedence: an explicit user +/// enable/disable always wins; otherwise, when adaptive stream is active for +/// the track, view visibility decides; otherwise the track is enabled. +@internal +bool resolveDisabled({ + required TrackEnabledPreference enabledPreference, + required bool adaptiveStreamActive, + required bool adaptiveStreamVisible, +}) { + switch (enabledPreference) { + case TrackEnabledPreference.enabled: + return false; + case TrackEnabledPreference.disabled: + return true; + case TrackEnabledPreference.unset: + return adaptiveStreamActive ? !adaptiveStreamVisible : false; + } +} + +/// Builds the [lk_rtc.UpdateTrackSettings] request sent to the server from the +/// already-resolved [disabled] flag and, for video, the resolved [dimensions] +/// or [quality] plus an optional [fps]. [dimensions] takes precedence over +/// [quality]; pass neither for non-video tracks. +@internal +lk_rtc.UpdateTrackSettings buildUpdateTrackSettings({ + required String sid, + required bool disabled, + VideoDimensions? dimensions, + lk_models.VideoQuality? quality, + int? fps, +}) { + final settings = lk_rtc.UpdateTrackSettings( + trackSids: [sid], + disabled: disabled, + ); + if (dimensions != null) { + settings.width = dimensions.width; + settings.height = dimensions.height; + } else if (quality != null) { + settings.quality = quality; + } + if (fps != null) settings.fps = fps; + return settings; +} diff --git a/test/publication/track_settings_test.dart b/test/publication/track_settings_test.dart new file mode 100644 index 000000000..e63f6a581 --- /dev/null +++ b/test/publication/track_settings_test.dart @@ -0,0 +1,265 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/src/proto/livekit_models.pb.dart' as lk_models; +import 'package:livekit_client/src/publication/track_settings.dart'; +import 'package:livekit_client/src/types/other.dart'; +import 'package:livekit_client/src/types/video_dimensions.dart'; + +/// Test helper: returns layer dimensions for a standard 3-layer SVC/simulcast track. +VideoDimensions? _testLayerDimensions(VideoQuality quality) { + return { + VideoQuality.LOW: VideoDimensions(320, 180), + VideoQuality.MEDIUM: VideoDimensions(640, 360), + VideoQuality.HIGH: VideoDimensions(1280, 720), + }[quality]; +} + +void main() { + group('resolveVideoSettings', () { + group('no adaptive stream', () { + test('defaults to HIGH quality when nothing set', () { + final r = resolveVideoSettings(); + expect(r.quality, VideoQuality.HIGH); + expect(r.dimensions, isNull); + }); + + test('uses preferred quality', () { + final r = resolveVideoSettings( + userPreference: VideoSettings.quality(VideoQuality.LOW), + ); + expect(r.quality, VideoQuality.LOW); + expect(r.dimensions, isNull); + }); + + test('uses preferred dimensions', () { + final r = resolveVideoSettings( + userPreference: VideoSettings.dimensions(VideoDimensions(800, 600)), + ); + expect(r.dimensions, VideoDimensions(800, 600)); + expect(r.quality, isNull); + }); + }); + + group('adaptive stream only', () { + test('uses adaptive stream dimensions', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(480, 270), + ); + expect(r.dimensions, VideoDimensions(480, 270)); + expect(r.quality, isNull); + }); + }); + + group('adaptive stream + preferred dimensions', () { + test('adaptive wins when smaller', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.dimensions(VideoDimensions(1280, 720)), + ); + expect(r.dimensions, VideoDimensions(320, 180)); + }); + + test('preferred wins when smaller', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(1920, 1080), + userPreference: VideoSettings.dimensions(VideoDimensions(640, 360)), + ); + expect(r.dimensions, VideoDimensions(640, 360)); + }); + + test('equal areas keep preferred', () { + // 720*320 == 640*360 == 230400. Distinct dimensions with equal area + // so the assertion can actually distinguish strict `<` (keep preferred) + // from `<=` (switch to adaptive), matching JS areDimensionsSmaller. + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(720, 320), + userPreference: VideoSettings.dimensions(VideoDimensions(640, 360)), + ); + expect(r.dimensions, VideoDimensions(640, 360)); + }); + + test('adaptive wins when area is one smaller', () { + // 639*360 = 230040 < 640*360 = 230400, so adaptive is strictly smaller. + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(639, 360), + userPreference: VideoSettings.dimensions(VideoDimensions(640, 360)), + ); + expect(r.dimensions, VideoDimensions(639, 360)); + }); + }); + + group('adaptive stream + preferred quality', () { + test('adaptive wins when smaller than quality layer', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.quality(VideoQuality.HIGH), + layerDimensionsForQuality: _testLayerDimensions, + ); + // adaptive 320*180 < HIGH 1280*720 → sends adaptive dimensions + expect(r.dimensions, VideoDimensions(320, 180)); + expect(r.quality, isNull); + }); + + test('quality wins when adaptive is larger than quality layer', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(1920, 1080), + userPreference: VideoSettings.quality(VideoQuality.LOW), + layerDimensionsForQuality: _testLayerDimensions, + ); + // adaptive 1920*1080 > LOW 320*180 → sends quality directly + expect(r.quality, VideoQuality.LOW); + expect(r.dimensions, isNull); + }); + + test('quality sent directly when no layer info available', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.quality(VideoQuality.LOW), + ); + expect(r.quality, VideoQuality.LOW); + expect(r.dimensions, isNull); + }); + + test('quality sent when layer lookup returns null', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.quality(VideoQuality.MEDIUM), + layerDimensionsForQuality: (_) => null, + ); + expect(r.quality, VideoQuality.MEDIUM); + expect(r.dimensions, isNull); + }); + }); + }); + + group('resolveDisabled', () { + test('not disabled by default (unset preference, adaptive inactive)', () { + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.unset, + adaptiveStreamActive: false, + adaptiveStreamVisible: true, + ), + isFalse, + ); + }); + + test('explicit disable wins even when visible', () { + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.disabled, + adaptiveStreamActive: true, + adaptiveStreamVisible: true, + ), + isTrue, + ); + }); + + test('explicit enable wins even when not visible (JS tri-state parity)', () { + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.enabled, + adaptiveStreamActive: true, + adaptiveStreamVisible: false, + ), + isFalse, + ); + }); + + test('adaptive visibility decides when preference is unset', () { + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.unset, + adaptiveStreamActive: true, + adaptiveStreamVisible: true, + ), + isFalse, + ); + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.unset, + adaptiveStreamActive: true, + adaptiveStreamVisible: false, + ), + isTrue, + ); + }); + + test('visibility is ignored when adaptive stream is inactive', () { + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.unset, + adaptiveStreamActive: false, + adaptiveStreamVisible: false, + ), + isFalse, + ); + }); + }); + + group('buildUpdateTrackSettings', () { + test('sets sid and disabled flag', () { + final s = buildUpdateTrackSettings(sid: 'TR_abc', disabled: true); + expect(s.trackSids, ['TR_abc']); + expect(s.disabled, isTrue); + expect(s.hasWidth(), isFalse); + expect(s.hasQuality(), isFalse); + expect(s.hasFps(), isFalse); + }); + + test('dimensions are written, quality is not', () { + final s = buildUpdateTrackSettings( + sid: 'TR_abc', + disabled: false, + dimensions: VideoDimensions(640, 360), + quality: lk_models.VideoQuality.LOW, + ); + expect(s.width, 640); + expect(s.height, 360); + expect(s.hasQuality(), isFalse); + }); + + test('quality is written when no dimensions', () { + final s = buildUpdateTrackSettings( + sid: 'TR_abc', + disabled: false, + quality: lk_models.VideoQuality.HIGH, + ); + expect(s.quality, lk_models.VideoQuality.HIGH); + expect(s.hasWidth(), isFalse); + expect(s.hasHeight(), isFalse); + }); + + test('fps is forwarded when set and omitted when null', () { + final withFps = buildUpdateTrackSettings( + sid: 'TR_abc', + disabled: false, + quality: lk_models.VideoQuality.HIGH, + fps: 30, + ); + expect(withFps.hasFps(), isTrue); + expect(withFps.fps, 30); + + final withoutFps = buildUpdateTrackSettings( + sid: 'TR_abc', + disabled: false, + quality: lk_models.VideoQuality.HIGH, + ); + expect(withoutFps.hasFps(), isFalse); + }); + }); +}