From f5081e1a03a567efbd4cec944acebc877ade15a0 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Fri, 12 Dec 2025 08:59:41 -0800 Subject: [PATCH 01/11] add structure for live session resumption --- .../firebase_ai/lib/src/live_api.dart | 164 ++++++++++++++++-- .../firebase_ai/lib/src/live_model.dart | 6 + 2 files changed, 158 insertions(+), 12 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart index 83c105b708f2..c23d98c268d6 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart @@ -77,21 +77,95 @@ class AudioTranscriptionConfig { Map toJson() => {}; } +/// Configures the sliding window context compression mechanism. +/// +/// The context window will be truncated by keeping only a suffix of it. +class SlidingWindow { + /// Creates a [SlidingWindow] instance. + /// + /// [targetTokens] (optional): The target number of tokens to keep in the + /// context window. + SlidingWindow({this.targetTokens}); + + /// The session reduction target, i.e., how many tokens we should keep. + final int? targetTokens; + // ignore: public_member_api_docs + Map toJson() => + {if (targetTokens case final targetTokens?) 'targetTokens': targetTokens}; +} + +/// Enables context window compression to manage the model's context window. +/// +/// This mechanism prevents the context from exceeding a given length. +class ContextWindowCompressionConfig { + /// Creates a [ContextWindowCompressionConfig] instance. + /// + /// [triggerTokens] (optional): The number of tokens that triggers the + /// compression mechanism. + /// [slidingWindow] (optional): The sliding window compression mechanism to + /// use. + ContextWindowCompressionConfig({this.triggerTokens, this.slidingWindow}); + + /// The number of tokens (before running a turn) that triggers the context + /// window compression. + final int? triggerTokens; + + /// The sliding window compression mechanism. + final SlidingWindow? slidingWindow; + // ignore: public_member_api_docs + Map toJson() => { + if (triggerTokens case final triggerTokens?) + 'triggerTokens': triggerTokens, + if (slidingWindow case final slidingWindow?) + 'slidingWindow': slidingWindow.toJson() + }; +} + +/// Configuration for the session resumption mechanism. +/// +/// When included in the session setup, the server will send +/// [SessionResumptionUpdate] messages. +class SessionResumptionConfig { + /// Creates a [SessionResumptionConfig] instance. + /// + /// [handle] (optional): The session resumption handle of the previous session + /// to restore. + /// [transparent] (optional): If set, requests the server to send updates with + /// the message index of the last client message included in the session + /// state. + SessionResumptionConfig({this.handle, this.transparent}); + + /// The session resumption handle of the previous session to restore. + /// + /// If not present, a new session will be started. + final String? handle; + + /// If set, requests the server to send updates with the message index of the + /// last client message included in the session state. + final bool? transparent; + // ignore: public_member_api_docs + Map toJson() => { + if (handle case final handle?) 'handle': handle, + if (transparent case final transparent?) 'transparent': transparent, + }; +} + /// Configures live generation settings. final class LiveGenerationConfig extends BaseGenerationConfig { // ignore: public_member_api_docs - LiveGenerationConfig({ - this.speechConfig, - this.inputAudioTranscription, - this.outputAudioTranscription, - super.responseModalities, - super.maxOutputTokens, - super.temperature, - super.topP, - super.topK, - super.presencePenalty, - super.frequencyPenalty, - }); + LiveGenerationConfig( + {this.speechConfig, + this.inputAudioTranscription, + this.outputAudioTranscription, + this.sessionResumption, + this.contextWindowCompression, + super.responseModalities, + super.maxOutputTokens, + super.temperature, + super.topP, + super.topK, + super.presencePenalty, + super.frequencyPenalty}); /// The speech configuration. final SpeechConfig? speechConfig; @@ -103,6 +177,12 @@ final class LiveGenerationConfig extends BaseGenerationConfig { /// the output audio. final AudioTranscriptionConfig? outputAudioTranscription; + /// The session resumption configuration. + final SessionResumptionConfig? sessionResumption; + + /// The context window compression configuration. + final ContextWindowCompressionConfig? contextWindowCompression; + @override Map toJson() => { ...super.toJson(), @@ -209,6 +289,47 @@ class LiveServerToolCallCancellation implements LiveServerMessage { final List? functionIds; } +/// A server message indicating that the server will not be able to service the +/// client soon. +class GoAway implements LiveServerMessage { + /// Creates a [GoAway] instance. + /// + /// [timeLeft] (optional): The remaining time before the connection will be + /// terminated. + GoAway({this.timeLeft}); + + /// The remaining time before the connection will be terminated as ABORTED. + final Duration? timeLeft; +} + +/// An update of the session resumption state. +/// +/// This message is only sent if [SessionResumptionConfig] was set in the +/// session setup. +class SessionResumptionUpdate implements LiveServerMessage { + /// Creates a [SessionResumptionUpdate] instance. + /// + /// [newHandle] (optional): The new handle that represents the state that can + /// be resumed. + /// [resumable] (optional): Indicates if the session can be resumed at this + /// point. + /// [lastConsumedClientMessageIndex] (optional): The index of the last client + /// message that is included in the state represented by this update. + SessionResumptionUpdate( + {this.newHandle, this.resumable, this.lastConsumedClientMessageIndex}); + + /// The new handle that represents the state that can be resumed. Empty if + /// `resumable` is false. + final String? newHandle; + + /// Indicates if the session can be resumed at this point. + final bool? resumable; + + /// The index of the last client message that is included in the state + /// represented by this update. + final int? lastConsumedClientMessageIndex; +} + /// A single response chunk received during a live content generation. /// /// It can contain generated content, function calls to be executed, or @@ -435,6 +556,25 @@ LiveServerMessage _parseServerMessage(Object jsonObject) { return LiveServerToolCallCancellation(functionIds: toolCancelJson['ids']); } else if (json.containsKey('setupComplete')) { return LiveServerSetupComplete(); + } else if (json.containsKey('goAway')) { + final goAwayJson = json['goAway'] as Map; + Duration? timeLeft; + if (goAwayJson.containsKey('timeLeft')) { + final timeLeftString = goAwayJson['timeLeft'] as String; + final seconds = + int.parse(timeLeftString.substring(0, timeLeftString.length - 1)); + timeLeft = Duration(seconds: seconds); + } + return GoAway(timeLeft: timeLeft); + } else if (json.containsKey('sessionResumptionUpdate')) { + final sessionResumptionUpdateJson = + json['sessionResumptionUpdate'] as Map; + return SessionResumptionUpdate( + newHandle: sessionResumptionUpdateJson['newHandle'] as String?, + resumable: sessionResumptionUpdateJson['resumable'] as bool?, + lastConsumedClientMessageIndex: + sessionResumptionUpdateJson['lastConsumedClientMessageIndex'] as int?, + ); } else { throw unhandledFormat('LiveServerMessage', json); } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart index 3414b1376af1..c0a11c1536aa 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart @@ -112,6 +112,12 @@ final class LiveGenerativeModel extends BaseModel { if (_liveGenerationConfig.outputAudioTranscription != null) 'output_audio_transcription': _liveGenerationConfig.outputAudioTranscription!.toJson(), + if (_liveGenerationConfig.sessionResumption + case final sessionResumption?) + 'sessionResumption': sessionResumption.toJson(), + if (_liveGenerationConfig.contextWindowCompression + case final contextWindowCompression?) + 'contextWindowCompression': contextWindowCompression.toJson() }, } }; From f4b7b4712f4c5c8ca9623e2c2389aca977e92ca3 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Fri, 12 Dec 2025 09:35:29 -0800 Subject: [PATCH 02/11] session resumption config should be session based --- packages/firebase_ai/firebase_ai/lib/src/live_api.dart | 4 ---- packages/firebase_ai/firebase_ai/lib/src/live_model.dart | 8 ++++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart index c23d98c268d6..a4aa50af476e 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart @@ -157,7 +157,6 @@ final class LiveGenerationConfig extends BaseGenerationConfig { {this.speechConfig, this.inputAudioTranscription, this.outputAudioTranscription, - this.sessionResumption, this.contextWindowCompression, super.responseModalities, super.maxOutputTokens, @@ -177,9 +176,6 @@ final class LiveGenerationConfig extends BaseGenerationConfig { /// the output audio. final AudioTranscriptionConfig? outputAudioTranscription; - /// The session resumption configuration. - final SessionResumptionConfig? sessionResumption; - /// The context window compression configuration. final ContextWindowCompressionConfig? contextWindowCompression; diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart index c0a11c1536aa..f1d2e8234f1a 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart @@ -93,7 +93,8 @@ final class LiveGenerativeModel extends BaseModel { /// /// Returns a [Future] that resolves to an [LiveSession] object upon successful /// connection. - Future connect() async { + Future connect( + {SessionResumptionConfig? sessionResumption}) async { final uri = _useVertexBackend ? _vertexAIUri() : _googleAIUri(); final modelString = _useVertexBackend ? _vertexAIModelString() : _googleAIModelString(); @@ -104,6 +105,8 @@ final class LiveGenerativeModel extends BaseModel { if (_systemInstruction != null) 'system_instruction': _systemInstruction.toJson(), if (_tools != null) 'tools': _tools.map((t) => t.toJson()).toList(), + if (sessionResumption != null) + 'session_resumption': sessionResumption.toJson(), if (_liveGenerationConfig != null) ...{ 'generation_config': _liveGenerationConfig.toJson(), if (_liveGenerationConfig.inputAudioTranscription != null) @@ -112,9 +115,6 @@ final class LiveGenerativeModel extends BaseModel { if (_liveGenerationConfig.outputAudioTranscription != null) 'output_audio_transcription': _liveGenerationConfig.outputAudioTranscription!.toJson(), - if (_liveGenerationConfig.sessionResumption - case final sessionResumption?) - 'sessionResumption': sessionResumption.toJson(), if (_liveGenerationConfig.contextWindowCompression case final contextWindowCompression?) 'contextWindowCompression': contextWindowCompression.toJson() From 0dc39e582522253a063e418eaaa7f05082ce65d1 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Mon, 5 Jan 2026 12:58:05 -0800 Subject: [PATCH 03/11] init setup --- .../example/lib/pages/bidi_page.dart | 44 ++++++++++++++++--- .../firebase_ai/lib/firebase_ai.dart | 7 ++- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 70c7007cdd5b..234721409b2f 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -51,12 +51,12 @@ class _BidiPageState extends State { final List _messages = []; bool _loading = false; bool _sessionOpening = false; - bool _recording = false; - late LiveGenerativeModel _liveModel; - late LiveSession _session; - StreamController _stopController = StreamController(); - final AudioOutput _audioOutput = AudioOutput(); - final AudioInput _audioInput = AudioInput(); + bool _recording = false; + late LiveGenerativeModel _liveModel; + late LiveSession _session; + String? _sessionId; + StreamController _stopController = StreamController(); + final AudioOutput _audioOutput = AudioOutput(); final AudioInput _audioInput = AudioInput(); int? _inputTranscriptionMessageIndex; int? _outputTranscriptionMessageIndex; @@ -253,7 +253,33 @@ class _BidiPageState extends State { }); if (!_sessionOpening) { - _session = await _liveModel.connect(); + try { + if (_sessionId != null) { + _session = await _liveModel.connect(sessionId: _sessionId!); + if (context.mounted) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar( + content: Text('Resumed session with ID: $_sessionId'), + ), + ); + } + } else { + _session = await _liveModel.connect(); + } + _sessionId = _session.sessionId; + } on Exception catch (e) { + developer.log('Error setting up session: $e, starting a new one.'); + _session = await _liveModel.connect(); + _sessionId = _session.sessionId; + if (context.mounted) { + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar( + content: Text('Previous session expired, started a new one.'), + ), + ); + } + } + _sessionOpening = true; _stopController = StreamController(); unawaited( @@ -405,6 +431,10 @@ class _BidiPageState extends State { } } else if (message is LiveServerToolCall && message.functionCalls != null) { await _handleLiveServerToolCall(message); + } else if (message is GoAway) { + developer.log('GoAway message received: $response'); + } else if (message is SessionResumptionUpdate) { + developer.log('SessionResumptionUpdate message received: $response'); } } diff --git a/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart b/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart index 4be52604e600..2d7ac277c13f 100644 --- a/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart +++ b/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart @@ -96,14 +96,17 @@ export 'src/imagen/imagen_reference.dart' ImagenControlReference; export 'src/live_api.dart' show - LiveGenerationConfig, - SpeechConfig, AudioTranscriptionConfig, + GoAway, + LiveGenerationConfig, LiveServerMessage, LiveServerContent, LiveServerToolCall, LiveServerToolCallCancellation, LiveServerResponse, + SessionResumptionConfig, + SessionResumptionUpdate, + SpeechConfig, Transcription; export 'src/live_session.dart' show LiveSession; export 'src/schema.dart' show Schema, SchemaType; From 7f52d8a6b3c84bd7c20085e58c1d004a38692ff9 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Thu, 29 Jan 2026 11:29:27 -0800 Subject: [PATCH 04/11] a bit session management fix --- .../example/lib/pages/bidi_page.dart | 19 ++++++++++--------- .../firebase_ai/lib/src/live_model.dart | 2 +- .../firebase_ai/lib/src/live_session.dart | 2 ++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 234721409b2f..c6dfcb14ffc2 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -51,12 +51,13 @@ class _BidiPageState extends State { final List _messages = []; bool _loading = false; bool _sessionOpening = false; - bool _recording = false; - late LiveGenerativeModel _liveModel; - late LiveSession _session; - String? _sessionId; - StreamController _stopController = StreamController(); - final AudioOutput _audioOutput = AudioOutput(); final AudioInput _audioInput = AudioInput(); + bool _recording = false; + late LiveGenerativeModel _liveModel; + late LiveSession _session; + String? _sessionId; + StreamController _stopController = StreamController(); + final AudioOutput _audioOutput = AudioOutput(); + final AudioInput _audioInput = AudioInput(); int? _inputTranscriptionMessageIndex; int? _outputTranscriptionMessageIndex; @@ -431,10 +432,10 @@ class _BidiPageState extends State { } } else if (message is LiveServerToolCall && message.functionCalls != null) { await _handleLiveServerToolCall(message); - } else if (message is GoAway) { - developer.log('GoAway message received: $response'); + } else if (message is GoingAwayNotice) { + developer.log('GoAway message received: $message'); } else if (message is SessionResumptionUpdate) { - developer.log('SessionResumptionUpdate message received: $response'); + developer.log('SessionResumptionUpdate message received: $message'); } } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart index f1d2e8234f1a..bb0fb6b4f1ef 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart @@ -94,7 +94,7 @@ final class LiveGenerativeModel extends BaseModel { /// Returns a [Future] that resolves to an [LiveSession] object upon successful /// connection. Future connect( - {SessionResumptionConfig? sessionResumption}) async { + {SessionResumptionConfig? sessionResumption, String? sessionId}) async { final uri = _useVertexBackend ? _vertexAIUri() : _googleAIUri(); final modelString = _useVertexBackend ? _vertexAIModelString() : _googleAIModelString(); diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index f136a644d03d..2f90c1c51dcf 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -48,6 +48,8 @@ class LiveSession { final _messageController = StreamController.broadcast(); late StreamSubscription _wsSubscription; + String? sessionId; + /// Sends content to the server. /// /// [input] (optional): The content to send. From a944f564362be4d7dc5a1567034a04d6b3a359ba Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Wed, 4 Feb 2026 08:58:27 -0800 Subject: [PATCH 05/11] some update for session management --- .../example/lib/pages/bidi_page.dart | 24 ++++++++++++------- .../firebase_ai/lib/src/live_model.dart | 2 +- .../firebase_ai/lib/src/live_session.dart | 8 ++++--- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index c6dfcb14ffc2..891c775bca86 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -54,7 +54,8 @@ class _BidiPageState extends State { bool _recording = false; late LiveGenerativeModel _liveModel; late LiveSession _session; - String? _sessionId; + String? _activeSessionHandle; + int? _lastProcessedIndex; StreamController _stopController = StreamController(); final AudioOutput _audioOutput = AudioOutput(); final AudioInput _audioInput = AudioInput(); @@ -255,23 +256,26 @@ class _BidiPageState extends State { if (!_sessionOpening) { try { - if (_sessionId != null) { - _session = await _liveModel.connect(sessionId: _sessionId!); + if (_activeSessionHandle != null) { + _session = await _liveModel.connect( + sessionResumption: SessionResumptionConfig( + handle: _activeSessionHandle, transparent: true), + ); if (context.mounted) { ScaffoldMessenger.of(context).showSnackBar( SnackBar( - content: Text('Resumed session with ID: $_sessionId'), + content: Text( + 'Resumed session with session handle: $_activeSessionHandle'), ), ); } } else { _session = await _liveModel.connect(); } - _sessionId = _session.sessionId; } on Exception catch (e) { developer.log('Error setting up session: $e, starting a new one.'); _session = await _liveModel.connect(); - _sessionId = _session.sessionId; + if (context.mounted) { ScaffoldMessenger.of(context).showSnackBar( const SnackBar( @@ -433,8 +437,12 @@ class _BidiPageState extends State { } else if (message is LiveServerToolCall && message.functionCalls != null) { await _handleLiveServerToolCall(message); } else if (message is GoingAwayNotice) { - developer.log('GoAway message received: $message'); - } else if (message is SessionResumptionUpdate) { + developer.log('GoAway message received, time left: ${message.timeLeft}'); + } else if (message is SessionResumptionUpdate && + message.resumable != null && + message.resumable!) { + _activeSessionHandle = message.newHandle; + _lastProcessedIndex = message.lastConsumedClientMessageIndex; developer.log('SessionResumptionUpdate message received: $message'); } } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart index bb0fb6b4f1ef..f1d2e8234f1a 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart @@ -94,7 +94,7 @@ final class LiveGenerativeModel extends BaseModel { /// Returns a [Future] that resolves to an [LiveSession] object upon successful /// connection. Future connect( - {SessionResumptionConfig? sessionResumption, String? sessionId}) async { + {SessionResumptionConfig? sessionResumption}) async { final uri = _useVertexBackend ? _vertexAIUri() : _googleAIUri(); final modelString = _useVertexBackend ? _vertexAIModelString() : _googleAIModelString(); diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index 2f90c1c51dcf..a1b6c016c00b 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -48,8 +48,6 @@ class LiveSession { final _messageController = StreamController.broadcast(); late StreamSubscription _wsSubscription; - String? sessionId; - /// Sends content to the server. /// /// [input] (optional): The content to send. @@ -168,7 +166,11 @@ class LiveSession { await for (final result in _messageController.stream) { yield result; - if (result case LiveServerContent(turnComplete: true)) { + final message = result.message; + + if (message is LiveServerContent && + message.turnComplete != null && + message.turnComplete!) { break; // Exit the loop when the turn is complete } } From cf4f29d6ce03e51d809e3b17457f46feee7b52b8 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Wed, 4 Feb 2026 21:49:47 -0800 Subject: [PATCH 06/11] refactor live_session connect and resume api --- .../example/lib/pages/bidi_page.dart | 15 +++ .../firebase_ai/lib/src/base_model.dart | 3 - .../firebase_ai/lib/src/live_model.dart | 41 ++----- .../firebase_ai/lib/src/live_session.dart | 108 +++++++++++++++++- 4 files changed, 129 insertions(+), 38 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 891c775bca86..b1343cd82968 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -438,6 +438,21 @@ class _BidiPageState extends State { await _handleLiveServerToolCall(message); } else if (message is GoingAwayNotice) { developer.log('GoAway message received, time left: ${message.timeLeft}'); + if (_activeSessionHandle != null) { + await _session.resumeSession( + sessionResumption: SessionResumptionConfig( + handle: _activeSessionHandle, + transparent: true, + ), + ); + if (mounted) { + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar( + content: Text('Session is going away, resuming...'), + ), + ); + } + } } else if (message is SessionResumptionUpdate && message.resumable != null && message.resumable!) { diff --git a/packages/firebase_ai/firebase_ai/lib/src/base_model.dart b/packages/firebase_ai/firebase_ai/lib/src/base_model.dart index 01ac7eb834b3..ea95fdb09918 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/base_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/base_model.dart @@ -18,11 +18,8 @@ import 'dart:convert'; import 'package:firebase_app_check/firebase_app_check.dart'; import 'package:firebase_auth/firebase_auth.dart'; import 'package:firebase_core/firebase_core.dart'; -import 'package:flutter/foundation.dart'; import 'package:http/http.dart' as http; import 'package:meta/meta.dart'; -import 'package:web_socket_channel/io.dart'; -import 'package:web_socket_channel/web_socket_channel.dart'; import 'api.dart'; import 'client.dart'; diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart index f1d2e8234f1a..8d6bfd478965 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart @@ -99,30 +99,6 @@ final class LiveGenerativeModel extends BaseModel { final modelString = _useVertexBackend ? _vertexAIModelString() : _googleAIModelString(); - final setupJson = { - 'setup': { - 'model': modelString, - if (_systemInstruction != null) - 'system_instruction': _systemInstruction.toJson(), - if (_tools != null) 'tools': _tools.map((t) => t.toJson()).toList(), - if (sessionResumption != null) - 'session_resumption': sessionResumption.toJson(), - if (_liveGenerationConfig != null) ...{ - 'generation_config': _liveGenerationConfig.toJson(), - if (_liveGenerationConfig.inputAudioTranscription != null) - 'input_audio_transcription': - _liveGenerationConfig.inputAudioTranscription!.toJson(), - if (_liveGenerationConfig.outputAudioTranscription != null) - 'output_audio_transcription': - _liveGenerationConfig.outputAudioTranscription!.toJson(), - if (_liveGenerationConfig.contextWindowCompression - case final contextWindowCompression?) - 'contextWindowCompression': contextWindowCompression.toJson() - }, - } - }; - - final request = jsonEncode(setupJson); final headers = await BaseModel.firebaseTokens( _appCheck, _auth, @@ -130,14 +106,15 @@ final class LiveGenerativeModel extends BaseModel { _useLimitedUseAppCheckTokens, )(); - var ws = kIsWeb - ? WebSocketChannel.connect(Uri.parse(uri)) - : IOWebSocketChannel.connect(Uri.parse(uri), headers: headers); - await ws.ready; - - ws.sink.add(request); - - return LiveSession(ws); + return LiveSession.connect( + uri: uri, + headers: headers, + modelString: modelString, + systemInstruction: _systemInstruction, + tools: _tools, + sessionResumption: sessionResumption, + liveGenerationConfig: _liveGenerationConfig, + ); } } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index a1b6c016c00b..3308845af15e 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -16,17 +16,34 @@ import 'dart:async'; import 'dart:convert'; import 'dart:developer'; +import 'package:flutter/foundation.dart'; +import 'package:web_socket_channel/io.dart'; import 'package:web_socket_channel/web_socket_channel.dart'; import 'content.dart'; import 'error.dart'; import 'live_api.dart'; +import 'tool.dart'; /// Manages asynchronous communication with Gemini model over a WebSocket /// connection. class LiveSession { // ignore: public_member_api_docs - LiveSession(this._ws) { + LiveSession._( + this._ws, + this._messageController, { + required String uri, + required Map headers, + required String modelString, + Content? systemInstruction, + List? tools, + LiveGenerationConfig? liveGenerationConfig, + }) : _uri = uri, + _headers = headers, + _modelString = modelString, + _systemInstruction = systemInstruction, + _tools = tools, + _liveGenerationConfig = liveGenerationConfig { _wsSubscription = _ws.stream.listen( (message) { try { @@ -44,10 +61,95 @@ class LiveSession { onDone: _messageController.close, ); } - final WebSocketChannel _ws; - final _messageController = StreamController.broadcast(); + + /// Establishes a connection to a live generation service. + /// + /// This function handles the WebSocket connection setup and returns an [LiveSession] + /// object that can be used to communicate with the service. + /// + /// Returns a [Future] that resolves to an [LiveSession] object upon successful + /// connection. + static Future connect({ + required String uri, + required Map headers, + required String modelString, + Content? systemInstruction, + List? tools, + SessionResumptionConfig? sessionResumption, + LiveGenerationConfig? liveGenerationConfig, + }) async { + final setupJson = { + 'setup': { + 'model': modelString, + if (systemInstruction != null) + 'system_instruction': systemInstruction.toJson(), + if (tools != null) 'tools': tools.map((t) => t.toJson()).toList(), + if (sessionResumption != null) + 'session_resumption': sessionResumption.toJson(), + if (liveGenerationConfig != null) ...{ + 'generation_config': liveGenerationConfig.toJson(), + if (liveGenerationConfig.inputAudioTranscription != null) + 'input_audio_transcription': + liveGenerationConfig.inputAudioTranscription!.toJson(), + if (liveGenerationConfig.outputAudioTranscription != null) + 'output_audio_transcription': + liveGenerationConfig.outputAudioTranscription!.toJson(), + if (liveGenerationConfig.contextWindowCompression + case final contextWindowCompression?) + 'contextWindowCompression': contextWindowCompression.toJson() + }, + } + }; + + final request = jsonEncode(setupJson); + final ws = kIsWeb + ? WebSocketChannel.connect(Uri.parse(uri)) + : IOWebSocketChannel.connect(Uri.parse(uri), headers: headers); + await ws.ready; + + ws.sink.add(request); + return LiveSession._( + ws, + StreamController.broadcast(), + uri: uri, + headers: headers, + modelString: modelString, + systemInstruction: systemInstruction, + tools: tools, + liveGenerationConfig: liveGenerationConfig, + ); + } + + // Persisted values for session resumption. + final String _uri; + final Map _headers; + final String _modelString; + final Content? _systemInstruction; + final List? _tools; + final LiveGenerationConfig? _liveGenerationConfig; + + WebSocketChannel _ws; + StreamController _messageController; late StreamSubscription _wsSubscription; + Future resumeSession( + {required SessionResumptionConfig sessionResumption, + LiveGenerationConfig? liveGenerationConfig}) async { + await close(); + final newSession = await connect( + uri: _uri, + headers: _headers, + modelString: _modelString, + systemInstruction: _systemInstruction, + tools: _tools, + sessionResumption: sessionResumption, + liveGenerationConfig: liveGenerationConfig ?? _liveGenerationConfig, + ); + _ws = newSession._ws; + _messageController = newSession._messageController; + _wsSubscription = newSession._wsSubscription; + } + /// Sends content to the server. /// /// [input] (optional): The content to send. From dadb0f889d235c7c152c53f8d5cfdc73636ed681 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Thu, 5 Feb 2026 11:19:50 -0800 Subject: [PATCH 07/11] refactor for websocket session open --- .../example/lib/pages/bidi_page.dart | 40 +++--- .../firebase_ai/lib/src/live_api.dart | 6 +- .../firebase_ai/lib/src/live_session.dart | 116 +++++++++++------- 3 files changed, 93 insertions(+), 69 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index b1343cd82968..5849ab8af3a7 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -256,22 +256,24 @@ class _BidiPageState extends State { if (!_sessionOpening) { try { - if (_activeSessionHandle != null) { - _session = await _liveModel.connect( - sessionResumption: SessionResumptionConfig( - handle: _activeSessionHandle, transparent: true), - ); - if (context.mounted) { - ScaffoldMessenger.of(context).showSnackBar( - SnackBar( - content: Text( - 'Resumed session with session handle: $_activeSessionHandle'), - ), - ); - } - } else { - _session = await _liveModel.connect(); - } + // if (_activeSessionHandle != null) { + + // if (context.mounted) { + // ScaffoldMessenger.of(context).showSnackBar( + // SnackBar( + // content: Text( + // 'Resumed session with session handle: $_activeSessionHandle'), + // ), + // ); + // } + // } else { + // _session = await _liveModel.connect(); + // } + _session = await _liveModel.connect( + sessionResumption: SessionResumptionConfig( + handle: _activeSessionHandle, + ), + ); } on Exception catch (e) { developer.log('Error setting up session: $e, starting a new one.'); _session = await _liveModel.connect(); @@ -439,10 +441,10 @@ class _BidiPageState extends State { } else if (message is GoingAwayNotice) { developer.log('GoAway message received, time left: ${message.timeLeft}'); if (_activeSessionHandle != null) { + developer.log('Resume Session with handle: $_activeSessionHandle'); await _session.resumeSession( sessionResumption: SessionResumptionConfig( handle: _activeSessionHandle, - transparent: true, ), ); if (mounted) { @@ -458,7 +460,9 @@ class _BidiPageState extends State { message.resumable!) { _activeSessionHandle = message.newHandle; _lastProcessedIndex = message.lastConsumedClientMessageIndex; - developer.log('SessionResumptionUpdate message received: $message'); + developer.log( + 'SessionResumptionUpdate message received: session handle ${message.newHandle}, processId ${message.lastConsumedClientMessageIndex}', + ); } } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart index 2180991b34b0..ba70050c7d3b 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart @@ -133,20 +133,16 @@ class SessionResumptionConfig { /// [transparent] (optional): If set, requests the server to send updates with /// the message index of the last client message included in the session /// state. - SessionResumptionConfig({this.handle, this.transparent}); + SessionResumptionConfig({this.handle}); /// The session resumption handle of the previous session to restore. /// /// If not present, a new session will be started. final String? handle; - /// If set, requests the server to send updates with the message index of the - /// last client message included in the session state. - final bool? transparent; // ignore: public_member_api_docs Map toJson() => { if (handle case final handle?) 'handle': handle, - if (transparent case final transparent?) 'transparent': transparent, }; } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index 3308845af15e..a153b127d614 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -30,8 +30,7 @@ import 'tool.dart'; class LiveSession { // ignore: public_member_api_docs LiveSession._( - this._ws, - this._messageController, { + this._ws, { required String uri, required Map headers, required String modelString, @@ -43,23 +42,9 @@ class LiveSession { _modelString = modelString, _systemInstruction = systemInstruction, _tools = tools, - _liveGenerationConfig = liveGenerationConfig { - _wsSubscription = _ws.stream.listen( - (message) { - try { - var jsonString = utf8.decode(message); - var response = json.decode(jsonString); - - _messageController.add(parseServerResponse(response)); - } catch (e) { - _messageController.addError(e); - } - }, - onError: (error) { - _messageController.addError(error); - }, - onDone: _messageController.close, - ); + _liveGenerationConfig = liveGenerationConfig, + _messageController = StreamController.broadcast() { + _listenToWebSocket(); } /// Establishes a connection to a live generation service. @@ -77,6 +62,47 @@ class LiveSession { List? tools, SessionResumptionConfig? sessionResumption, LiveGenerationConfig? liveGenerationConfig, + }) async { + final ws = await _performWebSocketSetup( + uri: uri, + headers: headers, + modelString: modelString, + systemInstruction: systemInstruction, + tools: tools, + sessionResumption: sessionResumption, + liveGenerationConfig: liveGenerationConfig, + ); + return LiveSession._( + ws, + uri: uri, + headers: headers, + modelString: modelString, + systemInstruction: systemInstruction, + tools: tools, + liveGenerationConfig: liveGenerationConfig, + ); + } + + // Persisted values for session resumption. + final String _uri; + final Map _headers; + final String _modelString; + final Content? _systemInstruction; + final List? _tools; + final LiveGenerationConfig? _liveGenerationConfig; + + WebSocketChannel _ws; + StreamController _messageController; + late StreamSubscription _wsSubscription; + + static Future _performWebSocketSetup({ + required String uri, + required Map headers, + required String modelString, + Content? systemInstruction, + List? tools, + SessionResumptionConfig? sessionResumption, + LiveGenerationConfig? liveGenerationConfig, }) async { final setupJson = { 'setup': { @@ -108,46 +134,44 @@ class LiveSession { await ws.ready; ws.sink.add(request); - return LiveSession._( - ws, - StreamController.broadcast(), - uri: uri, - headers: headers, - modelString: modelString, - systemInstruction: systemInstruction, - tools: tools, - liveGenerationConfig: liveGenerationConfig, - ); + return ws; } - // Persisted values for session resumption. - final String _uri; - final Map _headers; - final String _modelString; - final Content? _systemInstruction; - final List? _tools; - final LiveGenerationConfig? _liveGenerationConfig; + void _listenToWebSocket() { + _wsSubscription = _ws.stream.listen( + (message) { + try { + var jsonString = utf8.decode(message); + var response = json.decode(jsonString); - WebSocketChannel _ws; - StreamController _messageController; - late StreamSubscription _wsSubscription; + _messageController.add(parseServerResponse(response)); + } catch (e) { + _messageController.addError(e); + } + }, + onError: (error) { + _messageController.addError(error); + }, + onDone: _messageController.close, + ); + } Future resumeSession( - {required SessionResumptionConfig sessionResumption, - LiveGenerationConfig? liveGenerationConfig}) async { + {SessionResumptionConfig? sessionResumption}) async { await close(); - final newSession = await connect( + + _ws = await _performWebSocketSetup( uri: _uri, headers: _headers, modelString: _modelString, systemInstruction: _systemInstruction, tools: _tools, sessionResumption: sessionResumption, - liveGenerationConfig: liveGenerationConfig ?? _liveGenerationConfig, + liveGenerationConfig: _liveGenerationConfig, ); - _ws = newSession._ws; - _messageController = newSession._messageController; - _wsSubscription = newSession._wsSubscription; + + _messageController = StreamController.broadcast(); + _listenToWebSocket(); } /// Sends content to the server. From 81eb5ac4bd216f708585c5124dd7f184bf2457ce Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Wed, 25 Feb 2026 14:44:56 -0800 Subject: [PATCH 08/11] fix some error while handling the received message --- .../firebase_ai/example/lib/pages/bidi_page.dart | 3 ++- .../firebase_ai/firebase_ai/lib/src/live_session.dart | 8 +------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 5f411354a4bb..84658bdbb3de 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -262,6 +262,7 @@ class _BidiPageState extends State { ), if (!_loading) IconButton( + tooltip: 'Send Text', onPressed: () async { await _sendTextPrompt(_textController.text); }, @@ -513,7 +514,7 @@ class _BidiPageState extends State { Future _processMessagesContinuously() async { try { await for (final message in _session.receive()) { - if (!mounted) break; + //if (!mounted) break; await _handleLiveServerMessage(message); } } catch (e) { diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index a153b127d614..eda7898e3c56 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -233,6 +233,7 @@ class LiveSession { /// [text]: The text data to send. Future sendTextRealtime(String text) async { _checkWsStatus(); + log('sendTextRealtime: $text'); var clientMessage = LiveClientRealtimeInput.text(text); var clientJson = jsonEncode(clientMessage.toJson()); _ws.sink.add(clientJson); @@ -292,13 +293,6 @@ class LiveSession { await for (final result in _messageController.stream) { yield result; - final message = result.message; - - if (message is LiveServerContent && - message.turnComplete != null && - message.turnComplete!) { - break; // Exit the loop when the turn is complete - } } } From ac5568a26c766403113ed8958e8c00d1ff3bb5be Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Thu, 26 Feb 2026 15:23:25 -0800 Subject: [PATCH 09/11] some clean up, and more dev logs --- .../firebase_ai/example/lib/pages/bidi_page.dart | 14 +------------- .../firebase_ai/lib/src/live_session.dart | 7 +++++-- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 84658bdbb3de..7bc16f2ffcde 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -325,19 +325,6 @@ class _BidiPageState extends State { if (!_sessionOpening) { try { - // if (_activeSessionHandle != null) { - - // if (context.mounted) { - // ScaffoldMessenger.of(context).showSnackBar( - // SnackBar( - // content: Text( - // 'Resumed session with session handle: $_activeSessionHandle'), - // ), - // ); - // } - // } else { - // _session = await _liveModel.connect(); - // } _session = await _liveModel.connect( sessionResumption: SessionResumptionConfig( handle: _activeSessionHandle, @@ -585,6 +572,7 @@ class _BidiPageState extends State { } else if (message is GoingAwayNotice) { developer.log('GoAway message received, time left: ${message.timeLeft}'); if (_activeSessionHandle != null) { + developer.log('====================================================='); developer.log('Resume Session with handle: $_activeSessionHandle'); await _session.resumeSession( sessionResumption: SessionResumptionConfig( diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index eda7898e3c56..ed74a7e0d279 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -158,7 +158,8 @@ class LiveSession { Future resumeSession( {SessionResumptionConfig? sessionResumption}) async { - await close(); + await _wsSubscription.cancel(); + await _ws.sink.close(); _ws = await _performWebSocketSetup( uri: _uri, @@ -170,7 +171,6 @@ class LiveSession { liveGenerationConfig: _liveGenerationConfig, ); - _messageController = StreamController.broadcast(); _listenToWebSocket(); } @@ -209,6 +209,7 @@ class LiveSession { /// [audio]: The audio data to send. Future sendAudioRealtime(InlineDataPart audio) async { _checkWsStatus(); + log('send-AUDIO-Realtime: size: ${audio.bytes.length}, mime: ${audio.mimeType}'); var clientMessage = LiveClientRealtimeInput.audio(audio); var clientJson = jsonEncode(clientMessage.toJson()); _ws.sink.add(clientJson); @@ -221,6 +222,7 @@ class LiveSession { /// [video]: The video data to send. Future sendVideoRealtime(InlineDataPart video) async { _checkWsStatus(); + log('send-VIDEO-Realtime: size: ${video.bytes.length}, mime: ${video.mimeType}'); var clientMessage = LiveClientRealtimeInput.video(video); var clientJson = jsonEncode(clientMessage.toJson()); _ws.sink.add(clientJson); @@ -292,6 +294,7 @@ class LiveSession { _checkWsStatus(); await for (final result in _messageController.stream) { + log('live_session.received result, ${result.message.runtimeType}'); yield result; } } From abe62dbfd8e058be5f5eced1552be6359f53eaab Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Fri, 27 Feb 2026 10:22:35 -0800 Subject: [PATCH 10/11] session resume with toggle --- .../example/lib/pages/bidi_page.dart | 1036 +++++++++-------- .../firebase_ai/lib/src/live_session.dart | 2 - 2 files changed, 553 insertions(+), 485 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 7bc16f2ffcde..b5d1b01fb9cd 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -14,9 +14,9 @@ import 'dart:async'; import 'dart:developer' as developer; import 'package:flutter/foundation.dart'; - import 'package:flutter/material.dart'; import 'package:firebase_ai/firebase_ai.dart'; +import 'package:waveform_flutter/waveform_flutter.dart'; import '../utils/audio_input.dart'; import '../utils/audio_output.dart'; @@ -25,487 +25,373 @@ import '../widgets/message_widget.dart'; import '../widgets/audio_visualizer.dart'; import '../widgets/camera_previews.dart'; -class BidiPage extends StatefulWidget { - const BidiPage({ - super.key, - required this.title, +// ============================================================================ +// MEDIA MANAGER +// Isolates Audio and Video hardware stream setup, start, stop, and cleanup. +// ============================================================================ +class BidiMediaManager { + final AudioOutput _audioOutput = AudioOutput(); + final AudioInput _audioInput = AudioInput(); + final VideoInput _videoInput = VideoInput(); + + StreamSubscription? _audioSubscription; + StreamSubscription? _videoSubscription; + + bool videoIsInitialized = false; + + // Expose hardware state/streams to the Controller and UI + Stream? get amplitudeStream => _audioInput.amplitudeStream; + dynamic get cameraController => _videoInput.cameraController; + String? get selectedCameraId => _videoInput.selectedCameraId; + bool get controllerInitialized => _videoInput.controllerInitialized; + + void setMacOSController(dynamic controller) { + _videoInput.setMacOSController(controller); + } + + Future init() async { + try { + await _audioOutput.init(); + } catch (e) { + developer.log('Audio Output init error: $e'); + } + + try { + await _audioInput.init(); + } catch (e) { + developer.log('Audio Input init error: $e'); + } + + try { + await _videoInput.init(); + videoIsInitialized = true; + } catch (e) { + developer.log('Error during video initialization: $e'); + } + } + + Future startAudio(void Function(Uint8List) onData) async { + await stopAudio(); + try { + var inputStream = await _audioInput.startRecordingStream(); + await _audioOutput.playStream(); + if (inputStream != null) { + _audioSubscription = inputStream.listen( + onData, + onError: (e) { + developer.log('Audio Stream Error: $e'); + stopAudio(); + }, + cancelOnError: true, + ); + } + } catch (e) { + developer.log('BidiMediaManager.startAudio(): $e'); + rethrow; + } + } + + Future stopAudio() async { + await _audioSubscription?.cancel(); + _audioSubscription = null; + await _audioInput.stopRecording(); + } + + Future startVideo(void Function(Uint8List, String) onData) async { + if (!videoIsInitialized) return; + + if (!_videoInput.controllerInitialized || + _videoInput.cameraController == null) { + await _videoInput.initializeCameraController(); + } + + if (!kIsWeb && defaultTargetPlatform == TargetPlatform.macOS) { + int attempts = 0; + while (_videoInput.cameraController == null) { + if (attempts > 50) break; // 5 second timeout safety + await Future.delayed(const Duration(milliseconds: 100)); + attempts++; + } + } + + // Wait for Mac Camera to Settle (Prevent audio hijack) + await Future.delayed(const Duration(milliseconds: 1000)); + + _videoSubscription = _videoInput.startStreamingImages().listen( + (data) { + String mimeType = 'image/jpeg'; + if (!kIsWeb && defaultTargetPlatform == TargetPlatform.macOS) { + if (data.length > 3 && data[0] == 0x89 && data[1] == 0x50) { + mimeType = 'image/png'; + } + } + onData(data, mimeType); + }, + onError: (e) => developer.log('Video Stream Error: $e'), + ); + } + + Future stopVideo() async { + await _videoSubscription?.cancel(); + _videoSubscription = null; + await _videoInput.stopStreamingImages(); + } + + void playAudioChunk(Uint8List bytes) { + _audioOutput.addDataToAudioStream(bytes); + } +} + +// ============================================================================ +// BIDI SESSION CONTROLLER +// Isolates business logic, session start/stop, reconnection, and tool execution. +// ============================================================================ +class BidiSessionController extends ChangeNotifier { + BidiSessionController({ required this.model, required this.useVertexBackend, - }); + this.onShowError, + this.onScrollDown, + }) { + _initLiveModel(); + } - final String title; final GenerativeModel model; final bool useVertexBackend; + final void Function(String)? onShowError; + final VoidCallback? onScrollDown; - @override - State createState() => _BidiPageState(); -} + late LiveGenerativeModel _liveModel; + LiveSession? _session; + final BidiMediaManager mediaManager = BidiMediaManager(); -class LightControl { - final int? brightness; - final String? colorTemperature; + bool isLoading = false; + bool isSessionActive = false; + bool isMicOn = false; + bool isCameraOn = false; - LightControl({this.brightness, this.colorTemperature}); -} + // Intention state for robust stream reconnection + bool _intendedMicOn = false; + bool _intendedCameraOn = false; -class _BidiPageState extends State { - final ScrollController _scrollController = ScrollController(); - final TextEditingController _textController = TextEditingController(); - final FocusNode _textFieldFocus = FocusNode(); - final List _messages = []; - bool _loading = false; - bool _sessionOpening = false; - bool _recording = false; - late LiveGenerativeModel _liveModel; - late LiveSession _session; + final List messages = []; String? _activeSessionHandle; int? _lastProcessedIndex; - final AudioOutput _audioOutput = AudioOutput(); - final AudioInput _audioInput = AudioInput(); - final VideoInput _videoInput = VideoInput(); - StreamSubscription? _audioSubscription; int? _inputTranscriptionMessageIndex; int? _outputTranscriptionMessageIndex; - bool _isCameraOn = false; - bool _videoIsInitialized = false; - - @override - void initState() { - super.initState(); + void _initLiveModel() { final config = LiveGenerationConfig( speechConfig: SpeechConfig(voiceName: 'Fenrir'), - responseModalities: [ - ResponseModalities.audio, - ], + responseModalities: [ResponseModalities.audio], inputAudioTranscription: AudioTranscriptionConfig(), outputAudioTranscription: AudioTranscriptionConfig(), ); - _liveModel = widget.useVertexBackend + final tools = [ + Tool.functionDeclarations([_lightControlTool]) + ]; + + _liveModel = useVertexBackend ? FirebaseAI.vertexAI().liveGenerativeModel( model: 'gemini-live-2.5-flash-preview-native-audio-09-2025', liveGenerationConfig: config, - tools: [ - Tool.functionDeclarations([lightControlTool]), - ], + tools: tools, ) : FirebaseAI.googleAI().liveGenerativeModel( model: 'gemini-2.5-flash-native-audio-preview-09-2025', liveGenerationConfig: config, - tools: [ - Tool.functionDeclarations([lightControlTool]), - ], + tools: tools, ); } - Future _initAudio() async { - try { - await _audioOutput.init(); - } catch (e) { - developer.log('Audio Output init error: $e'); - } + Future initialize() async { + isLoading = true; + notifyListeners(); + await mediaManager.init(); + isLoading = false; + notifyListeners(); + } - try { - await _audioInput.init(); - } catch (e) { - developer.log('Audio Input init error: $e'); + Future toggleSession() async { + if (isSessionActive) { + await _stopSession(explicit: true); + } else { + await _startSession(explicit: true); } } - Future _initVideo() async { + Future _startSession({required bool explicit}) async { + isLoading = true; + notifyListeners(); + try { - await _videoInput.init(); - setState(() { - _videoIsInitialized = true; - }); - } catch (e) { - developer.log('Error during video initialization: $e'); + _session = await _liveModel.connect( + sessionResumption: + SessionResumptionConfig(handle: _activeSessionHandle), + ); + } on Exception catch (e) { + developer.log('Error setting up session: $e, starting a new one.'); + _session = await _liveModel.connect(); } - } - - void _scrollDown() { - if (!_scrollController.hasClients) return; - _scrollController.jumpTo( - _scrollController.position.maxScrollExtent, - ); - } + isSessionActive = true; + unawaited(_processMessagesContinuously()); - @override - void dispose() { - if (_sessionOpening) { - _sessionOpening = false; - _session.close(); + if (explicit) { + // Reconnect previously active hardware seamlessly into the new session + if (_intendedMicOn) await _startMicStream(); + if (_intendedCameraOn) await _startCameraStream(); } - super.dispose(); - } - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.all(8), - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - if (_isCameraOn) - Container( - height: 200, - color: Colors.black, - alignment: Alignment.center, - child: (!kIsWeb && defaultTargetPlatform == TargetPlatform.macOS) - ? FullCameraPreview( - controller: _videoInput.cameraController, - deviceId: _videoInput.selectedCameraId, - onInitialized: (controller) { - // This is where the controller actually gets born on macOS - _videoInput.setMacOSController(controller); - }, - ) - : (_videoInput.cameraController != null && - _videoInput.controllerInitialized) - ? FullCameraPreview( - controller: _videoInput.cameraController, - deviceId: _videoInput.selectedCameraId, - onInitialized: (controller) { - // Web/Mobile callback (often unused if controller passed in) - }, - ) - : const Center(child: CircularProgressIndicator()), - ), - Expanded( - child: ListView.builder( - controller: _scrollController, - itemBuilder: (context, idx) { - return MessageWidget( - text: _messages[idx].text, - image: _messages[idx].imageBytes != null - ? Image.memory( - _messages[idx].imageBytes!, - cacheWidth: 400, - cacheHeight: 400, - ) - : null, - isFromUser: _messages[idx].fromUser ?? false, - isThought: _messages[idx].isThought, - ); - }, - itemCount: _messages.length, - ), - ), - Padding( - padding: const EdgeInsets.symmetric( - vertical: 25, - horizontal: 15, - ), - child: Row( - children: [ - Expanded( - child: TextField( - focusNode: _textFieldFocus, - controller: _textController, - onSubmitted: _sendTextPrompt, - ), - ), - const SizedBox.square( - dimension: 15, - ), - AudioVisualizer( - audioStreamIsActive: _recording, - amplitudeStream: _audioInput.amplitudeStream, - ), - const SizedBox.square( - dimension: 15, - ), - IconButton( - tooltip: 'Start Streaming', - onPressed: !_loading - ? () async { - await _setupSession(); - } - : null, - icon: Icon( - Icons.network_wifi, - color: _sessionOpening - ? Theme.of(context).colorScheme.secondary - : Theme.of(context).colorScheme.primary, - ), - ), - IconButton( - tooltip: 'Send Stream Message', - onPressed: !_loading - ? () async { - if (_recording) { - await _stopRecording(); - } else { - await _startRecording(); - } - } - : null, - icon: Icon( - _recording ? Icons.stop : Icons.mic, - color: _loading - ? Theme.of(context).colorScheme.secondary - : Theme.of(context).colorScheme.primary, - ), - ), - IconButton( - tooltip: 'Toggle Camera', - onPressed: _isCameraOn ? _stopVideoStream : _startVideoStream, - icon: Icon( - _isCameraOn ? Icons.videocam_off : Icons.videocam, - color: _loading - ? Theme.of(context).colorScheme.secondary - : Theme.of(context).colorScheme.primary, - ), - ), - if (!_loading) - IconButton( - tooltip: 'Send Text', - onPressed: () async { - await _sendTextPrompt(_textController.text); - }, - icon: Icon( - Icons.send, - color: Theme.of(context).colorScheme.primary, - ), - ) - else - const CircularProgressIndicator(), - ], - ), - ), - ], - ), - ); + isLoading = false; + notifyListeners(); } - final lightControlTool = FunctionDeclaration( - 'setLightValues', - 'Set the brightness and color temperature of a room light.', - parameters: { - 'brightness': Schema.integer( - description: 'Light level from 0 to 100. ' - 'Zero is off and 100 is full brightness.', - ), - 'colorTemperature': Schema.string( - description: 'Color temperature of the light fixture, ' - 'which can be `daylight`, `cool` or `warm`.', - ), - }, - ); + Future _stopSession({required bool explicit}) async { + isLoading = true; + notifyListeners(); + + if (explicit) { + await mediaManager.stopAudio(); + await mediaManager.stopVideo(); + isMicOn = false; + isCameraOn = false; + // We purposefully DO NOT reset _intendedMicOn/CameraOn so we know what + // the user had active when they reconnect! + } - Future> _setLightValues({ - int? brightness, - String? colorTemperature, - }) async { - final apiResponse = { - 'colorTemprature': 'warm', - 'brightness': brightness, - }; - return apiResponse; + await _session?.close(); + _session = null; + isSessionActive = false; + + isLoading = false; + notifyListeners(); } - Future _setupSession() async { - setState(() { - _loading = true; - }); - await _initAudio(); + Future _sessionResume() async { + if (isSessionActive) { + // Explicit false means we keep the hardware pipelines active + // so they immediately route data to the new session object + await _stopSession(explicit: false); + await _startSession(explicit: false); + } + } - try { - if (!_videoIsInitialized) { - await _initVideo(); - } else { - await _videoInput.initializeCameraController(); - } - } catch (e) { - developer.log('Video Hardware init error: $e'); + void _onAudioData(Uint8List data) { + if (isSessionActive && _session != null) { + _session!.sendAudioRealtime(InlineDataPart('audio/pcm', data)); } + } - if (!_sessionOpening) { - try { - _session = await _liveModel.connect( - sessionResumption: SessionResumptionConfig( - handle: _activeSessionHandle, - ), - ); - } on Exception catch (e) { - developer.log('Error setting up session: $e, starting a new one.'); - _session = await _liveModel.connect(); - - if (context.mounted) { - ScaffoldMessenger.of(context).showSnackBar( - const SnackBar( - content: Text('Previous session expired, started a new one.'), - ), - ); - } - } + void _onVideoData(Uint8List data, String mimeType) { + if (isSessionActive && _session != null) { + _session!.sendVideoRealtime(InlineDataPart(mimeType, data)); + } + } - _sessionOpening = true; - unawaited( - _processMessagesContinuously(), - ); + Future toggleMic() async { + _intendedMicOn = !_intendedMicOn; + if (_intendedMicOn) { + await _startMicStream(); } else { - await _session.close(); - _sessionOpening = false; + await mediaManager.stopAudio(); + isMicOn = false; + notifyListeners(); } - - setState(() { - _loading = false; - }); } - Future _startRecording() async { - await _audioSubscription?.cancel(); - _audioSubscription = null; - setState(() { - _recording = true; - }); + Future _startMicStream() async { + if (!isSessionActive) { + isMicOn = true; + notifyListeners(); + return; + } try { - var inputStream = await _audioInput.startRecordingStream(); - await _audioOutput.playStream(); - if (inputStream != null) { - _audioSubscription = inputStream.listen( - (data) { - _session.sendAudioRealtime(InlineDataPart('audio/pcm', data)); - }, - onError: (e) { - developer.log('Audio Stream Error: $e'); - _stopRecording(); - }, - cancelOnError: true, - ); - } + await mediaManager.startAudio(_onAudioData); + isMicOn = true; + notifyListeners(); } catch (e) { - developer.log('bidi_page._startRecording(): $e'); - _showError('bidi_page._startRecording(): $e'); - setState(() => _recording = false); + onShowError?.call(e.toString()); + isMicOn = false; + notifyListeners(); } } - Future _stopRecording() async { - await _audioSubscription?.cancel(); - _audioSubscription = null; + Future _startCameraStream() async { + if (!isSessionActive) { + isCameraOn = true; + notifyListeners(); + return; + } try { - await _audioInput.stopRecording(); + await mediaManager.startVideo(_onVideoData); + isCameraOn = true; + notifyListeners(); } catch (e) { - _showError(e.toString()); + developer.log('Error starting video stream: $e'); + onShowError?.call(e.toString()); + isCameraOn = false; + notifyListeners(); } - - setState(() { - _recording = false; - }); } - Future _startVideoStream() async { - // 1. Re-entry Guard: Prevent multiple clicks while switching - if (_loading || !_videoIsInitialized) return; - - // 2. Capture the current recording state - bool wasRecording = _recording; + Future toggleCamera() async { + if (isLoading) return; // Prevent multiple clicks + _intendedCameraOn = !_intendedCameraOn; - setState(() { - _loading = true; // Lock the UI during the switch - }); + isLoading = true; + notifyListeners(); try { - if (wasRecording) { - await _stopRecording(); - } - - // 4. Wait for ripple/UI (Prevent freeze) - await Future.delayed(const Duration(milliseconds: 250)); - - // 5. Initialize Camera if needed - if (!_videoInput.controllerInitialized || - _videoInput.cameraController == null) { - await _videoInput.initializeCameraController(); - } + if (!_intendedCameraOn) { + await mediaManager.stopVideo(); + isCameraOn = false; + } else { + // Stop audio momentarily to prevent hijacking (Mac quirk workaround) + bool wasMicOn = isMicOn; + if (wasMicOn) await mediaManager.stopAudio(); - // 6. Mount Camera UI - setState(() { - _isCameraOn = true; - }); - - if (!kIsWeb && defaultTargetPlatform == TargetPlatform.macOS) { - // ✅ Because we set _cameraController to null in stopStreamingImages, - // this loop will now CORRECTLY wait for the new View to initialize. - int attempts = 0; - while (_videoInput.cameraController == null) { - if (attempts > 50) break; // 5 second timeout safety - await Future.delayed(const Duration(milliseconds: 100)); - attempts++; - } - } + await Future.delayed(const Duration(milliseconds: 250)); - // 7. Wait for Mac Camera to Settle (Prevent audio hijack) - await Future.delayed(const Duration(milliseconds: 1000)); + await _startCameraStream(); - // 8. CLEAN RESTART: Use the helper method! - // Only restart if we were recording before. - if (wasRecording) { - developer.log('Resuming audio session...'); - await _startRecording(); + // Restart Audio + if (wasMicOn) await _startMicStream(); } - - // 9. Start Video Stream - _videoInput.startStreamingImages().listen( - (data) { - String mimeType = 'image/jpeg'; - if (!kIsWeb && defaultTargetPlatform == TargetPlatform.macOS) { - if (data.length > 3 && data[0] == 0x89 && data[1] == 0x50) { - mimeType = 'image/png'; - } - } - _session.sendVideoRealtime(InlineDataPart(mimeType, data)); - }, - onError: (e) => developer.log('Video Stream Error: $e'), - ); } catch (e) { developer.log('Error switching to video: $e'); - _showError(e.toString()); + onShowError?.call(e.toString()); + isCameraOn = false; } finally { - // 10. Always unlock the UI - setState(() { - _loading = false; - }); + isLoading = false; + notifyListeners(); } } - Future _stopVideoStream() async { - await _videoInput.stopStreamingImages(); - setState(() { - _isCameraOn = false; - }); - } + Future sendTextPrompt(String textPrompt) async { + if (!isSessionActive || _session == null) return; + isLoading = true; + notifyListeners(); - Future _sendTextPrompt(String textPrompt) async { - setState(() { - _loading = true; - }); try { - //final prompt = Content.text(textPrompt); - // await _session.send(input: prompt, turnComplete: true); - await _session.sendTextRealtime(textPrompt); + await _session!.sendTextRealtime(textPrompt); } catch (e) { - _showError(e.toString()); + onShowError?.call(e.toString()); } - setState(() { - _loading = false; - }); + isLoading = false; + notifyListeners(); } Future _processMessagesContinuously() async { + if (_session == null) return; try { - await for (final message in _session.receive()) { - //if (!mounted) break; + await for (final message in _session!.receive()) { await _handleLiveServerMessage(message); } } catch (e) { - _showError(e.toString()); + onShowError?.call(e.toString()); } } @@ -517,40 +403,6 @@ class _BidiPageState extends State { await _handleLiveServerContent(message); } - int? _handleTranscription( - Transcription? transcription, - int? messageIndex, - String prefix, - bool fromUser, - ) { - int? currentIndex = messageIndex; - if (transcription?.text != null) { - if (currentIndex != null) { - _messages[currentIndex] = _messages[currentIndex].copyWith( - text: '${_messages[currentIndex].text}${transcription!.text!}', - ); - } else { - _messages.add( - MessageData( - text: '$prefix${transcription!.text!}', - fromUser: fromUser, - ), - ); - currentIndex = _messages.length - 1; - } - if (transcription.finished ?? false) { - currentIndex = null; - setState(_scrollDown); - } else { - // Use a scheduled frame instead of an immediate setState - WidgetsBinding.instance.addPostFrameCallback((_) { - if (mounted) setState(() {}); - }); - } - } - return currentIndex; - } - _inputTranscriptionMessageIndex = _handleTranscription( message.inputTranscription, _inputTranscriptionMessageIndex, @@ -574,18 +426,7 @@ class _BidiPageState extends State { if (_activeSessionHandle != null) { developer.log('====================================================='); developer.log('Resume Session with handle: $_activeSessionHandle'); - await _session.resumeSession( - sessionResumption: SessionResumptionConfig( - handle: _activeSessionHandle, - ), - ); - if (mounted) { - ScaffoldMessenger.of(context).showSnackBar( - const SnackBar( - content: Text('Session is going away, resuming...'), - ), - ); - } + await _sessionResume(); } } else if (message is SessionResumptionUpdate && message.resumable != null && @@ -593,19 +434,56 @@ class _BidiPageState extends State { _activeSessionHandle = message.newHandle; _lastProcessedIndex = message.lastConsumedClientMessageIndex; developer.log( - 'SessionResumptionUpdate message received: session handle ${message.newHandle}, processId ${message.lastConsumedClientMessageIndex}', - ); + 'SessionResumptionUpdate: handle ${message.newHandle}, index $_lastProcessedIndex'); } } + int? _handleTranscription(Transcription? transcription, int? messageIndex, + String prefix, bool fromUser) { + int? currentIndex = messageIndex; + if (transcription?.text != null) { + if (currentIndex != null) { + messages[currentIndex] = messages[currentIndex].copyWith( + text: '${messages[currentIndex].text}${transcription!.text!}', + ); + } else { + messages.add( + MessageData( + text: '$prefix${transcription!.text!}', + fromUser: fromUser, + ), + ); + currentIndex = messages.length - 1; + } + + if (transcription.finished ?? false) { + currentIndex = null; + onScrollDown?.call(); + } else { + notifyListeners(); // Trigger UI rebuild for streaming text + } + } + return currentIndex; + } + Future _handleLiveServerContent(LiveServerContent response) async { final partList = response.modelTurn?.parts; if (partList != null) { for (final part in partList) { if (part is TextPart) { - await _handleTextPart(part); + messages.add( + MessageData( + text: part.text, + fromUser: false, + isThought: part.isThought ?? false, + ), + ); + onScrollDown?.call(); + notifyListeners(); } else if (part is InlineDataPart) { - await _handleInlineDataPart(part); + if (part.mimeType.startsWith('audio')) { + mediaManager.playAudioChunk(part.bytes); + } } else { developer.log('receive part with type ${part.runtimeType}'); } @@ -613,31 +491,6 @@ class _BidiPageState extends State { } } - Future _handleTextPart(TextPart part) async { - if (!_loading) { - setState(() { - _loading = true; - }); - } - _messages.add( - MessageData( - text: part.text, - fromUser: false, - isThought: part.isThought ?? false, - ), - ); - setState(() { - _loading = false; - _scrollDown(); - }); - } - - Future _handleInlineDataPart(InlineDataPart part) async { - if (part.mimeType.startsWith('audio')) { - _audioOutput.addDataToAudioStream(part.bytes); - } - } - Future _handleLiveServerToolCall(LiveServerToolCall response) async { final functionCalls = response.functionCalls!.toList(); if (functionCalls.isNotEmpty) { @@ -645,11 +498,15 @@ class _BidiPageState extends State { if (functionCall.name == 'setLightValues') { var color = functionCall.args['colorTemperature']! as String; var brightness = functionCall.args['brightness']! as int; - final functionResult = await _setLightValues( - brightness: brightness, - colorTemperature: color, - ); - await _session.sendToolResponse([ + + // Mock Tool Execution + final functionResult = { + 'colorTemprature': + color, // original had a typo, keeping to preserve functionality intent + 'brightness': brightness, + }; + + await _session?.sendToolResponse([ FunctionResponse( functionCall.name, functionResult, @@ -657,27 +514,99 @@ class _BidiPageState extends State { ), ]); } else { - throw UnimplementedError( - 'Function not declared to the model: ${functionCall.name}', - ); + throw UnimplementedError('Function not declared: ${functionCall.name}'); } } } + @override + void dispose() { + _stopSession(explicit: true); + super.dispose(); + } + + static final _lightControlTool = FunctionDeclaration( + 'setLightValues', + 'Set the brightness and color temperature of a room light.', + parameters: { + 'brightness': Schema.integer( + description: + 'Light level from 0 to 100. Zero is off and 100 is full brightness.', + ), + 'colorTemperature': Schema.string( + description: + 'Color temperature of the light fixture, which can be `daylight`, `cool` or `warm`.', + ), + }, + ); +} + +// ============================================================================ +// UI WIDGET +// Isolates presentation, keeping state out of the visual hierarchy. +// ============================================================================ +class BidiPage extends StatefulWidget { + const BidiPage({ + super.key, + required this.title, + required this.model, + required this.useVertexBackend, + }); + + final String title; + final GenerativeModel model; + final bool useVertexBackend; + + @override + State createState() => _BidiPageState(); +} + +class _BidiPageState extends State { + late final BidiSessionController _controller; + final ScrollController _scrollController = ScrollController(); + final TextEditingController _textController = TextEditingController(); + final FocusNode _textFieldFocus = FocusNode(); + + @override + void initState() { + super.initState(); + _controller = BidiSessionController( + model: widget.model, + useVertexBackend: widget.useVertexBackend, + onShowError: _showError, + onScrollDown: _scrollDown, + ); + _controller.initialize(); + } + + @override + void dispose() { + _controller.dispose(); + _scrollController.dispose(); + _textController.dispose(); + _textFieldFocus.dispose(); + super.dispose(); + } + + void _scrollDown() { + if (!_scrollController.hasClients) return; + WidgetsBinding.instance.addPostFrameCallback((_) { + if (mounted) { + _scrollController.jumpTo(_scrollController.position.maxScrollExtent); + } + }); + } + void _showError(String message) { showDialog( context: context, builder: (context) { return AlertDialog( title: const Text('Something went wrong'), - content: SingleChildScrollView( - child: SelectableText(message), - ), + content: SingleChildScrollView(child: SelectableText(message)), actions: [ TextButton( - onPressed: () { - Navigator.of(context).pop(); - }, + onPressed: () => Navigator.of(context).pop(), child: const Text('OK'), ), ], @@ -685,4 +614,145 @@ class _BidiPageState extends State { }, ); } + + @override + Widget build(BuildContext context) { + return AnimatedBuilder( + animation: _controller, + builder: (context, child) { + return Padding( + padding: const EdgeInsets.all(8), + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + if (_controller.isCameraOn) + Container( + height: 200, + color: Colors.black, + alignment: Alignment.center, + child: (!kIsWeb && + defaultTargetPlatform == TargetPlatform.macOS) + ? FullCameraPreview( + controller: _controller.mediaManager.cameraController, + deviceId: _controller.mediaManager.selectedCameraId, + onInitialized: (controller) { + _controller.mediaManager + .setMacOSController(controller); + }, + ) + : (_controller.mediaManager.cameraController != null && + _controller.mediaManager.controllerInitialized) + ? FullCameraPreview( + controller: + _controller.mediaManager.cameraController, + deviceId: + _controller.mediaManager.selectedCameraId, + onInitialized: (controller) {}, + ) + : const Center(child: CircularProgressIndicator()), + ), + Expanded( + child: ListView.builder( + controller: _scrollController, + itemCount: _controller.messages.length, + itemBuilder: (context, idx) { + final message = _controller.messages[idx]; + return MessageWidget( + text: message.text, + image: message.imageBytes != null + ? Image.memory( + message.imageBytes!, + cacheWidth: 400, + cacheHeight: 400, + ) + : null, + isFromUser: message.fromUser ?? false, + isThought: message.isThought, + ); + }, + ), + ), + Padding( + padding: + const EdgeInsets.symmetric(vertical: 25, horizontal: 15), + child: Row( + children: [ + Expanded( + child: TextField( + focusNode: _textFieldFocus, + controller: _textController, + onSubmitted: (text) { + _controller.sendTextPrompt(text); + _textController.clear(); + }, + ), + ), + const SizedBox.square(dimension: 15), + AudioVisualizer( + audioStreamIsActive: _controller.isMicOn, + amplitudeStream: _controller.mediaManager.amplitudeStream, + ), + const SizedBox.square(dimension: 15), + IconButton( + tooltip: 'Start Streaming', + onPressed: !_controller.isLoading + ? () => _controller.toggleSession() + : null, + icon: Icon( + Icons.network_wifi, + color: _controller.isSessionActive + ? Theme.of(context).colorScheme.secondary + : Theme.of(context).colorScheme.primary, + ), + ), + IconButton( + tooltip: 'Send Stream Message', + onPressed: !_controller.isLoading + ? () => _controller.toggleMic() + : null, + icon: Icon( + _controller.isMicOn ? Icons.stop : Icons.mic, + color: _controller.isLoading + ? Theme.of(context).colorScheme.secondary + : Theme.of(context).colorScheme.primary, + ), + ), + IconButton( + tooltip: 'Toggle Camera', + onPressed: !_controller.isLoading + ? () => _controller.toggleCamera() + : null, + icon: Icon( + _controller.isCameraOn + ? Icons.videocam_off + : Icons.videocam, + color: _controller.isLoading + ? Theme.of(context).colorScheme.secondary + : Theme.of(context).colorScheme.primary, + ), + ), + if (!_controller.isLoading) + IconButton( + tooltip: 'Send Text', + onPressed: () { + _controller.sendTextPrompt(_textController.text); + _textController.clear(); + }, + icon: Icon( + Icons.send, + color: Theme.of(context).colorScheme.primary, + ), + ) + else + const CircularProgressIndicator(), + ], + ), + ), + ], + ), + ); + }, + ); + } } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index ed74a7e0d279..a682c1d75f3f 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -209,7 +209,6 @@ class LiveSession { /// [audio]: The audio data to send. Future sendAudioRealtime(InlineDataPart audio) async { _checkWsStatus(); - log('send-AUDIO-Realtime: size: ${audio.bytes.length}, mime: ${audio.mimeType}'); var clientMessage = LiveClientRealtimeInput.audio(audio); var clientJson = jsonEncode(clientMessage.toJson()); _ws.sink.add(clientJson); @@ -222,7 +221,6 @@ class LiveSession { /// [video]: The video data to send. Future sendVideoRealtime(InlineDataPart video) async { _checkWsStatus(); - log('send-VIDEO-Realtime: size: ${video.bytes.length}, mime: ${video.mimeType}'); var clientMessage = LiveClientRealtimeInput.video(video); var clientJson = jsonEncode(clientMessage.toJson()); _ws.sink.add(clientJson); From 9e6b95d2823a01f4bc9e4d87ff636b0a72fb9bd7 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Wed, 4 Mar 2026 19:34:45 -0800 Subject: [PATCH 11/11] some minor updates --- packages/firebase_ai/firebase_ai/lib/firebase_ai.dart | 4 +++- .../firebase_ai/firebase_ai/lib/src/live_session.dart | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart b/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart index a7e4e6087c05..d2586202923b 100644 --- a/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart +++ b/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart @@ -100,6 +100,8 @@ export 'src/imagen/imagen_reference.dart' export 'src/live_api.dart' show AudioTranscriptionConfig, + ContextWindowCompressionConfig, + GoingAwayNotice, LiveGenerationConfig, LiveServerMessage, LiveServerContent, @@ -108,8 +110,8 @@ export 'src/live_api.dart' LiveServerResponse, SessionResumptionConfig, SessionResumptionUpdate, + SlidingWindow, SpeechConfig, - GoingAwayNotice, Transcription; export 'src/live_session.dart' show LiveSession; export 'src/schema.dart' show Schema, SchemaType; diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index a682c1d75f3f..9e21ea530bb2 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -156,6 +156,14 @@ class LiveSession { ); } + /// Resumes an existing live session with the server. + /// + /// This closes the current WebSocket connection and establishes a new one using + /// the same configuration (URI, headers, model, system instruction, tools, etc.) + /// as the original session. + /// + /// [sessionResumption] (optional): The configuration for session resumption, + /// such as the handle to the previous session state to restore. Future resumeSession( {SessionResumptionConfig? sessionResumption}) async { await _wsSubscription.cancel();