From 7926fcb091069f5e2cd6395805b8e197126a69b3 Mon Sep 17 00:00:00 2001 From: KoushikReddy Date: Sat, 11 Apr 2026 13:02:27 -0700 Subject: [PATCH 1/3] fix(evaluation): skip invocations without user events in convert_events_to_eval_invocations Sessions can contain invocation_ids whose events are all authored by agents or tools (e.g. internal/background turns with no corresponding user message). Previously, convert_events_to_eval_invocations left user_content as an empty Content(parts=[]) for such invocations, and earlier versions used an empty string, which caused a Pydantic ValidationError because Invocation.user_content requires a genai_types.Content object. Invocations without a user-authored event are not meaningful for evaluation, so skip them instead of constructing an Invocation with a placeholder user_content. A debug log line is emitted for each skipped invocation to aid troubleshooting. Fixes #3760 --- .../adk/evaluation/evaluation_generator.py | 14 +++++- .../evaluation/test_evaluation_generator.py | 46 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/google/adk/evaluation/evaluation_generator.py b/src/google/adk/evaluation/evaluation_generator.py index f8fb6795aa..74a93f330b 100644 --- a/src/google/adk/evaluation/evaluation_generator.py +++ b/src/google/adk/evaluation/evaluation_generator.py @@ -281,7 +281,7 @@ def convert_events_to_eval_invocations( for invocation_id, events in events_by_invocation_id.items(): final_response = None final_event = None - user_content = Content(parts=[]) + user_content = None invocation_timestamp = 0 app_details = None if ( @@ -312,6 +312,18 @@ def convert_events_to_eval_invocations( events_to_add.append(event) break + if user_content is None: + # Skip invocations that have no user-authored event. Such invocations + # arise from internal/system-driven turns (e.g. background agent tasks) + # and are not meaningful for evaluation purposes. Including them would + # also cause a Pydantic ValidationError because Invocation.user_content + # requires a Content object. + logger.debug( + 'Skipping invocation %s: no user-authored event found.', + invocation_id, + ) + continue + invocation_events = [ InvocationEvent(author=e.author, content=e.content) for e in events_to_add diff --git a/tests/unittests/evaluation/test_evaluation_generator.py b/tests/unittests/evaluation/test_evaluation_generator.py index a4aa8691fd..0ecdb1b64f 100644 --- a/tests/unittests/evaluation/test_evaluation_generator.py +++ b/tests/unittests/evaluation/test_evaluation_generator.py @@ -226,6 +226,52 @@ def test_convert_multi_agent_final_responses( assert intermediate_events[0].author == "agent1" assert intermediate_events[0].content.parts[0].text == "First response" + def test_invocation_without_user_event_is_skipped(self): + """Invocations with no user-authored event must be skipped. + + Regression test for https://github.com/google/adk-python/issues/3760. + When a session contains an invocation_id whose events are all authored by + agents or tools (no 'user' event), convert_events_to_eval_invocations used + to leave user_content as a bare string, causing a Pydantic ValidationError + from Invocation.user_content which requires genai_types.Content. + The fix skips such invocations because they represent internal/system-driven + turns that are not meaningful for evaluation. + """ + events = [ + _build_event("agent", [types.Part(text="agent-only event")], "inv1"), + ] + + # Must not raise a Pydantic ValidationError. + invocations = EvaluationGenerator.convert_events_to_eval_invocations(events) + + assert invocations == [], ( + "Invocations without a user event should be skipped." + ) + + def test_mixed_invocations_skips_only_agent_only_ones(self): + """Only agent-only invocations are skipped; normal invocations are kept. + + Regression test for https://github.com/google/adk-python/issues/3760. + """ + events = [ + # inv1: normal user+agent turn — should be kept. + _build_event("user", [types.Part(text="Hello")], "inv1"), + _build_event("agent", [types.Part(text="Hi there!")], "inv1"), + # inv2: agent-only turn (e.g. background/system task) — should be skipped. + _build_event("agent", [types.Part(text="Internal work")], "inv2"), + # inv3: normal user+agent turn — should be kept. + _build_event("user", [types.Part(text="Follow-up")], "inv3"), + _build_event("agent", [types.Part(text="Sure!")], "inv3"), + ] + + invocations = EvaluationGenerator.convert_events_to_eval_invocations(events) + + assert len(invocations) == 2 + assert invocations[0].invocation_id == "inv1" + assert invocations[0].user_content.parts[0].text == "Hello" + assert invocations[1].invocation_id == "inv3" + assert invocations[1].user_content.parts[0].text == "Follow-up" + class TestGetAppDetailsByInvocationId: """Test cases for EvaluationGenerator._get_app_details_by_invocation_id method.""" From e1350bbe23e4f48bf06ac9a252f1a17d1dd02efb Mon Sep 17 00:00:00 2001 From: KoushikReddy Date: Mon, 13 Apr 2026 14:19:41 -0700 Subject: [PATCH 2/3] style: apply pyink formatting to evaluation files --- src/google/adk/evaluation/evaluation_generator.py | 2 +- tests/unittests/evaluation/test_evaluation_generator.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/google/adk/evaluation/evaluation_generator.py b/src/google/adk/evaluation/evaluation_generator.py index 74a93f330b..6fac2decdd 100644 --- a/src/google/adk/evaluation/evaluation_generator.py +++ b/src/google/adk/evaluation/evaluation_generator.py @@ -319,7 +319,7 @@ def convert_events_to_eval_invocations( # also cause a Pydantic ValidationError because Invocation.user_content # requires a Content object. logger.debug( - 'Skipping invocation %s: no user-authored event found.', + "Skipping invocation %s: no user-authored event found.", invocation_id, ) continue diff --git a/tests/unittests/evaluation/test_evaluation_generator.py b/tests/unittests/evaluation/test_evaluation_generator.py index 0ecdb1b64f..db30bfac75 100644 --- a/tests/unittests/evaluation/test_evaluation_generator.py +++ b/tests/unittests/evaluation/test_evaluation_generator.py @@ -244,9 +244,9 @@ def test_invocation_without_user_event_is_skipped(self): # Must not raise a Pydantic ValidationError. invocations = EvaluationGenerator.convert_events_to_eval_invocations(events) - assert invocations == [], ( - "Invocations without a user event should be skipped." - ) + assert ( + invocations == [] + ), "Invocations without a user event should be skipped." def test_mixed_invocations_skips_only_agent_only_ones(self): """Only agent-only invocations are skipped; normal invocations are kept. From 1aed984b449780a7b9349fee341d05a29a273e69 Mon Sep 17 00:00:00 2001 From: KoushikReddy Date: Mon, 13 Apr 2026 15:39:13 -0700 Subject: [PATCH 3/3] style: fix import ordering via autoformat.sh --- contributing/samples/gepa/experiment.py | 1 - contributing/samples/gepa/run_experiment.py | 1 - 2 files changed, 2 deletions(-) diff --git a/contributing/samples/gepa/experiment.py b/contributing/samples/gepa/experiment.py index f3751206a8..2710c3894c 100644 --- a/contributing/samples/gepa/experiment.py +++ b/contributing/samples/gepa/experiment.py @@ -43,7 +43,6 @@ from tau_bench.types import EnvRunResult from tau_bench.types import RunConfig import tau_bench_agent as tau_bench_agent_lib - import utils diff --git a/contributing/samples/gepa/run_experiment.py b/contributing/samples/gepa/run_experiment.py index d857da9635..e31db15788 100644 --- a/contributing/samples/gepa/run_experiment.py +++ b/contributing/samples/gepa/run_experiment.py @@ -25,7 +25,6 @@ from absl import flags import experiment from google.genai import types - import utils _OUTPUT_DIR = flags.DEFINE_string(