From 326ffbeb89df9a72831a7f22709d77771e5371a2 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 20 Mar 2026 15:16:38 +0100 Subject: [PATCH 1/2] test: fix flaky adaptive playwright timeout sub-crawler test Increase request_handler_timeout from 1s to 10s so the static pipeline has enough time to reach the handler on slow Windows CI before the timeout fires. Replace calibrated asyncio.sleep with asyncio.Event().wait() to block indefinitely instead of relying on a sleep > timeout duration. Name the mocks for easier debugging if the test fails again. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../test_adaptive_playwright_crawler.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index b6f0289c02..8aed535c8a 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -576,15 +576,17 @@ async def test_adaptive_playwright_crawler_timeout_in_sub_crawler(test_urls: lis crawler. """ static_only_predictor_no_detection = _SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0])) - request_handler_timeout = timedelta(seconds=1) + # Use a generous timeout so the static pipeline has enough time to reach the handler even on slow CI. + # The handler will block indefinitely, so the timeout will always fire during the handler's wait. + request_handler_timeout = timedelta(seconds=10) crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( max_request_retries=0, rendering_type_predictor=static_only_predictor_no_detection, request_handler_timeout=request_handler_timeout, ) - mocked_static_handler = Mock() - mocked_browser_handler = Mock() + mocked_static_handler = Mock(name='static_handler') + mocked_browser_handler = Mock(name='browser_handler') @crawler.router.default_handler async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: @@ -596,13 +598,13 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: mocked_static_handler() # Relax timeout for the fallback browser request to allow for slow browser startup on CI crawler._request_handler_timeout = timedelta(seconds=120) - # Sleep for time obviously larger than top crawler timeout. - await asyncio.sleep(request_handler_timeout.total_seconds() * 3) + # Block indefinitely - will be cancelled when the request_handler_timeout fires. + await asyncio.Event().wait() await crawler.run(test_urls[:1]) mocked_static_handler.assert_called_once_with() - # Browser handler was capable of running despite static handler having sleep time larger than top handler timeout. + # Browser handler was capable of running despite static handler blocking longer than the handler timeout. mocked_browser_handler.assert_called_once_with() From 44b6eeeb5058b2f824e3a4fe80450b49a1b243c5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 20 Mar 2026 15:33:04 +0100 Subject: [PATCH 2/2] test: increase browser pool operation timeout for Firefox on Windows CI The default 15s operation_timeout is not enough for Firefox's launch_persistent_context on busy Windows CI runners. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/unit/browsers/test_browser_pool.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/browsers/test_browser_pool.py b/tests/unit/browsers/test_browser_pool.py index 56e4706adc..a8e054379f 100644 --- a/tests/unit/browsers/test_browser_pool.py +++ b/tests/unit/browsers/test_browser_pool.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import timedelta from typing import TYPE_CHECKING from unittest.mock import AsyncMock @@ -103,7 +104,10 @@ async def test_new_page_with_each_plugin(server_url: URL) -> None: @run_alone_on_mac async def test_with_default_plugin_constructor(server_url: URL) -> None: - async with BrowserPool.with_default_plugin(headless=True, browser_type='firefox') as browser_pool: + # Use a generous operation timeout so that Firefox has enough time to launch on slow Windows CI. + async with BrowserPool.with_default_plugin( + headless=True, browser_type='firefox', operation_timeout=timedelta(seconds=60) + ) as browser_pool: assert len(browser_pool.plugins) == 1 assert isinstance(browser_pool.plugins[0], PlaywrightBrowserPlugin)