From 8f0b99737dc910346ab411468ed75ae8cdb34280 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Feb 2026 01:44:48 +0000 Subject: [PATCH 1/4] feat: add rate limit logging to HttpClient and WaitTimeFromHeaderBackoffStrategy Co-Authored-By: alexandre@airbyte.io --- .../wait_time_from_header_backoff_strategy.py | 10 ++++++++++ airbyte_cdk/sources/streams/http/http_client.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py index 5cda96a4d..be83d06c7 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging import re from dataclasses import InitVar, dataclass from typing import Any, Mapping, Optional, Union @@ -19,6 +20,8 @@ from airbyte_cdk.sources.types import Config from airbyte_cdk.utils import AirbyteTracedException +logger = logging.getLogger("airbyte") + @dataclass class WaitTimeFromHeaderBackoffStrategy(BackoffStrategy): @@ -57,6 +60,13 @@ def backoff_time( header_value = None if isinstance(response_or_exception, requests.Response): header_value = get_numeric_value_from_header(response_or_exception, header, regex) + if header_value is not None: + logger.info( + "Rate limit header '%s' detected with value: %s (status code: %d)", + header, + header_value, + response_or_exception.status_code, + ) if ( self.max_waiting_time_in_seconds and header_value diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index e7a5715ac..c86ed3b95 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -440,6 +440,12 @@ def _handle_error_resolution( # Emit stream status RUNNING with the reason RATE_LIMITED to log that the rate limit has been reached if error_resolution.response_action == ResponseAction.RATE_LIMITED: + self._logger.info( + "Rate limited: emitting RATE_LIMITED stream status for stream '%s' (status code: %s, url: %s)", + self._name, + response.status_code if response is not None else "N/A", + request.url, + ) # TODO: Update to handle with message repository when concurrent message repository is ready reasons = [AirbyteStreamStatusReason(type=AirbyteStreamStatusReasonType.RATE_LIMITED)] message = orjson.dumps( @@ -524,6 +530,15 @@ def _handle_error_resolution( if backoff_time: user_defined_backoff_time = backoff_time break + if user_defined_backoff_time is not None: + self._logger.info( + "Rate limit backoff: waiting %.2f seconds before retry (attempt %d, status code: %s, action: %s, url: %s)", + user_defined_backoff_time, + self._request_attempt_count[request], + response.status_code if response is not None else "N/A", + error_resolution.response_action.value, + request.url, + ) error_message = ( error_resolution.error_message or f"Request to {request.url} failed with failure type {error_resolution.failure_type}, response action {error_resolution.response_action}." From 8ed04ef579dc9320414c188013fceb486973ec5a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Feb 2026 01:47:05 +0000 Subject: [PATCH 2/4] feat: include error message in rate limit and backoff log lines Co-Authored-By: alexandre@airbyte.io --- airbyte_cdk/sources/streams/http/http_client.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index c86ed3b95..aed16b9fd 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -441,10 +441,16 @@ def _handle_error_resolution( # Emit stream status RUNNING with the reason RATE_LIMITED to log that the rate limit has been reached if error_resolution.response_action == ResponseAction.RATE_LIMITED: self._logger.info( - "Rate limited: emitting RATE_LIMITED stream status for stream '%s' (status code: %s, url: %s)", + "Rate limited: emitting RATE_LIMITED stream status for stream '%s' (status code: %s, url: %s, error: %s)", self._name, response.status_code if response is not None else "N/A", request.url, + error_resolution.error_message + or ( + self._error_message_parser.parse_response_error_message(response) + if response is not None + else str(exc) + ), ) # TODO: Update to handle with message repository when concurrent message repository is ready reasons = [AirbyteStreamStatusReason(type=AirbyteStreamStatusReasonType.RATE_LIMITED)] @@ -532,12 +538,18 @@ def _handle_error_resolution( break if user_defined_backoff_time is not None: self._logger.info( - "Rate limit backoff: waiting %.2f seconds before retry (attempt %d, status code: %s, action: %s, url: %s)", + "Retrying with backoff: waiting %.2f seconds (attempt %d, status code: %s, action: %s, url: %s, error: %s)", user_defined_backoff_time, self._request_attempt_count[request], response.status_code if response is not None else "N/A", error_resolution.response_action.value, request.url, + error_resolution.error_message + or ( + self._error_message_parser.parse_response_error_message(response) + if response is not None + else str(exc) + ), ) error_message = ( error_resolution.error_message From 0496deef671aeae5ffda8f9c068765fd19f56ef8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Feb 2026 01:54:29 +0000 Subject: [PATCH 3/4] fix: handle missing status_code attribute in backoff strategy logging Co-Authored-By: alexandre@airbyte.io --- .../wait_time_from_header_backoff_strategy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py index be83d06c7..851f75869 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py @@ -62,10 +62,10 @@ def backoff_time( header_value = get_numeric_value_from_header(response_or_exception, header, regex) if header_value is not None: logger.info( - "Rate limit header '%s' detected with value: %s (status code: %d)", + "Rate limit header '%s' detected with value: %s (status code: %s)", header, header_value, - response_or_exception.status_code, + response_or_exception.status_code if hasattr(response_or_exception, "status_code") else "N/A", ) if ( self.max_waiting_time_in_seconds From e0b8b487f00ee7c760d9e66eb60ff3371f3cbe82 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Feb 2026 01:55:44 +0000 Subject: [PATCH 4/4] style: fix ruff formatting Co-Authored-By: alexandre@airbyte.io --- .../wait_time_from_header_backoff_strategy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py index 851f75869..e4e44883e 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py @@ -65,7 +65,9 @@ def backoff_time( "Rate limit header '%s' detected with value: %s (status code: %s)", header, header_value, - response_or_exception.status_code if hasattr(response_or_exception, "status_code") else "N/A", + response_or_exception.status_code + if hasattr(response_or_exception, "status_code") + else "N/A", ) if ( self.max_waiting_time_in_seconds