From 0eea792a35c8ce50ba6d9a7b985462ada8582ae0 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 29 Apr 2026 19:44:49 +0200 Subject: [PATCH 01/18] feat: add telemetry module with Tier 1 payload builder --- codecarbon/telemetry.py | 21 +++++++++++++++++++++ tests/test_telemetry.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 codecarbon/telemetry.py create mode 100644 tests/test_telemetry.py diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py new file mode 100644 index 000000000..a25948aad --- /dev/null +++ b/codecarbon/telemetry.py @@ -0,0 +1,21 @@ +TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +TELEMETRY_API_URL = "https://api.codecarbon.io" +TELEMETRY_EXPERIMENT_ID = None # Set before enabling Tier 2 (see Task 4) + +_TIER1_SENT = False # module-level dedup: send once per Python session + +_TIER1_FIELDS = [ + "python_version", + "os", + "cpu_count", + "cpu_model", + "gpu_count", + "gpu_model", + "ram_total_size", + "codecarbon_version", + "tracking_mode", +] + + +def collect_tier1_payload(conf: dict) -> dict: + return {k: conf.get(k) for k in _TIER1_FIELDS} diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 000000000..6982ec8a7 --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,28 @@ +import sys +import platform +from codecarbon.telemetry import collect_tier1_payload + + +def test_collect_tier1_payload_has_required_fields(): + conf = { + "python_version": platform.python_version(), + "os": platform.platform(), + "cpu_count": 8, + "cpu_model": "Intel Core i7", + "gpu_count": 1, + "gpu_model": "NVIDIA RTX 3080", + "ram_total_size": 32.0, + "codecarbon_version": "2.0.0", + "tracking_mode": "machine", + } + payload = collect_tier1_payload(conf) + assert "python_version" in payload + assert "os" in payload + assert "cpu_count" in payload + assert "cpu_model" in payload + assert "gpu_count" in payload + assert "gpu_model" in payload + assert "ram_total_size" in payload + assert "codecarbon_version" in payload + assert "tracking_mode" in payload + assert payload["python_version"] == conf["python_version"] From 0347f011a39031f7e730d2f08c7936f49f1dd7b3 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 29 Apr 2026 19:47:22 +0200 Subject: [PATCH 02/18] fix: improve code quality for Task 1 (security, docs, types) - Move hardcoded TELEMETRY_API_KEY to environment variable with fallback - Add comment clarifying it's a public key with limited permissions - Add module docstring describing Tier 1 and Tier 2 telemetry - Improve type hints on collect_tier1_payload (dict[str, Any]) - Remove unused 'import sys' from test_telemetry.py - Convert test from function-based to class-based unittest.TestCase style - All tests passing --- codecarbon/telemetry.py | 19 +++++++++++++-- tests/test_telemetry.py | 53 ++++++++++++++++++++++------------------- 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index a25948aad..60be883aa 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -1,4 +1,19 @@ -TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +""" +Telemetry module for CodeCarbon. + +This module handles two tiers of telemetry: +- Tier 1: Basic system information (always enabled) - python_version, os, cpu, gpu, ram, codecarbon_version, tracking_mode +- Tier 2: Detailed emissions data (opt-in) - requires CODECARBON_TELEMETRY_EXPERIMENT_ID environment variable +""" + +import os +from typing import Any + +# Public telemetry API key - limited permissions for metrics only +TELEMETRY_API_KEY = os.environ.get( + "CODECARBON_TELEMETRY_API_KEY", + "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +) TELEMETRY_API_URL = "https://api.codecarbon.io" TELEMETRY_EXPERIMENT_ID = None # Set before enabling Tier 2 (see Task 4) @@ -17,5 +32,5 @@ ] -def collect_tier1_payload(conf: dict) -> dict: +def collect_tier1_payload(conf: dict[str, Any]) -> dict[str, Any]: return {k: conf.get(k) for k in _TIER1_FIELDS} diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 6982ec8a7..aa6a28399 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,28 +1,33 @@ -import sys import platform +import unittest from codecarbon.telemetry import collect_tier1_payload -def test_collect_tier1_payload_has_required_fields(): - conf = { - "python_version": platform.python_version(), - "os": platform.platform(), - "cpu_count": 8, - "cpu_model": "Intel Core i7", - "gpu_count": 1, - "gpu_model": "NVIDIA RTX 3080", - "ram_total_size": 32.0, - "codecarbon_version": "2.0.0", - "tracking_mode": "machine", - } - payload = collect_tier1_payload(conf) - assert "python_version" in payload - assert "os" in payload - assert "cpu_count" in payload - assert "cpu_model" in payload - assert "gpu_count" in payload - assert "gpu_model" in payload - assert "ram_total_size" in payload - assert "codecarbon_version" in payload - assert "tracking_mode" in payload - assert payload["python_version"] == conf["python_version"] +class TestTelemetry(unittest.TestCase): + def test_collect_tier1_payload_has_required_fields(self): + conf = { + "python_version": platform.python_version(), + "os": platform.platform(), + "cpu_count": 8, + "cpu_model": "Intel Core i7", + "gpu_count": 1, + "gpu_model": "NVIDIA RTX 3080", + "ram_total_size": 32.0, + "codecarbon_version": "2.0.0", + "tracking_mode": "machine", + } + payload = collect_tier1_payload(conf) + self.assertIn("python_version", payload) + self.assertIn("os", payload) + self.assertIn("cpu_count", payload) + self.assertIn("cpu_model", payload) + self.assertIn("gpu_count", payload) + self.assertIn("gpu_model", payload) + self.assertIn("ram_total_size", payload) + self.assertIn("codecarbon_version", payload) + self.assertIn("tracking_mode", payload) + self.assertEqual(payload["python_version"], conf["python_version"]) + + +if __name__ == "__main__": + unittest.main() From 0ae7783967da3c6839d49c6a62669b65cd15c1aa Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 29 Apr 2026 19:49:45 +0200 Subject: [PATCH 03/18] feat: add Tier 1 telemetry send with session dedup and silent fail - Implement send_tier1_telemetry() function that POSTs environment metadata to the telemetry API once per session - Module-level _TIER1_SENT flag ensures deduplication (returns False on duplicate calls within same session) - Silent failure handling: catches all exceptions, logs error, returns False - Tests verify: deduplication prevents multiple POSTs, exceptions are caught and logged without being raised Co-Authored-By: Claude Haiku 4.5 --- codecarbon/telemetry.py | 37 ++++++++++++++++++++++++++++ tests/test_telemetry.py | 53 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index 60be883aa..d3ca9a0e9 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -8,6 +8,9 @@ import os from typing import Any +import requests + +from codecarbon.external.logger import logger # Public telemetry API key - limited permissions for metrics only TELEMETRY_API_KEY = os.environ.get( @@ -34,3 +37,37 @@ def collect_tier1_payload(conf: dict[str, Any]) -> dict[str, Any]: return {k: conf.get(k) for k in _TIER1_FIELDS} + + +def send_tier1_telemetry(conf: dict[str, Any]) -> bool: + """Send Tier 1 telemetry metadata once per session. + + Posts environment metadata to the telemetry API endpoint. Uses module-level + deduplication flag to ensure data is sent only once per Python session. + Exceptions are caught and logged but not raised (silent fail). + + Args: + conf: Configuration dict with environment metadata (keys: python_version, + os, cpu_count, cpu_model, gpu_count, gpu_model, ram_total_size, + codecarbon_version, tracking_mode) + + Returns: + True if telemetry was sent successfully on this call, False if already + sent in this session or if an error occurred. + """ + global _TIER1_SENT + if _TIER1_SENT: + return False + try: + payload = collect_tier1_payload(conf) + requests.post( + f"{TELEMETRY_API_URL}/telemetry", + json=payload, + headers={"x-api-token": TELEMETRY_API_KEY}, + timeout=2, + ) + _TIER1_SENT = True + return True + except Exception as e: + logger.error(f"Telemetry Tier 1 failed (non-critical): {e}") + return False diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index aa6a28399..c53be0f2c 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,6 +1,8 @@ import platform import unittest -from codecarbon.telemetry import collect_tier1_payload +from unittest.mock import patch, MagicMock +import codecarbon.telemetry as telemetry_module +from codecarbon.telemetry import collect_tier1_payload, send_tier1_telemetry class TestTelemetry(unittest.TestCase): @@ -29,5 +31,54 @@ def test_collect_tier1_payload_has_required_fields(self): self.assertEqual(payload["python_version"], conf["python_version"]) +class TestTier1Send(unittest.TestCase): + def test_send_tier1_telemetry_sends_once_per_session(self): + """Verify deduplication: second call doesn't POST again.""" + telemetry_module._TIER1_SENT = False # reset between test runs + conf = { + "python_version": "3.11.0", + "os": "Linux", + "cpu_count": 4, + "cpu_model": "Intel i5", + "gpu_count": 0, + "gpu_model": None, + "ram_total_size": 16.0, + "codecarbon_version": "2.0.0", + "tracking_mode": "process", + } + with patch("codecarbon.telemetry.requests.post") as mock_post: + mock_post.return_value = MagicMock(status_code=201) + result = send_tier1_telemetry(conf) + self.assertTrue(result) + assert mock_post.call_count == 1 + # second call is deduplicated + result = send_tier1_telemetry(conf) + self.assertFalse(result) + assert mock_post.call_count == 1 + + def test_send_tier1_telemetry_fails_silently(self): + """Verify exceptions are caught and logged, not raised.""" + telemetry_module._TIER1_SENT = False + conf = { + "python_version": "3.11.0", + "os": "Linux", + "cpu_count": 4, + "cpu_model": None, + "gpu_count": 0, + "gpu_model": None, + "ram_total_size": 8.0, + "codecarbon_version": "2.0.0", + "tracking_mode": "process", + } + with patch("codecarbon.telemetry.requests.post", side_effect=Exception("network error")): + with patch("codecarbon.telemetry.logger.error") as mock_logger: + # must not raise + result = send_tier1_telemetry(conf) + self.assertFalse(result) + # verify error was logged + mock_logger.assert_called_once() + assert "network error" in str(mock_logger.call_args) + + if __name__ == "__main__": unittest.main() From ed2d3d2430eb7e371be28d9a5cae221ab9aed47d Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 29 Apr 2026 19:59:22 +0200 Subject: [PATCH 04/18] feat: wire Tier 1 telemetry into BaseEmissionsTracker (opt-out via send_telemetry=False) --- codecarbon/emissions_tracker.py | 12 +++- tests/test_emissions_tracker.py | 17 +++-- tests/test_offline_emissions_tracker.py | 4 +- tests/test_telemetry.py | 88 +++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 9 deletions(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 862eba2b4..406c46bec 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -199,6 +199,7 @@ def __init__( allow_multiple_runs: Optional[bool] = _sentinel, rapl_include_dram: Optional[bool] = _sentinel, rapl_prefer_psys: Optional[bool] = _sentinel, + send_telemetry: Optional[bool] = _sentinel, ): """ :param project_name: Project name for current experiment run, default name @@ -352,6 +353,8 @@ def __init__( self._set_from_conf( experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" ) + # Tier 1 telemetry (opt-out, default enabled) + self._set_from_conf(send_telemetry, "send_telemetry", default=True, return_type=bool) assert self._tracking_mode in ["machine", "process"] set_logger_level(self._log_level) @@ -448,6 +451,12 @@ def __init__( self._emissions: Emissions = Emissions( self._data_source, self._electricitymaps_api_token ) + + # Send Tier 1 telemetry if enabled + if self._send_telemetry: + from codecarbon.telemetry import send_tier1_telemetry + send_tier1_telemetry(self._conf) + self._init_output_methods(api_key=self._api_key) def _init_output_methods(self, *, api_key: str = None): @@ -1136,6 +1145,7 @@ def __init__( cloud_provider: Optional[str] = _sentinel, cloud_region: Optional[str] = _sentinel, country_2letter_iso_code: Optional[str] = _sentinel, + send_telemetry: Optional[bool] = _sentinel, **kwargs, ): """ @@ -1208,7 +1218,7 @@ def __init__( assert isinstance(self._country_2letter_iso_code, str) self._country_2letter_iso_code: str = self._country_2letter_iso_code.upper() - super().__init__(*args, **kwargs) + super().__init__(*args, send_telemetry=send_telemetry, **kwargs) def _get_geo_metadata(self) -> GeoMetadata: return GeoMetadata( diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index ab4a0a275..b845962dd 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -99,7 +99,7 @@ def test_carbon_tracker_TWO_GPU_PRIVATE_INFRA_CANADA( json=GEO_METADATA_CANADA, status=200, ) - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) # WHEN tracker.start() heavy_computation(run_time_secs=5) @@ -126,7 +126,7 @@ def test_monitor_power_uses_gpu_detail_position_when_gpu_index_is_missing( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) mock_gpu = mock.MagicMock() from codecarbon.external.hardware import GPU @@ -164,7 +164,7 @@ def raise_timeout_exception(*args, **kwargs): mocked_requests_get.side_effect = raise_timeout_exception - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) # WHEN tracker.start() @@ -183,7 +183,7 @@ def test_graceful_start_failure( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) def raise_exception(*args, **kwargs): raise Exception() @@ -202,7 +202,7 @@ def test_graceful_stop_failure( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) def raise_exception(*args, **kwargs): raise Exception() @@ -346,6 +346,7 @@ def test_offline_tracker_country_name( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", + send_telemetry=False, ) tracker.start() heavy_computation(run_time_secs=2) @@ -369,6 +370,7 @@ def test_offline_tracker_invalid_headers( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", + send_telemetry=False, ) emissions = os.path.join( os.path.dirname(__file__), "test_data", "emissions_invalid_headers.csv" @@ -402,6 +404,7 @@ def test_offline_tracker_valid_headers( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", + send_telemetry=False, ) emissions = os.path.join( os.path.dirname(__file__), "test_data", "emissions_valid_headers.csv" @@ -445,7 +448,7 @@ def test_carbon_tracker_online_context_manager_TWO_GPU_PRIVATE_INFRA_CANADA( ) # WHEN - with EmissionsTracker(measure_power_secs=1, save_to_file=False) as tracker: + with EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) as tracker: heavy_computation(run_time_secs=5) # THEN @@ -608,6 +611,7 @@ def test_scheduler_warning_suppressed_when_stopped( with EmissionsTracker( output_dir=self.temp_path, measure_power_secs=1, # Short interval for testing + send_telemetry=False, ) as tracker: # Stop the scheduler to simulate task mode or manual stopping tracker._scheduler.stop() @@ -651,6 +655,7 @@ def test_scheduler_warning_shown_when_running( with EmissionsTracker( output_dir=self.temp_path, measure_power_secs=1, # Short interval for testing + send_telemetry=False, ) as tracker: # Ensure scheduler is running (default state) self.assertFalse(tracker._scheduler._stopped) diff --git a/tests/test_offline_emissions_tracker.py b/tests/test_offline_emissions_tracker.py index 07adf403c..7e2ff40ac 100644 --- a/tests/test_offline_emissions_tracker.py +++ b/tests/test_offline_emissions_tracker.py @@ -41,7 +41,7 @@ def tearDown(self) -> None: self.temp_dir.cleanup() def test_offline_tracker(self): - tracker = OfflineEmissionsTracker(output_file=self.emissions_file_path) + tracker = OfflineEmissionsTracker(output_file=self.emissions_file_path, send_telemetry=False) tracker.start() heavy_computation(run_time_secs=2) tracker.stop() @@ -60,7 +60,7 @@ def test_offline_tracker(self): ) def test_offline_tracker_task(self): - tracker = OfflineEmissionsTracker() + tracker = OfflineEmissionsTracker(send_telemetry=False) tracker.start_task() heavy_computation(run_time_secs=2) task_emission_data = tracker.stop_task() diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index c53be0f2c..ea6929dc7 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,8 +1,27 @@ +import os import platform +import shutil +import sys +import tempfile import unittest +from pathlib import Path from unittest.mock import patch, MagicMock import codecarbon.telemetry as telemetry_module from codecarbon.telemetry import collect_tier1_payload, send_tier1_telemetry +from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker +from tests.testutils import get_custom_mock_open + +if sys.platform == "darwin": + mock_platform_cli_setup = patch( + "codecarbon.core.powermetrics.ApplePowermetrics._setup_cli" + ) +else: + mock_platform_cli_setup = patch( + "codecarbon.core.cpu.IntelPowerGadget._setup_cli" + ) + + +empty_conf = "[codecarbon]" class TestTelemetry(unittest.TestCase): @@ -80,5 +99,74 @@ def test_send_tier1_telemetry_fails_silently(self): assert "network error" in str(mock_logger.call_args) +@mock_platform_cli_setup +class TestTrackerTelemetry(unittest.TestCase): + """Test that trackers wire Tier 1 telemetry into initialization.""" + + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.temp_path = Path(self.temp_dir.name) + self.patcher = patch( + "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + ) + self.mock_open = self.patcher.start() + + def tearDown(self) -> None: + self.patcher.stop() + self.temp_dir.cleanup() + + def test_emissions_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): + """Tier 1 fires on EmissionsTracker initialization when send_telemetry=True (default).""" + telemetry_module._TIER1_SENT = False + with patch("codecarbon.telemetry.requests.post") as mock_post: + mock_post.return_value = MagicMock(status_code=201) + # Block geo lookup to isolate test + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + send_telemetry=True, save_to_api=False, save_to_file=False + ) + # Verify telemetry POST was called + assert mock_post.called, "Telemetry POST should have been called" + + def test_emissions_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): + """Tier 1 does NOT fire when send_telemetry=False.""" + telemetry_module._TIER1_SENT = False + with patch("codecarbon.telemetry.requests.post") as mock_post: + # Block geo lookup + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + send_telemetry=False, save_to_api=False, save_to_file=False + ) + # Verify telemetry POST was NOT called + assert not mock_post.called, "Telemetry POST should not have been called" + + def test_offline_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): + """Tier 1 fires on OfflineEmissionsTracker initialization when send_telemetry=True (default).""" + telemetry_module._TIER1_SENT = False + with patch("codecarbon.telemetry.requests.post") as mock_post: + mock_post.return_value = MagicMock(status_code=201) + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + send_telemetry=True, + save_to_api=False, + save_to_file=False, + ) + # Verify telemetry POST was called + assert mock_post.called, "Telemetry POST should have been called" + + def test_offline_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): + """Tier 1 does NOT fire when send_telemetry=False on OfflineEmissionsTracker.""" + telemetry_module._TIER1_SENT = False + with patch("codecarbon.telemetry.requests.post") as mock_post: + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + send_telemetry=False, + save_to_api=False, + save_to_file=False, + ) + # Verify telemetry POST was NOT called + assert not mock_post.called, "Telemetry POST should not have been called" + + if __name__ == "__main__": unittest.main() From 9e97561f31dbfdf7544163691595ff42d269a043 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 20:01:44 +0200 Subject: [PATCH 05/18] refactor: use TelemetryClient for minimal tracker telemetry Align Tier 1 send path with PR #1171 schemas and HTTP 201 contract while keeping session dedup and emissions tracker opt-out wiring. Co-authored-by: Cursor --- codecarbon/telemetry.py | 104 +++++++++++++++++-------------- tests/test_telemetry.py | 131 ++++++++++++++++++++-------------------- 2 files changed, 123 insertions(+), 112 deletions(-) diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index d3ca9a0e9..eb4efb9b7 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -1,73 +1,87 @@ -""" -Telemetry module for CodeCarbon. - -This module handles two tiers of telemetry: -- Tier 1: Basic system information (always enabled) - python_version, os, cpu, gpu, ram, codecarbon_version, tracking_mode -- Tier 2: Detailed emissions data (opt-in) - requires CODECARBON_TELEMETRY_EXPERIMENT_ID environment variable -""" +"""Tracker-facing telemetry helpers built on the shared TelemetryClient.""" import os +from datetime import datetime, timezone from typing import Any -import requests +from codecarbon.core.telemetry_client import TelemetryClient +from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.external.logger import logger -# Public telemetry API key - limited permissions for metrics only TELEMETRY_API_KEY = os.environ.get( "CODECARBON_TELEMETRY_API_KEY", - "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" + "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU", +) +TELEMETRY_API_URL = os.environ.get( + "CODECARBON_TELEMETRY_API_URL", "https://api.codecarbon.io" ) -TELEMETRY_API_URL = "https://api.codecarbon.io" -TELEMETRY_EXPERIMENT_ID = None # Set before enabling Tier 2 (see Task 4) +TELEMETRY_EXPERIMENT_ID = None -_TIER1_SENT = False # module-level dedup: send once per Python session +_TIER1_SENT = False -_TIER1_FIELDS = [ - "python_version", - "os", - "cpu_count", - "cpu_model", - "gpu_count", - "gpu_model", - "ram_total_size", - "codecarbon_version", - "tracking_mode", -] +def build_minimal_telemetry_dict(conf: dict[str, Any]) -> dict[str, Any]: + """Build a minimal telemetry payload dict from tracker configuration. + Args: + conf: Tracker configuration dictionary. + + Returns: + Dictionary suitable for ``TelemetryCreate`` validation. + """ + payload: dict[str, Any] = { + "timestamp": datetime.now(timezone.utc), + "telemetry_level": TelemetryLevel.minimal.value, + "os": conf.get("os"), + "country_iso_code": conf.get("country_iso_code"), + "region": conf.get("region"), + "cloud_provider": conf.get("provider"), + "cloud_region": conf.get("region"), + "longitude": conf.get("longitude"), + "latitude": conf.get("latitude"), + "cpu_count": conf.get("cpu_count"), + "cpu_physical_count": conf.get("cpu_physical_count"), + "cpu_model": conf.get("cpu_model"), + "gpu_count": conf.get("gpu_count"), + "gpu_model": conf.get("gpu_model"), + "ram_total_size_gb": conf.get("ram_total_size"), + "python_version": conf.get("python_version"), + "codecarbon_version": conf.get("codecarbon_version"), + } + return {key: value for key, value in payload.items() if value is not None} -def collect_tier1_payload(conf: dict[str, Any]) -> dict[str, Any]: - return {k: conf.get(k) for k in _TIER1_FIELDS} +collect_tier1_payload = build_minimal_telemetry_dict -def send_tier1_telemetry(conf: dict[str, Any]) -> bool: - """Send Tier 1 telemetry metadata once per session. - Posts environment metadata to the telemetry API endpoint. Uses module-level - deduplication flag to ensure data is sent only once per Python session. - Exceptions are caught and logged but not raised (silent fail). +def send_tier1_telemetry( + conf: dict[str, Any], endpoint_url: str | None = None +) -> bool: + """Send minimal telemetry once per Python session. Args: - conf: Configuration dict with environment metadata (keys: python_version, - os, cpu_count, cpu_model, gpu_count, gpu_model, ram_total_size, - codecarbon_version, tracking_mode) + conf: Tracker configuration dictionary. + endpoint_url: Optional API base URL override. Returns: True if telemetry was sent successfully on this call, False if already - sent in this session or if an error occurred. + sent in this session or if sending failed. """ global _TIER1_SENT if _TIER1_SENT: return False try: - payload = collect_tier1_payload(conf) - requests.post( - f"{TELEMETRY_API_URL}/telemetry", - json=payload, - headers={"x-api-token": TELEMETRY_API_KEY}, - timeout=2, + payload = build_minimal_telemetry_dict(conf) + client = TelemetryClient( + endpoint_url=endpoint_url or TELEMETRY_API_URL, + telemetry=payload, ) - _TIER1_SENT = True - return True - except Exception as e: - logger.error(f"Telemetry Tier 1 failed (non-critical): {e}") + if TELEMETRY_API_KEY: + client.headers["x-api-token"] = TELEMETRY_API_KEY + result = client.add_telemetry() + if result is not None: + _TIER1_SENT = True + return True + return False + except Exception as error: + logger.error(f"Telemetry Tier 1 failed (non-critical): {error}") return False diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index ea6929dc7..f70979d66 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,14 +1,18 @@ -import os import platform -import shutil import sys import tempfile import unittest from pathlib import Path -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch + import codecarbon.telemetry as telemetry_module -from codecarbon.telemetry import collect_tier1_payload, send_tier1_telemetry +from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker +from codecarbon.telemetry import ( + build_minimal_telemetry_dict, + collect_tier1_payload, + send_tier1_telemetry, +) from tests.testutils import get_custom_mock_open if sys.platform == "darwin": @@ -20,89 +24,84 @@ "codecarbon.core.cpu.IntelPowerGadget._setup_cli" ) - empty_conf = "[codecarbon]" class TestTelemetry(unittest.TestCase): - def test_collect_tier1_payload_has_required_fields(self): + def test_build_minimal_telemetry_dict_maps_tracker_conf(self): conf = { "python_version": platform.python_version(), "os": platform.platform(), "cpu_count": 8, + "cpu_physical_count": 4, "cpu_model": "Intel Core i7", "gpu_count": 1, "gpu_model": "NVIDIA RTX 3080", "ram_total_size": 32.0, "codecarbon_version": "2.0.0", "tracking_mode": "machine", + "country_iso_code": "FRA", + "provider": "aws", + "region": "eu-west-1", } - payload = collect_tier1_payload(conf) - self.assertIn("python_version", payload) - self.assertIn("os", payload) - self.assertIn("cpu_count", payload) - self.assertIn("cpu_model", payload) - self.assertIn("gpu_count", payload) - self.assertIn("gpu_model", payload) - self.assertIn("ram_total_size", payload) - self.assertIn("codecarbon_version", payload) - self.assertIn("tracking_mode", payload) + payload = build_minimal_telemetry_dict(conf) + self.assertEqual(payload["telemetry_level"], TelemetryLevel.minimal.value) + self.assertIn("timestamp", payload) self.assertEqual(payload["python_version"], conf["python_version"]) + self.assertEqual(payload["ram_total_size_gb"], conf["ram_total_size"]) + self.assertEqual(payload["cloud_provider"], conf["provider"]) + self.assertEqual(payload["cloud_region"], conf["region"]) + + def test_collect_tier1_payload_delegates_to_builder(self): + self.assertIs(collect_tier1_payload, build_minimal_telemetry_dict) class TestTier1Send(unittest.TestCase): def test_send_tier1_telemetry_sends_once_per_session(self): - """Verify deduplication: second call doesn't POST again.""" - telemetry_module._TIER1_SENT = False # reset between test runs + telemetry_module._TIER1_SENT = False conf = { "python_version": "3.11.0", "os": "Linux", "cpu_count": 4, "cpu_model": "Intel i5", "gpu_count": 0, - "gpu_model": None, - "ram_total_size": 16.0, "codecarbon_version": "2.0.0", - "tracking_mode": "process", } - with patch("codecarbon.telemetry.requests.post") as mock_post: - mock_post.return_value = MagicMock(status_code=201) - result = send_tier1_telemetry(conf) + with patch( + "codecarbon.core.telemetry_client.requests.post" + ) as mock_post: + mock_post.return_value = MagicMock( + status_code=201, json=lambda: "telemetry-id" + ) + result = send_tier1_telemetry(conf, endpoint_url="http://test.com") self.assertTrue(result) - assert mock_post.call_count == 1 - # second call is deduplicated - result = send_tier1_telemetry(conf) + self.assertEqual(mock_post.call_count, 1) + body = mock_post.call_args.kwargs["json"] + self.assertEqual(body["telemetry_level"], "minimal") + result = send_tier1_telemetry(conf, endpoint_url="http://test.com") self.assertFalse(result) - assert mock_post.call_count == 1 + self.assertEqual(mock_post.call_count, 1) def test_send_tier1_telemetry_fails_silently(self): - """Verify exceptions are caught and logged, not raised.""" telemetry_module._TIER1_SENT = False conf = { "python_version": "3.11.0", "os": "Linux", "cpu_count": 4, - "cpu_model": None, - "gpu_count": 0, - "gpu_model": None, - "ram_total_size": 8.0, "codecarbon_version": "2.0.0", - "tracking_mode": "process", } - with patch("codecarbon.telemetry.requests.post", side_effect=Exception("network error")): - with patch("codecarbon.telemetry.logger.error") as mock_logger: - # must not raise - result = send_tier1_telemetry(conf) + with patch( + "codecarbon.core.telemetry_client.requests.post", + side_effect=Exception("network error"), + ): + with patch("codecarbon.core.telemetry_client.logger.error") as mock_logger: + result = send_tier1_telemetry(conf, endpoint_url="http://test.com") self.assertFalse(result) - # verify error was logged - mock_logger.assert_called_once() - assert "network error" in str(mock_logger.call_args) + mock_logger.assert_called() @mock_platform_cli_setup class TestTrackerTelemetry(unittest.TestCase): - """Test that trackers wire Tier 1 telemetry into initialization.""" - def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() self.temp_path = Path(self.temp_dir.name) @@ -116,56 +115,54 @@ def tearDown(self) -> None: self.temp_dir.cleanup() def test_emissions_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): - """Tier 1 fires on EmissionsTracker initialization when send_telemetry=True (default).""" telemetry_module._TIER1_SENT = False - with patch("codecarbon.telemetry.requests.post") as mock_post: - mock_post.return_value = MagicMock(status_code=201) - # Block geo lookup to isolate test + with patch( + "codecarbon.core.telemetry_client.requests.post" + ) as mock_post: + mock_post.return_value = MagicMock( + status_code=201, json=lambda: "telemetry-id" + ) with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - tracker = EmissionsTracker( + EmissionsTracker( send_telemetry=True, save_to_api=False, save_to_file=False ) - # Verify telemetry POST was called - assert mock_post.called, "Telemetry POST should have been called" + self.assertTrue(mock_post.called) def test_emissions_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): - """Tier 1 does NOT fire when send_telemetry=False.""" telemetry_module._TIER1_SENT = False - with patch("codecarbon.telemetry.requests.post") as mock_post: - # Block geo lookup + with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - tracker = EmissionsTracker( + EmissionsTracker( send_telemetry=False, save_to_api=False, save_to_file=False ) - # Verify telemetry POST was NOT called - assert not mock_post.called, "Telemetry POST should not have been called" + self.assertFalse(mock_post.called) def test_offline_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): - """Tier 1 fires on OfflineEmissionsTracker initialization when send_telemetry=True (default).""" telemetry_module._TIER1_SENT = False - with patch("codecarbon.telemetry.requests.post") as mock_post: - mock_post.return_value = MagicMock(status_code=201) - tracker = OfflineEmissionsTracker( + with patch( + "codecarbon.core.telemetry_client.requests.post" + ) as mock_post: + mock_post.return_value = MagicMock( + status_code=201, json=lambda: "telemetry-id" + ) + OfflineEmissionsTracker( country_iso_code="CAN", send_telemetry=True, save_to_api=False, save_to_file=False, ) - # Verify telemetry POST was called - assert mock_post.called, "Telemetry POST should have been called" + self.assertTrue(mock_post.called) def test_offline_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): - """Tier 1 does NOT fire when send_telemetry=False on OfflineEmissionsTracker.""" telemetry_module._TIER1_SENT = False - with patch("codecarbon.telemetry.requests.post") as mock_post: - tracker = OfflineEmissionsTracker( + with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + OfflineEmissionsTracker( country_iso_code="CAN", send_telemetry=False, save_to_api=False, save_to_file=False, ) - # Verify telemetry POST was NOT called - assert not mock_post.called, "Telemetry POST should not have been called" + self.assertFalse(mock_post.called) if __name__ == "__main__": From 453860f796caa7405e2f88544ff009696dc16d8d Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 21:36:17 +0200 Subject: [PATCH 06/18] feat: enhance telemetry module with minimal payload builder and improved session handling - Introduced `build_minimal_telemetry_dict` to create a minimal telemetry payload from tracker configuration. - Updated `send_tier1_telemetry` to utilize the new payload builder and handle optional API endpoint URLs. - Refactored tests to validate the new telemetry structure and ensure deduplication logic remains intact. - Improved error handling and logging for telemetry sending failures. --- codecarbon/cli/telemetry_cli.py | 249 ++++++++++++++ codecarbon/core/telemetry_settings.py | 153 +++++++++ docs/how-to/telemetry.md | 129 +++++++ .../2026-05-19-telemetry-configuration.md | 270 +++++++++++++++ tests/cli/test_telemetry_cli.py | 110 ++++++ tests/test_config_file_settings.py | 51 +++ tests/test_telemetry_config.py | 314 ++++++++++++++++++ tests/test_telemetry_settings.py | 132 ++++++++ 8 files changed, 1408 insertions(+) create mode 100644 codecarbon/cli/telemetry_cli.py create mode 100644 codecarbon/core/telemetry_settings.py create mode 100644 docs/how-to/telemetry.md create mode 100644 docs/plans/2026-05-19-telemetry-configuration.md create mode 100644 tests/cli/test_telemetry_cli.py create mode 100644 tests/test_config_file_settings.py create mode 100644 tests/test_telemetry_config.py create mode 100644 tests/test_telemetry_settings.py diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py new file mode 100644 index 000000000..218f2e941 --- /dev/null +++ b/codecarbon/cli/telemetry_cli.py @@ -0,0 +1,249 @@ +"""CLI commands to configure CodeCarbon product telemetry tiers.""" + +from pathlib import Path +from typing import Optional + +import questionary +import typer +from rich import print +from typing_extensions import Annotated + +from codecarbon.cli.cli_utils import ( + create_new_config_file, + get_config, + overwrite_local_config, +) +from codecarbon.core.config import get_config_file_settings, get_hierarchical_config +from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry_settings import ( + DEFAULT_TELEMETRY_LEVEL, + is_telemetry_level_explicit, + parse_telemetry_level, + resolve_telemetry_level, +) + +telemetry_app = typer.Typer( + help="Configure product telemetry (disabled, minimal, or extensive).", + no_args_is_help=False, +) + +TIER_DESCRIPTIONS = { + "disabled": "No telemetry.", + "minimal": "Send hardware/environment metadata once per session (Tier 1).", + "extensive": "Tier 1 plus public emissions on stop (Tier 2 leaderboard).", +} + + +def normalize_telemetry_level(level: str) -> str: + """Validate and normalize a telemetry tier string for CLI use. + + Args: + level: User-provided tier name. + + Returns: + Canonical tier value. + + Raises: + typer.BadParameter: If the level is not a valid ``TelemetryLevel``. + """ + try: + return parse_telemetry_level(level).value + except ValueError as error: + raise typer.BadParameter(str(error)) from error + + +def resolve_config_path(config: Optional[Path], *, create: bool = False) -> Path: + """Resolve which config file to read or write. + + Args: + config: Explicit path from ``--config``, if any. + create: When True and no file exists, create ``./.codecarbon.config``. + + Returns: + Resolved config file path. + """ + if config is not None: + path = config.expanduser().resolve() + if create and not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("[codecarbon]\n", encoding="utf-8") + return path + local_path = Path.cwd().resolve() / ".codecarbon.config" + if local_path.exists(): + return local_path + global_path = (Path.home() / ".codecarbon.config").expanduser().resolve() + if global_path.exists(): + return global_path + if create: + local_path.write_text("[codecarbon]\n", encoding="utf-8") + return local_path + return local_path + + +def pick_config_path_interactive() -> Path: + """Prompt for which config file to update. + + Returns: + Path chosen by the user. + """ + home = Path.home() + global_path = (home / ".codecarbon.config").expanduser().resolve() + local_path = Path.cwd().resolve() / ".codecarbon.config" + options = [] + if global_path.exists(): + options.append(str(global_path)) + if local_path.exists() and local_path not in options: + options.append(str(local_path)) + options.append("Create new config file") + if not options: + options = ["Create new config file"] + choice = questionary.select( + "Which configuration file should store telemetry_level?", + choices=options, + ).ask() + if choice == "Create new config file": + return create_new_config_file() + return Path(choice).expanduser().resolve() + + +def write_telemetry_level(path: Path, level: str) -> None: + """Persist ``telemetry_level`` to a config file. + + Args: + path: Target ``.codecarbon.config`` path. + level: Validated tier value. + """ + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("[codecarbon]\n", encoding="utf-8") + overwrite_local_config("telemetry_level", level, path=path) + + +def print_telemetry_status(config_path: Optional[Path] = None) -> None: + """Print resolved telemetry settings. + + Without ``config_path``, uses the same merged file settings and env overlay + as ``EmissionsTracker``. With ``config_path``, inspects that file only. + + Args: + config_path: Optional single config file to inspect. + """ + if config_path is not None: + path = config_path.expanduser().resolve() + if not path.exists(): + print(f"[yellow]Config file not found:[/yellow] {path}") + print(f"Default tier: {DEFAULT_TELEMETRY_LEVEL.value} (not explicit)") + return + file_settings = get_config(path) + external_conf: dict[str, str] = {} + source_label = str(path) + else: + file_settings = get_config_file_settings() + external_conf = get_hierarchical_config() + source_label = "merged ~/.codecarbon.config + ./.codecarbon.config" + + level = resolve_telemetry_level(file_settings) + explicit = is_telemetry_level_explicit(file_settings, external_conf=external_conf) + stored = file_settings.get("telemetry_level") + print(f"Config source: {source_label}") + print(f"telemetry_level in file(s): {stored!r}") + print(f"Resolved tier: {level.value}") + print(f"Explicitly configured: {explicit}") + if not explicit: + print( + "[yellow]Tier 1 minimal telemetry will be sent once per session " + "until you set telemetry_level.[/yellow]" + ) + + +def run_telemetry_interactive(config: Optional[Path] = None) -> None: + """Run the interactive telemetry configuration wizard. + + Args: + config: Optional fixed config path; otherwise prompt for file choice. + """ + print("CodeCarbon product telemetry") + print( + "Separate from your dashboard experiment (codecarbon config). " + "Controls optional usage analytics and public leaderboard data.\n" + ) + path = resolve_config_path(config) if config else None + if path is None or (config is None and not path.exists()): + path = pick_config_path_interactive() + else: + path = resolve_config_path(config, create=True) + + choices = [ + questionary.Choice("disabled — " + TIER_DESCRIPTIONS["disabled"], "disabled"), + questionary.Choice("minimal — " + TIER_DESCRIPTIONS["minimal"], "minimal"), + questionary.Choice( + "extensive — " + TIER_DESCRIPTIONS["extensive"], "extensive" + ), + ] + try: + current = get_config(path).get("telemetry_level") + except FileNotFoundError: + current = None + valid_levels = {member.value for member in TelemetryLevel} + default = current if current in valid_levels else "minimal" + level = questionary.select( + "Select telemetry_level:", + choices=choices, + default=default, + ).ask() + if level is None: + raise typer.Exit(0) + level = normalize_telemetry_level(level) + write_telemetry_level(path, level) + print(f"[green]Saved[/green] telemetry_level = {level} in {path}") + + +@telemetry_app.callback(invoke_without_command=True) +def telemetry_entry( + ctx: typer.Context, + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Path to .codecarbon.config (default: local then global).", + ), + ] = None, +) -> None: + """Configure telemetry interactively when no subcommand is given.""" + if ctx.invoked_subcommand is None: + run_telemetry_interactive(config=config) + + +@telemetry_app.command("show") +def show( + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Inspect one file only; default matches EmissionsTracker merge.", + ), + ] = None, +) -> None: + """Show the resolved telemetry tier.""" + print_telemetry_status(config_path=config) + + +@telemetry_app.command("set") +def set_level( + level: Annotated[ + str, + typer.Argument(help="Telemetry tier: disabled, minimal, or extensive."), + ], + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Path to .codecarbon.config (creates ./.codecarbon.config if missing).", + ), + ] = None, +) -> None: + """Write telemetry_level to a config file.""" + path = resolve_config_path(config, create=True) + normalized = normalize_telemetry_level(level) + write_telemetry_level(path, normalized) + print(f"[green]Saved[/green] telemetry_level = {normalized} in {path}") diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py new file mode 100644 index 000000000..ee78970c4 --- /dev/null +++ b/codecarbon/core/telemetry_settings.py @@ -0,0 +1,153 @@ +"""Resolve telemetry tier and API settings from config and environment.""" + +import os +from typing import Any + +from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.external.logger import logger + +DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" +DEFAULT_TELEMETRY_API_KEY = "cpt_JZhj-vJdEVG28qErZL5mh1ftiqbnDIBYjWSxwvX3rfI" +DEFAULT_TELEMETRY_EXPERIMENT_ID = "aa69b440-014a-4562-ac06-ba7eecb023f9" +DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal + +TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" + + +def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: + """Parse a telemetry tier from a string or enum value. + + Args: + raw: Tier name or ``TelemetryLevel`` member. + + Returns: + Parsed ``TelemetryLevel``. + + Raises: + ValueError: If ``raw`` is not a valid tier name. + """ + if isinstance(raw, TelemetryLevel): + return raw + try: + return TelemetryLevel(str(raw).lower()) + except ValueError as error: + raise ValueError( + f"Invalid telemetry_level {raw!r}. Choose: disabled, minimal, or extensive." + ) from error + + +def is_telemetry_level_explicit( + config_file_conf: dict[str, Any], + *, + override: str | TelemetryLevel | None = None, + external_conf: dict[str, Any] | None = None, +) -> bool: + """Return whether the user explicitly chose a telemetry tier. + + Explicit sources (in order): tracker ``telemetry_level`` argument, config file + ``telemetry_level``, environment ``CODECARBON_TELEMETRY_LEVEL`` / + ``telemetry_level``. Legacy ``CODECARBON_TELEMETRY`` / ``telemetry`` counts as + explicit for this check only. + + Args: + config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). + override: Value passed to ``EmissionsTracker(telemetry_level=...)``. + external_conf: Merged config from file and environment. + + Returns: + True if any explicit source is set. + """ + if override is not None: + return True + if config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: + return True + if external_conf is None: + return False + if external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: + return True + return external_conf.get("telemetry") is not None + + +def resolve_telemetry_level( + config_file_conf: dict[str, Any], + *, + override: str | TelemetryLevel | None = None, +) -> TelemetryLevel: + """Resolve the active telemetry tier. + + Precedence: tracker ``telemetry_level`` argument, then ``telemetry_level`` in + ``.codecarbon.config``. Environment variables do not change the tier unless + passed as ``override`` from a future CLI integration. + + Args: + config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). + override: Optional tier from ``EmissionsTracker(telemetry_level=...)``. + + Returns: + Resolved ``TelemetryLevel``. + """ + raw = ( + override + if override is not None + else config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) + ) + if raw is None: + return DEFAULT_TELEMETRY_LEVEL + try: + return parse_telemetry_level(raw) + except ValueError: + logger.error( + "Invalid telemetry_level %r; falling back to %r", + raw, + DEFAULT_TELEMETRY_LEVEL.value, + ) + return DEFAULT_TELEMETRY_LEVEL + + +def get_telemetry_api_url( + external_conf: dict[str, Any], + default: str = DEFAULT_TELEMETRY_API_URL, +) -> str: + """Return telemetry API base URL from config or environment. + + Args: + external_conf: Merged config from file and environment. + default: URL used when unset. + + Returns: + API base URL without trailing slash. + """ + url = external_conf.get("telemetry_api_url") or os.environ.get( + "CODECARBON_TELEMETRY_API_URL" + ) + return (url or default).rstrip("/") + + +def get_telemetry_api_key(external_conf: dict[str, Any]) -> str: + """Return telemetry API token from config, environment, or public default. + + Args: + external_conf: Merged config from file and environment. + + Returns: + API token string. + """ + key = external_conf.get("telemetry_api_key") or os.environ.get( + "CODECARBON_TELEMETRY_API_KEY" + ) + return key or DEFAULT_TELEMETRY_API_KEY + + +def get_telemetry_experiment_id(external_conf: dict[str, Any]) -> str: + """Return telemetry experiment id from config, environment, or public default. + + Args: + external_conf: Merged config from file and environment. + + Returns: + Experiment UUID string for Tier 2 / leaderboard linkage. + """ + experiment_id = external_conf.get("telemetry_experiment_id") or os.environ.get( + "CODECARBON_TELEMETRY_EXPERIMENT_ID" + ) + return experiment_id or DEFAULT_TELEMETRY_EXPERIMENT_ID diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md new file mode 100644 index 000000000..7a19bafa9 --- /dev/null +++ b/docs/how-to/telemetry.md @@ -0,0 +1,129 @@ +# Product telemetry + +CodeCarbon can send **optional product telemetry** to help improve the library: which hardware and environments people use, and (if you opt in) anonymous run emissions on a public leaderboard. + +This is **separate from** your own dashboard setup (`codecarbon config`, `codecarbon login`, `save_to_api`). Those commands configure **your** project and experiments. Product telemetry uses the shared settings below. + +## Telemetry tiers + +| Tier | `telemetry_level` | When | What is sent | +|------|-------------------|------|----------------| +| **0** | `disabled` | — | Nothing | +| **1** | `minimal` (default) | Once per Python process, when the tracker starts | Minimal hardware / environment metadata (see below) | +| **2** | `extensive` | Tier 1 on start **and** Tier 2 on `stop()` | Tier 1 plus one public emissions row for the run | + +If you never set `telemetry_level`, CodeCarbon uses **`minimal`** and logs a **one-time warning** per Python session telling you that Tier 1 will be sent. + +## Tier 1: what we collect today + +Tier 1 sends a single `POST` to `{telemetry_api_url}/telemetry` the first time an `EmissionsTracker` or `OfflineEmissionsTracker` is created in a process (not on every run). + +Only the fields below are included. Any value that is unknown is **omitted** from the payload (not sent as `null`). + +| Field | Description | +|-------|-------------| +| `timestamp` | UTC time when the tracker was initialized | +| `telemetry_level` | Always `minimal` for this tier | +| `os` | Operating system string (e.g. platform description) | +| `country_iso_code` | ISO country code when known (e.g. offline mode or geo lookup) | +| `region` | Region or cloud region when known | +| `cloud_provider` | Cloud provider name when known | +| `cloud_region` | Same as `region` when cloud metadata is available | +| `longitude` | Approximate longitude when known (degrees) | +| `latitude` | Approximate latitude when known (degrees) | +| `cpu_count` | Logical CPU count | +| `cpu_physical_count` | Physical CPU count | +| `cpu_model` | CPU model name | +| `gpu_count` | Number of GPUs detected | +| `gpu_model` | GPU model name(s) | +| `ram_total_size_gb` | Total RAM in GB | +| `python_version` | Python version string | +| `codecarbon_version` | Installed CodeCarbon version | + +### Tier 1: what we do **not** collect yet + +The API schema supports more “minimal” fields (framework versions, install method, executable hash, etc.). **The client does not send them today.** In particular, Tier 1 does **not** include: + +- Run duration, energy, or CO₂ emissions +- CPU/GPU utilization or power samples +- Project, experiment, or user identifiers from your dashboard config +- Python executable hash, virtualenv type, or ML framework versions +- IDE, CI, or notebook environment metadata + +If we add fields later, this page will be updated; the payload will stay limited to the minimal tier rules on the server. + +## Tier 2: extensive (public leaderboard) + +When `telemetry_level = extensive`, CodeCarbon still sends Tier 1 once per process, and on **`stop()`** posts **one** emissions summary to the shared telemetry experiment via the same API used for leaderboard data (`add_emission`). That is independent of `save_to_api` (your private dashboard uploads). + +Use this only if you are comfortable contributing anonymous run-level emissions to the public experiment. + +## Configure telemetry + +### Config file + +Add to `~/.codecarbon.config` and/or `./.codecarbon.config`: + +```ini +[codecarbon] +telemetry_level = minimal +``` + +Allowed values: `disabled`, `minimal`, `extensive`. + +Optional API overrides (defaults point at the public telemetry project): + +```ini +telemetry_api_url = https://api.codecarbon.io +telemetry_api_key = cpt_... +telemetry_experiment_id = aa69b440-014a-4562-ac06-ba7eecb023f9 +``` + +Environment variables for URL, key, and experiment id: `CODECARBON_TELEMETRY_API_URL`, `CODECARBON_TELEMETRY_API_KEY`, `CODECARBON_TELEMETRY_EXPERIMENT_ID`. + +**Tier resolution:** `telemetry_level` in the config file, or the tracker / CLI override below. Environment variables such as `CODECARBON_TELEMETRY_LEVEL` or legacy `CODECARBON_TELEMETRY` can mark your choice as “explicit” (so the default warning is skipped) but **do not** change the tier unless the same value is also in the config file or passed to the tracker. + +### CLI + +```bash +# Interactive wizard +codecarbon telemetry + +# Set tier in a config file +codecarbon telemetry set disabled +codecarbon telemetry show + +# One-run override (does not write the config file) +codecarbon monitor --telemetry-level minimal -- python train.py +``` + +See the [CLI reference](../reference/cli.md#codecarbon-telemetry) for details. + +### Python + +```python +from codecarbon import EmissionsTracker + +tracker = EmissionsTracker(telemetry_level="disabled") +``` + +`OfflineEmissionsTracker`, `@track_emissions`, and `codecarbon monitor` accept the same `telemetry_level` argument. + +### Disable telemetry in tests + +```ini +[codecarbon] +telemetry_level = disabled +``` + +## Privacy notes + +- Tier 1 is **best-effort**: failures are logged and do not block tracking. +- Coordinates are only sent when the tracker already resolved them; they are not precise location tracking by themselves. +- Choose `disabled` if you do not want any product telemetry. +- Choose `extensive` only if you accept publishing one emissions row per process to the public telemetry experiment on stop. + +## Related + +- [Configure CodeCarbon](configuration.md) — general config file and environment variables +- [Use the Cloud API & Dashboard](cloud-api.md) — your own projects and experiments diff --git a/docs/plans/2026-05-19-telemetry-configuration.md b/docs/plans/2026-05-19-telemetry-configuration.md new file mode 100644 index 000000000..82bc5e6c1 --- /dev/null +++ b/docs/plans/2026-05-19-telemetry-configuration.md @@ -0,0 +1,270 @@ +# Telemetry Tier Configuration Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use executing-plans to implement this plan task-by-task. + +**Goal:** Configure telemetry tiers (`disabled`, `minimal`, `extensive`) with clear defaults, explicit opt-in/out, and optional public emissions on stop. Aligned with issue #1106 and `TelemetryLevel` from PR #1171. + +**Architecture (shipped):** Tier resolution: **tracker `telemetry_level` kwarg** (if set) → else **`telemetry_level` in `.codecarbon.config`**. Env `CODECARBON_TELEMETRY` / `telemetry` does **not** change the tier (only counts as “explicit” for the configure warning). API URL/key/experiment: config → env → `DEFAULT_*` in `telemetry_settings.py`. **Tier 1:** `TelemetryClient` → `POST /telemetry` once per session on init (404 logged as warning until prod deploy). **Tier 2:** `ApiClient.add_emission` to public experiment on `stop()` when `extensive`. One-time **warning** if tier was not set explicitly. No `send_telemetry` bool. + +**Tech Stack:** Python 3.11+, Pydantic v2, `configparser`, Typer CLI, `pytest`, `requests` / `requests_mock`. + +**Base branch:** `feat/add-telemetry` (stacked on `feat/telemetry-backend`, PR #1200) + +**Related:** +- Issue: [#1106](https://github.com/mlco2/codecarbon/issues/1106) +- Backend: [PR #1171](https://github.com/mlco2/codecarbon/pull/1171) +- Client integration: [PR #1200](https://github.com/mlco2/codecarbon/pull/1200) +- CLI import fix (separate, to `master`): [PR #1201](https://github.com/mlco2/codecarbon/pull/1201) — `amdsmi` / `libamd_smi.so` on macOS + +--- + +## Configuration contract (current) + +### What users can configure where + +| Setting | Config file | Env | Python kwarg | CLI | Default | +|---------|-------------|-----|--------------|-----|---------| +| **`telemetry_level`** (tier) | ✅ | ❌ (tier) / ✅ (explicit warning only)¹ | ✅ `telemetry_level=` | ✅ see below | `minimal` | +| `telemetry_api_url` | ✅ | `CODECARBON_TELEMETRY_API_URL` | ❌ | ❌ | `DEFAULT_TELEMETRY_API_URL` | +| `telemetry_api_key` | ✅ | `CODECARBON_TELEMETRY_API_KEY` | ❌ | ❌ | `DEFAULT_TELEMETRY_API_KEY` | +| `telemetry_experiment_id` | ✅ | `CODECARBON_TELEMETRY_EXPERIMENT_ID` | ❌ | ❌ | `DEFAULT_TELEMETRY_EXPERIMENT_ID` | + +¹ `CODECARBON_TELEMETRY_LEVEL` or legacy `CODECARBON_TELEMETRY` suppress the “not configured” warning but do **not** change the resolved tier unless also in config or passed as kwarg. + +**CLI for `telemetry_level`:** + +| Command | Persists to config? | Notes | +|---------|---------------------|--------| +| `codecarbon telemetry` | ✅ | Interactive wizard | +| `codecarbon telemetry set ` | ✅ | `disabled` \| `minimal` \| `extensive` | +| `codecarbon telemetry show` | — | Read resolved tier | +| `codecarbon monitor --telemetry-level ` | ❌ | One-run override only (tracker kwarg) | + +Separate from `codecarbon config` (dashboard org/project/experiment). + +### Tier values + +| Value | Tier | On init | On stop | Visibility (product) | +|-------|------|---------|---------|----------------------| +| `disabled` | 0 | — | — | None | +| `minimal` | 1 | Tier 1 `POST /telemetry` (once/session) | — | Private metadata | +| `extensive` | 2 | Tier 1 `POST /telemetry` (once/session) | Tier 2 `add_emission` | Public leaderboard | + +### Default when not explicit + +If the user never sets `telemetry_level` (no config key, no kwarg, no env key counted as explicit): + +1. **Warning** (once per Python session): default is `minimal`; **Tier 1 minimal telemetry will be sent** once per session. +2. **Behaviour:** same as `telemetry_level = minimal` (Tier 1 HTTP on init). + +### Explicit configuration (no warning) + +Any of: + +- `telemetry_level = …` in `.codecarbon.config` (local or global) +- `EmissionsTracker(telemetry_level="…")` / `OfflineEmissionsTracker` / `@track_emissions` +- `codecarbon telemetry set …` (writes config) +- `codecarbon monitor --telemetry-level …` (run override; counts as explicit for that run) +- `CODECARBON_TELEMETRY_LEVEL` or `CODECARBON_TELEMETRY` in the environment + +### Tier resolution precedence + +1. Tracker kwarg `telemetry_level` (CLI `--telemetry-level` or Python arg) +2. `telemetry_level` in config file (local overrides global) +3. Built-in default: `minimal` + +### `.codecarbon.config` example + +```ini +[codecarbon] +telemetry_level = minimal +telemetry_api_url = https://api.codecarbon.io +telemetry_api_key = cpt_... +telemetry_experiment_id = aa69b440-014a-4562-ac06-ba7eecb023f9 +``` + +### Tests disabling telemetry + +```ini +[codecarbon] +telemetry_level = disabled +``` + +Pattern: `get_custom_mock_open("[codecarbon]\ntelemetry_level = disabled\n", ...)`. + +--- + +## Out of scope + +- `codecarbon config` wizard step for telemetry (use `codecarbon telemetry` instead) +- `CODECARBON_TELEMETRY` env var **changing** the tier (legacy key only affects “explicit” warning) +- First-run consent prompt, Alembic migration +- Leaderboard UI / server-side public visibility enforcement + +--- + +## Task 1: Telemetry settings module ✅ + +**Files:** `codecarbon/core/telemetry_settings.py`, `codecarbon/core/config.py`, `tests/test_telemetry_settings.py`, `tests/test_config_file_settings.py` + +**Done:** +- `DEFAULT_TELEMETRY_*` constants and `get_telemetry_api_*` helpers +- `get_config_file_settings()` — file only +- `resolve_telemetry_level(config_file_conf, override=…)` — kwarg overrides config +- `is_telemetry_level_explicit(config_file_conf, override=…, external_conf=…)` + +--- + +## Task 2: `codecarbon/telemetry.py` ✅ + +**Done:** +- `build_minimal_telemetry_dict` / `send_tier1_telemetry` — `TelemetryClient` → `POST /telemetry` + `x-api-token`; session dedup `_TIER1_SENT`; HTTP 404 → warning +- `send_tier2_public_emission` — `ApiClient` + `add_emission` to public experiment; `_TIER2_SENT` +- `warn_if_telemetry_not_configured` — one-time warning; message states Tier 1 will be sent +- Tests in `tests/test_telemetry.py` + +--- + +## Task 3: Tracker wiring ✅ + +**Files:** `codecarbon/emissions_tracker.py` + +```mermaid +flowchart LR + subgraph init [Tracker __init__] + W[warn if not explicit] + T1[Tier 1 POST /telemetry] + end + subgraph stop [Tracker stop] + T2[Tier 2 add_emission] + end + W --> T1 + T1 --> TelAPI[TelemetryClient] + T2 --> CCApi[ApiClient public experiment] +``` + +```python +self._config_file_conf = get_config_file_settings() +telemetry_override = None if telemetry_level is _sentinel else telemetry_level +self._telemetry_level = resolve_telemetry_level(self._config_file_conf, override=telemetry_override) +self._apply_init_telemetry(telemetry_override) # warn + Tier 1 when minimal/extensive +# stop(): +self._maybe_send_tier2_telemetry(emissions_data_delta) # extensive only +``` + +| Config | Init | Stop | +|--------|------|------| +| `disabled` | — | — | +| `minimal` | Tier 1 HTTP | — | +| `extensive` | Tier 1 HTTP | Tier 2 `add_emission` | + +Independent of `save_to_api`. Best-effort; never blocks tracker. + +--- + +## Task 4: Extensive `POST /telemetry` payload ⏸ + +Deferred — Tier 2 is public `add_emission` only. + +--- + +## Task 5: Config integration tests ✅ + +**File:** `tests/test_telemetry_config.py` + +- Tier behaviour per config (`disabled` / `minimal` / `extensive`) +- Env does not change tier; env API URL for Tier 2 +- Warning when config empty; no warning when config or kwarg explicit +- Tier 1 asserts `TelemetryClient` / `POST /telemetry` (mocked in unit tests) + +--- + +## Task 6: Telemetry CLI ✅ + +**Files:** `codecarbon/cli/telemetry_cli.py`, `codecarbon/cli/main.py`, `tests/cli/test_telemetry_cli.py` + +**Done:** +- `codecarbon.add_typer(telemetry_app, name="telemetry")` +- `codecarbon telemetry` — interactive (`questionary`): pick config path + tier +- `codecarbon telemetry set [--config PATH]` — write `telemetry_level` +- `codecarbon telemetry show [--config PATH]` — resolved tier + explicit flag +- `codecarbon monitor --telemetry-level ` — one-run override via tracker kwarg +- `parse_telemetry_level()` (core) + `normalize_telemetry_level()` (CLI Typer wrapper) +- `telemetry show` uses merged file settings by default (matches tracker) + +**Not done:** optional hook at end of `codecarbon config` wizard (deferred). + +--- + +## Task 7: Documentation ✅ + +**Files:** `docs/how-to/telemetry.md`, `docs/reference/cli.md`, `mkdocs.yml`, link from `docs/how-to/configuration.md` + +**Done:** +- User-facing telemetry how-to with tiers, config, CLI, Python opt-out +- Tier 1 documents **only fields sent today** by `build_minimal_telemetry_dict` plus explicit “not collected yet” list +- CLI reference: `codecarbon telemetry`, `--telemetry-level` + +--- + +## Task 8: PR hygiene ✅ + +```bash +uv run pytest tests/test_telemetry_settings.py tests/test_config_file_settings.py \ + tests/test_telemetry.py tests/test_telemetry_config.py tests/test_telemetry_client.py \ + tests/cli/test_telemetry_cli.py -v +uv run pytest tests/test_emissions_tracker.py tests/test_offline_emissions_tracker.py -q +uv run pytest carbonserver/tests/api/test_telemetry_schema_drift.py -v +uv run task test-package +``` + +**Results (2026-05-19):** 62 telemetry tests, 22 tracker tests, schema drift, **527** package unit tests passed. + +- [x] Public `DEFAULT_TELEMETRY_API_KEY` / `DEFAULT_TELEMETRY_EXPERIMENT_ID` +- [x] `telemetry_level` kwarg on tracker + `@track_emissions` +- [x] Configure warning + Tier 1 HTTP when not explicit +- [x] Telemetry CLI + `monitor --telemetry-level` +- [x] No `send_telemetry`; tests use config mocks +- [x] Full package test suite green (`uv run task test-package`) +- [x] Schema drift test passes +- [x] `test_config.py` isolates `get_config_file_settings` (no leak from real `~/.codecarbon.config`) +- [x] `test_gpu.py` amdsmi warning message aligned with `gpu_amd.py` +- [x] `ApiClient._create_run` handles `longitude`/`latitude` `None` + +--- + +## Related fix (separate PR, not telemetry feature) + +**[PR #1201](https://github.com/mlco2/codecarbon/pull/1201)** → `master`: `codecarbon/core/gpu_amd.py` catches `KeyError` / `OSError` when `amdsmi` is installed but `libamd_smi.so` is missing (macOS CLI crash). Branch: `fix/amdsmi-import-keyerror-macos`. + +--- + +## Risk notes + +| Risk | Mitigation | +|------|------------| +| Users surprised by default Tier 1 logging | One-time warning names Tier 1 explicitly | +| Env `CODECARBON_TELEMETRY` confusion | Document: warning-only, not tier resolution | +| Tier 1 log vs future HTTP | Doc + comment in `send_tier1_telemetry`; flip when API ready | +| CLI import before amdsmi fix merged | PR #1201 to master; rebase `feat/add-telemetry` after merge | + +--- + +## Progress summary + +| Task | Status | +|------|--------| +| 1. Telemetry settings | ✅ | +| 2. `telemetry.py` | ✅ | +| 3. Tracker wiring | ✅ | +| 4. Extensive `/telemetry` POST | ⏸ | +| 5. Integration tests | ✅ | +| 6. Telemetry CLI | ✅ | +| 7. Documentation | ✅ | +| 8. Full CI / PR hygiene | ✅ | + +--- + +## Execution handoff + +**Next step:** Merge PR #1200; rebase after [PR #1201](https://github.com/mlco2/codecarbon/pull/1201); deploy [#1171](https://github.com/mlco2/codecarbon/pull/1171) so prod `/telemetry` stops returning 404. diff --git a/tests/cli/test_telemetry_cli.py b/tests/cli/test_telemetry_cli.py new file mode 100644 index 000000000..871bd13c3 --- /dev/null +++ b/tests/cli/test_telemetry_cli.py @@ -0,0 +1,110 @@ +"""Tests for codecarbon telemetry CLI commands.""" + +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from codecarbon.cli import main as cli_main +from codecarbon.cli.telemetry_cli import normalize_telemetry_level, telemetry_app + + +def test_normalize_telemetry_level_accepts_valid_values(): + assert normalize_telemetry_level("MINIMAL") == "minimal" + assert normalize_telemetry_level("disabled") == "disabled" + + +def test_normalize_telemetry_level_rejects_invalid(): + with pytest.raises(Exception): + normalize_telemetry_level("bogus") + + +def test_telemetry_set_writes_config(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / ".codecarbon.config" + result = runner.invoke( + telemetry_app, + ["set", "disabled", "--config", str(config_path)], + ) + assert result.exit_code == 0 + assert "telemetry_level = disabled" in result.output + content = config_path.read_text() + assert "telemetry_level = disabled" in content + + +def test_telemetry_show_reports_stored_level(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / ".codecarbon.config" + config_path.write_text("[codecarbon]\ntelemetry_level = extensive\n") + result = runner.invoke( + telemetry_app, + ["show", "--config", str(config_path)], + ) + assert result.exit_code == 0 + assert "extensive" in result.output + assert "Explicitly configured: True" in result.output + + +def test_telemetry_show_merged_matches_tracker_precedence(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + global_path = Path(tmp) / "global.config" + local_path = Path(tmp) / "local.config" + global_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + local_path.write_text("[codecarbon]\ntelemetry_level = disabled\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=(str(global_path), str(local_path)), + ): + result = runner.invoke(telemetry_app, ["show"]) + assert result.exit_code == 0 + assert "Resolved tier: disabled" in result.output + assert "merged" in result.output + + +def test_monitor_passes_telemetry_level_override(monkeypatch): + from codecarbon.cli import monitor as monitor_module + + captured = {} + + class FakeTracker: + def __init__(self, **kwargs): + captured.update(kwargs) + self._conf = {"output_file": "emissions.csv"} + + def start(self): + return None + + def stop(self): + return 0.0 + + monkeypatch.setattr(monitor_module, "EmissionsTracker", FakeTracker) + + runner = CliRunner() + result = runner.invoke( + cli_main.codecarbon, + [ + "monitor", + "--no-api", + "--telemetry-level", + "disabled", + "--", + "echo", + "ok", + ], + ) + assert result.exit_code == 0 + assert captured.get("telemetry_level") == "disabled" + + +def test_monitor_rejects_invalid_telemetry_level(): + runner = CliRunner() + result = runner.invoke( + cli_main.codecarbon, + ["monitor", "--telemetry-level", "invalid"], + ) + assert result.exit_code != 0 diff --git a/tests/test_config_file_settings.py b/tests/test_config_file_settings.py new file mode 100644 index 000000000..6a155ef7e --- /dev/null +++ b/tests/test_config_file_settings.py @@ -0,0 +1,51 @@ +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from codecarbon.core.config import get_config_file_settings, get_hierarchical_config + + +class TestGetConfigFileSettings(unittest.TestCase): + def test_returns_empty_when_no_config_files(self): + with patch("codecarbon.core.config._config_file_paths") as mock_paths: + mock_paths.return_value = ("/nonexistent/global", "/nonexistent/local") + settings = get_config_file_settings() + self.assertEqual(settings, {}) + + def test_local_overrides_global_telemetry_level(self): + with tempfile.TemporaryDirectory() as tmp: + global_path = Path(tmp) / "global.config" + local_path = Path(tmp) / "local.config" + global_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + local_path.write_text("[codecarbon]\ntelemetry_level = disabled\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=(str(global_path), str(local_path)), + ): + settings = get_config_file_settings() + self.assertEqual(settings["telemetry_level"], "disabled") + + def test_hierarchical_config_includes_env_but_file_settings_do_not(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY": "disabled"}, + clear=False, + ): + file_settings = get_config_file_settings() + hierarchical = get_hierarchical_config() + self.assertEqual(file_settings.get("telemetry_level"), "minimal") + self.assertNotIn("telemetry", file_settings) + self.assertEqual(hierarchical.get("telemetry"), "disabled") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py new file mode 100644 index 000000000..41cbc26d0 --- /dev/null +++ b/tests/test_telemetry_config.py @@ -0,0 +1,314 @@ +"""Integration tests for telemetry tier resolution and config contract (Task 5).""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +import codecarbon.telemetry as telemetry_module +from codecarbon.core.config import get_config_file_settings +from codecarbon.core.telemetry_settings import resolve_telemetry_level +from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry_settings import get_telemetry_api_url +from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker +from codecarbon.telemetry import ( + send_tier1_telemetry, + send_tier2_public_emission, + warn_if_telemetry_not_configured, +) +from tests.testutils import get_custom_mock_open + +if sys.platform == "darwin": + mock_platform_cli_setup = patch( + "codecarbon.core.powermetrics.ApplePowermetrics._setup_cli" + ) +else: + mock_platform_cli_setup = patch("codecarbon.core.cpu.IntelPowerGadget._setup_cli") + + +def _conf(level: str) -> str: + return f"[codecarbon]\ntelemetry_level = {level}\n" + + +class TestTelemetryConfigContract(unittest.TestCase): + def setUp(self) -> None: + telemetry_module._TELEMETRY_CONFIGURE_WARNED = False + + def test_warns_once_when_telemetry_not_explicit(self): + with patch("codecarbon.telemetry.logger.warning") as mock_warning: + warn_if_telemetry_not_configured({}, TelemetryLevel.minimal) + warn_if_telemetry_not_configured({}, TelemetryLevel.minimal) + self.assertEqual(mock_warning.call_count, 1) + self.assertIn("Tier 1", mock_warning.call_args[0][0]) + + def test_no_warn_when_config_explicit(self): + with patch("codecarbon.telemetry.logger.warning") as mock_warning: + warn_if_telemetry_not_configured( + {"telemetry_level": "disabled"}, TelemetryLevel.disabled + ) + mock_warning.assert_not_called() + + def test_minimal_tier1_posts_once_per_session(self): + telemetry_module._TIER1_SENT = False + conf = {"python_version": "3.11", "os": "Linux", "codecarbon_version": "2.0"} + with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_client_cls.return_value = mock_client + self.assertTrue(send_tier1_telemetry(conf)) + self.assertFalse(send_tier1_telemetry(conf)) + mock_client_cls.assert_called_once() + payload = mock_client_cls.call_args.kwargs["telemetry"] + self.assertEqual(payload["telemetry_level"], TelemetryLevel.minimal.value) + + def test_tier1_posts_to_telemetry_endpoint(self): + telemetry_module._TIER1_SENT = False + with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + mock_post.return_value.status_code = 201 + mock_post.return_value.json.return_value = "telemetry-id" + send_tier1_telemetry( + {"python_version": "3.11", "os": "Linux", "codecarbon_version": "2.0"}, + external_conf={"telemetry_api_url": "http://tier1.example"}, + ) + mock_post.assert_called_once() + self.assertEqual( + mock_post.call_args.kwargs["url"], "http://tier1.example/telemetry" + ) + self.assertEqual( + mock_post.call_args.kwargs["json"]["telemetry_level"], + TelemetryLevel.minimal.value, + ) + + def test_env_codecarbon_telemetry_does_not_change_tier(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text(_conf("minimal")) + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY": "disabled"}, + clear=False, + ): + file_settings = get_config_file_settings() + level = resolve_telemetry_level(file_settings) + self.assertEqual(level, TelemetryLevel.minimal) + + def test_telemetry_api_url_env_used_for_tier2_client(self): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_API_URL": "http://env-telemetry.example"}, + clear=False, + ): + url = get_telemetry_api_url({}) + self.assertEqual(url, "http://env-telemetry.example") + + def test_telemetry_api_url_from_config_overrides_default(self): + url = get_telemetry_api_url( + {"telemetry_api_url": "http://config-telemetry.example"} + ) + self.assertEqual(url, "http://config-telemetry.example") + + +@mock_platform_cli_setup +class TestTrackerTelemetryFromConfig(unittest.TestCase): + def setUp(self) -> None: + telemetry_module._TIER1_SENT = False + telemetry_module._TIER2_SENT = False + telemetry_module._TELEMETRY_CONFIGURE_WARNED = False + self._config_patcher = None + + def tearDown(self) -> None: + if self._config_patcher: + self._config_patcher.stop() + + def _mock_config(self, conf: str) -> None: + self._config_patcher = patch( + "builtins.open", new_callable=get_custom_mock_open(conf, conf) + ) + self._config_patcher.start() + + def test_disabled_no_tier1_post_or_tier2_emission(self, mock_cli_setup): + self._mock_config(_conf("disabled")) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_not_called() + mock_api_cls.assert_not_called() + + def test_minimal_posts_tier1_on_init_not_tier2_on_stop(self, mock_cli_setup): + self._mock_config(_conf("minimal")) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_called_once() + mock_client.add_telemetry.assert_called_once() + mock_api_cls.assert_not_called() + + def test_extensive_posts_tier1_and_tier2_on_stop(self, mock_cli_setup): + self._mock_config(_conf("extensive")) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_called_once() + mock_client.add_telemetry.assert_called_once() + mock_api.add_emission.assert_called_once() + + def test_offline_minimal_with_save_to_api_false_still_posts_tier1( + self, mock_cli_setup + ): + self._mock_config(_conf("minimal")) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + OfflineEmissionsTracker( + country_iso_code="CAN", + save_to_api=False, + save_to_file=False, + ) + mock_telemetry_cls.assert_called_once() + + def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup): + self._mock_config("[codecarbon]\n") + env_without_telemetry = { + key: value + for key, value in os.environ.items() + if key.lower() + not in ( + "codecarbon_telemetry", + "codecarbon_telemetry_level", + ) + } + with patch.dict(os.environ, env_without_telemetry, clear=True): + with patch("codecarbon.telemetry.logger.warning") as mock_warning: + with patch( + "codecarbon.telemetry.TelemetryClient" + ) as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker(save_to_api=False, save_to_file=False) + configure_warnings = [ + c + for c in mock_warning.call_args_list + if c[0] and "telemetry" in str(c[0][0]).lower() + ] + self.assertEqual(len(configure_warnings), 1) + + def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): + self._mock_config("[codecarbon]\n") + with patch("codecarbon.telemetry.logger.warning") as mock_warning: + with patch("codecarbon.telemetry.TelemetryClient"): + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker( + telemetry_level="disabled", + save_to_api=False, + save_to_file=False, + ) + configure_warnings = [ + c for c in mock_warning.call_args_list if c[0] and "Tier 1" in str(c[0][0]) + ] + self.assertEqual(len(configure_warnings), 0) + + def test_env_telemetry_disabled_still_minimal_tier1_when_config_minimal( + self, mock_cli_setup + ): + self._mock_config(_conf("minimal")) + with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker(save_to_api=False, save_to_file=False) + mock_telemetry_cls.assert_called_once() + + +class TestTier2ApiSettings(unittest.TestCase): + def test_tier2_uses_resolved_api_url(self): + telemetry_module._TIER2_SENT = False + from codecarbon.output_methods.emissions_data import EmissionsData + + emissions = EmissionsData( + timestamp="2020-01-01T00:00:00", + project_name="test", + run_id="run-1", + experiment_id="exp-1", + duration=1.0, + emissions=0.001, + emissions_rate=0.001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="", + country_iso_code="", + region="", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="2.0", + cpu_count=1.0, + cpu_model="", + gpu_count=0.0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=8.0, + tracking_mode="machine", + ) + external_conf = {"telemetry_api_url": "http://custom.example"} + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + send_tier2_public_emission({}, emissions, external_conf=external_conf) + mock_api_cls.assert_called_once() + self.assertEqual( + mock_api_cls.call_args.kwargs["endpoint_url"], "http://custom.example" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py new file mode 100644 index 000000000..68254999c --- /dev/null +++ b/tests/test_telemetry_settings.py @@ -0,0 +1,132 @@ +import os +import unittest +from unittest.mock import patch + +from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry_settings import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_API_URL, + DEFAULT_TELEMETRY_EXPERIMENT_ID, + get_telemetry_api_key, + get_telemetry_api_url, + get_telemetry_experiment_id, + is_telemetry_level_explicit, + parse_telemetry_level, + resolve_telemetry_level, +) + + +class TestParseTelemetryLevel(unittest.TestCase): + def test_parse_accepts_enum(self): + self.assertEqual( + parse_telemetry_level(TelemetryLevel.minimal), TelemetryLevel.minimal + ) + + def test_parse_normalizes_case(self): + self.assertEqual(parse_telemetry_level("EXTENSIVE"), TelemetryLevel.extensive) + + def test_parse_rejects_invalid(self): + with self.assertRaises(ValueError): + parse_telemetry_level("bogus") + + +class TestResolveTelemetryLevel(unittest.TestCase): + def test_default_is_minimal_when_unset(self): + level = resolve_telemetry_level({}) + self.assertEqual(level, TelemetryLevel.minimal) + + def test_telemetry_level_from_config_file(self): + level = resolve_telemetry_level({"telemetry_level": "disabled"}) + self.assertEqual(level, TelemetryLevel.disabled) + + def test_telemetry_level_extensive(self): + level = resolve_telemetry_level({"telemetry_level": "extensive"}) + self.assertEqual(level, TelemetryLevel.extensive) + + def test_env_telemetry_key_ignored(self): + with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): + level = resolve_telemetry_level({"telemetry": "extensive"}) + self.assertEqual(level, TelemetryLevel.minimal) + + def test_invalid_level_falls_back_to_minimal(self): + with patch("codecarbon.core.telemetry_settings.logger.error") as mock_error: + level = resolve_telemetry_level({"telemetry_level": "bogus"}) + self.assertEqual(level, TelemetryLevel.minimal) + mock_error.assert_called_once() + + def test_override_kwarg_takes_precedence_over_config_file(self): + level = resolve_telemetry_level( + {"telemetry_level": "minimal"}, override="disabled" + ) + self.assertEqual(level, TelemetryLevel.disabled) + + def test_is_explicit_with_config_file(self): + self.assertTrue(is_telemetry_level_explicit({"telemetry_level": "minimal"})) + + def test_is_explicit_with_override(self): + self.assertTrue(is_telemetry_level_explicit({}, override="disabled")) + + def test_is_explicit_with_env_telemetry_level(self): + self.assertTrue( + is_telemetry_level_explicit( + {}, external_conf={"telemetry_level": "minimal"} + ) + ) + + def test_is_not_explicit_when_unset(self): + self.assertFalse(is_telemetry_level_explicit({})) + + +class TestTelemetryApiSettings(unittest.TestCase): + def test_get_telemetry_api_url_from_conf(self): + url = get_telemetry_api_url({"telemetry_api_url": "http://test.example"}) + self.assertEqual(url, "http://test.example") + + def test_get_telemetry_api_url_default(self): + env = { + k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_URL" + } + with patch.dict(os.environ, env, clear=True): + url = get_telemetry_api_url({}) + self.assertEqual(url, DEFAULT_TELEMETRY_API_URL) + + def test_get_telemetry_api_url_env_fallback(self): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_API_URL": "http://env.example"}, + clear=False, + ): + url = get_telemetry_api_url({}) + self.assertEqual(url, "http://env.example") + + def test_get_telemetry_api_key_from_conf(self): + key = get_telemetry_api_key({"telemetry_api_key": "cpt_test"}) + self.assertEqual(key, "cpt_test") + + def test_get_telemetry_api_key_uses_public_default_when_unset(self): + env = { + k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_KEY" + } + with patch.dict(os.environ, env, clear=True): + key = get_telemetry_api_key({}) + self.assertEqual(key, DEFAULT_TELEMETRY_API_KEY) + + def test_get_telemetry_experiment_id_from_conf(self): + experiment_id = get_telemetry_experiment_id( + {"telemetry_experiment_id": "00000000-0000-0000-0000-000000000001"} + ) + self.assertEqual(experiment_id, "00000000-0000-0000-0000-000000000001") + + def test_get_telemetry_experiment_id_uses_public_default_when_unset(self): + env = { + k: v + for k, v in os.environ.items() + if k != "CODECARBON_TELEMETRY_EXPERIMENT_ID" + } + with patch.dict(os.environ, env, clear=True): + experiment_id = get_telemetry_experiment_id({}) + self.assertEqual(experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) + + +if __name__ == "__main__": + unittest.main() From 954910ef28ce2477367ecc153a6c61f50574b361 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 21:40:50 +0200 Subject: [PATCH 07/18] feat: enhance telemetry functionality with new configuration and session handling - Added support for telemetry level configuration via `.codecarbon.config`, allowing users to specify tiers (disabled, minimal, extensive). - Introduced `get_config_file_settings` to read configuration settings from global and local files. - Updated `BaseEmissionsTracker` to utilize the new telemetry level parameter, improving flexibility in telemetry data collection. - Refactored telemetry sending methods to handle new configurations and ensure proper logging and error handling. - Enhanced tests to cover new telemetry configurations and ensure correct behavior across different scenarios. --- .gitignore | 3 + codecarbon/cli/main.py | 13 ++ codecarbon/core/api_client.py | 12 +- codecarbon/core/config.py | 40 +++-- codecarbon/core/gpu_amd.py | 10 +- codecarbon/core/telemetry_client.py | 19 ++- codecarbon/emissions_tracker.py | 60 +++++-- codecarbon/telemetry.py | 129 +++++++++++--- docs/how-to/configuration.md | 6 + docs/how-to/telemetry.md | 149 ++++++++-------- docs/reference/cli.md | 29 ++++ mkdocs.yml | 1 + tests/test_api_call.py | 16 ++ tests/test_config.py | 33 +++- tests/test_emissions_tracker.py | 22 +-- tests/test_gpu.py | 2 +- tests/test_offline_emissions_tracker.py | 9 +- tests/test_telemetry.py | 215 ++++++++++++++++-------- tests/test_telemetry_client.py | 39 +++++ 19 files changed, 587 insertions(+), 220 deletions(-) diff --git a/.gitignore b/.gitignore index 86958c842..b88e0109f 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,6 @@ tests/test_data/rapl/* credentials* .codecarbon.config* scripts/agent-vm.personal.config.sh + +# Added by ggshield +.cache_ggshield diff --git a/codecarbon/cli/main.py b/codecarbon/cli/main.py index d5175ea9c..5174d6475 100644 --- a/codecarbon/cli/main.py +++ b/codecarbon/cli/main.py @@ -22,6 +22,7 @@ overwrite_local_config, ) from codecarbon.cli.monitor import run_and_monitor +from codecarbon.cli.telemetry_cli import normalize_telemetry_level, telemetry_app from codecarbon.core.api_client import ApiClient, get_datetime_with_timezone from codecarbon.core.schemas import ExperimentCreate, OrganizationCreate, ProjectCreate from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker @@ -32,6 +33,7 @@ DEFAULT_ORGANIzATION_ID = "e60afa92-17b7-4720-91a0-1ae91e409ba1" codecarbon = typer.Typer(no_args_is_help=True) +codecarbon.add_typer(telemetry_app, name="telemetry") def main(): @@ -342,6 +344,15 @@ def monitor( str, typer.Option(help="Region/province for offline mode"), ] = None, + telemetry_level: Annotated[ + Optional[str], + typer.Option( + help=( + "Override telemetry tier for this run only " + "(disabled, minimal, or extensive)." + ), + ), + ] = None, ): """Monitor your machine's carbon emissions.""" @@ -350,6 +361,8 @@ def monitor( "measure_power_secs": measure_power_secs, "api_call_interval": api_call_interval, } + if telemetry_level is not None: + tracker_args["telemetry_level"] = normalize_telemetry_level(telemetry_level) # Set up the tracker arguments based on mode (offline vs online) and validate required args for each mode if offline: if not country_iso_code: diff --git a/codecarbon/core/api_client.py b/codecarbon/core/api_client.py index 34067c71c..de1988d34 100644 --- a/codecarbon/core/api_client.py +++ b/codecarbon/core/api_client.py @@ -30,6 +30,13 @@ def get_datetime_with_timezone(): return timestamp +def _round_coordinate(value, decimals: int = 1) -> float: + """Round a geographic coordinate for API payloads, treating None as zero.""" + if value is None: + return round(0.0, decimals) + return round(float(value), decimals) + + class ApiClient: # (AsyncClient) """ This class call the Code Carbon API @@ -263,9 +270,8 @@ def _create_run(self, experiment_id: str): cpu_model=self.conf.get("cpu_model"), gpu_count=self.conf.get("gpu_count"), gpu_model=self.conf.get("gpu_model"), - # Reduce precision for Privacy - longitude=round(self.conf.get("longitude", 0), 1), - latitude=round(self.conf.get("latitude", 0), 1), + longitude=_round_coordinate(self.conf.get("longitude")), + latitude=_round_coordinate(self.conf.get("latitude")), region=self.conf.get("region"), provider=self.conf.get("provider"), ram_total_size=self.conf.get("ram_total_size"), diff --git a/codecarbon/core/config.py b/codecarbon/core/config.py index 7cacea41d..65330f7a0 100644 --- a/codecarbon/core/config.py +++ b/codecarbon/core/config.py @@ -110,6 +110,31 @@ def normalize_gpu_ids( return None +def _config_file_paths() -> tuple[str, str]: + """Return resolved paths for global and local CodeCarbon config files.""" + cwd = Path.cwd() + home = Path.home() + global_path = str((home / ".codecarbon.config").expanduser().resolve()) + local_path = str((cwd / ".codecarbon.config").expanduser().resolve()) + return global_path, local_path + + +def get_config_file_settings() -> dict[str, str]: + """Return the ``[codecarbon]`` section from config files without environment overlay. + + Reads ``~/.codecarbon.config`` then ``./.codecarbon.config`` (local overrides global). + + Returns: + Configuration dict from files only. Empty when no file or section exists. + """ + config = configparser.ConfigParser() + global_path, local_path = _config_file_paths() + config.read([global_path, local_path]) + if "codecarbon" not in config: + return {} + return dict(config["codecarbon"]) + + def get_hierarchical_config(): """ Get the user-defined codecarbon configuration ConfigParser dictionnary @@ -137,13 +162,7 @@ def get_hierarchical_config(): dict: The final configuration dict parsed from global, local and environment configurations. **All values are strings**. """ - - config = configparser.ConfigParser() - - cwd = Path.cwd() - home = Path.home() - global_path = str((home / ".codecarbon.config").expanduser().resolve()) - local_path = str((cwd / ".codecarbon.config").expanduser().resolve()) + global_path, local_path = _config_file_paths() if Path(global_path).exists(): logger.info( f"Codecarbon is taking the configuration from global file: {global_path}" @@ -155,7 +174,6 @@ def get_hierarchical_config(): f"Codecarbon is taking the configuration from the local file {local_path}" ) - config.read([global_path, local_path]) - config.read_dict(parse_env_config()) - - return dict(config["codecarbon"]) + conf = get_config_file_settings() + conf.update(parse_env_config().get("codecarbon", {})) + return conf diff --git a/codecarbon/core/gpu_amd.py b/codecarbon/core/gpu_amd.py index 6cff5a287..9b1b3ddc1 100644 --- a/codecarbon/core/gpu_amd.py +++ b/codecarbon/core/gpu_amd.py @@ -28,14 +28,12 @@ def is_rocm_system(): "Please install amdsmi to get GPU metrics." ) AMDSMI_AVAILABLE = False -except AttributeError as e: +except (AttributeError, OSError, KeyError) as error: amdsmi = None - # In some environments, amdsmi may be present but not properly configured, leading to AttributeError when importing logger.warning( - "AMD GPU detected but amdsmi is not properly configured. " - "Please ensure amdsmi is correctly installed to get GPU metrics." - "Tips : check consistency between Python amdsmi package and ROCm versions, and ensure AMD drivers are up to date." - f" Error: {e}" + "amdsmi is installed but could not load libamd_smi (ROCm drivers missing " + "or incompatible). AMD GPU metrics will be unavailable. Error: %s", + error, ) AMDSMI_AVAILABLE = False diff --git a/codecarbon/core/telemetry_client.py b/codecarbon/core/telemetry_client.py index 8dfdad05a..6cf821f28 100644 --- a/codecarbon/core/telemetry_client.py +++ b/codecarbon/core/telemetry_client.py @@ -14,14 +14,23 @@ class TelemetryClient: def __init__( self, - endpoint_url="https://api.codecarbon.io", + endpoint_url: str = "https://api.codecarbon.io", telemetry: Optional[Union[TelemetryCreate, dict]] = None, + api_key: Optional[str] = None, ): self.endpoint_url = endpoint_url.rstrip("/") self.telemetry_url = self.endpoint_url + "/telemetry" - self.headers = {"Content-Type": "application/json"} + self.api_key = api_key + self.headers = self._build_headers(api_key) self.telemetry = self._validate_telemetry(telemetry) if telemetry else None + @staticmethod + def _build_headers(api_key: Optional[str]) -> dict[str, str]: + headers = {"Content-Type": "application/json"} + if api_key: + headers["x-api-token"] = api_key + return headers + def add_telemetry(self, telemetry: Optional[Union[TelemetryCreate, dict]] = None): telemetry_payload = ( self._validate_telemetry(telemetry) if telemetry else self.telemetry @@ -38,6 +47,12 @@ def add_telemetry(self, telemetry: Optional[Union[TelemetryCreate, dict]] = None timeout=2, headers=self.headers, ) + if response.status_code == 404: + logger.warning( + "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", + self.telemetry_url, + ) + return None if response.status_code != 201: self._log_error(payload, response) return None diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 406c46bec..46d79759f 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -17,7 +17,18 @@ import psutil from codecarbon._version import __version__ -from codecarbon.core.config import get_hierarchical_config, normalize_gpu_ids +from codecarbon.core.config import ( + get_config_file_settings, + get_hierarchical_config, + normalize_gpu_ids, +) +from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry_settings import resolve_telemetry_level +from codecarbon.telemetry import ( + send_tier1_telemetry, + send_tier2_public_emission, + warn_if_telemetry_not_configured, +) from codecarbon.core.emissions import Emissions from codecarbon.core.resource_tracker import ResourceTracker from codecarbon.core.units import Energy, Power, Time, Water @@ -199,7 +210,7 @@ def __init__( allow_multiple_runs: Optional[bool] = _sentinel, rapl_include_dram: Optional[bool] = _sentinel, rapl_prefer_psys: Optional[bool] = _sentinel, - send_telemetry: Optional[bool] = _sentinel, + telemetry_level: Optional[str] = _sentinel, ): """ :param project_name: Project name for current experiment run, default name @@ -273,10 +284,16 @@ def __init__( (CPU + chipset + PCIe). When False, uses package domains which are more reliable. Note: psys can report higher values than CPU TDP and may be unreliable on older systems. + :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). + Overrides ``telemetry_level`` in ``.codecarbon.config`` when set. """ - # logger.info("base tracker init") self._external_conf = get_hierarchical_config() + self._config_file_conf = get_config_file_settings() + telemetry_override = None if telemetry_level is _sentinel else telemetry_level + self._telemetry_level = resolve_telemetry_level( + self._config_file_conf, override=telemetry_override + ) self._set_from_conf(allow_multiple_runs, "allow_multiple_runs", True, bool) if self._allow_multiple_runs: logger.warning( @@ -353,9 +370,6 @@ def __init__( self._set_from_conf( experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" ) - # Tier 1 telemetry (opt-out, default enabled) - self._set_from_conf(send_telemetry, "send_telemetry", default=True, return_type=bool) - assert self._tracking_mode in ["machine", "process"] set_logger_level(self._log_level) set_logger_format(self._logger_preamble) @@ -452,13 +466,29 @@ def __init__( self._data_source, self._electricitymaps_api_token ) - # Send Tier 1 telemetry if enabled - if self._send_telemetry: - from codecarbon.telemetry import send_tier1_telemetry - send_tier1_telemetry(self._conf) - + self._apply_init_telemetry(telemetry_override) self._init_output_methods(api_key=self._api_key) + @suppress(Exception) + def _apply_init_telemetry(self, telemetry_override: str | None) -> None: + warn_if_telemetry_not_configured( + self._config_file_conf, + self._telemetry_level, + override=telemetry_override, + external_conf=self._external_conf, + ) + if self._telemetry_level in (TelemetryLevel.minimal, TelemetryLevel.extensive): + send_tier1_telemetry(self._conf, external_conf=self._external_conf) + + @suppress(Exception) + def _maybe_send_tier2_telemetry(self, emissions_data_delta: EmissionsData) -> None: + if self._telemetry_level == TelemetryLevel.extensive: + send_tier2_public_emission( + self._conf, + emissions_data_delta, + external_conf=self._external_conf, + ) + def _init_output_methods(self, *, api_key: str = None): """ Prepare the different output methods @@ -760,6 +790,7 @@ def stop(self) -> Optional[float]: emissions_data = self._prepare_emissions_data() emissions_data_delta = self._compute_emissions_delta(emissions_data) + self._maybe_send_tier2_telemetry(emissions_data_delta) self._persist_data( total_emissions=emissions_data, @@ -1145,7 +1176,6 @@ def __init__( cloud_provider: Optional[str] = _sentinel, cloud_region: Optional[str] = _sentinel, country_2letter_iso_code: Optional[str] = _sentinel, - send_telemetry: Optional[bool] = _sentinel, **kwargs, ): """ @@ -1218,7 +1248,7 @@ def __init__( assert isinstance(self._country_2letter_iso_code, str) self._country_2letter_iso_code: str = self._country_2letter_iso_code.upper() - super().__init__(*args, send_telemetry=send_telemetry, **kwargs) + super().__init__(*args, **kwargs) def _get_geo_metadata(self) -> GeoMetadata: return GeoMetadata( @@ -1323,6 +1353,7 @@ def track_emissions( allow_multiple_runs: Optional[bool] = _sentinel, rapl_include_dram: Optional[bool] = _sentinel, rapl_prefer_psys: Optional[bool] = _sentinel, + telemetry_level: Optional[str] = _sentinel, ): """ Decorator that supports both `EmissionsTracker` and `OfflineEmissionsTracker` @@ -1406,6 +1437,7 @@ def track_emissions( When True, measures CPU package + DRAM. :param rapl_prefer_psys: Prefer psys over package domains for RAPL on Linux (default: False). When True, uses total platform power. + :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). :return: The decorated function """ @@ -1460,6 +1492,7 @@ def wrapped_fn(*args, **kwargs): allow_multiple_runs=allow_multiple_runs, rapl_include_dram=rapl_include_dram, rapl_prefer_psys=rapl_prefer_psys, + telemetry_level=telemetry_level, ) else: tracker = EmissionsTracker( @@ -1494,6 +1527,7 @@ def wrapped_fn(*args, **kwargs): allow_multiple_runs=allow_multiple_runs, rapl_include_dram=rapl_include_dram, rapl_prefer_psys=rapl_prefer_psys, + telemetry_level=telemetry_level, ) tracker.start() try: diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index eb4efb9b7..7bcbd6373 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -1,23 +1,64 @@ -"""Tracker-facing telemetry helpers built on the shared TelemetryClient.""" +"""Tracker-facing telemetry helpers (Tier 1 HTTP, Tier 2 public emissions).""" -import os +import dataclasses from datetime import datetime, timezone from typing import Any +from codecarbon.core.api_client import ApiClient from codecarbon.core.telemetry_client import TelemetryClient from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry_settings import ( + get_telemetry_api_key, + get_telemetry_api_url, + get_telemetry_experiment_id, + is_telemetry_level_explicit, +) from codecarbon.external.logger import logger +from codecarbon.output_methods.emissions_data import EmissionsData -TELEMETRY_API_KEY = os.environ.get( - "CODECARBON_TELEMETRY_API_KEY", - "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU", -) -TELEMETRY_API_URL = os.environ.get( - "CODECARBON_TELEMETRY_API_URL", "https://api.codecarbon.io" +_TIER1_SENT = False +_TIER2_SENT = False +_TELEMETRY_CONFIGURE_WARNED = False + +TELEMETRY_NOT_CONFIGURED_MESSAGE = ( + "CodeCarbon telemetry_level was not set explicitly; using default %r. " + "Tier 1 minimal telemetry (hardware and environment metadata) will be " + "sent once per Python session. Set telemetry_level in .codecarbon.config " + "(disabled, minimal, or extensive), pass telemetry_level=... to " + "EmissionsTracker / OfflineEmissionsTracker, or set CODECARBON_TELEMETRY_LEVEL " + "to opt out (disabled), keep minimal telemetry, or enable extensive (public " + "emissions on stop)." ) -TELEMETRY_EXPERIMENT_ID = None -_TIER1_SENT = False + +def warn_if_telemetry_not_configured( + config_file_conf: dict[str, Any], + active_level: TelemetryLevel, + *, + override: str | TelemetryLevel | None = None, + external_conf: dict[str, Any] | None = None, +) -> None: + """Log a one-time warning when telemetry tier was not set explicitly. + + Args: + config_file_conf: File-only settings from ``get_config_file_settings()``. + active_level: Resolved tier in use for this tracker. + override: Optional ``telemetry_level`` tracker argument. + external_conf: Merged file/env settings for explicit-env detection. + """ + global _TELEMETRY_CONFIGURE_WARNED + if _TELEMETRY_CONFIGURE_WARNED: + return + if is_telemetry_level_explicit( + config_file_conf, override=override, external_conf=external_conf + ): + return + logger.warning( + TELEMETRY_NOT_CONFIGURED_MESSAGE, + active_level.value, + ) + _TELEMETRY_CONFIGURE_WARNED = True + def build_minimal_telemetry_dict(conf: dict[str, Any]) -> dict[str, Any]: """Build a minimal telemetry payload dict from tracker configuration. @@ -50,38 +91,80 @@ def build_minimal_telemetry_dict(conf: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in payload.items() if value is not None} -collect_tier1_payload = build_minimal_telemetry_dict - - def send_tier1_telemetry( - conf: dict[str, Any], endpoint_url: str | None = None + conf: dict[str, Any], + external_conf: dict[str, Any] | None = None, ) -> bool: - """Send minimal telemetry once per Python session. + """POST minimal telemetry once per Python session via ``TelemetryClient``. + + Sends ``POST {telemetry_api_url}/telemetry`` with a ``TelemetryCreate`` payload + built from tracker configuration. Best-effort: failures are logged and not raised. Args: conf: Tracker configuration dictionary. - endpoint_url: Optional API base URL override. + external_conf: Merged file/env config for telemetry API URL and key resolution. Returns: - True if telemetry was sent successfully on this call, False if already - sent in this session or if sending failed. + True if telemetry was posted successfully on this call, False if already sent + in this session or if the request failed. """ global _TIER1_SENT if _TIER1_SENT: return False + settings_conf = external_conf or {} try: payload = build_minimal_telemetry_dict(conf) + endpoint_url = get_telemetry_api_url(settings_conf) client = TelemetryClient( - endpoint_url=endpoint_url or TELEMETRY_API_URL, + endpoint_url=endpoint_url, telemetry=payload, + api_key=get_telemetry_api_key(settings_conf), ) - if TELEMETRY_API_KEY: - client.headers["x-api-token"] = TELEMETRY_API_KEY - result = client.add_telemetry() - if result is not None: + response = client.add_telemetry() + if response is not None: _TIER1_SENT = True return True return False except Exception as error: logger.error(f"Telemetry Tier 1 failed (non-critical): {error}") return False + + +def send_tier2_public_emission( + conf: dict[str, Any], + emissions_data: EmissionsData, + external_conf: dict[str, Any] | None = None, +) -> bool: + """Send run emissions to the public telemetry experiment via ApiClient. + + Mirrors ``CodeCarbonAPIOutput`` / ``add_emission`` for the shared leaderboard + project. Best-effort: errors are logged and not raised. + + Args: + conf: Tracker configuration dictionary. + emissions_data: Delta or total emissions row for the run. + external_conf: Merged file/env config for API URL, key, and experiment id. + + Returns: + True if emission was posted successfully on this call, False otherwise. + """ + global _TIER2_SENT + if _TIER2_SENT: + return False + settings_conf = external_conf or {} + try: + api = ApiClient( + endpoint_url=get_telemetry_api_url(settings_conf), + experiment_id=get_telemetry_experiment_id(settings_conf), + api_key=get_telemetry_api_key(settings_conf), + conf=conf, + create_run_automatically=True, + ) + posted = api.add_emission(dataclasses.asdict(emissions_data)) + if posted: + _TIER2_SENT = True + return True + return False + except Exception as error: + logger.error(f"Telemetry Tier 2 failed (non-critical): {error}") + return False diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index e8b42b929..029135d65 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -107,3 +107,9 @@ os.environ["HTTPS_PROXY"] = "http://0.0.0.0:0000" For more information, please read the [requests library proxy documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies) + +## Product telemetry + +Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Tier 1 (`minimal`) sends only hardware and environment metadata once per process—not your emissions or project ids. + +See [Product telemetry](telemetry.md) for tiers, what is collected, and how to opt out. diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index 7a19bafa9..c5a0dda36 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -1,104 +1,107 @@ # Product telemetry -CodeCarbon can send **optional product telemetry** to help improve the library: which hardware and environments people use, and (if you opt in) anonymous run emissions on a public leaderboard. +CodeCarbon can send **optional product telemetry** to help improve the library: which hardware and environments people run on, not what your code does. This is separate from sending **your** emissions to the [dashboard](cloud-api.md) with `save_to_api=True`. -This is **separate from** your own dashboard setup (`codecarbon config`, `codecarbon login`, `save_to_api`). Those commands configure **your** project and experiments. Product telemetry uses the shared settings below. +## Telemetry vs your dashboard data -## Telemetry tiers +| | Product telemetry | Your emissions (`save_to_api`) | +|--|-------------------|--------------------------------| +| Purpose | Improve CodeCarbon (aggregate usage) | Your projects and experiments | +| Config | `telemetry_level`, `codecarbon telemetry` | `codecarbon config`, `experiment_id` | +| Default experiment | Public telemetry project (built-in defaults) | Your account / experiment | -| Tier | `telemetry_level` | When | What is sent | -|------|-------------------|------|----------------| -| **0** | `disabled` | — | Nothing | -| **1** | `minimal` (default) | Once per Python process, when the tracker starts | Minimal hardware / environment metadata (see below) | -| **2** | `extensive` | Tier 1 on start **and** Tier 2 on `stop()` | Tier 1 plus one public emissions row for the run | +You can use one without the other. -If you never set `telemetry_level`, CodeCarbon uses **`minimal`** and logs a **one-time warning** per Python session telling you that Tier 1 will be sent. +## Tiers -## Tier 1: what we collect today +| `telemetry_level` | What happens | +|-------------------|--------------| +| `disabled` | No product telemetry | +| `minimal` (default) | **Tier 1** once per Python process: minimal hardware/environment metadata | +| `extensive` | Tier 1 + **Tier 2** on tracker `stop()`: one public emissions row (leaderboard-style) | -Tier 1 sends a single `POST` to `{telemetry_api_url}/telemetry` the first time an `EmissionsTracker` or `OfflineEmissionsTracker` is created in a process (not on every run). +Tier is resolved in this order: -Only the fields below are included. Any value that is unknown is **omitted** from the payload (not sent as `null`). +1. `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` +2. `telemetry_level` in `.codecarbon.config` (local overrides global) +3. Default: `minimal` + +Environment variables `CODECARBON_TELEMETRY` / `CODECARBON_TELEMETRY_LEVEL` count as “explicit” configuration (they suppress the one-time setup warning) but **do not** change the tier unless you also set `telemetry_level` in a config file or pass the tracker argument. + +## Tier 1: what we collect (and what we do not) + +Tier 1 is intentionally small. The client only builds a **minimal** payload; the API schema rejects “extensive” fields when `telemetry_level` is `minimal`. + +### Sent at most once per process (if known) + +Only non-empty values are included: | Field | Description | |-------|-------------| -| `timestamp` | UTC time when the tracker was initialized | +| `timestamp` | UTC time of the send | | `telemetry_level` | Always `minimal` for this tier | -| `os` | Operating system string (e.g. platform description) | -| `country_iso_code` | ISO country code when known (e.g. offline mode or geo lookup) | -| `region` | Region or cloud region when known | -| `cloud_provider` | Cloud provider name when known | -| `cloud_region` | Same as `region` when cloud metadata is available | -| `longitude` | Approximate longitude when known (degrees) | -| `latitude` | Approximate latitude when known (degrees) | -| `cpu_count` | Logical CPU count | -| `cpu_physical_count` | Physical CPU count | -| `cpu_model` | CPU model name | -| `gpu_count` | Number of GPUs detected | -| `gpu_model` | GPU model name(s) | -| `ram_total_size_gb` | Total RAM in GB | -| `python_version` | Python version string | +| `os` | Platform string | +| `country_iso_code` | Country (e.g. from offline mode or geo) | +| `region` | Region / province | +| `cloud_provider` | Cloud provider name, if detected | +| `cloud_region` | Cloud region, if detected | +| `longitude`, `latitude` | Approximate location, if geo resolution ran | +| `cpu_count`, `cpu_physical_count`, `cpu_model` | CPU metadata | +| `gpu_count`, `gpu_model` | GPU metadata | +| `ram_total_size_gb` | Total RAM | +| `python_version` | Python version | | `codecarbon_version` | Installed CodeCarbon version | -### Tier 1: what we do **not** collect yet - -The API schema supports more “minimal” fields (framework versions, install method, executable hash, etc.). **The client does not send them today.** In particular, Tier 1 does **not** include: +### Not sent in Tier 1 -- Run duration, energy, or CO₂ emissions -- CPU/GPU utilization or power samples -- Project, experiment, or user identifiers from your dashboard config -- Python executable hash, virtualenv type, or ML framework versions -- IDE, CI, or notebook environment metadata +Tier 1 does **not** include: -If we add fields later, this page will be updated; the payload will stay limited to the minimal tier rules on the server. +- Emissions, energy, power, duration, or utilization +- Project name, experiment id, run id, or API keys +- Source code, file paths, hostnames, or user ids +- ML stack versions (PyTorch, TensorFlow, etc.) +- Output methods, tracking mode, or internal diagnostics +- Anything else defined as “extensive” in the telemetry schema -## Tier 2: extensive (public leaderboard) +Tier 2 (`extensive`) adds a single **public** emissions summary via the same mechanism as `add_emission` to the shared telemetry experiment—not a full extensive telemetry document. -When `telemetry_level = extensive`, CodeCarbon still sends Tier 1 once per process, and on **`stop()`** posts **one** emissions summary to the shared telemetry experiment via the same API used for leaderboard data (`add_emission`). That is independent of `save_to_api` (your private dashboard uploads). +### Transport -Use this only if you are comfortable contributing anonymous run-level emissions to the public experiment. +- HTTP `POST` to `{telemetry_api_url}/telemetry` +- Best-effort: failures are logged and do not stop your tracker +- If the endpoint is not deployed yet, you may see a warning (HTTP 404) ## Configure telemetry ### Config file -Add to `~/.codecarbon.config` and/or `./.codecarbon.config`: - ```ini [codecarbon] telemetry_level = minimal +# Optional overrides (defaults point at the public telemetry API): +# telemetry_api_url = https://api.codecarbon.io +# telemetry_api_key = ... +# telemetry_experiment_id = ... ``` -Allowed values: `disabled`, `minimal`, `extensive`. - -Optional API overrides (defaults point at the public telemetry project): - -```ini -telemetry_api_url = https://api.codecarbon.io -telemetry_api_key = cpt_... -telemetry_experiment_id = aa69b440-014a-4562-ac06-ba7eecb023f9 -``` - -Environment variables for URL, key, and experiment id: `CODECARBON_TELEMETRY_API_URL`, `CODECARBON_TELEMETRY_API_KEY`, `CODECARBON_TELEMETRY_EXPERIMENT_ID`. - -**Tier resolution:** `telemetry_level` in the config file, or the tracker / CLI override below. Environment variables such as `CODECARBON_TELEMETRY_LEVEL` or legacy `CODECARBON_TELEMETRY` can mark your choice as “explicit” (so the default warning is skipped) but **do not** change the tier unless the same value is also in the config file or passed to the tracker. - ### CLI ```bash -# Interactive wizard +# Interactive wizard (pick config file + tier) codecarbon telemetry -# Set tier in a config file +# Set tier in config codecarbon telemetry set disabled +codecarbon telemetry set minimal +codecarbon telemetry set extensive + +# Show resolved tier (merged global + local config) codecarbon telemetry show -# One-run override (does not write the config file) -codecarbon monitor --telemetry-level minimal -- python train.py +# One-run override (does not write config) +codecarbon monitor --telemetry-level disabled -- python train.py ``` -See the [CLI reference](../reference/cli.md#codecarbon-telemetry) for details. - ### Python ```python @@ -107,23 +110,29 @@ from codecarbon import EmissionsTracker tracker = EmissionsTracker(telemetry_level="disabled") ``` -`OfflineEmissionsTracker`, `@track_emissions`, and `codecarbon monitor` accept the same `telemetry_level` argument. +## Opt out -### Disable telemetry in tests +Set any of: ```ini [codecarbon] telemetry_level = disabled ``` -## Privacy notes +```bash +codecarbon telemetry set disabled +``` + +```python +EmissionsTracker(telemetry_level="disabled") +``` + +## First run without explicit configuration -- Tier 1 is **best-effort**: failures are logged and do not block tracking. -- Coordinates are only sent when the tracker already resolved them; they are not precise location tracking by themselves. -- Choose `disabled` if you do not want any product telemetry. -- Choose `extensive` only if you accept publishing one emissions row per process to the public telemetry experiment on stop. +If you never set `telemetry_level`, CodeCarbon uses `minimal` and logs a **one-time warning** per Python session that Tier 1 will be sent. Set `telemetry_level` explicitly (config, CLI, or tracker argument) to silence it. ## Related -- [Configure CodeCarbon](configuration.md) — general config file and environment variables -- [Use the Cloud API & Dashboard](cloud-api.md) — your own projects and experiments +- [Configure CodeCarbon](configuration.md) — general `.codecarbon.config` hierarchy +- [CLI reference](../reference/cli.md#codecarbon-telemetry) — command flags +- [Cloud API & dashboard](cloud-api.md) — your own emissions data diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 1e93a588f..6c550f625 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -45,6 +45,7 @@ Displays real-time emissions data for all processes on your machine. Press `Ctrl | `--offline` | flag | false | Run without internet access | | `--country-iso-code` | string | - | ISO 3166-1 alpha-3 country code (required in offline mode) | | `--log-level` | choice | INFO | Log level: DEBUG, INFO, WARNING, ERROR | +| `--telemetry-level` | string | - | One-run tier: `disabled`, `minimal`, or `extensive` | **Examples:** ```bash @@ -88,6 +89,34 @@ codecarbon monitor -- node app.js --port 8080 Same options as `codecarbon monitor` apply (see above). +### `codecarbon telemetry` + +Configure **product telemetry** (library usage metadata), separate from `codecarbon config` (dashboard org/project/experiment). + +**Usage:** + +```bash +codecarbon telemetry # interactive wizard +codecarbon telemetry set # disabled | minimal | extensive +codecarbon telemetry show # resolved tier and whether it was set explicitly +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--config PATH` | Use a specific `.codecarbon.config` (default: local then global) | + +See [Product telemetry](../how-to/telemetry.md) for what Tier 1 collects and how to opt out. + +### `codecarbon monitor --telemetry-level` + +Override the telemetry tier for a single monitor run (does not update config). + +```bash +codecarbon monitor --telemetry-level disabled -- python train.py +``` + ### `codecarbon detect` Detect and print hardware information. diff --git a/mkdocs.yml b/mkdocs.yml index b82641fb7..1de7399a8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -149,6 +149,7 @@ nav: - LLMs and Agents: how-to/agents.md - How-to Guides: - Configure CodeCarbon: how-to/configuration.md + - Product telemetry: how-to/telemetry.md - Compare Model Efficiency: tutorials/comparing-model-efficiency.md - Dashboard & Visualization: - Use the Cloud API & Dashboard: how-to/cloud-api.md diff --git a/tests/test_api_call.py b/tests/test_api_call.py index a4bb4cd7f..e0b61d01c 100644 --- a/tests/test_api_call.py +++ b/tests/test_api_call.py @@ -244,6 +244,22 @@ def test_add_emission_returns_false_on_unsuccessful_post(self): ) ) + def test_create_run_handles_none_coordinates(self): + conf_with_none_coords = {**conf, "longitude": None, "latitude": None} + with requests_mock.Mocker() as m: + m.post("http://test.com/runs", json={"id": "run-id"}, status_code=201) + api = ApiClient( + endpoint_url="http://test.com", + experiment_id="experiment_id", + api_key="Toto", + conf=conf_with_none_coords, + create_run_automatically=False, + ) + run_id = api._create_run("experiment_id") + self.assertEqual(run_id, "run-id") + self.assertEqual(m.last_request.json()["longitude"], 0.0) + self.assertEqual(m.last_request.json()["latitude"], 0.0) + def test_create_run_returns_none_on_unsuccessful_status(self): with requests_mock.Mocker() as m: m.post("http://test.com/runs", text="bad", status_code=400) diff --git a/tests/test_config.py b/tests/test_config.py index 181913c6c..1f431e236 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +import configparser import os import unittest from textwrap import dedent @@ -16,6 +17,19 @@ from tests.testutils import get_custom_mock_open +def _file_settings_from_ini(global_conf: str, local_conf: str) -> dict[str, str]: + """Merge mocked global and local ``[codecarbon]`` sections like config files do.""" + merged: dict[str, str] = {} + for text in (global_conf, local_conf): + if not text.strip(): + continue + parser = configparser.ConfigParser() + parser.read_string(text) + if "codecarbon" in parser: + merged.update(dict(parser["codecarbon"])) + return merged + + class TestConfig(unittest.TestCase): def setUp(self): self._original_environ = os.environ.copy() @@ -23,6 +37,12 @@ def setUp(self): "CODECARBON_API_KEY", "CODECARBON_EXPERIMENT_ID", "CODECARBON_API_ENDPOINT", + "CODECARBON_TELEMETRY", + "CODECARBON_TELEMETRY_LEVEL", + "CODECARBON_TELEMETRY_PROJECT_TOKEN", + "CODECARBON_TELEMETRY_API_URL", + "CODECARBON_TELEMETRY_API_KEY", + "CODECARBON_TELEMETRY_EXPERIMENT_ID", "codecarbon_api_key", "codecarbon_experiment_id", "codecarbon_api_endpoint", @@ -115,7 +135,8 @@ def test_read_confs(self): ) with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) + "codecarbon.core.config.get_config_file_settings", + return_value=_file_settings_from_ini(global_conf, local_conf), ): conf = dict(get_hierarchical_config()) target = { @@ -156,7 +177,8 @@ def test_read_confs_and_parse_envs(self): ) with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) + "codecarbon.core.config.get_config_file_settings", + return_value=_file_settings_from_ini(global_conf, local_conf), ): conf = dict(get_hierarchical_config()) target = { @@ -171,12 +193,7 @@ def test_read_confs_and_parse_envs(self): self.assertDictEqual(conf, target) def test_empty_conf(self): - global_conf = "" - local_conf = "" - - with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) - ): + with patch("codecarbon.core.config.get_config_file_settings", return_value={}): conf = dict(get_hierarchical_config()) # allow_multiple_runs is set in pytest.ini and not mocked, so it's visible here. target = {"allow_multiple_runs": "True"} diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index b845962dd..2fe5a1199 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -35,7 +35,7 @@ def heavy_computation(run_time_secs: float = 3): pass -empty_conf = "[codecarbon]" +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" if sys.platform == "darwin": @@ -73,7 +73,8 @@ def setUp(self) -> None: # ./.codecarbon.config so that the user's local configuration does not # alter tests patcher = mock.patch( - "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + "builtins.open", + new_callable=get_custom_mock_open(disabled_conf, disabled_conf), ) self.addCleanup(patcher.stop) patcher.start() @@ -99,7 +100,7 @@ def test_carbon_tracker_TWO_GPU_PRIVATE_INFRA_CANADA( json=GEO_METADATA_CANADA, status=200, ) - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) # WHEN tracker.start() heavy_computation(run_time_secs=5) @@ -126,7 +127,7 @@ def test_monitor_power_uses_gpu_detail_position_when_gpu_index_is_missing( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) mock_gpu = mock.MagicMock() from codecarbon.external.hardware import GPU @@ -164,7 +165,7 @@ def raise_timeout_exception(*args, **kwargs): mocked_requests_get.side_effect = raise_timeout_exception - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) # WHEN tracker.start() @@ -183,7 +184,7 @@ def test_graceful_start_failure( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) def raise_exception(*args, **kwargs): raise Exception() @@ -202,7 +203,7 @@ def test_graceful_stop_failure( mocked_is_gpu_details_available, mocked_is_nvidia_system, ): - tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) def raise_exception(*args, **kwargs): raise Exception() @@ -346,7 +347,6 @@ def test_offline_tracker_country_name( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", - send_telemetry=False, ) tracker.start() heavy_computation(run_time_secs=2) @@ -370,7 +370,6 @@ def test_offline_tracker_invalid_headers( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", - send_telemetry=False, ) emissions = os.path.join( os.path.dirname(__file__), "test_data", "emissions_invalid_headers.csv" @@ -404,7 +403,6 @@ def test_offline_tracker_valid_headers( country_iso_code="USA", output_dir=self.temp_path, experiment_id="test", - send_telemetry=False, ) emissions = os.path.join( os.path.dirname(__file__), "test_data", "emissions_valid_headers.csv" @@ -448,7 +446,7 @@ def test_carbon_tracker_online_context_manager_TWO_GPU_PRIVATE_INFRA_CANADA( ) # WHEN - with EmissionsTracker(measure_power_secs=1, save_to_file=False, send_telemetry=False) as tracker: + with EmissionsTracker(measure_power_secs=1, save_to_file=False) as tracker: heavy_computation(run_time_secs=5) # THEN @@ -611,7 +609,6 @@ def test_scheduler_warning_suppressed_when_stopped( with EmissionsTracker( output_dir=self.temp_path, measure_power_secs=1, # Short interval for testing - send_telemetry=False, ) as tracker: # Stop the scheduler to simulate task mode or manual stopping tracker._scheduler.stop() @@ -655,7 +652,6 @@ def test_scheduler_warning_shown_when_running( with EmissionsTracker( output_dir=self.temp_path, measure_power_secs=1, # Short interval for testing - send_telemetry=False, ) as tracker: # Ensure scheduler is running (default state) self.assertFalse(tracker._scheduler._stopped) diff --git a/tests/test_gpu.py b/tests/test_gpu.py index bfbc8e603..913c05017 100644 --- a/tests/test_gpu.py +++ b/tests/test_gpu.py @@ -153,7 +153,7 @@ def check_output(cmd, *args, **kwargs): sys.modules["pynvml"] = old_pynvml assert any( - "amdsmi is not properly configured" in str(c.args[0]) + "could not load libamd_smi" in str(c.args[0]) for c in warning_mock.call_args_list ) diff --git a/tests/test_offline_emissions_tracker.py b/tests/test_offline_emissions_tracker.py index 7e2ff40ac..4b9b4b4ec 100644 --- a/tests/test_offline_emissions_tracker.py +++ b/tests/test_offline_emissions_tracker.py @@ -18,7 +18,7 @@ def heavy_computation(run_time_secs: float = 3): pass -empty_conf = "[codecarbon]" +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" class TestOfflineEmissionsTracker(unittest.TestCase): @@ -32,7 +32,8 @@ def setUp(self) -> None: # ./.codecarbon.config so that the user's local configuration does not # alter tests patcher = mock.patch( - "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + "builtins.open", + new_callable=get_custom_mock_open(disabled_conf, disabled_conf), ) self.addCleanup(patcher.stop) patcher.start() @@ -41,7 +42,7 @@ def tearDown(self) -> None: self.temp_dir.cleanup() def test_offline_tracker(self): - tracker = OfflineEmissionsTracker(output_file=self.emissions_file_path, send_telemetry=False) + tracker = OfflineEmissionsTracker(output_file=self.emissions_file_path) tracker.start() heavy_computation(run_time_secs=2) tracker.stop() @@ -60,7 +61,7 @@ def test_offline_tracker(self): ) def test_offline_tracker_task(self): - tracker = OfflineEmissionsTracker(send_telemetry=False) + tracker = OfflineEmissionsTracker() tracker.start_task() heavy_computation(run_time_secs=2) task_emission_data = tracker.stop_task() diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index f70979d66..7a235f76c 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -10,8 +10,8 @@ from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.telemetry import ( build_minimal_telemetry_dict, - collect_tier1_payload, send_tier1_telemetry, + send_tier2_public_emission, ) from tests.testutils import get_custom_mock_open @@ -20,11 +20,11 @@ "codecarbon.core.powermetrics.ApplePowermetrics._setup_cli" ) else: - mock_platform_cli_setup = patch( - "codecarbon.core.cpu.IntelPowerGadget._setup_cli" - ) + mock_platform_cli_setup = patch("codecarbon.core.cpu.IntelPowerGadget._setup_cli") -empty_conf = "[codecarbon]" +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" +minimal_conf = "[codecarbon]\ntelemetry_level = minimal\n" +extensive_conf = "[codecarbon]\ntelemetry_level = extensive\n" class TestTelemetry(unittest.TestCase): @@ -52,12 +52,9 @@ def test_build_minimal_telemetry_dict_maps_tracker_conf(self): self.assertEqual(payload["cloud_provider"], conf["provider"]) self.assertEqual(payload["cloud_region"], conf["region"]) - def test_collect_tier1_payload_delegates_to_builder(self): - self.assertIs(collect_tier1_payload, build_minimal_telemetry_dict) - class TestTier1Send(unittest.TestCase): - def test_send_tier1_telemetry_sends_once_per_session(self): + def test_send_tier1_telemetry_posts_once_per_session(self): telemetry_module._TIER1_SENT = False conf = { "python_version": "3.11.0", @@ -67,102 +64,188 @@ def test_send_tier1_telemetry_sends_once_per_session(self): "gpu_count": 0, "codecarbon_version": "2.0.0", } - with patch( - "codecarbon.core.telemetry_client.requests.post" - ) as mock_post: - mock_post.return_value = MagicMock( - status_code=201, json=lambda: "telemetry-id" - ) - result = send_tier1_telemetry(conf, endpoint_url="http://test.com") + with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_client_cls.return_value = mock_client + result = send_tier1_telemetry(conf) self.assertTrue(result) - self.assertEqual(mock_post.call_count, 1) - body = mock_post.call_args.kwargs["json"] - self.assertEqual(body["telemetry_level"], "minimal") - result = send_tier1_telemetry(conf, endpoint_url="http://test.com") + result = send_tier1_telemetry(conf) self.assertFalse(result) - self.assertEqual(mock_post.call_count, 1) + mock_client_cls.assert_called_once() + mock_client.add_telemetry.assert_called_once() + call_kwargs = mock_client_cls.call_args.kwargs + self.assertEqual( + call_kwargs["telemetry"]["telemetry_level"], TelemetryLevel.minimal.value + ) - def test_send_tier1_telemetry_fails_silently(self): + def test_send_tier1_telemetry_uses_resolved_api_url_and_key(self): telemetry_module._TIER1_SENT = False - conf = { - "python_version": "3.11.0", - "os": "Linux", - "cpu_count": 4, - "codecarbon_version": "2.0.0", + external_conf = { + "telemetry_api_url": "http://custom-tier1.example", + "telemetry_api_key": "cpt_custom", } + with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_client_cls.return_value = mock_client + send_tier1_telemetry( + {"codecarbon_version": "2.0"}, external_conf=external_conf + ) + self.assertEqual( + mock_client_cls.call_args.kwargs["endpoint_url"], + "http://custom-tier1.example", + ) + self.assertEqual(mock_client_cls.call_args.kwargs["api_key"], "cpt_custom") + + def test_send_tier1_telemetry_fails_silently(self): + telemetry_module._TIER1_SENT = False with patch( - "codecarbon.core.telemetry_client.requests.post", - side_effect=Exception("network error"), + "codecarbon.telemetry.build_minimal_telemetry_dict", + side_effect=RuntimeError("build failed"), ): - with patch("codecarbon.core.telemetry_client.logger.error") as mock_logger: - result = send_tier1_telemetry(conf, endpoint_url="http://test.com") + with patch("codecarbon.telemetry.logger.error") as mock_logger: + result = send_tier1_telemetry({}) self.assertFalse(result) mock_logger.assert_called() +class TestTier2Send(unittest.TestCase): + def test_send_tier2_public_emission_sends_once_per_session(self): + telemetry_module._TIER2_SENT = False + from codecarbon.output_methods.emissions_data import EmissionsData + + emissions = EmissionsData( + timestamp="2020-01-01T00:00:00", + project_name="test", + run_id="run-1", + experiment_id="exp-1", + duration=1.0, + emissions=0.001, + emissions_rate=0.001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="", + country_iso_code="", + region="", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="2.0", + cpu_count=1.0, + cpu_model="", + gpu_count=0.0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=8.0, + tracking_mode="machine", + ) + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + result = send_tier2_public_emission({}, emissions) + self.assertTrue(result) + mock_api.add_emission.assert_called_once() + result = send_tier2_public_emission({}, emissions) + self.assertFalse(result) + mock_api.add_emission.assert_called_once() + + @mock_platform_cli_setup class TestTrackerTelemetry(unittest.TestCase): def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() self.temp_path = Path(self.temp_dir.name) - self.patcher = patch( - "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) - ) - self.mock_open = self.patcher.start() + self.patcher = None def tearDown(self) -> None: - self.patcher.stop() + if self.patcher: + self.patcher.stop() self.temp_dir.cleanup() - def test_emissions_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): + def _start_config_mock(self, conf: str) -> None: + self.patcher = patch( + "builtins.open", new_callable=get_custom_mock_open(conf, conf) + ) + self.patcher.start() + + def test_emissions_tracker_sends_tier1_when_minimal_config(self, mock_cli_setup): telemetry_module._TIER1_SENT = False - with patch( - "codecarbon.core.telemetry_client.requests.post" - ) as mock_post: - mock_post.return_value = MagicMock( - status_code=201, json=lambda: "telemetry-id" - ) + self._start_config_mock(minimal_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - EmissionsTracker( - send_telemetry=True, save_to_api=False, save_to_file=False - ) - self.assertTrue(mock_post.called) + EmissionsTracker(save_to_api=False, save_to_file=False) + mock_telemetry_cls.assert_called_once() + mock_client.add_telemetry.assert_called_once() - def test_emissions_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): + def test_emissions_tracker_skips_tier1_when_disabled_config(self, mock_cli_setup): telemetry_module._TIER1_SENT = False - with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + self._start_config_mock(disabled_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - EmissionsTracker( - send_telemetry=False, save_to_api=False, save_to_file=False - ) - self.assertFalse(mock_post.called) + EmissionsTracker(save_to_api=False, save_to_file=False) + mock_telemetry_cls.assert_not_called() - def test_offline_tracker_sends_tier1_telemetry_by_default(self, mock_cli_setup): + def test_offline_tracker_sends_tier1_when_minimal_config(self, mock_cli_setup): telemetry_module._TIER1_SENT = False - with patch( - "codecarbon.core.telemetry_client.requests.post" - ) as mock_post: - mock_post.return_value = MagicMock( - status_code=201, json=lambda: "telemetry-id" - ) + self._start_config_mock(minimal_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client OfflineEmissionsTracker( country_iso_code="CAN", - send_telemetry=True, save_to_api=False, save_to_file=False, ) - self.assertTrue(mock_post.called) + mock_telemetry_cls.assert_called_once() - def test_offline_tracker_skips_tier1_when_opted_out(self, mock_cli_setup): + def test_offline_tracker_skips_tier1_when_disabled_config(self, mock_cli_setup): telemetry_module._TIER1_SENT = False - with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + self._start_config_mock(disabled_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: OfflineEmissionsTracker( country_iso_code="CAN", - send_telemetry=False, save_to_api=False, save_to_file=False, ) - self.assertFalse(mock_post.called) + mock_telemetry_cls.assert_not_called() + + def test_extensive_sends_tier1_on_init_and_tier2_on_stop(self, mock_cli_setup): + telemetry_module._TIER1_SENT = False + telemetry_module._TIER2_SENT = False + self._start_config_mock(extensive_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_telemetry = MagicMock() + mock_telemetry.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_telemetry + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_called_once() + mock_telemetry.add_telemetry.assert_called_once() + mock_api.add_emission.assert_called_once() if __name__ == "__main__": diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 14ade8b7f..5ce145370 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import patch import requests_mock from pydantic import ValidationError @@ -102,3 +103,41 @@ def test_add_telemetry_returns_none_without_payload(self): client = TelemetryClient(endpoint_url="http://test.com") self.assertIsNone(client.add_telemetry()) + + def test_add_telemetry_logs_warning_on_404(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + with requests_mock.Mocker() as m: + m.post( + "http://test.com/telemetry", + text='{"detail":"Not Found"}', + status_code=404, + ) + with patch("codecarbon.core.telemetry_client.logger") as mock_logger: + client = TelemetryClient( + endpoint_url="http://test.com", telemetry=telemetry + ) + result = client.add_telemetry() + self.assertIsNone(result) + mock_logger.warning.assert_called_once() + + def test_add_telemetry_sends_api_key_header_when_configured(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + with requests_mock.Mocker() as m: + m.post( + "http://test.com/telemetry", + json="telemetry-id", + status_code=201, + ) + client = TelemetryClient( + endpoint_url="http://test.com", + telemetry=telemetry, + api_key="cpt_test_key", + ) + client.add_telemetry() + self.assertEqual(m.last_request.headers["x-api-token"], "cpt_test_key") From 2a865f6abfebc054ee7cc891e345b74a72e6acb6 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 21:44:00 +0200 Subject: [PATCH 08/18] refactor: enhance telemetry level resolution and configuration handling - Updated `resolve_telemetry_level` to incorporate external configuration, allowing for more flexible telemetry tier resolution. - Improved handling of telemetry level overrides from environment variables and configuration files. - Enhanced documentation to clarify telemetry configuration options and resolution order. - Added tests to validate new behavior for telemetry level resolution and ensure correct precedence of settings. --- codecarbon/cli/telemetry_cli.py | 5 +++- codecarbon/core/telemetry_settings.py | 37 ++++++++++++++++++--------- codecarbon/emissions_tracker.py | 6 +++-- codecarbon/telemetry.py | 9 +++---- docs/how-to/configuration.md | 2 +- docs/how-to/telemetry.md | 8 +++--- docs/reference/api.md | 14 ++++++++++ tests/test_telemetry_config.py | 31 +++++++++++++++++++--- tests/test_telemetry_settings.py | 23 +++++++++++++++++ 9 files changed, 107 insertions(+), 28 deletions(-) diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py index 218f2e941..e32cd730b 100644 --- a/codecarbon/cli/telemetry_cli.py +++ b/codecarbon/cli/telemetry_cli.py @@ -142,7 +142,10 @@ def print_telemetry_status(config_path: Optional[Path] = None) -> None: external_conf = get_hierarchical_config() source_label = "merged ~/.codecarbon.config + ./.codecarbon.config" - level = resolve_telemetry_level(file_settings) + level = resolve_telemetry_level( + file_settings, + external_conf=external_conf or None, + ) explicit = is_telemetry_level_explicit(file_settings, external_conf=external_conf) stored = file_settings.get("telemetry_level") print(f"Config source: {source_label}") diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py index ee78970c4..9e6d5ad91 100644 --- a/codecarbon/core/telemetry_settings.py +++ b/codecarbon/core/telemetry_settings.py @@ -69,29 +69,42 @@ def is_telemetry_level_explicit( def resolve_telemetry_level( - config_file_conf: dict[str, Any], + config_file_conf: dict[str, Any] | None = None, *, override: str | TelemetryLevel | None = None, + external_conf: dict[str, Any] | None = None, ) -> TelemetryLevel: """Resolve the active telemetry tier. - Precedence: tracker ``telemetry_level`` argument, then ``telemetry_level`` in - ``.codecarbon.config``. Environment variables do not change the tier unless - passed as ``override`` from a future CLI integration. + Precedence: + + 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or + ``codecarbon monitor --telemetry-level`` + 2. ``external_conf`` — merged ``.codecarbon.config`` and ``CODECARBON_*`` env + (environment overrides file for the same key) + 3. ``config_file_conf`` — file-only settings when ``external_conf`` is omitted + 4. Default: ``minimal`` + + Legacy ``CODECARBON_TELEMETRY`` / config key ``telemetry`` only affect whether + the tier counts as explicitly configured, not tier resolution. Args: - config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). - override: Optional tier from ``EmissionsTracker(telemetry_level=...)``. + config_file_conf: Settings from ``get_config_file_settings()`` (optional). + override: Optional tier from tracker or CLI. + external_conf: Merged settings from ``get_hierarchical_config()`` (optional). Returns: Resolved ``TelemetryLevel``. """ - raw = ( - override - if override is not None - else config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) - ) - if raw is None: + if override is not None: + raw = override + elif external_conf is not None and external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: + raw = external_conf[TELEMETRY_LEVEL_CONFIG_KEY] + elif config_file_conf is not None and config_file_conf.get( + TELEMETRY_LEVEL_CONFIG_KEY + ) is not None: + raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] + else: return DEFAULT_TELEMETRY_LEVEL try: return parse_telemetry_level(raw) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 46d79759f..d88892d66 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -285,14 +285,16 @@ def __init__( are more reliable. Note: psys can report higher values than CPU TDP and may be unreliable on older systems. :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). - Overrides ``telemetry_level`` in ``.codecarbon.config`` when set. + Overrides config file and ``CODECARBON_TELEMETRY_LEVEL`` when set. """ self._external_conf = get_hierarchical_config() self._config_file_conf = get_config_file_settings() telemetry_override = None if telemetry_level is _sentinel else telemetry_level self._telemetry_level = resolve_telemetry_level( - self._config_file_conf, override=telemetry_override + self._config_file_conf, + override=telemetry_override, + external_conf=self._external_conf, ) self._set_from_conf(allow_multiple_runs, "allow_multiple_runs", True, bool) if self._allow_multiple_runs: diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index 7bcbd6373..ad4946e0c 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -23,11 +23,10 @@ TELEMETRY_NOT_CONFIGURED_MESSAGE = ( "CodeCarbon telemetry_level was not set explicitly; using default %r. " "Tier 1 minimal telemetry (hardware and environment metadata) will be " - "sent once per Python session. Set telemetry_level in .codecarbon.config " - "(disabled, minimal, or extensive), pass telemetry_level=... to " - "EmissionsTracker / OfflineEmissionsTracker, or set CODECARBON_TELEMETRY_LEVEL " - "to opt out (disabled), keep minimal telemetry, or enable extensive (public " - "emissions on stop)." + "sent once per Python session. Set telemetry_level in .codecarbon.config, " + "set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... to " + "EmissionsTracker / OfflineEmissionsTracker, or run " + "codecarbon telemetry set ." ) diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index 029135d65..34e32b2bb 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -110,6 +110,6 @@ documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies) ## Product telemetry -Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Tier 1 (`minimal`) sends only hardware and environment metadata once per process—not your emissions or project ids. +Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Set it in `.codecarbon.config`, via `CODECARBON_TELEMETRY_LEVEL`, or with `EmissionsTracker(telemetry_level=...)` (argument wins). Tier 1 (`minimal`) sends only hardware and environment metadata once per process—not your emissions or project ids. See [Product telemetry](telemetry.md) for tiers, what is collected, and how to opt out. diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index c5a0dda36..a99d42930 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -22,11 +22,11 @@ You can use one without the other. Tier is resolved in this order: -1. `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` -2. `telemetry_level` in `.codecarbon.config` (local overrides global) -3. Default: `minimal` +1. **Tracker or CLI argument** — `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` (highest priority) +2. **Config + environment** — `telemetry_level` in `.codecarbon.config` (local overrides global), then `CODECARBON_TELEMETRY_LEVEL` overrides the file value when both are set (same rules as other `CODECARBON_*` settings) +3. **Default:** `minimal` -Environment variables `CODECARBON_TELEMETRY` / `CODECARBON_TELEMETRY_LEVEL` count as “explicit” configuration (they suppress the one-time setup warning) but **do not** change the tier unless you also set `telemetry_level` in a config file or pass the tracker argument. +Legacy `CODECARBON_TELEMETRY` / config key `telemetry` suppress the one-time setup warning when set, but do **not** set the tier (use `telemetry_level` or `CODECARBON_TELEMETRY_LEVEL`). ## Tier 1: what we collect (and what we do not) diff --git a/docs/reference/api.md b/docs/reference/api.md index 15117b123..37d2abea0 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -12,6 +12,20 @@ Parameters can be set via `EmissionsTracker()`, `OfflineEmissionsTracker()`, the If you use `CUDA_VISIBLE_DEVICES` or `ROCR_VISIBLE_DEVICES` to set GPUs, CodeCarbon will automatically populate `gpu_ids`. Manual `gpu_ids` overrides this. +## Product telemetry + +Optional library telemetry is controlled by **`telemetry_level`** on the tracker (same parameter on `OfflineEmissionsTracker` and `@track_emissions`): + +| Value | Behavior | +|-------|----------| +| `disabled` | No product telemetry | +| `minimal` (default) | Tier 1 hardware/environment metadata once per process | +| `extensive` | Tier 1 + public emissions summary on `stop()` | + +**Resolution order:** tracker argument → `.codecarbon.config` → `CODECARBON_TELEMETRY_LEVEL` → default `minimal`. The tracker argument overrides config and environment. + +This is separate from `save_to_api` (your dashboard experiment). See [Product telemetry](../how-to/telemetry.md). + ## EmissionsTracker / BaseEmissionsTracker `EmissionsTracker` and `OfflineEmissionsTracker` inherit from `BaseEmissionsTracker`. diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index 41cbc26d0..57845f32f 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -81,7 +81,7 @@ def test_tier1_posts_to_telemetry_endpoint(self): TelemetryLevel.minimal.value, ) - def test_env_codecarbon_telemetry_does_not_change_tier(self): + def test_legacy_env_codecarbon_telemetry_does_not_change_tier(self): with tempfile.TemporaryDirectory() as tmp: local_path = Path(tmp) / ".codecarbon.config" local_path.write_text(_conf("minimal")) @@ -94,10 +94,35 @@ def test_env_codecarbon_telemetry_does_not_change_tier(self): {"CODECARBON_TELEMETRY": "disabled"}, clear=False, ): - file_settings = get_config_file_settings() - level = resolve_telemetry_level(file_settings) + from codecarbon.core.config import get_hierarchical_config + + level = resolve_telemetry_level( + get_config_file_settings(), + external_conf=get_hierarchical_config(), + ) self.assertEqual(level, TelemetryLevel.minimal) + def test_env_codecarbon_telemetry_level_overrides_file(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text(_conf("minimal")) + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_LEVEL": "disabled"}, + clear=False, + ): + from codecarbon.core.config import get_hierarchical_config + + level = resolve_telemetry_level( + get_config_file_settings(), + external_conf=get_hierarchical_config(), + ) + self.assertEqual(level, TelemetryLevel.disabled) + def test_telemetry_api_url_env_used_for_tier2_client(self): with patch.dict( os.environ, diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py index 68254999c..c09582c62 100644 --- a/tests/test_telemetry_settings.py +++ b/tests/test_telemetry_settings.py @@ -60,6 +60,29 @@ def test_override_kwarg_takes_precedence_over_config_file(self): ) self.assertEqual(level, TelemetryLevel.disabled) + def test_override_kwarg_takes_precedence_over_external_conf(self): + level = resolve_telemetry_level( + external_conf={"telemetry_level": "extensive"}, + override="disabled", + ) + self.assertEqual(level, TelemetryLevel.disabled) + + def test_external_conf_env_overrides_file_when_merged(self): + level = resolve_telemetry_level( + {"telemetry_level": "minimal"}, + external_conf={"telemetry_level": "disabled"}, + ) + self.assertEqual(level, TelemetryLevel.disabled) + + def test_env_telemetry_level_via_external_conf(self): + with patch.dict( + os.environ, {"CODECARBON_TELEMETRY_LEVEL": "disabled"}, clear=False + ): + from codecarbon.core.config import get_hierarchical_config + + level = resolve_telemetry_level(external_conf=get_hierarchical_config()) + self.assertEqual(level, TelemetryLevel.disabled) + def test_is_explicit_with_config_file(self): self.assertTrue(is_telemetry_level_explicit({"telemetry_level": "minimal"})) From 1ffc45dd2b8554943e1ad973657756e9bb520689 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Tue, 19 May 2026 21:46:57 +0200 Subject: [PATCH 09/18] Delete docs/plans/2026-05-19-telemetry-configuration.md --- .../2026-05-19-telemetry-configuration.md | 270 ------------------ 1 file changed, 270 deletions(-) delete mode 100644 docs/plans/2026-05-19-telemetry-configuration.md diff --git a/docs/plans/2026-05-19-telemetry-configuration.md b/docs/plans/2026-05-19-telemetry-configuration.md deleted file mode 100644 index 82bc5e6c1..000000000 --- a/docs/plans/2026-05-19-telemetry-configuration.md +++ /dev/null @@ -1,270 +0,0 @@ -# Telemetry Tier Configuration Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use executing-plans to implement this plan task-by-task. - -**Goal:** Configure telemetry tiers (`disabled`, `minimal`, `extensive`) with clear defaults, explicit opt-in/out, and optional public emissions on stop. Aligned with issue #1106 and `TelemetryLevel` from PR #1171. - -**Architecture (shipped):** Tier resolution: **tracker `telemetry_level` kwarg** (if set) → else **`telemetry_level` in `.codecarbon.config`**. Env `CODECARBON_TELEMETRY` / `telemetry` does **not** change the tier (only counts as “explicit” for the configure warning). API URL/key/experiment: config → env → `DEFAULT_*` in `telemetry_settings.py`. **Tier 1:** `TelemetryClient` → `POST /telemetry` once per session on init (404 logged as warning until prod deploy). **Tier 2:** `ApiClient.add_emission` to public experiment on `stop()` when `extensive`. One-time **warning** if tier was not set explicitly. No `send_telemetry` bool. - -**Tech Stack:** Python 3.11+, Pydantic v2, `configparser`, Typer CLI, `pytest`, `requests` / `requests_mock`. - -**Base branch:** `feat/add-telemetry` (stacked on `feat/telemetry-backend`, PR #1200) - -**Related:** -- Issue: [#1106](https://github.com/mlco2/codecarbon/issues/1106) -- Backend: [PR #1171](https://github.com/mlco2/codecarbon/pull/1171) -- Client integration: [PR #1200](https://github.com/mlco2/codecarbon/pull/1200) -- CLI import fix (separate, to `master`): [PR #1201](https://github.com/mlco2/codecarbon/pull/1201) — `amdsmi` / `libamd_smi.so` on macOS - ---- - -## Configuration contract (current) - -### What users can configure where - -| Setting | Config file | Env | Python kwarg | CLI | Default | -|---------|-------------|-----|--------------|-----|---------| -| **`telemetry_level`** (tier) | ✅ | ❌ (tier) / ✅ (explicit warning only)¹ | ✅ `telemetry_level=` | ✅ see below | `minimal` | -| `telemetry_api_url` | ✅ | `CODECARBON_TELEMETRY_API_URL` | ❌ | ❌ | `DEFAULT_TELEMETRY_API_URL` | -| `telemetry_api_key` | ✅ | `CODECARBON_TELEMETRY_API_KEY` | ❌ | ❌ | `DEFAULT_TELEMETRY_API_KEY` | -| `telemetry_experiment_id` | ✅ | `CODECARBON_TELEMETRY_EXPERIMENT_ID` | ❌ | ❌ | `DEFAULT_TELEMETRY_EXPERIMENT_ID` | - -¹ `CODECARBON_TELEMETRY_LEVEL` or legacy `CODECARBON_TELEMETRY` suppress the “not configured” warning but do **not** change the resolved tier unless also in config or passed as kwarg. - -**CLI for `telemetry_level`:** - -| Command | Persists to config? | Notes | -|---------|---------------------|--------| -| `codecarbon telemetry` | ✅ | Interactive wizard | -| `codecarbon telemetry set ` | ✅ | `disabled` \| `minimal` \| `extensive` | -| `codecarbon telemetry show` | — | Read resolved tier | -| `codecarbon monitor --telemetry-level ` | ❌ | One-run override only (tracker kwarg) | - -Separate from `codecarbon config` (dashboard org/project/experiment). - -### Tier values - -| Value | Tier | On init | On stop | Visibility (product) | -|-------|------|---------|---------|----------------------| -| `disabled` | 0 | — | — | None | -| `minimal` | 1 | Tier 1 `POST /telemetry` (once/session) | — | Private metadata | -| `extensive` | 2 | Tier 1 `POST /telemetry` (once/session) | Tier 2 `add_emission` | Public leaderboard | - -### Default when not explicit - -If the user never sets `telemetry_level` (no config key, no kwarg, no env key counted as explicit): - -1. **Warning** (once per Python session): default is `minimal`; **Tier 1 minimal telemetry will be sent** once per session. -2. **Behaviour:** same as `telemetry_level = minimal` (Tier 1 HTTP on init). - -### Explicit configuration (no warning) - -Any of: - -- `telemetry_level = …` in `.codecarbon.config` (local or global) -- `EmissionsTracker(telemetry_level="…")` / `OfflineEmissionsTracker` / `@track_emissions` -- `codecarbon telemetry set …` (writes config) -- `codecarbon monitor --telemetry-level …` (run override; counts as explicit for that run) -- `CODECARBON_TELEMETRY_LEVEL` or `CODECARBON_TELEMETRY` in the environment - -### Tier resolution precedence - -1. Tracker kwarg `telemetry_level` (CLI `--telemetry-level` or Python arg) -2. `telemetry_level` in config file (local overrides global) -3. Built-in default: `minimal` - -### `.codecarbon.config` example - -```ini -[codecarbon] -telemetry_level = minimal -telemetry_api_url = https://api.codecarbon.io -telemetry_api_key = cpt_... -telemetry_experiment_id = aa69b440-014a-4562-ac06-ba7eecb023f9 -``` - -### Tests disabling telemetry - -```ini -[codecarbon] -telemetry_level = disabled -``` - -Pattern: `get_custom_mock_open("[codecarbon]\ntelemetry_level = disabled\n", ...)`. - ---- - -## Out of scope - -- `codecarbon config` wizard step for telemetry (use `codecarbon telemetry` instead) -- `CODECARBON_TELEMETRY` env var **changing** the tier (legacy key only affects “explicit” warning) -- First-run consent prompt, Alembic migration -- Leaderboard UI / server-side public visibility enforcement - ---- - -## Task 1: Telemetry settings module ✅ - -**Files:** `codecarbon/core/telemetry_settings.py`, `codecarbon/core/config.py`, `tests/test_telemetry_settings.py`, `tests/test_config_file_settings.py` - -**Done:** -- `DEFAULT_TELEMETRY_*` constants and `get_telemetry_api_*` helpers -- `get_config_file_settings()` — file only -- `resolve_telemetry_level(config_file_conf, override=…)` — kwarg overrides config -- `is_telemetry_level_explicit(config_file_conf, override=…, external_conf=…)` - ---- - -## Task 2: `codecarbon/telemetry.py` ✅ - -**Done:** -- `build_minimal_telemetry_dict` / `send_tier1_telemetry` — `TelemetryClient` → `POST /telemetry` + `x-api-token`; session dedup `_TIER1_SENT`; HTTP 404 → warning -- `send_tier2_public_emission` — `ApiClient` + `add_emission` to public experiment; `_TIER2_SENT` -- `warn_if_telemetry_not_configured` — one-time warning; message states Tier 1 will be sent -- Tests in `tests/test_telemetry.py` - ---- - -## Task 3: Tracker wiring ✅ - -**Files:** `codecarbon/emissions_tracker.py` - -```mermaid -flowchart LR - subgraph init [Tracker __init__] - W[warn if not explicit] - T1[Tier 1 POST /telemetry] - end - subgraph stop [Tracker stop] - T2[Tier 2 add_emission] - end - W --> T1 - T1 --> TelAPI[TelemetryClient] - T2 --> CCApi[ApiClient public experiment] -``` - -```python -self._config_file_conf = get_config_file_settings() -telemetry_override = None if telemetry_level is _sentinel else telemetry_level -self._telemetry_level = resolve_telemetry_level(self._config_file_conf, override=telemetry_override) -self._apply_init_telemetry(telemetry_override) # warn + Tier 1 when minimal/extensive -# stop(): -self._maybe_send_tier2_telemetry(emissions_data_delta) # extensive only -``` - -| Config | Init | Stop | -|--------|------|------| -| `disabled` | — | — | -| `minimal` | Tier 1 HTTP | — | -| `extensive` | Tier 1 HTTP | Tier 2 `add_emission` | - -Independent of `save_to_api`. Best-effort; never blocks tracker. - ---- - -## Task 4: Extensive `POST /telemetry` payload ⏸ - -Deferred — Tier 2 is public `add_emission` only. - ---- - -## Task 5: Config integration tests ✅ - -**File:** `tests/test_telemetry_config.py` - -- Tier behaviour per config (`disabled` / `minimal` / `extensive`) -- Env does not change tier; env API URL for Tier 2 -- Warning when config empty; no warning when config or kwarg explicit -- Tier 1 asserts `TelemetryClient` / `POST /telemetry` (mocked in unit tests) - ---- - -## Task 6: Telemetry CLI ✅ - -**Files:** `codecarbon/cli/telemetry_cli.py`, `codecarbon/cli/main.py`, `tests/cli/test_telemetry_cli.py` - -**Done:** -- `codecarbon.add_typer(telemetry_app, name="telemetry")` -- `codecarbon telemetry` — interactive (`questionary`): pick config path + tier -- `codecarbon telemetry set [--config PATH]` — write `telemetry_level` -- `codecarbon telemetry show [--config PATH]` — resolved tier + explicit flag -- `codecarbon monitor --telemetry-level ` — one-run override via tracker kwarg -- `parse_telemetry_level()` (core) + `normalize_telemetry_level()` (CLI Typer wrapper) -- `telemetry show` uses merged file settings by default (matches tracker) - -**Not done:** optional hook at end of `codecarbon config` wizard (deferred). - ---- - -## Task 7: Documentation ✅ - -**Files:** `docs/how-to/telemetry.md`, `docs/reference/cli.md`, `mkdocs.yml`, link from `docs/how-to/configuration.md` - -**Done:** -- User-facing telemetry how-to with tiers, config, CLI, Python opt-out -- Tier 1 documents **only fields sent today** by `build_minimal_telemetry_dict` plus explicit “not collected yet” list -- CLI reference: `codecarbon telemetry`, `--telemetry-level` - ---- - -## Task 8: PR hygiene ✅ - -```bash -uv run pytest tests/test_telemetry_settings.py tests/test_config_file_settings.py \ - tests/test_telemetry.py tests/test_telemetry_config.py tests/test_telemetry_client.py \ - tests/cli/test_telemetry_cli.py -v -uv run pytest tests/test_emissions_tracker.py tests/test_offline_emissions_tracker.py -q -uv run pytest carbonserver/tests/api/test_telemetry_schema_drift.py -v -uv run task test-package -``` - -**Results (2026-05-19):** 62 telemetry tests, 22 tracker tests, schema drift, **527** package unit tests passed. - -- [x] Public `DEFAULT_TELEMETRY_API_KEY` / `DEFAULT_TELEMETRY_EXPERIMENT_ID` -- [x] `telemetry_level` kwarg on tracker + `@track_emissions` -- [x] Configure warning + Tier 1 HTTP when not explicit -- [x] Telemetry CLI + `monitor --telemetry-level` -- [x] No `send_telemetry`; tests use config mocks -- [x] Full package test suite green (`uv run task test-package`) -- [x] Schema drift test passes -- [x] `test_config.py` isolates `get_config_file_settings` (no leak from real `~/.codecarbon.config`) -- [x] `test_gpu.py` amdsmi warning message aligned with `gpu_amd.py` -- [x] `ApiClient._create_run` handles `longitude`/`latitude` `None` - ---- - -## Related fix (separate PR, not telemetry feature) - -**[PR #1201](https://github.com/mlco2/codecarbon/pull/1201)** → `master`: `codecarbon/core/gpu_amd.py` catches `KeyError` / `OSError` when `amdsmi` is installed but `libamd_smi.so` is missing (macOS CLI crash). Branch: `fix/amdsmi-import-keyerror-macos`. - ---- - -## Risk notes - -| Risk | Mitigation | -|------|------------| -| Users surprised by default Tier 1 logging | One-time warning names Tier 1 explicitly | -| Env `CODECARBON_TELEMETRY` confusion | Document: warning-only, not tier resolution | -| Tier 1 log vs future HTTP | Doc + comment in `send_tier1_telemetry`; flip when API ready | -| CLI import before amdsmi fix merged | PR #1201 to master; rebase `feat/add-telemetry` after merge | - ---- - -## Progress summary - -| Task | Status | -|------|--------| -| 1. Telemetry settings | ✅ | -| 2. `telemetry.py` | ✅ | -| 3. Tracker wiring | ✅ | -| 4. Extensive `/telemetry` POST | ⏸ | -| 5. Integration tests | ✅ | -| 6. Telemetry CLI | ✅ | -| 7. Documentation | ✅ | -| 8. Full CI / PR hygiene | ✅ | - ---- - -## Execution handoff - -**Next step:** Merge PR #1200; rebase after [PR #1201](https://github.com/mlco2/codecarbon/pull/1201); deploy [#1171](https://github.com/mlco2/codecarbon/pull/1171) so prod `/telemetry` stops returning 404. From 0a319bcb480c4146a3dc16ec8453e2e7ad7c9f93 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 21:49:41 +0200 Subject: [PATCH 10/18] fix: improve error handling and logging for AMD GPU metrics - Updated warning message for amdsmi configuration issues to provide clearer guidance on potential causes and solutions. - Adjusted test to reflect the new warning message format, ensuring accurate validation of GPU import warnings. --- codecarbon/core/gpu_amd.py | 10 ++++++---- tests/test_gpu.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/codecarbon/core/gpu_amd.py b/codecarbon/core/gpu_amd.py index 9b1b3ddc1..bd8eeb226 100644 --- a/codecarbon/core/gpu_amd.py +++ b/codecarbon/core/gpu_amd.py @@ -28,12 +28,14 @@ def is_rocm_system(): "Please install amdsmi to get GPU metrics." ) AMDSMI_AVAILABLE = False -except (AttributeError, OSError, KeyError) as error: +except (AttributeError, OSError, KeyError) as e: amdsmi = None + # In some environments, amdsmi may be present but not properly configured, leading to AttributeError when importing logger.warning( - "amdsmi is installed but could not load libamd_smi (ROCm drivers missing " - "or incompatible). AMD GPU metrics will be unavailable. Error: %s", - error, + "AMD GPU detected but amdsmi is not properly configured. " + "Please ensure amdsmi is correctly installed to get GPU metrics." + "Tips : check consistency between Python amdsmi package and ROCm versions, and ensure AMD drivers are up to date." + f" Error: {e}" ) AMDSMI_AVAILABLE = False diff --git a/tests/test_gpu.py b/tests/test_gpu.py index 913c05017..bfbc8e603 100644 --- a/tests/test_gpu.py +++ b/tests/test_gpu.py @@ -153,7 +153,7 @@ def check_output(cmd, *args, **kwargs): sys.modules["pynvml"] = old_pynvml assert any( - "could not load libamd_smi" in str(c.args[0]) + "amdsmi is not properly configured" in str(c.args[0]) for c in warning_mock.call_args_list ) From dcec0c4e8b4544c3d9f73ff529f73d7b66296e21 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 23:09:46 +0200 Subject: [PATCH 11/18] feat: enhance telemetry schema and collection methods - Added new fields to the TelemetryBase schema, including `on_cloud`, `python_package_manager`, `integration_surface`, `offline_mode`, and `save_to_api_enabled`. - Updated MINIMAL_TELEMETRY_FIELDS to include new fields for improved telemetry data collection. - Refactored telemetry sending methods to utilize the new schema and ensure accurate data transmission. - Enhanced tests to validate the new telemetry fields and their integration into the existing telemetry framework. --- .../carbonserver/api/schemas_telemetry.py | 40 ++- .../tests/api/routers/test_telemetry.py | 2 +- codecarbon/cli/telemetry_cli.py | 6 +- codecarbon/core/telemetry_collect.py | 313 ++++++++++++++++++ codecarbon/core/telemetry_schemas.py | 41 ++- codecarbon/core/telemetry_settings.py | 33 +- codecarbon/emissions_tracker.py | 25 +- codecarbon/telemetry.py | 141 ++++---- docs/how-to/configuration.md | 2 +- docs/how-to/telemetry.md | 110 +++--- docs/reference/api.md | 4 +- tests/test_telemetry.py | 276 +++++++-------- tests/test_telemetry_client.py | 2 +- tests/test_telemetry_collect.py | 113 +++++++ tests/test_telemetry_config.py | 257 +++++++------- tests/testutils.py | 21 ++ 16 files changed, 887 insertions(+), 499 deletions(-) create mode 100644 codecarbon/core/telemetry_collect.py create mode 100644 tests/test_telemetry_collect.py diff --git a/carbonserver/carbonserver/api/schemas_telemetry.py b/carbonserver/carbonserver/api/schemas_telemetry.py index 5517c2ff4..dec7e0dfa 100644 --- a/carbonserver/carbonserver/api/schemas_telemetry.py +++ b/carbonserver/carbonserver/api/schemas_telemetry.py @@ -41,6 +41,7 @@ class TelemetryBase(BaseModel): region: Optional[str] = None cloud_provider: Optional[str] = None cloud_region: Optional[str] = None + on_cloud: Optional[bool] = None longitude: Optional[float] = Field(default=None, ge=-180, le=180) latitude: Optional[float] = Field(default=None, ge=-90, le=90) @@ -64,6 +65,7 @@ class TelemetryBase(BaseModel): python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None + python_package_manager: Optional[str] = None total_emissions_kg: Optional[float] = Field(default=None, ge=0) emissions_rate_kg_per_sec: Optional[float] = Field(default=None, ge=0) @@ -83,6 +85,9 @@ class TelemetryBase(BaseModel): task_tracking_used: Optional[bool] = None decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) + integration_surface: Optional[str] = None + offline_mode: Optional[bool] = None + save_to_api_enabled: Optional[bool] = None hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None @@ -148,25 +153,50 @@ def validate_telemetry_level(self): "region", "cloud_provider", "cloud_region", - "longitude", - "latitude", + "on_cloud", "cpu_count", "cpu_physical_count", "cpu_model", "cpu_architecture", "gpu_count", "gpu_model", - "gpu_driver_version", "gpu_memory_total_gb", "ram_total_size_gb", "cuda_version", - "cudnn_version", "python_version", "python_implementation", - "python_executable_hash", "python_env_type", + "python_package_manager", "codecarbon_version", "codecarbon_install_method", + "tracking_mode", + "integration_surface", + "offline_mode", + "output_methods", + "save_to_api_enabled", + "task_tracking_used", + "measure_power_interval_secs", + "in_container", + "ci_environment", + "notebook_environment", + "has_torch", + "has_transformers", + "has_tensorflow", + "has_keras", + "has_diffusers", + "has_pytorch_lightning", + "has_fastai", + "ml_framework_primary", + "total_emissions_kg", + "emissions_rate_kg_per_sec", + "energy_consumed_kwh", + "cpu_energy_kwh", + "gpu_energy_kwh", + "ram_energy_kwh", + "duration_seconds", + "cpu_utilization_avg", + "gpu_utilization_avg", + "ram_utilization_avg", } diff --git a/carbonserver/tests/api/routers/test_telemetry.py b/carbonserver/tests/api/routers/test_telemetry.py index a405b59e8..733269bb0 100644 --- a/carbonserver/tests/api/routers/test_telemetry.py +++ b/carbonserver/tests/api/routers/test_telemetry.py @@ -56,7 +56,7 @@ def test_minimal_telemetry_rejects_extensive_fields(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) telemetry_with_extensive_field = { **MINIMAL_TELEMETRY_TO_CREATE, - "total_emissions_kg": 0.42, + "torch_version": "2.2.0", } with custom_test_server.container.telemetry_repository.override(repository_mock): diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py index e32cd730b..0cf7a8d77 100644 --- a/codecarbon/cli/telemetry_cli.py +++ b/codecarbon/cli/telemetry_cli.py @@ -28,9 +28,9 @@ ) TIER_DESCRIPTIONS = { - "disabled": "No telemetry.", - "minimal": "Send hardware/environment metadata once per session (Tier 1).", - "extensive": "Tier 1 plus public emissions on stop (Tier 2 leaderboard).", + "disabled": "No product telemetry.", + "minimal": "Tier 1: private telemetry per run at stop (environment, usage, frameworks, emissions).", + "extensive": "Tier 2: Tier 1 plus run emissions summary to the shared telemetry experiment (ApiClient).", } diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry_collect.py new file mode 100644 index 000000000..bdd3ed676 --- /dev/null +++ b/codecarbon/core/telemetry_collect.py @@ -0,0 +1,313 @@ +"""Collect and project private product telemetry (Tier 1 / Tier 2) from tracker state.""" + +from __future__ import annotations + +import importlib.util +import os +import platform +import sys +from datetime import datetime, timezone +from typing import Any, Optional + +from codecarbon.core.cloud import get_env_cloud_details +from codecarbon.core.gpu import is_nvidia_system +from codecarbon.core.telemetry_schemas import ( + MINIMAL_TELEMETRY_FIELDS, + TelemetryLevel, +) +from codecarbon.output_methods.emissions_data import EmissionsData + +FRAMEWORK_PACKAGES = ( + ("torch", "has_torch", "torch_version"), + ("transformers", "has_transformers", "transformers_version"), + ("tensorflow", "has_tensorflow", "tensorflow_version"), + ("keras", "has_keras", "keras_version"), + ("diffusers", "has_diffusers", "diffusers_version"), + ("pytorch_lightning", "has_pytorch_lightning", "pytorch_lightning_version"), + ("fastai", "has_fastai", "fastai_version"), +) + +def _non_empty(value: Any) -> bool: + if value is None: + return False + if value == [] or value == {}: + return False + if isinstance(value, str) and value == "": + return False + return True + + +def _strip_none(data: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in data.items() if _non_empty(value)} + + +def _package_installed(name: str) -> bool: + return importlib.util.find_spec(name) is not None + + +def _package_version(name: str) -> Optional[str]: + if not _package_installed(name): + return None + try: + from importlib.metadata import version + + return version(name) + except Exception: + return None + + +def _detect_codecarbon_install_method() -> Optional[str]: + try: + from importlib.metadata import distribution + + dist = distribution("codecarbon") + if getattr(dist, "editable", False): + return "editable" + installer = (dist.metadata.get("Installer") or "").lower() + if "uv" in installer: + return "uv" + if "pip" in installer: + return "pip" + except Exception: + pass + return None + + +def _detect_python_env_type() -> Optional[str]: + if os.environ.get("CONDA_DEFAULT_ENV"): + return "conda" + if os.environ.get("VIRTUAL_ENV"): + return "venv" + if sys.prefix != getattr(sys, "base_prefix", sys.prefix): + return "venv" + return "system" + + +def _detect_python_package_manager() -> Optional[str]: + if os.environ.get("UV"): + return "uv" + if os.environ.get("POETRY_ACTIVE"): + return "poetry" + if os.environ.get("PIP_RUN"): + return "pip" + return None + + +def _detect_ci_environment() -> Optional[str]: + if os.environ.get("GITHUB_ACTIONS"): + return "github_actions" + if os.environ.get("GITLAB_CI"): + return "gitlab_ci" + if os.environ.get("CIRCLECI"): + return "circleci" + if os.environ.get("JENKINS_URL"): + return "jenkins" + if os.environ.get("CI"): + return "ci" + return None + + +def _detect_notebook_environment() -> Optional[str]: + if os.environ.get("COLAB_GPU") is not None or "google.colab" in sys.modules: + return "colab" + try: + from IPython import get_ipython + + shell = get_ipython().__class__.__name__ + if "ZMQInteractiveShell" in shell: + return "jupyter" + except Exception: + pass + return None + + +def _detect_in_container() -> bool: + if os.path.exists("/.dockerenv"): + return True + if os.environ.get("KUBERNETES_SERVICE_HOST"): + return True + return False + + +def _detect_integration_surface(tracker: Any) -> str: + from codecarbon.emissions_tracker import OfflineEmissionsTracker + + if isinstance(tracker, OfflineEmissionsTracker): + return "offline_tracker" + argv = " ".join(sys.argv) + if "codecarbon" in argv and "monitor" in argv: + return "cli_monitor" + return "library" + + +def _collect_output_methods(tracker: Any) -> list[str]: + methods: list[str] = [] + if getattr(tracker, "_save_to_file", False): + methods.append("file") + if getattr(tracker, "_save_to_api", False): + methods.append("api") + if getattr(tracker, "_save_to_logger", False): + methods.append("logger") + if getattr(tracker, "_emissions_endpoint", None): + methods.append("http") + if getattr(tracker, "_save_to_prometheus", False): + methods.append("prometheus") + if getattr(tracker, "_save_to_logfire", False): + methods.append("logfire") + return methods + + +def _raw_cloud_provider_and_region() -> tuple[Optional[str], Optional[str]]: + details = get_env_cloud_details() + if not details or not details.get("metadata"): + return None, None + provider = (details.get("provider") or "").lower() or None + metadata = details.get("metadata") or {} + region: Optional[str] = None + if provider == "aws": + region = metadata.get("region") + elif provider == "azure": + region = (metadata.get("compute") or {}).get("location") + elif provider == "gcp": + zone = metadata.get("zone") or "" + parts = zone.split("/") + region = parts[-1].rsplit("-", 1)[0] if parts else None + return provider, region + + +def _collect_framework_fields(include_versions: bool) -> dict[str, Any]: + fields: dict[str, Any] = {} + primary: Optional[str] = None + for package, has_field, version_field in FRAMEWORK_PACKAGES: + installed = _package_installed(package) + fields[has_field] = installed + if include_versions and installed: + fields[version_field] = _package_version(package) + if installed and primary is None: + primary = package + if primary: + fields["ml_framework_primary"] = primary + return fields + + +def _gpu_static_fields() -> dict[str, Any]: + fields: dict[str, Any] = {} + if not is_nvidia_system(): + return fields + try: + import pynvml + + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + mem = pynvml.nvmlDeviceGetMemoryInfo(handle) + fields["gpu_memory_total_gb"] = mem.total / (1024**3) + fields["gpu_driver_version"] = pynvml.nvmlSystemGetDriverVersion() + fields["cuda_version"] = pynvml.nvmlSystemGetCudaDriverVersion_v2() + if isinstance(fields["cuda_version"], int): + v = fields["cuda_version"] + fields["cuda_version"] = f"{v // 1000}.{(v % 1000) // 10}" + except Exception: + pass + return fields + + +def collect_telemetry_context( + tracker: Any, + emissions: EmissionsData, +) -> dict[str, Any]: + """Build a flat telemetry context from tracker state and emissions at stop. + + Args: + tracker: Active ``BaseEmissionsTracker`` instance. + emissions: Total emissions row from ``_prepare_emissions_data()``. + + Returns: + Flat dictionary for ``project_tier1``. + """ + conf = getattr(tracker, "_conf", {}) + raw_provider, raw_region = _raw_cloud_provider_and_region() + on_cloud = emissions.on_cloud == "Y" + cloud_provider = emissions.cloud_provider or raw_provider + cloud_region = emissions.cloud_region or raw_region + region = emissions.region or conf.get("region") + if on_cloud and cloud_region: + region = region or cloud_region + + integration_surface = _detect_integration_surface(tracker) + gpu_fields = _gpu_static_fields() + + context: dict[str, Any] = { + "timestamp": datetime.now(timezone.utc), + "os": conf.get("os") or platform.platform(), + "python_version": conf.get("python_version") or platform.python_version(), + "python_implementation": platform.python_implementation(), + "python_env_type": _detect_python_env_type(), + "python_package_manager": _detect_python_package_manager(), + "codecarbon_version": conf.get("codecarbon_version"), + "codecarbon_install_method": _detect_codecarbon_install_method(), + "country_name": emissions.country_name, + "country_iso_code": emissions.country_iso_code, + "region": region, + "cloud_provider": cloud_provider, + "cloud_region": cloud_region, + "on_cloud": on_cloud, + "cpu_count": conf.get("cpu_count"), + "cpu_physical_count": conf.get("cpu_physical_count"), + "cpu_model": conf.get("cpu_model"), + "cpu_architecture": platform.machine(), + "gpu_count": conf.get("gpu_count"), + "gpu_model": conf.get("gpu_model"), + "ram_total_size_gb": conf.get("ram_total_size"), + "tracking_mode": conf.get("tracking_mode"), + "integration_surface": integration_surface, + "offline_mode": integration_surface == "offline_tracker", + "output_methods": _collect_output_methods(tracker), + "save_to_api_enabled": bool(getattr(tracker, "_save_to_api", False)), + "task_tracking_used": bool(getattr(tracker, "_tasks", {})), + "measure_power_interval_secs": getattr(tracker, "_measure_power_secs", None), + "in_container": _detect_in_container(), + "ci_environment": _detect_ci_environment(), + "notebook_environment": _detect_notebook_environment(), + "duration_seconds": float(emissions.duration) if emissions.duration else None, + "total_emissions_kg": emissions.emissions, + "emissions_rate_kg_per_sec": emissions.emissions_rate, + "energy_consumed_kwh": emissions.energy_consumed, + "cpu_energy_kwh": emissions.cpu_energy, + "gpu_energy_kwh": emissions.gpu_energy, + "ram_energy_kwh": emissions.ram_energy, + "cpu_utilization_avg": emissions.cpu_utilization_percent, + "gpu_utilization_avg": emissions.gpu_utilization_percent, + "ram_utilization_avg": emissions.ram_utilization_percent, + **_collect_framework_fields(include_versions=False), + } + + for key in ("gpu_memory_total_gb", "cuda_version"): + if key in gpu_fields: + context[key] = gpu_fields[key] + + return _strip_none(context) + + +def project_tier1(context: dict[str, Any]) -> dict[str, Any]: + """Project context to Tier 1 (``telemetry_level=minimal``) fields.""" + payload = { + key: context[key] + for key in MINIMAL_TELEMETRY_FIELDS + if key in context + } + payload["telemetry_level"] = TelemetryLevel.minimal.value + return _strip_none(payload) + + +def build_tier1_payload(tracker: Any, emissions: EmissionsData) -> dict[str, Any]: + """Build a Tier 1 payload dict for ``TelemetryCreate``. + + Args: + tracker: Active emissions tracker. + emissions: Run emissions data. + + Returns: + Payload dict for ``POST /telemetry``. + """ + context = collect_telemetry_context(tracker, emissions) + return project_tier1(context) diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry_schemas.py index ea6249b65..9dc3e2997 100644 --- a/codecarbon/core/telemetry_schemas.py +++ b/codecarbon/core/telemetry_schemas.py @@ -23,6 +23,7 @@ class TelemetryBase(BaseModel): region: Optional[str] = None cloud_provider: Optional[str] = None cloud_region: Optional[str] = None + on_cloud: Optional[bool] = None longitude: Optional[float] = Field(default=None, ge=-180, le=180) latitude: Optional[float] = Field(default=None, ge=-90, le=90) @@ -46,6 +47,7 @@ class TelemetryBase(BaseModel): python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None + python_package_manager: Optional[str] = None total_emissions_kg: Optional[float] = Field(default=None, ge=0) emissions_rate_kg_per_sec: Optional[float] = Field(default=None, ge=0) @@ -65,6 +67,9 @@ class TelemetryBase(BaseModel): task_tracking_used: Optional[bool] = None decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) + integration_surface: Optional[str] = None + offline_mode: Optional[bool] = None + save_to_api_enabled: Optional[bool] = None hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None @@ -77,7 +82,6 @@ class TelemetryBase(BaseModel): ide_used: Optional[str] = None notebook_environment: Optional[str] = None ci_environment: Optional[str] = None - python_package_manager: Optional[str] = None framework_detected: Optional[str] = None has_torch: Optional[bool] = None @@ -130,25 +134,50 @@ def validate_telemetry_level(self): "region", "cloud_provider", "cloud_region", - "longitude", - "latitude", + "on_cloud", "cpu_count", "cpu_physical_count", "cpu_model", "cpu_architecture", "gpu_count", "gpu_model", - "gpu_driver_version", "gpu_memory_total_gb", "ram_total_size_gb", "cuda_version", - "cudnn_version", "python_version", "python_implementation", - "python_executable_hash", "python_env_type", + "python_package_manager", "codecarbon_version", "codecarbon_install_method", + "tracking_mode", + "integration_surface", + "offline_mode", + "output_methods", + "save_to_api_enabled", + "task_tracking_used", + "measure_power_interval_secs", + "in_container", + "ci_environment", + "notebook_environment", + "has_torch", + "has_transformers", + "has_tensorflow", + "has_keras", + "has_diffusers", + "has_pytorch_lightning", + "has_fastai", + "ml_framework_primary", + "total_emissions_kg", + "emissions_rate_kg_per_sec", + "energy_consumed_kwh", + "cpu_energy_kwh", + "gpu_energy_kwh", + "ram_energy_kwh", + "duration_seconds", + "cpu_utilization_avg", + "gpu_utilization_avg", + "ram_utilization_avg", } diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py index 9e6d5ad91..1d2f65c87 100644 --- a/codecarbon/core/telemetry_settings.py +++ b/codecarbon/core/telemetry_settings.py @@ -44,10 +44,8 @@ def is_telemetry_level_explicit( ) -> bool: """Return whether the user explicitly chose a telemetry tier. - Explicit sources (in order): tracker ``telemetry_level`` argument, config file - ``telemetry_level``, environment ``CODECARBON_TELEMETRY_LEVEL`` / - ``telemetry_level``. Legacy ``CODECARBON_TELEMETRY`` / ``telemetry`` counts as - explicit for this check only. + Explicit sources: tracker ``telemetry_level`` argument, config file + ``telemetry_level``, or environment ``CODECARBON_TELEMETRY_LEVEL``. Args: config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). @@ -63,9 +61,7 @@ def is_telemetry_level_explicit( return True if external_conf is None: return False - if external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: - return True - return external_conf.get("telemetry") is not None + return external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None def resolve_telemetry_level( @@ -78,15 +74,10 @@ def resolve_telemetry_level( Precedence: - 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or - ``codecarbon monitor --telemetry-level`` + 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or CLI 2. ``external_conf`` — merged ``.codecarbon.config`` and ``CODECARBON_*`` env - (environment overrides file for the same key) 3. ``config_file_conf`` — file-only settings when ``external_conf`` is omitted - 4. Default: ``minimal`` - - Legacy ``CODECARBON_TELEMETRY`` / config key ``telemetry`` only affect whether - the tier counts as explicitly configured, not tier resolution. + 4. Default: ``minimal`` (Tier 1) Args: config_file_conf: Settings from ``get_config_file_settings()`` (optional). @@ -130,8 +121,10 @@ def get_telemetry_api_url( Returns: API base URL without trailing slash. """ - url = external_conf.get("telemetry_api_url") or os.environ.get( - "CODECARBON_TELEMETRY_API_URL" + url = ( + external_conf.get("telemetry_api_url") + or external_conf.get("api_endpoint") + or os.environ.get("CODECARBON_TELEMETRY_API_URL") ) return (url or default).rstrip("/") @@ -145,8 +138,10 @@ def get_telemetry_api_key(external_conf: dict[str, Any]) -> str: Returns: API token string. """ - key = external_conf.get("telemetry_api_key") or os.environ.get( - "CODECARBON_TELEMETRY_API_KEY" + key = ( + external_conf.get("telemetry_api_key") + or external_conf.get("api_key") + or os.environ.get("CODECARBON_TELEMETRY_API_KEY") ) return key or DEFAULT_TELEMETRY_API_KEY @@ -158,7 +153,7 @@ def get_telemetry_experiment_id(external_conf: dict[str, Any]) -> str: external_conf: Merged config from file and environment. Returns: - Experiment UUID string for Tier 2 / leaderboard linkage. + Experiment UUID string (legacy leaderboard / API helpers). """ experiment_id = external_conf.get("telemetry_experiment_id") or os.environ.get( "CODECARBON_TELEMETRY_EXPERIMENT_ID" diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index d88892d66..fa4eeb166 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -25,8 +25,7 @@ from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.core.telemetry_settings import resolve_telemetry_level from codecarbon.telemetry import ( - send_tier1_telemetry, - send_tier2_public_emission, + send_product_telemetry_at_stop, warn_if_telemetry_not_configured, ) from codecarbon.core.emissions import Emissions @@ -415,6 +414,7 @@ def __init__( self._hardware = [] resource_tracker = ResourceTracker(self) resource_tracker.set_CPU_GPU_ram_tracking() + self._resource_tracker = resource_tracker self._conf["hardware"] = list(map(lambda x: x.description(), self._hardware)) @@ -458,7 +458,7 @@ def __init__( if cloud.is_on_private_infra: self._conf["longitude"] = self._geo.longitude self._conf["latitude"] = self._geo.latitude - self._conf["region"] = cloud.region + self._conf["region"] = self._geo.region self._conf["provider"] = cloud.provider else: self._conf["region"] = cloud.region @@ -479,17 +479,14 @@ def _apply_init_telemetry(self, telemetry_override: str | None) -> None: override=telemetry_override, external_conf=self._external_conf, ) - if self._telemetry_level in (TelemetryLevel.minimal, TelemetryLevel.extensive): - send_tier1_telemetry(self._conf, external_conf=self._external_conf) - @suppress(Exception) - def _maybe_send_tier2_telemetry(self, emissions_data_delta: EmissionsData) -> None: - if self._telemetry_level == TelemetryLevel.extensive: - send_tier2_public_emission( - self._conf, - emissions_data_delta, - external_conf=self._external_conf, - ) + def _send_product_telemetry_at_stop(self, emissions_data: EmissionsData) -> None: + send_product_telemetry_at_stop( + self, + emissions_data, + self._telemetry_level, + external_conf=self._external_conf, + ) def _init_output_methods(self, *, api_key: str = None): """ @@ -792,7 +789,7 @@ def stop(self) -> Optional[float]: emissions_data = self._prepare_emissions_data() emissions_data_delta = self._compute_emissions_delta(emissions_data) - self._maybe_send_tier2_telemetry(emissions_data_delta) + self._send_product_telemetry_at_stop(emissions_data) self._persist_data( total_emissions=emissions_data, diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index ad4946e0c..397be674b 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -1,11 +1,11 @@ -"""Tracker-facing telemetry helpers (Tier 1 HTTP, Tier 2 public emissions).""" +"""Product telemetry sent at tracker stop (Tier 1 private, Tier 2 public emissions).""" import dataclasses -from datetime import datetime, timezone from typing import Any from codecarbon.core.api_client import ApiClient from codecarbon.core.telemetry_client import TelemetryClient +from codecarbon.core.telemetry_collect import build_tier1_payload from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.core.telemetry_settings import ( get_telemetry_api_key, @@ -16,16 +16,13 @@ from codecarbon.external.logger import logger from codecarbon.output_methods.emissions_data import EmissionsData -_TIER1_SENT = False -_TIER2_SENT = False _TELEMETRY_CONFIGURE_WARNED = False TELEMETRY_NOT_CONFIGURED_MESSAGE = ( "CodeCarbon telemetry_level was not set explicitly; using default %r. " - "Tier 1 minimal telemetry (hardware and environment metadata) will be " - "sent once per Python session. Set telemetry_level in .codecarbon.config, " - "set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... to " - "EmissionsTracker / OfflineEmissionsTracker, or run " + "Tier 1 private telemetry (per run at stop) will be sent. Set telemetry_level " + "in .codecarbon.config, set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... " + "to EmissionsTracker / OfflineEmissionsTracker, or run " "codecarbon telemetry set ." ) @@ -59,98 +56,66 @@ def warn_if_telemetry_not_configured( _TELEMETRY_CONFIGURE_WARNED = True -def build_minimal_telemetry_dict(conf: dict[str, Any]) -> dict[str, Any]: - """Build a minimal telemetry payload dict from tracker configuration. +def _run_too_short_for_telemetry(emissions: EmissionsData) -> bool: + return emissions.duration is not None and emissions.duration < 1 - Args: - conf: Tracker configuration dictionary. - Returns: - Dictionary suitable for ``TelemetryCreate`` validation. - """ - payload: dict[str, Any] = { - "timestamp": datetime.now(timezone.utc), - "telemetry_level": TelemetryLevel.minimal.value, - "os": conf.get("os"), - "country_iso_code": conf.get("country_iso_code"), - "region": conf.get("region"), - "cloud_provider": conf.get("provider"), - "cloud_region": conf.get("region"), - "longitude": conf.get("longitude"), - "latitude": conf.get("latitude"), - "cpu_count": conf.get("cpu_count"), - "cpu_physical_count": conf.get("cpu_physical_count"), - "cpu_model": conf.get("cpu_model"), - "gpu_count": conf.get("gpu_count"), - "gpu_model": conf.get("gpu_model"), - "ram_total_size_gb": conf.get("ram_total_size"), - "python_version": conf.get("python_version"), - "codecarbon_version": conf.get("codecarbon_version"), - } - return {key: value for key, value in payload.items() if value is not None} - - -def send_tier1_telemetry( - conf: dict[str, Any], +def send_tier1_at_stop( + tracker: Any, + emissions: EmissionsData, external_conf: dict[str, Any] | None = None, ) -> bool: - """POST minimal telemetry once per Python session via ``TelemetryClient``. - - Sends ``POST {telemetry_api_url}/telemetry`` with a ``TelemetryCreate`` payload - built from tracker configuration. Best-effort: failures are logged and not raised. + """Send Tier 1 telemetry: private hardware/usage/run summary via ``POST /telemetry``. Args: - conf: Tracker configuration dictionary. - external_conf: Merged file/env config for telemetry API URL and key resolution. + tracker: Active emissions tracker instance. + emissions: Total emissions from ``_prepare_emissions_data()``. + external_conf: Merged config for telemetry API URL and key resolution. Returns: - True if telemetry was posted successfully on this call, False if already sent - in this session or if the request failed. + True if Tier 1 was accepted, False otherwise. """ - global _TIER1_SENT - if _TIER1_SENT: + if _run_too_short_for_telemetry(emissions): + logger.debug( + "Tier 1 telemetry not sent because run duration is shorter than 1 second." + ) return False settings_conf = external_conf or {} try: - payload = build_minimal_telemetry_dict(conf) - endpoint_url = get_telemetry_api_url(settings_conf) + payload = build_tier1_payload(tracker, emissions) client = TelemetryClient( - endpoint_url=endpoint_url, + endpoint_url=get_telemetry_api_url(settings_conf), telemetry=payload, api_key=get_telemetry_api_key(settings_conf), ) - response = client.add_telemetry() - if response is not None: - _TIER1_SENT = True - return True - return False + return client.add_telemetry() is not None except Exception as error: - logger.error(f"Telemetry Tier 1 failed (non-critical): {error}") + logger.error(f"Tier 1 telemetry failed (non-critical): {error}") return False -def send_tier2_public_emission( - conf: dict[str, Any], - emissions_data: EmissionsData, +def send_tier2_at_stop( + tracker: Any, + emissions: EmissionsData, external_conf: dict[str, Any] | None = None, ) -> bool: - """Send run emissions to the public telemetry experiment via ApiClient. - - Mirrors ``CodeCarbonAPIOutput`` / ``add_emission`` for the shared leaderboard - project. Best-effort: errors are logged and not raised. + """Send Tier 2 telemetry: run emissions to the shared experiment via ``ApiClient``. Args: - conf: Tracker configuration dictionary. - emissions_data: Delta or total emissions row for the run. - external_conf: Merged file/env config for API URL, key, and experiment id. + tracker: Active emissions tracker instance. + emissions: Total emissions from ``_prepare_emissions_data()``. + external_conf: Merged config for API URL, key, and experiment resolution. Returns: - True if emission was posted successfully on this call, False otherwise. + True if Tier 2 was posted successfully, False otherwise. """ - global _TIER2_SENT - if _TIER2_SENT: + if _run_too_short_for_telemetry(emissions): + logger.debug( + "Tier 2 telemetry not sent because run duration is shorter than 1 second." + ) return False settings_conf = external_conf or {} + conf = getattr(tracker, "_conf", {}) try: api = ApiClient( endpoint_url=get_telemetry_api_url(settings_conf), @@ -159,11 +124,33 @@ def send_tier2_public_emission( conf=conf, create_run_automatically=True, ) - posted = api.add_emission(dataclasses.asdict(emissions_data)) - if posted: - _TIER2_SENT = True - return True - return False + return bool(api.add_emission(dataclasses.asdict(emissions))) except Exception as error: - logger.error(f"Telemetry Tier 2 failed (non-critical): {error}") + logger.error(f"Tier 2 telemetry failed (non-critical): {error}") return False + + +def send_product_telemetry_at_stop( + tracker: Any, + emissions: EmissionsData, + level: TelemetryLevel, + external_conf: dict[str, Any] | None = None, +) -> None: + """Send product telemetry for the resolved tier at tracker ``stop()``. + + Tier 1 (``minimal``): private ``POST /telemetry`` only. + Tier 2 (``extensive``): Tier 1 plus ``ApiClient`` run summary. + + Args: + tracker: Active emissions tracker instance. + emissions: Total emissions from ``_prepare_emissions_data()``. + level: Resolved ``TelemetryLevel``. + external_conf: Merged config for API settings. + """ + if level == TelemetryLevel.disabled: + return + settings = external_conf or {} + if level in (TelemetryLevel.minimal, TelemetryLevel.extensive): + send_tier1_at_stop(tracker, emissions, settings) + if level == TelemetryLevel.extensive: + send_tier2_at_stop(tracker, emissions, settings) diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index 34e32b2bb..5cabf13ee 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -110,6 +110,6 @@ documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies) ## Product telemetry -Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Set it in `.codecarbon.config`, via `CODECARBON_TELEMETRY_LEVEL`, or with `EmissionsTracker(telemetry_level=...)` (argument wins). Tier 1 (`minimal`) sends only hardware and environment metadata once per process—not your emissions or project ids. +Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Set it in `.codecarbon.config`, via `CODECARBON_TELEMETRY_LEVEL`, or with `EmissionsTracker(telemetry_level=...)` (argument wins). Tier 1 (`minimal`) sends private product telemetry once per run at `stop()`—see [Product telemetry](telemetry.md). See [Product telemetry](telemetry.md) for tiers, what is collected, and how to opt out. diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index a99d42930..181605047 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -1,6 +1,6 @@ # Product telemetry -CodeCarbon can send **optional product telemetry** to help improve the library: which hardware and environments people run on, not what your code does. This is separate from sending **your** emissions to the [dashboard](cloud-api.md) with `save_to_api=True`. +CodeCarbon can send **optional private product telemetry** to help improve the library: hardware, environment, how the package is used, and per-run carbon/energy summaries. This is separate from sending **your** emissions to the [dashboard](cloud-api.md) with `save_to_api=True`. ## Telemetry vs your dashboard data @@ -8,68 +8,53 @@ CodeCarbon can send **optional product telemetry** to help improve the library: |--|-------------------|--------------------------------| | Purpose | Improve CodeCarbon (aggregate usage) | Your projects and experiments | | Config | `telemetry_level`, `codecarbon telemetry` | `codecarbon config`, `experiment_id` | -| Default experiment | Public telemetry project (built-in defaults) | Your account / experiment | +| Default API target | Built-in telemetry project (private) | Your account / experiment | You can use one without the other. ## Tiers -| `telemetry_level` | What happens | -|-------------------|--------------| -| `disabled` | No product telemetry | -| `minimal` (default) | **Tier 1** once per Python process: minimal hardware/environment metadata | -| `extensive` | Tier 1 + **Tier 2** on tracker `stop()`: one public emissions row (leaderboard-style) | +| `telemetry_level` | Name | When | Transport | +|-------------------|------|------|-----------| +| `disabled` | — | — | Nothing | +| `minimal` | **Tier 1** | Each `stop()` | `POST /telemetry` (private) | +| `extensive` | **Tier 2** | Each `stop()` | Tier 1 (`POST /telemetry`) **and** Tier 2 (`ApiClient` → `/emissions`) | Tier is resolved in this order: -1. **Tracker or CLI argument** — `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` (highest priority) -2. **Config + environment** — `telemetry_level` in `.codecarbon.config` (local overrides global), then `CODECARBON_TELEMETRY_LEVEL` overrides the file value when both are set (same rules as other `CODECARBON_*` settings) -3. **Default:** `minimal` +1. **Tracker or CLI argument** — `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` +2. **Config + environment** — `telemetry_level` in `.codecarbon.config`, then `CODECARBON_TELEMETRY_LEVEL` when both are set +3. **Default:** `minimal` (Tier 1) -Legacy `CODECARBON_TELEMETRY` / config key `telemetry` suppress the one-time setup warning when set, but do **not** set the tier (use `telemetry_level` or `CODECARBON_TELEMETRY_LEVEL`). +## Lifecycle -## Tier 1: what we collect (and what we do not) - -Tier 1 is intentionally small. The client only builds a **minimal** payload; the API schema rejects “extensive” fields when `telemetry_level` is `minimal`. +```text +EmissionsTracker.__init__ → collect hardware/geo (no POST) +EmissionsTracker.stop() → minimal: Tier 1 only | extensive: Tier 1 + Tier 2 +``` -### Sent at most once per process (if known) +If the run lasts less than one second, telemetry is not sent. -Only non-empty values are included: +## Tier 1 (`minimal`) — per run -| Field | Description | -|-------|-------------| -| `timestamp` | UTC time of the send | -| `telemetry_level` | Always `minimal` for this tier | -| `os` | Platform string | -| `country_iso_code` | Country (e.g. from offline mode or geo) | -| `region` | Region / province | -| `cloud_provider` | Cloud provider name, if detected | -| `cloud_region` | Cloud region, if detected | -| `longitude`, `latitude` | Approximate location, if geo resolution ran | -| `cpu_count`, `cpu_physical_count`, `cpu_model` | CPU metadata | -| `gpu_count`, `gpu_model` | GPU metadata | -| `ram_total_size_gb` | Total RAM | -| `python_version` | Python version | -| `codecarbon_version` | Installed CodeCarbon version | +One private row per tracker run with: -### Not sent in Tier 1 +- **Environment:** OS, Python, CPU/GPU/RAM, country/region, cloud provider/region +- **Usage:** tracking mode, output methods, integration surface (library / CLI / offline), task tracking, CI/notebook/container hints +- **ML stack (presence):** `has_torch`, `has_transformers`, `has_tensorflow`, and related flags +- **Run outcome:** duration, emissions, energy (total and per component), utilization averages -Tier 1 does **not** include: +Tier 1 does **not** include project names, experiment ids, API keys, file paths, or survey demographics (role, industry, etc.). -- Emissions, energy, power, duration, or utilization -- Project name, experiment id, run id, or API keys -- Source code, file paths, hostnames, or user ids -- ML stack versions (PyTorch, TensorFlow, etc.) -- Output methods, tracking mode, or internal diagnostics -- Anything else defined as “extensive” in the telemetry schema +## Tier 2 (`extensive`) — per run -Tier 2 (`extensive`) adds a single **public** emissions summary via the same mechanism as `add_emission` to the shared telemetry experiment—not a full extensive telemetry document. +**Always sends Tier 1 first**, then adds a **run emissions summary** to the shared CodeCarbon telemetry experiment via `ApiClient` (`/runs` then `/emissions`). Endpoint, API key, and experiment id come from `telemetry_api_url` / `telemetry_api_key` / `telemetry_experiment_id` (or `CODECARBON_TELEMETRY_*` env vars), falling back to the built-in defaults and your `api_endpoint` / `api_key` when set. -### Transport +## Never collected -- HTTP `POST` to `{telemetry_api_url}/telemetry` -- Best-effort: failures are logged and do not stop your tracker -- If the endpoint is not deployed yet, you may see a warning (HTTP 404) +- Project name, experiment id, run id, API keys +- Source code, file paths, hostnames +- Voluntary [user survey](https://docs.google.com/forms/d/e/1FAIpQLSeQ5Tu_rdrpDhBJvh5R1-_iB4Ld-kgh6iNMjgaMXa8AEVPxqA/viewform) demographics (role, industry, experience) ## Configure telemetry @@ -78,27 +63,13 @@ Tier 2 (`extensive`) adds a single **public** emissions summary via the same mec ```ini [codecarbon] telemetry_level = minimal -# Optional overrides (defaults point at the public telemetry API): -# telemetry_api_url = https://api.codecarbon.io -# telemetry_api_key = ... -# telemetry_experiment_id = ... ``` ### CLI ```bash -# Interactive wizard (pick config file + tier) -codecarbon telemetry - -# Set tier in config -codecarbon telemetry set disabled codecarbon telemetry set minimal -codecarbon telemetry set extensive - -# Show resolved tier (merged global + local config) codecarbon telemetry show - -# One-run override (does not write config) codecarbon monitor --telemetry-level disabled -- python train.py ``` @@ -107,32 +78,25 @@ codecarbon monitor --telemetry-level disabled -- python train.py ```python from codecarbon import EmissionsTracker -tracker = EmissionsTracker(telemetry_level="disabled") +tracker = EmissionsTracker(telemetry_level="minimal") +tracker.start() +# ... +tracker.stop() ``` ## Opt out -Set any of: - ```ini [codecarbon] telemetry_level = disabled ``` -```bash -codecarbon telemetry set disabled -``` - -```python -EmissionsTracker(telemetry_level="disabled") -``` - ## First run without explicit configuration -If you never set `telemetry_level`, CodeCarbon uses `minimal` and logs a **one-time warning** per Python session that Tier 1 will be sent. Set `telemetry_level` explicitly (config, CLI, or tracker argument) to silence it. +If you never set `telemetry_level`, CodeCarbon uses `minimal` (Tier 1) and logs a **one-time warning** per Python session. Set `telemetry_level` explicitly to silence it. ## Related -- [Configure CodeCarbon](configuration.md) — general `.codecarbon.config` hierarchy -- [CLI reference](../reference/cli.md#codecarbon-telemetry) — command flags -- [Cloud API & dashboard](cloud-api.md) — your own emissions data +- [Configure CodeCarbon](configuration.md) +- [CLI reference](../reference/cli.md#codecarbon-telemetry) +- [Cloud API & dashboard](cloud-api.md) diff --git a/docs/reference/api.md b/docs/reference/api.md index 37d2abea0..8b8bea306 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -19,8 +19,8 @@ Optional library telemetry is controlled by **`telemetry_level`** on the tracker | Value | Behavior | |-------|----------| | `disabled` | No product telemetry | -| `minimal` (default) | Tier 1 hardware/environment metadata once per process | -| `extensive` | Tier 1 + public emissions summary on `stop()` | +| `minimal` (default) | Tier 1 private telemetry at each `stop()` | +| `extensive` | Tier 1 + Tier 2 (private telemetry and public run summary) at each `stop()` | **Resolution order:** tracker argument → `.codecarbon.config` → `CODECARBON_TELEMETRY_LEVEL` → default `minimal`. The tracker argument overrides config and environment. diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 7a235f76c..741182d8f 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,19 +1,14 @@ -import platform import sys import tempfile import unittest from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import ANY, MagicMock, patch -import codecarbon.telemetry as telemetry_module from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker -from codecarbon.telemetry import ( - build_minimal_telemetry_dict, - send_tier1_telemetry, - send_tier2_public_emission, -) -from tests.testutils import get_custom_mock_open +from codecarbon.output_methods.emissions_data import EmissionsData +from codecarbon.telemetry import send_tier1_at_stop, send_tier2_at_stop +from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": mock_platform_cli_setup = patch( @@ -27,102 +22,16 @@ extensive_conf = "[codecarbon]\ntelemetry_level = extensive\n" -class TestTelemetry(unittest.TestCase): - def test_build_minimal_telemetry_dict_maps_tracker_conf(self): - conf = { - "python_version": platform.python_version(), - "os": platform.platform(), - "cpu_count": 8, - "cpu_physical_count": 4, - "cpu_model": "Intel Core i7", - "gpu_count": 1, - "gpu_model": "NVIDIA RTX 3080", - "ram_total_size": 32.0, - "codecarbon_version": "2.0.0", - "tracking_mode": "machine", - "country_iso_code": "FRA", - "provider": "aws", - "region": "eu-west-1", - } - payload = build_minimal_telemetry_dict(conf) - self.assertEqual(payload["telemetry_level"], TelemetryLevel.minimal.value) - self.assertIn("timestamp", payload) - self.assertEqual(payload["python_version"], conf["python_version"]) - self.assertEqual(payload["ram_total_size_gb"], conf["ram_total_size"]) - self.assertEqual(payload["cloud_provider"], conf["provider"]) - self.assertEqual(payload["cloud_region"], conf["region"]) - - -class TestTier1Send(unittest.TestCase): - def test_send_tier1_telemetry_posts_once_per_session(self): - telemetry_module._TIER1_SENT = False - conf = { - "python_version": "3.11.0", - "os": "Linux", - "cpu_count": 4, - "cpu_model": "Intel i5", - "gpu_count": 0, - "codecarbon_version": "2.0.0", - } - with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_client_cls.return_value = mock_client - result = send_tier1_telemetry(conf) - self.assertTrue(result) - result = send_tier1_telemetry(conf) - self.assertFalse(result) - mock_client_cls.assert_called_once() - mock_client.add_telemetry.assert_called_once() - call_kwargs = mock_client_cls.call_args.kwargs - self.assertEqual( - call_kwargs["telemetry"]["telemetry_level"], TelemetryLevel.minimal.value - ) - - def test_send_tier1_telemetry_uses_resolved_api_url_and_key(self): - telemetry_module._TIER1_SENT = False - external_conf = { - "telemetry_api_url": "http://custom-tier1.example", - "telemetry_api_key": "cpt_custom", - } - with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_client_cls.return_value = mock_client - send_tier1_telemetry( - {"codecarbon_version": "2.0"}, external_conf=external_conf - ) - self.assertEqual( - mock_client_cls.call_args.kwargs["endpoint_url"], - "http://custom-tier1.example", - ) - self.assertEqual(mock_client_cls.call_args.kwargs["api_key"], "cpt_custom") - - def test_send_tier1_telemetry_fails_silently(self): - telemetry_module._TIER1_SENT = False - with patch( - "codecarbon.telemetry.build_minimal_telemetry_dict", - side_effect=RuntimeError("build failed"), - ): - with patch("codecarbon.telemetry.logger.error") as mock_logger: - result = send_tier1_telemetry({}) - self.assertFalse(result) - mock_logger.assert_called() - - -class TestTier2Send(unittest.TestCase): - def test_send_tier2_public_emission_sends_once_per_session(self): - telemetry_module._TIER2_SENT = False - from codecarbon.output_methods.emissions_data import EmissionsData - - emissions = EmissionsData( +class TestTelemetryTiersAtStop(unittest.TestCase): + def _emissions(self) -> EmissionsData: + return EmissionsData( timestamp="2020-01-01T00:00:00", project_name="test", run_id="run-1", experiment_id="exp-1", - duration=1.0, + duration=10.0, emissions=0.001, - emissions_rate=0.001, + emissions_rate=0.0001, cpu_power=0.0, gpu_power=0.0, ram_power=0.0, @@ -131,16 +40,16 @@ def test_send_tier2_public_emission_sends_once_per_session(self): ram_energy=0.0, energy_consumed=0.01, water_consumed=0.0, - country_name="", - country_iso_code="", - region="", + country_name="France", + country_iso_code="FRA", + region="idf", cloud_provider="", cloud_region="", os="Linux", python_version="3.11", codecarbon_version="2.0", cpu_count=1.0, - cpu_model="", + cpu_model="cpu", gpu_count=0.0, gpu_model="", longitude=0.0, @@ -148,23 +57,54 @@ def test_send_tier2_public_emission_sends_once_per_session(self): ram_total_size=8.0, tracking_mode="machine", ) + + def test_tier1_posts_private_telemetry(self): + tracker = MagicMock() + emissions = self._emissions() + with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = {"id": "ok"} + mock_client_cls.return_value = mock_client + result = send_tier1_at_stop(tracker, emissions, external_conf={}) + self.assertTrue(result) + posted = mock_client_cls.call_args.kwargs["telemetry"] + self.assertEqual(posted["telemetry_level"], "minimal") + self.assertEqual(posted["total_emissions_kg"], 0.001) + + def test_tier1_skips_short_duration(self): + tracker = MagicMock() + emissions = self._emissions() + emissions.duration = 0.5 + with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: + result = send_tier1_at_stop(tracker, emissions, external_conf={}) + self.assertFalse(result) + mock_client_cls.assert_not_called() + + def test_tier2_uses_api_client(self): + tracker = MagicMock() + tracker._conf = {"os": "Linux"} + emissions = self._emissions() with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api - result = send_tier2_public_emission({}, emissions) - self.assertTrue(result) - mock_api.add_emission.assert_called_once() - result = send_tier2_public_emission({}, emissions) - self.assertFalse(result) - mock_api.add_emission.assert_called_once() + result = send_tier2_at_stop(tracker, emissions, external_conf={}) + self.assertTrue(result) + mock_api_cls.assert_called_once() + mock_api.add_emission.assert_called_once() + mock_api_cls.assert_called_with( + endpoint_url=ANY, + experiment_id=ANY, + api_key=ANY, + conf=tracker._conf, + create_run_automatically=True, + ) @mock_platform_cli_setup class TestTrackerTelemetry(unittest.TestCase): def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() - self.temp_path = Path(self.temp_dir.name) self.patcher = None def tearDown(self) -> None: @@ -178,63 +118,22 @@ def _start_config_mock(self, conf: str) -> None: ) self.patcher.start() - def test_emissions_tracker_sends_tier1_when_minimal_config(self, mock_cli_setup): - telemetry_module._TIER1_SENT = False + def test_emissions_tracker_does_not_send_telemetry_on_init(self, mock_cli_setup): self._start_config_mock(minimal_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - EmissionsTracker(save_to_api=False, save_to_file=False) - mock_telemetry_cls.assert_called_once() - mock_client.add_telemetry.assert_called_once() - - def test_emissions_tracker_skips_tier1_when_disabled_config(self, mock_cli_setup): - telemetry_module._TIER1_SENT = False - self._start_config_mock(disabled_conf) with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) mock_telemetry_cls.assert_not_called() - def test_offline_tracker_sends_tier1_when_minimal_config(self, mock_cli_setup): - telemetry_module._TIER1_SENT = False + def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( + self, mock_cli_setup + ): self._start_config_mock(minimal_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - OfflineEmissionsTracker( - country_iso_code="CAN", - save_to_api=False, - save_to_file=False, - ) - mock_telemetry_cls.assert_called_once() - - def test_offline_tracker_skips_tier1_when_disabled_config(self, mock_cli_setup): - telemetry_module._TIER1_SENT = False - self._start_config_mock(disabled_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - OfflineEmissionsTracker( - country_iso_code="CAN", - save_to_api=False, - save_to_file=False, - ) - mock_telemetry_cls.assert_not_called() - - def test_extensive_sends_tier1_on_init_and_tier2_on_stop(self, mock_cli_setup): - telemetry_module._TIER1_SENT = False - telemetry_module._TIER2_SENT = False - self._start_config_mock(extensive_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_telemetry = MagicMock() - mock_telemetry.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_telemetry - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: - mock_api = MagicMock() - mock_api.add_emission.return_value = True - mock_api_cls.return_value = mock_api + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = {"id": "ok"} + mock_telemetry_cls.return_value = mock_client with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -244,9 +143,64 @@ def test_extensive_sends_tier1_on_init_and_tier2_on_stop(self, mock_cli_setup): tracker.start() tracker.stop() mock_telemetry_cls.assert_called_once() + mock_client.add_telemetry.assert_called_once() + payload = mock_telemetry_cls.call_args.kwargs["telemetry"] + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertIn("total_emissions_kg", payload) + + def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): + self._start_config_mock(disabled_conf) + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_not_called() + + def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): + self._start_config_mock(extensive_conf) + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_telemetry = MagicMock() + mock_telemetry.add_telemetry.return_value = {"id": "ok"} + mock_telemetry_cls.return_value = mock_telemetry + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_telemetry_cls.assert_called_once() mock_telemetry.add_telemetry.assert_called_once() + mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() + def test_offline_tracker_sends_on_stop(self, mock_cli_setup): + self._start_config_mock(minimal_conf) + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = {"id": "ok"} + mock_telemetry_cls.return_value = mock_client + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_client.add_telemetry.assert_called_once() + if __name__ == "__main__": unittest.main() diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 5ce145370..faefc555a 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -93,7 +93,7 @@ def test_init_rejects_invalid_telemetry_without_calling_api(self): telemetry={ "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", - "total_emissions_kg": 0.42, + "torch_version": "2.2.0", }, ) diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py new file mode 100644 index 000000000..774808d94 --- /dev/null +++ b/tests/test_telemetry_collect.py @@ -0,0 +1,113 @@ +import unittest +from unittest.mock import MagicMock, patch + +from codecarbon.core.telemetry_collect import ( + build_tier1_payload, + collect_telemetry_context, + project_tier1, +) +from codecarbon.output_methods.emissions_data import EmissionsData + + +def _sample_emissions(**overrides): + base = dict( + timestamp="2026-01-01T00:00:00", + project_name="p", + run_id="r", + experiment_id="e", + duration=10.0, + emissions=0.5, + emissions_rate=0.05, + cpu_power=1.0, + gpu_power=2.0, + ram_power=0.5, + cpu_energy=0.01, + gpu_energy=0.02, + ram_energy=0.001, + energy_consumed=0.031, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="3.0", + cpu_count=4, + cpu_model="cpu", + gpu_count=1, + gpu_model="gpu", + longitude=0.0, + latitude=0.0, + ram_total_size=16.0, + tracking_mode="machine", + ) + base.update(overrides) + return EmissionsData(**base) + + +class TestTelemetryCollect(unittest.TestCase): + def test_project_tier1_includes_run_and_framework_flags(self): + tracker = MagicMock() + tracker._conf = { + "os": "Linux", + "codecarbon_version": "3.0", + "cpu_count": 4, + "tracking_mode": "machine", + } + tracker._geo = None + tracker._save_to_file = True + tracker._save_to_api = False + tracker._save_to_logger = False + tracker._emissions_endpoint = None + tracker._save_to_prometheus = False + tracker._save_to_logfire = False + tracker._tasks = {} + tracker._measure_power_secs = 15 + tracker._hardware = [] + tracker._resource_tracker = None + + emissions = _sample_emissions() + with patch( + "codecarbon.core.telemetry_collect._package_installed", + side_effect=lambda name: name == "torch", + ): + context = collect_telemetry_context(tracker, emissions) + payload = project_tier1(context) + + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertEqual(payload["total_emissions_kg"], 0.5) + self.assertEqual(payload["duration_seconds"], 10.0) + self.assertTrue(payload["has_torch"]) + self.assertNotIn("torch_version", payload) + self.assertIn("file", payload["output_methods"]) + + def test_build_tier1_payload_excludes_framework_versions(self): + tracker = MagicMock() + tracker._conf = {"codecarbon_version": "3.0", "hardware": ["cpu"]} + tracker._geo = None + tracker._save_to_file = False + tracker._save_to_api = False + tracker._save_to_logger = False + tracker._emissions_endpoint = None + tracker._save_to_prometheus = False + tracker._save_to_logfire = False + tracker._tasks = {} + tracker._measure_power_secs = 15 + tracker._hardware = [] + tracker._resource_tracker = None + + emissions = _sample_emissions() + with patch( + "codecarbon.core.telemetry_collect._package_installed", + return_value=True, + ), patch( + "codecarbon.core.telemetry_collect._package_version", + return_value="2.0.0", + ): + payload = build_tier1_payload(tracker, emissions) + + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertTrue(payload["has_torch"]) + self.assertNotIn("torch_version", payload) diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index 57845f32f..e90b5987f 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -4,6 +4,7 @@ import sys import tempfile import unittest +from datetime import datetime, timezone from pathlib import Path from unittest.mock import MagicMock, patch @@ -13,12 +14,9 @@ from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.core.telemetry_settings import get_telemetry_api_url from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker -from codecarbon.telemetry import ( - send_tier1_telemetry, - send_tier2_public_emission, - warn_if_telemetry_not_configured, -) -from tests.testutils import get_custom_mock_open +from codecarbon.output_methods.emissions_data import EmissionsData +from codecarbon.telemetry import send_tier1_at_stop, warn_if_telemetry_not_configured +from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": mock_platform_cli_setup = patch( @@ -50,28 +48,58 @@ def test_no_warn_when_config_explicit(self): ) mock_warning.assert_not_called() - def test_minimal_tier1_posts_once_per_session(self): - telemetry_module._TIER1_SENT = False - conf = {"python_version": "3.11", "os": "Linux", "codecarbon_version": "2.0"} - with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_client_cls.return_value = mock_client - self.assertTrue(send_tier1_telemetry(conf)) - self.assertFalse(send_tier1_telemetry(conf)) - mock_client_cls.assert_called_once() - payload = mock_client_cls.call_args.kwargs["telemetry"] - self.assertEqual(payload["telemetry_level"], TelemetryLevel.minimal.value) - def test_tier1_posts_to_telemetry_endpoint(self): - telemetry_module._TIER1_SENT = False - with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: - mock_post.return_value.status_code = 201 - mock_post.return_value.json.return_value = "telemetry-id" - send_tier1_telemetry( - {"python_version": "3.11", "os": "Linux", "codecarbon_version": "2.0"}, - external_conf={"telemetry_api_url": "http://tier1.example"}, - ) + tier1_payload = { + "timestamp": datetime(2020, 1, 1, tzinfo=timezone.utc), + "telemetry_level": "minimal", + "total_emissions_kg": 0.001, + "os": "Linux", + } + tracker = MagicMock() + emissions = EmissionsData( + timestamp="2020-01-01T00:00:00", + project_name="test", + run_id="run-1", + experiment_id="exp-1", + duration=10.0, + emissions=0.001, + emissions_rate=0.0001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="2.0", + cpu_count=1.0, + cpu_model="cpu", + gpu_count=0.0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=8.0, + tracking_mode="machine", + ) + with patch( + "codecarbon.telemetry.build_tier1_payload", return_value=tier1_payload + ): + with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + mock_post.return_value.status_code = 201 + mock_post.return_value.json.return_value = "telemetry-id" + send_tier1_at_stop( + tracker, + emissions, + external_conf={"telemetry_api_url": "http://tier1.example"}, + ) mock_post.assert_called_once() self.assertEqual( mock_post.call_args.kwargs["url"], "http://tier1.example/telemetry" @@ -80,6 +108,7 @@ def test_tier1_posts_to_telemetry_endpoint(self): mock_post.call_args.kwargs["json"]["telemetry_level"], TelemetryLevel.minimal.value, ) + self.assertIn("total_emissions_kg", mock_post.call_args.kwargs["json"]) def test_legacy_env_codecarbon_telemetry_does_not_change_tier(self): with tempfile.TemporaryDirectory() as tmp: @@ -142,8 +171,6 @@ def test_telemetry_api_url_from_config_overrides_default(self): @mock_platform_cli_setup class TestTrackerTelemetryFromConfig(unittest.TestCase): def setUp(self) -> None: - telemetry_module._TIER1_SENT = False - telemetry_module._TIER2_SENT = False telemetry_module._TELEMETRY_CONFIGURE_WARNED = False self._config_patcher = None @@ -157,28 +184,26 @@ def _mock_config(self, conf: str) -> None: ) self._config_patcher.start() - def test_disabled_no_tier1_post_or_tier2_emission(self, mock_cli_setup): + def test_disabled_no_telemetry_on_stop(self, mock_cli_setup): self._mock_config(_conf("disabled")) with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: - with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - tracker = EmissionsTracker( - measure_power_secs=1, - save_to_api=False, - save_to_file=False, - ) - tracker.start() - tracker.stop() + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() mock_telemetry_cls.assert_not_called() - mock_api_cls.assert_not_called() - def test_minimal_posts_tier1_on_init_not_tier2_on_stop(self, mock_cli_setup): + def test_minimal_posts_tier1_on_stop_not_on_init(self, mock_cli_setup): self._mock_config(_conf("minimal")) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -189,44 +214,50 @@ def test_minimal_posts_tier1_on_init_not_tier2_on_stop(self, mock_cli_setup): tracker.stop() mock_telemetry_cls.assert_called_once() mock_client.add_telemetry.assert_called_once() - mock_api_cls.assert_not_called() + self.assertEqual( + mock_telemetry_cls.call_args.kwargs["telemetry"]["telemetry_level"], + TelemetryLevel.minimal.value, + ) - def test_extensive_posts_tier1_and_tier2_on_stop(self, mock_cli_setup): + def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup): self._mock_config(_conf("extensive")) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: - mock_api = MagicMock() - mock_api.add_emission.return_value = True - mock_api_cls.return_value = mock_api - with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - tracker = EmissionsTracker( - measure_power_secs=1, - save_to_api=False, - save_to_file=False, - ) - tracker.start() - tracker.stop() + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_telemetry = MagicMock() + mock_telemetry.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_telemetry + with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() mock_telemetry_cls.assert_called_once() - mock_client.add_telemetry.assert_called_once() + mock_telemetry.add_telemetry.assert_called_once() + mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() - def test_offline_minimal_with_save_to_api_false_still_posts_tier1( - self, mock_cli_setup - ): + def test_offline_minimal_posts_tier1_on_stop(self, mock_cli_setup): self._mock_config(_conf("minimal")) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - OfflineEmissionsTracker( - country_iso_code="CAN", - save_to_api=False, - save_to_file=False, - ) - mock_telemetry_cls.assert_called_once() + with ensure_telemetry_run_duration(): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + mock_client = MagicMock() + mock_client.add_telemetry.return_value = "telemetry-id" + mock_telemetry_cls.return_value = mock_client + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_client.add_telemetry.assert_called_once() def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup): self._mock_config("[codecarbon]\n") @@ -271,69 +302,23 @@ def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): ] self.assertEqual(len(configure_warnings), 0) - def test_env_telemetry_disabled_still_minimal_tier1_when_config_minimal( + def test_env_telemetry_disabled_does_not_change_resolved_level( self, mock_cli_setup ): self._mock_config(_conf("minimal")) - with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client - with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): - EmissionsTracker(save_to_api=False, save_to_file=False) + with ensure_telemetry_run_duration(): + with patch.dict( + os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False + ): + with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + save_to_api=False, save_to_file=False + ) + tracker.start() + tracker.stop() mock_telemetry_cls.assert_called_once() -class TestTier2ApiSettings(unittest.TestCase): - def test_tier2_uses_resolved_api_url(self): - telemetry_module._TIER2_SENT = False - from codecarbon.output_methods.emissions_data import EmissionsData - - emissions = EmissionsData( - timestamp="2020-01-01T00:00:00", - project_name="test", - run_id="run-1", - experiment_id="exp-1", - duration=1.0, - emissions=0.001, - emissions_rate=0.001, - cpu_power=0.0, - gpu_power=0.0, - ram_power=0.0, - cpu_energy=0.0, - gpu_energy=0.0, - ram_energy=0.0, - energy_consumed=0.01, - water_consumed=0.0, - country_name="", - country_iso_code="", - region="", - cloud_provider="", - cloud_region="", - os="Linux", - python_version="3.11", - codecarbon_version="2.0", - cpu_count=1.0, - cpu_model="", - gpu_count=0.0, - gpu_model="", - longitude=0.0, - latitude=0.0, - ram_total_size=8.0, - tracking_mode="machine", - ) - external_conf = {"telemetry_api_url": "http://custom.example"} - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: - mock_api = MagicMock() - mock_api.add_emission.return_value = True - mock_api_cls.return_value = mock_api - send_tier2_public_emission({}, emissions, external_conf=external_conf) - mock_api_cls.assert_called_once() - self.assertEqual( - mock_api_cls.call_args.kwargs["endpoint_url"], "http://custom.example" - ) - - if __name__ == "__main__": unittest.main() diff --git a/tests/testutils.py b/tests/testutils.py index e3d1dc2d1..57d94fa46 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -1,6 +1,8 @@ import builtins import unittest +from contextlib import contextmanager from pathlib import Path +from unittest.mock import patch from codecarbon.input import DataSource @@ -33,3 +35,22 @@ def conditional_open_func(path, *args, **kwargs): return conditional_open_func return mocked_open + + +@contextmanager +def ensure_telemetry_run_duration(min_seconds: float = 10.0): + """Force tracker stop emissions duration above telemetry's 1s minimum.""" + from codecarbon.emissions_tracker import BaseEmissionsTracker + + original_prepare = BaseEmissionsTracker._prepare_emissions_data + + def prepare_with_min_duration(self): + data = original_prepare(self) + if data is not None and (data.duration is None or data.duration < min_seconds): + data.duration = min_seconds + return data + + with patch.object( + BaseEmissionsTracker, "_prepare_emissions_data", prepare_with_min_duration + ): + yield From 1ecc23775cb55d4e0dda92748b861f77b822724e Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 23:18:37 +0200 Subject: [PATCH 12/18] refactor: update telemetry schema and enhance data collection - Replaced MINIMAL_TELEMETRY_FIELDS with a dynamic frozenset that excludes privacy-related fields. - Introduced EXCLUDED_PRIVACY_TELEMETRY_FIELDS to ensure sensitive data is not collected. - Updated telemetry payload building methods to accept a telemetry level parameter, allowing for more flexible data handling. - Enhanced tests to validate the acceptance of framework versions and rejection of privacy fields in telemetry submissions. - Improved documentation to clarify the changes in telemetry data collection and privacy considerations. --- .../carbonserver/api/schemas_telemetry.py | 68 +++----------- .../tests/api/routers/test_telemetry.py | 21 ++++- codecarbon/core/telemetry_collect.py | 93 +++++++++++++++++-- codecarbon/core/telemetry_schemas.py | 68 +++----------- codecarbon/telemetry.py | 6 +- docs/how-to/telemetry.md | 25 ++--- tests/test_telemetry_client.py | 2 +- tests/test_telemetry_collect.py | 26 +++++- 8 files changed, 171 insertions(+), 138 deletions(-) diff --git a/carbonserver/carbonserver/api/schemas_telemetry.py b/carbonserver/carbonserver/api/schemas_telemetry.py index dec7e0dfa..9c8c860ee 100644 --- a/carbonserver/carbonserver/api/schemas_telemetry.py +++ b/carbonserver/carbonserver/api/schemas_telemetry.py @@ -144,60 +144,20 @@ def validate_telemetry_level(self): return self -MINIMAL_TELEMETRY_FIELDS = { - "timestamp", - "telemetry_level", - "os", - "country_name", - "country_iso_code", - "region", - "cloud_provider", - "cloud_region", - "on_cloud", - "cpu_count", - "cpu_physical_count", - "cpu_model", - "cpu_architecture", - "gpu_count", - "gpu_model", - "gpu_memory_total_gb", - "ram_total_size_gb", - "cuda_version", - "python_version", - "python_implementation", - "python_env_type", - "python_package_manager", - "codecarbon_version", - "codecarbon_install_method", - "tracking_mode", - "integration_surface", - "offline_mode", - "output_methods", - "save_to_api_enabled", - "task_tracking_used", - "measure_power_interval_secs", - "in_container", - "ci_environment", - "notebook_environment", - "has_torch", - "has_transformers", - "has_tensorflow", - "has_keras", - "has_diffusers", - "has_pytorch_lightning", - "has_fastai", - "ml_framework_primary", - "total_emissions_kg", - "emissions_rate_kg_per_sec", - "energy_consumed_kwh", - "cpu_energy_kwh", - "gpu_energy_kwh", - "ram_energy_kwh", - "duration_seconds", - "cpu_utilization_avg", - "gpu_utilization_avg", - "ram_utilization_avg", -} +EXCLUDED_PRIVACY_TELEMETRY_FIELDS = frozenset( + { + "longitude", + "latitude", + "python_executable_hash", + "host_machine_hash", + } +) + +MINIMAL_TELEMETRY_FIELDS = frozenset( + field_name + for field_name in TelemetryBase.model_fields + if field_name not in EXCLUDED_PRIVACY_TELEMETRY_FIELDS +) class TelemetryCreate(TelemetryBase): diff --git a/carbonserver/tests/api/routers/test_telemetry.py b/carbonserver/tests/api/routers/test_telemetry.py index 733269bb0..f8ce065ea 100644 --- a/carbonserver/tests/api/routers/test_telemetry.py +++ b/carbonserver/tests/api/routers/test_telemetry.py @@ -52,15 +52,30 @@ def test_add_telemetry(client, custom_test_server): assert response.json() == TELEMETRY_ID -def test_minimal_telemetry_rejects_extensive_fields(client, custom_test_server): +def test_minimal_telemetry_accepts_framework_versions(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) - telemetry_with_extensive_field = { + repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) + telemetry_with_framework_version = { **MINIMAL_TELEMETRY_TO_CREATE, "torch_version": "2.2.0", } with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=telemetry_with_extensive_field) + response = client.post("/telemetry", json=telemetry_with_framework_version) + + assert response.status_code == status.HTTP_201_CREATED + repository_mock.add_telemetry.assert_called_once() + + +def test_minimal_telemetry_rejects_privacy_fields(client, custom_test_server): + repository_mock = mock.Mock(spec=TelemetryRepository) + telemetry_with_privacy_field = { + **MINIMAL_TELEMETRY_TO_CREATE, + "longitude": 2.35, + } + + with custom_test_server.container.telemetry_repository.override(repository_mock): + response = client.post("/telemetry", json=telemetry_with_privacy_field) assert response.status_code == 422 repository_mock.add_telemetry.assert_not_called() diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry_collect.py index bdd3ed676..cd40da6f6 100644 --- a/codecarbon/core/telemetry_collect.py +++ b/codecarbon/core/telemetry_collect.py @@ -129,6 +129,67 @@ def _detect_in_container() -> bool: return False +def _detect_container_runtime() -> Optional[str]: + if os.environ.get("KUBERNETES_SERVICE_HOST"): + return "kubernetes" + if os.path.exists("/.dockerenv"): + return "docker" + return None + + +def _detect_ide() -> Optional[str]: + if os.environ.get("CURSOR_TRACE_ID") or os.environ.get("CURSOR_SESSION"): + return "cursor" + if os.environ.get("VSCODE_PID") or os.environ.get("TERM_PROGRAM") == "vscode": + return "vscode" + if os.environ.get("PYCHARM_HOSTED"): + return "pycharm" + return None + + +def _cudnn_version() -> Optional[str]: + if not _package_installed("torch"): + return None + try: + import torch + + version = torch.backends.cudnn.version() + return str(version) if version is not None else None + except Exception: + return None + + +def _collect_hardware_diagnostics(tracker: Any) -> dict[str, Any]: + from codecarbon.core import cpu + + hardware_tracked: list[str] = [] + for item in getattr(tracker, "_hardware", []) or []: + try: + hardware_tracked.append(item.description()) + except Exception: + pass + + resource_tracker = getattr(tracker, "_resource_tracker", None) + gpu_detection_method: Optional[str] = None + if resource_tracker is not None: + gpu_tracker = getattr(resource_tracker, "gpu_tracker", None) + if gpu_tracker and gpu_tracker != "Unspecified": + gpu_detection_method = gpu_tracker + + rapl_available: Optional[bool] = None + if platform.system() == "Linux": + rapl_available = cpu.is_rapl_available() + + save_to_api = bool(getattr(tracker, "_save_to_api", False)) + return { + "hardware_tracked": hardware_tracked or None, + "hardware_detection_success": bool(hardware_tracked), + "rapl_available": rapl_available, + "gpu_detection_method": gpu_detection_method, + "api_mode": "online" if save_to_api else "offline", + } + + def _detect_integration_surface(tracker: Any) -> str: from codecarbon.emissions_tracker import OfflineEmissionsTracker @@ -266,8 +327,11 @@ def collect_telemetry_context( "task_tracking_used": bool(getattr(tracker, "_tasks", {})), "measure_power_interval_secs": getattr(tracker, "_measure_power_secs", None), "in_container": _detect_in_container(), + "container_runtime": _detect_container_runtime(), "ci_environment": _detect_ci_environment(), "notebook_environment": _detect_notebook_environment(), + "ide_used": _detect_ide(), + "cudnn_version": _cudnn_version(), "duration_seconds": float(emissions.duration) if emissions.duration else None, "total_emissions_kg": emissions.emissions, "emissions_rate_kg_per_sec": emissions.emissions_rate, @@ -278,36 +342,45 @@ def collect_telemetry_context( "cpu_utilization_avg": emissions.cpu_utilization_percent, "gpu_utilization_avg": emissions.gpu_utilization_percent, "ram_utilization_avg": emissions.ram_utilization_percent, - **_collect_framework_fields(include_versions=False), + **_collect_framework_fields(include_versions=True), + **_collect_hardware_diagnostics(tracker), } - for key in ("gpu_memory_total_gb", "cuda_version"): - if key in gpu_fields: - context[key] = gpu_fields[key] + context.update(gpu_fields) + if context.get("ml_framework_primary"): + context["framework_detected"] = context["ml_framework_primary"] return _strip_none(context) -def project_tier1(context: dict[str, Any]) -> dict[str, Any]: - """Project context to Tier 1 (``telemetry_level=minimal``) fields.""" +def project_tier1( + context: dict[str, Any], + level: TelemetryLevel = TelemetryLevel.minimal, +) -> dict[str, Any]: + """Project context to private ``POST /telemetry`` fields for the resolved tier.""" payload = { key: context[key] for key in MINIMAL_TELEMETRY_FIELDS if key in context } - payload["telemetry_level"] = TelemetryLevel.minimal.value + payload["telemetry_level"] = level.value return _strip_none(payload) -def build_tier1_payload(tracker: Any, emissions: EmissionsData) -> dict[str, Any]: - """Build a Tier 1 payload dict for ``TelemetryCreate``. +def build_tier1_payload( + tracker: Any, + emissions: EmissionsData, + level: TelemetryLevel = TelemetryLevel.minimal, +) -> dict[str, Any]: + """Build a private telemetry payload dict for ``TelemetryCreate``. Args: tracker: Active emissions tracker. emissions: Run emissions data. + level: Resolved ``TelemetryLevel`` (``minimal`` or ``extensive``). Returns: Payload dict for ``POST /telemetry``. """ context = collect_telemetry_context(tracker, emissions) - return project_tier1(context) + return project_tier1(context, level=level) diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry_schemas.py index 9dc3e2997..6fcb28446 100644 --- a/codecarbon/core/telemetry_schemas.py +++ b/codecarbon/core/telemetry_schemas.py @@ -125,60 +125,20 @@ def validate_telemetry_level(self): return self -MINIMAL_TELEMETRY_FIELDS = { - "timestamp", - "telemetry_level", - "os", - "country_name", - "country_iso_code", - "region", - "cloud_provider", - "cloud_region", - "on_cloud", - "cpu_count", - "cpu_physical_count", - "cpu_model", - "cpu_architecture", - "gpu_count", - "gpu_model", - "gpu_memory_total_gb", - "ram_total_size_gb", - "cuda_version", - "python_version", - "python_implementation", - "python_env_type", - "python_package_manager", - "codecarbon_version", - "codecarbon_install_method", - "tracking_mode", - "integration_surface", - "offline_mode", - "output_methods", - "save_to_api_enabled", - "task_tracking_used", - "measure_power_interval_secs", - "in_container", - "ci_environment", - "notebook_environment", - "has_torch", - "has_transformers", - "has_tensorflow", - "has_keras", - "has_diffusers", - "has_pytorch_lightning", - "has_fastai", - "ml_framework_primary", - "total_emissions_kg", - "emissions_rate_kg_per_sec", - "energy_consumed_kwh", - "cpu_energy_kwh", - "gpu_energy_kwh", - "ram_energy_kwh", - "duration_seconds", - "cpu_utilization_avg", - "gpu_utilization_avg", - "ram_utilization_avg", -} +EXCLUDED_PRIVACY_TELEMETRY_FIELDS = frozenset( + { + "longitude", + "latitude", + "python_executable_hash", + "host_machine_hash", + } +) + +MINIMAL_TELEMETRY_FIELDS = frozenset( + field_name + for field_name in TelemetryBase.model_fields + if field_name not in EXCLUDED_PRIVACY_TELEMETRY_FIELDS +) class TelemetryCreate(TelemetryBase): diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index 397be674b..20cdbeb65 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -64,6 +64,7 @@ def send_tier1_at_stop( tracker: Any, emissions: EmissionsData, external_conf: dict[str, Any] | None = None, + level: TelemetryLevel = TelemetryLevel.minimal, ) -> bool: """Send Tier 1 telemetry: private hardware/usage/run summary via ``POST /telemetry``. @@ -71,6 +72,7 @@ def send_tier1_at_stop( tracker: Active emissions tracker instance. emissions: Total emissions from ``_prepare_emissions_data()``. external_conf: Merged config for telemetry API URL and key resolution. + level: Resolved ``TelemetryLevel`` for the ``telemetry_level`` field. Returns: True if Tier 1 was accepted, False otherwise. @@ -82,7 +84,7 @@ def send_tier1_at_stop( return False settings_conf = external_conf or {} try: - payload = build_tier1_payload(tracker, emissions) + payload = build_tier1_payload(tracker, emissions, level=level) client = TelemetryClient( endpoint_url=get_telemetry_api_url(settings_conf), telemetry=payload, @@ -151,6 +153,6 @@ def send_product_telemetry_at_stop( return settings = external_conf or {} if level in (TelemetryLevel.minimal, TelemetryLevel.extensive): - send_tier1_at_stop(tracker, emissions, settings) + send_tier1_at_stop(tracker, emissions, settings, level=level) if level == TelemetryLevel.extensive: send_tier2_at_stop(tracker, emissions, settings) diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index 181605047..82ea56f7d 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -17,8 +17,8 @@ You can use one without the other. | `telemetry_level` | Name | When | Transport | |-------------------|------|------|-----------| | `disabled` | — | — | Nothing | -| `minimal` | **Tier 1** | Each `stop()` | `POST /telemetry` (private) | -| `extensive` | **Tier 2** | Each `stop()` | Tier 1 (`POST /telemetry`) **and** Tier 2 (`ApiClient` → `/emissions`) | +| `minimal` | Private product telemetry | Each `stop()` | `POST /telemetry` (private) | +| `extensive` | Private telemetry + shared run summary | Each `stop()` | Same private `POST /telemetry` **and** `ApiClient` → `/emissions` | Tier is resolved in this order: @@ -30,30 +30,33 @@ Tier is resolved in this order: ```text EmissionsTracker.__init__ → collect hardware/geo (no POST) -EmissionsTracker.stop() → minimal: Tier 1 only | extensive: Tier 1 + Tier 2 +EmissionsTracker.stop() → minimal: private POST only | extensive: private POST + /emissions ``` If the run lasts less than one second, telemetry is not sent. -## Tier 1 (`minimal`) — per run +## Private telemetry (`minimal` and `extensive`) — per run -One private row per tracker run with: +Both levels send the **same private payload** to `POST /telemetry` at each `stop()`. The `telemetry_level` field records which setting was used (`minimal` or `extensive`). -- **Environment:** OS, Python, CPU/GPU/RAM, country/region, cloud provider/region -- **Usage:** tracking mode, output methods, integration surface (library / CLI / offline), task tracking, CI/notebook/container hints -- **ML stack (presence):** `has_torch`, `has_transformers`, `has_tensorflow`, and related flags +The payload includes: + +- **Environment:** OS, Python, CPU/GPU/RAM, country/region, cloud provider/region, GPU driver/CUDA/cuDNN when available +- **Usage:** tracking mode, output methods, integration surface (library / CLI / offline), task tracking, CI/notebook/container/IDE hints, hardware diagnostics +- **ML stack:** framework presence flags **and** installed versions when detected - **Run outcome:** duration, emissions, energy (total and per component), utilization averages -Tier 1 does **not** include project names, experiment ids, API keys, file paths, or survey demographics (role, industry, etc.). +Private telemetry does **not** include project names, experiment ids, API keys, file paths, exact coordinates, executable/host hashes, or survey demographics (role, industry, etc.). -## Tier 2 (`extensive`) — per run +## `extensive` — additional public run summary -**Always sends Tier 1 first**, then adds a **run emissions summary** to the shared CodeCarbon telemetry experiment via `ApiClient` (`/runs` then `/emissions`). Endpoint, API key, and experiment id come from `telemetry_api_url` / `telemetry_api_key` / `telemetry_experiment_id` (or `CODECARBON_TELEMETRY_*` env vars), falling back to the built-in defaults and your `api_endpoint` / `api_key` when set. +**Also** posts a **run emissions summary** to the shared CodeCarbon telemetry experiment via `ApiClient` (`/runs` then `/emissions`). Endpoint, API key, and experiment id come from `telemetry_api_url` / `telemetry_api_key` / `telemetry_experiment_id` (or `CODECARBON_TELEMETRY_*` env vars), falling back to the built-in defaults and your `api_endpoint` / `api_key` when set. ## Never collected - Project name, experiment id, run id, API keys - Source code, file paths, hostnames +- Exact GPS coordinates (`longitude` / `latitude`), `python_executable_hash`, `host_machine_hash` - Voluntary [user survey](https://docs.google.com/forms/d/e/1FAIpQLSeQ5Tu_rdrpDhBJvh5R1-_iB4Ld-kgh6iNMjgaMXa8AEVPxqA/viewform) demographics (role, industry, experience) ## Configure telemetry diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index faefc555a..4d2b24b32 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -93,7 +93,7 @@ def test_init_rejects_invalid_telemetry_without_calling_api(self): telemetry={ "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", - "torch_version": "2.2.0", + "longitude": 2.35, }, ) diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 774808d94..1b711ad66 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -6,6 +6,7 @@ collect_telemetry_context, project_tier1, ) +from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.output_methods.emissions_data import EmissionsData @@ -80,10 +81,9 @@ def test_project_tier1_includes_run_and_framework_flags(self): self.assertEqual(payload["total_emissions_kg"], 0.5) self.assertEqual(payload["duration_seconds"], 10.0) self.assertTrue(payload["has_torch"]) - self.assertNotIn("torch_version", payload) self.assertIn("file", payload["output_methods"]) - def test_build_tier1_payload_excludes_framework_versions(self): + def test_build_tier1_payload_includes_framework_versions(self): tracker = MagicMock() tracker._conf = {"codecarbon_version": "3.0", "hardware": ["cpu"]} tracker._geo = None @@ -110,4 +110,24 @@ def test_build_tier1_payload_excludes_framework_versions(self): self.assertEqual(payload["telemetry_level"], "minimal") self.assertTrue(payload["has_torch"]) - self.assertNotIn("torch_version", payload) + self.assertEqual(payload["torch_version"], "2.0.0") + + def test_build_tier1_payload_uses_resolved_level(self): + tracker = MagicMock() + tracker._conf = {"codecarbon_version": "3.0"} + tracker._save_to_file = False + tracker._save_to_api = False + tracker._save_to_logger = False + tracker._emissions_endpoint = None + tracker._save_to_prometheus = False + tracker._save_to_logfire = False + tracker._tasks = {} + tracker._measure_power_secs = 15 + tracker._hardware = [] + tracker._resource_tracker = None + + emissions = _sample_emissions() + payload = build_tier1_payload( + tracker, emissions, level=TelemetryLevel.extensive + ) + self.assertEqual(payload["telemetry_level"], "extensive") From 6c873eb29551749842acf80d0dc5afa0c2f02885 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Tue, 19 May 2026 23:24:02 +0200 Subject: [PATCH 13/18] refactor: streamline telemetry schema and enhance privacy measures - Removed sensitive fields from the TelemetryBase schema, including `longitude`, `latitude`, and `python_executable_hash`. - Consolidated privacy-related field management into a single frozenset, `PRIVATE_TELEMETRY_FIELDS`. - Updated telemetry payload building methods to reflect the new schema and ensure compliance with privacy standards. - Refactored tests to validate the rejection of unknown fields and ensure proper handling of telemetry submissions. - Improved documentation to clarify the changes in telemetry data collection and privacy considerations. --- .../carbonserver/api/schemas_telemetry.py | 41 +------------------ .../tests/api/routers/test_telemetry.py | 18 ++++---- codecarbon/core/telemetry_collect.py | 15 +++---- codecarbon/core/telemetry_schemas.py | 40 +----------------- codecarbon/telemetry.py | 32 +++++++-------- docs/how-to/telemetry.md | 2 +- tests/test_telemetry.py | 17 ++++++-- tests/test_telemetry_client.py | 2 +- tests/test_telemetry_collect.py | 16 ++++---- tests/test_telemetry_config.py | 9 ++-- 10 files changed, 62 insertions(+), 130 deletions(-) diff --git a/carbonserver/carbonserver/api/schemas_telemetry.py b/carbonserver/carbonserver/api/schemas_telemetry.py index 9c8c860ee..ced8c13fd 100644 --- a/carbonserver/carbonserver/api/schemas_telemetry.py +++ b/carbonserver/carbonserver/api/schemas_telemetry.py @@ -42,8 +42,6 @@ class TelemetryBase(BaseModel): cloud_provider: Optional[str] = None cloud_region: Optional[str] = None on_cloud: Optional[bool] = None - longitude: Optional[float] = Field(default=None, ge=-180, le=180) - latitude: Optional[float] = Field(default=None, ge=-90, le=90) cpu_count: Optional[int] = Field(default=None, ge=0) cpu_physical_count: Optional[int] = Field(default=None, ge=0) @@ -59,9 +57,6 @@ class TelemetryBase(BaseModel): python_version: Optional[str] = None python_implementation: Optional[str] = None - python_executable_hash: Optional[str] = Field( - default=None, min_length=64, max_length=64 - ) python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None @@ -83,7 +78,6 @@ class TelemetryBase(BaseModel): output_methods: Optional[List[str]] = None hardware_tracked: Optional[List[str]] = None task_tracking_used: Optional[bool] = None - decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) integration_surface: Optional[str] = None offline_mode: Optional[bool] = None @@ -92,15 +86,10 @@ class TelemetryBase(BaseModel): hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None gpu_detection_method: Optional[str] = None - first_measurement_time_ms: Optional[float] = Field(default=None, ge=0) - tracking_overhead_percent: Optional[float] = Field(default=None, ge=0) - errors_encountered: Optional[List[str]] = None - warning_count: Optional[int] = Field(default=None, ge=0) ide_used: Optional[str] = None notebook_environment: Optional[str] = None ci_environment: Optional[str] = None - python_package_manager: Optional[str] = None framework_detected: Optional[str] = None has_torch: Optional[bool] = None @@ -121,43 +110,15 @@ class TelemetryBase(BaseModel): container_runtime: Optional[str] = None in_container: Optional[bool] = None - host_machine_hash: Optional[str] = None @model_validator(mode="after") def validate_telemetry_level(self): if self.telemetry_level == TelemetryLevel.disabled: raise ValueError("Disabled telemetry must not be submitted") - - if self.telemetry_level == TelemetryLevel.minimal: - extensive_fields = set(type(self).model_fields) - MINIMAL_TELEMETRY_FIELDS - submitted_extensive_fields = [ - field - for field in extensive_fields - if getattr(self, field) not in (None, [], {}) - ] - if submitted_extensive_fields: - fields = ", ".join(sorted(submitted_extensive_fields)) - raise ValueError( - f"Minimal telemetry cannot include extensive fields: {fields}" - ) - return self -EXCLUDED_PRIVACY_TELEMETRY_FIELDS = frozenset( - { - "longitude", - "latitude", - "python_executable_hash", - "host_machine_hash", - } -) - -MINIMAL_TELEMETRY_FIELDS = frozenset( - field_name - for field_name in TelemetryBase.model_fields - if field_name not in EXCLUDED_PRIVACY_TELEMETRY_FIELDS -) +PRIVATE_TELEMETRY_FIELDS = frozenset(TelemetryBase.model_fields) class TelemetryCreate(TelemetryBase): diff --git a/carbonserver/tests/api/routers/test_telemetry.py b/carbonserver/tests/api/routers/test_telemetry.py index f8ce065ea..a51a19ecd 100644 --- a/carbonserver/tests/api/routers/test_telemetry.py +++ b/carbonserver/tests/api/routers/test_telemetry.py @@ -14,7 +14,7 @@ TELEMETRY_ID = "f52fe339-164d-4c2b-a8c0-f562dfce066d" -MINIMAL_TELEMETRY_TO_CREATE = { +SAMPLE_PRIVATE_TELEMETRY = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", "os": "Linux-5.10.0-x86_64", @@ -46,7 +46,7 @@ def test_add_telemetry(client, custom_test_server): repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=MINIMAL_TELEMETRY_TO_CREATE) + response = client.post("/telemetry", json=SAMPLE_PRIVATE_TELEMETRY) assert response.status_code == status.HTTP_201_CREATED assert response.json() == TELEMETRY_ID @@ -56,7 +56,7 @@ def test_minimal_telemetry_accepts_framework_versions(client, custom_test_server repository_mock = mock.Mock(spec=TelemetryRepository) repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) telemetry_with_framework_version = { - **MINIMAL_TELEMETRY_TO_CREATE, + **SAMPLE_PRIVATE_TELEMETRY, "torch_version": "2.2.0", } @@ -67,15 +67,15 @@ def test_minimal_telemetry_accepts_framework_versions(client, custom_test_server repository_mock.add_telemetry.assert_called_once() -def test_minimal_telemetry_rejects_privacy_fields(client, custom_test_server): +def test_telemetry_rejects_unknown_fields(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) - telemetry_with_privacy_field = { - **MINIMAL_TELEMETRY_TO_CREATE, - "longitude": 2.35, + telemetry_with_unknown_field = { + **SAMPLE_PRIVATE_TELEMETRY, + "unknown_field": "value", } with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=telemetry_with_privacy_field) + response = client.post("/telemetry", json=telemetry_with_unknown_field) assert response.status_code == 422 repository_mock.add_telemetry.assert_not_called() @@ -84,7 +84,7 @@ def test_minimal_telemetry_rejects_privacy_fields(client, custom_test_server): def test_disabled_telemetry_is_rejected(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) disabled_telemetry = { - **MINIMAL_TELEMETRY_TO_CREATE, + **SAMPLE_PRIVATE_TELEMETRY, "telemetry_level": "disabled", } diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry_collect.py index cd40da6f6..4122e91dc 100644 --- a/codecarbon/core/telemetry_collect.py +++ b/codecarbon/core/telemetry_collect.py @@ -11,10 +11,7 @@ from codecarbon.core.cloud import get_env_cloud_details from codecarbon.core.gpu import is_nvidia_system -from codecarbon.core.telemetry_schemas import ( - MINIMAL_TELEMETRY_FIELDS, - TelemetryLevel, -) +from codecarbon.core.telemetry_schemas import PRIVATE_TELEMETRY_FIELDS, TelemetryLevel from codecarbon.output_methods.emissions_data import EmissionsData FRAMEWORK_PACKAGES = ( @@ -283,7 +280,7 @@ def collect_telemetry_context( emissions: Total emissions row from ``_prepare_emissions_data()``. Returns: - Flat dictionary for ``project_tier1``. + Flat dictionary for ``project_private_telemetry``. """ conf = getattr(tracker, "_conf", {}) raw_provider, raw_region = _raw_cloud_provider_and_region() @@ -353,21 +350,21 @@ def collect_telemetry_context( return _strip_none(context) -def project_tier1( +def project_private_telemetry( context: dict[str, Any], level: TelemetryLevel = TelemetryLevel.minimal, ) -> dict[str, Any]: """Project context to private ``POST /telemetry`` fields for the resolved tier.""" payload = { key: context[key] - for key in MINIMAL_TELEMETRY_FIELDS + for key in PRIVATE_TELEMETRY_FIELDS if key in context } payload["telemetry_level"] = level.value return _strip_none(payload) -def build_tier1_payload( +def build_telemetry_payload( tracker: Any, emissions: EmissionsData, level: TelemetryLevel = TelemetryLevel.minimal, @@ -383,4 +380,4 @@ def build_tier1_payload( Payload dict for ``POST /telemetry``. """ context = collect_telemetry_context(tracker, emissions) - return project_tier1(context, level=level) + return project_private_telemetry(context, level=level) diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry_schemas.py index 6fcb28446..bc6ac2dc2 100644 --- a/codecarbon/core/telemetry_schemas.py +++ b/codecarbon/core/telemetry_schemas.py @@ -24,8 +24,6 @@ class TelemetryBase(BaseModel): cloud_provider: Optional[str] = None cloud_region: Optional[str] = None on_cloud: Optional[bool] = None - longitude: Optional[float] = Field(default=None, ge=-180, le=180) - latitude: Optional[float] = Field(default=None, ge=-90, le=90) cpu_count: Optional[int] = Field(default=None, ge=0) cpu_physical_count: Optional[int] = Field(default=None, ge=0) @@ -41,9 +39,6 @@ class TelemetryBase(BaseModel): python_version: Optional[str] = None python_implementation: Optional[str] = None - python_executable_hash: Optional[str] = Field( - default=None, min_length=64, max_length=64 - ) python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None @@ -65,7 +60,6 @@ class TelemetryBase(BaseModel): output_methods: Optional[List[str]] = None hardware_tracked: Optional[List[str]] = None task_tracking_used: Optional[bool] = None - decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) integration_surface: Optional[str] = None offline_mode: Optional[bool] = None @@ -74,10 +68,6 @@ class TelemetryBase(BaseModel): hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None gpu_detection_method: Optional[str] = None - first_measurement_time_ms: Optional[float] = Field(default=None, ge=0) - tracking_overhead_percent: Optional[float] = Field(default=None, ge=0) - errors_encountered: Optional[List[str]] = None - warning_count: Optional[int] = Field(default=None, ge=0) ide_used: Optional[str] = None notebook_environment: Optional[str] = None @@ -102,43 +92,15 @@ class TelemetryBase(BaseModel): container_runtime: Optional[str] = None in_container: Optional[bool] = None - host_machine_hash: Optional[str] = None @model_validator(mode="after") def validate_telemetry_level(self): if self.telemetry_level == TelemetryLevel.disabled: raise ValueError("Disabled telemetry must not be submitted") - - if self.telemetry_level == TelemetryLevel.minimal: - extensive_fields = set(type(self).model_fields) - MINIMAL_TELEMETRY_FIELDS - submitted_extensive_fields = [ - field - for field in extensive_fields - if getattr(self, field) not in (None, [], {}) - ] - if submitted_extensive_fields: - fields = ", ".join(sorted(submitted_extensive_fields)) - raise ValueError( - f"Minimal telemetry cannot include extensive fields: {fields}" - ) - return self -EXCLUDED_PRIVACY_TELEMETRY_FIELDS = frozenset( - { - "longitude", - "latitude", - "python_executable_hash", - "host_machine_hash", - } -) - -MINIMAL_TELEMETRY_FIELDS = frozenset( - field_name - for field_name in TelemetryBase.model_fields - if field_name not in EXCLUDED_PRIVACY_TELEMETRY_FIELDS -) +PRIVATE_TELEMETRY_FIELDS = frozenset(TelemetryBase.model_fields) class TelemetryCreate(TelemetryBase): diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index 20cdbeb65..ed73537b4 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -1,11 +1,11 @@ -"""Product telemetry sent at tracker stop (Tier 1 private, Tier 2 public emissions).""" +"""Product telemetry sent at tracker stop (Tier 1 / Tier 2).""" import dataclasses from typing import Any from codecarbon.core.api_client import ApiClient from codecarbon.core.telemetry_client import TelemetryClient -from codecarbon.core.telemetry_collect import build_tier1_payload +from codecarbon.core.telemetry_collect import build_telemetry_payload from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.core.telemetry_settings import ( get_telemetry_api_key, @@ -60,13 +60,13 @@ def _run_too_short_for_telemetry(emissions: EmissionsData) -> bool: return emissions.duration is not None and emissions.duration < 1 -def send_tier1_at_stop( +def send_private_telemetry_at_stop( tracker: Any, emissions: EmissionsData, external_conf: dict[str, Any] | None = None, level: TelemetryLevel = TelemetryLevel.minimal, ) -> bool: - """Send Tier 1 telemetry: private hardware/usage/run summary via ``POST /telemetry``. + """Send Tier 1 private telemetry via ``POST /telemetry``. Args: tracker: Active emissions tracker instance. @@ -75,16 +75,16 @@ def send_tier1_at_stop( level: Resolved ``TelemetryLevel`` for the ``telemetry_level`` field. Returns: - True if Tier 1 was accepted, False otherwise. + True if the private telemetry POST was accepted, False otherwise. """ if _run_too_short_for_telemetry(emissions): logger.debug( - "Tier 1 telemetry not sent because run duration is shorter than 1 second." + "Private telemetry not sent because run duration is shorter than 1 second." ) return False settings_conf = external_conf or {} try: - payload = build_tier1_payload(tracker, emissions, level=level) + payload = build_telemetry_payload(tracker, emissions, level=level) client = TelemetryClient( endpoint_url=get_telemetry_api_url(settings_conf), telemetry=payload, @@ -92,16 +92,16 @@ def send_tier1_at_stop( ) return client.add_telemetry() is not None except Exception as error: - logger.error(f"Tier 1 telemetry failed (non-critical): {error}") + logger.error(f"Private telemetry failed (non-critical): {error}") return False -def send_tier2_at_stop( +def send_public_run_summary_at_stop( tracker: Any, emissions: EmissionsData, external_conf: dict[str, Any] | None = None, ) -> bool: - """Send Tier 2 telemetry: run emissions to the shared experiment via ``ApiClient``. + """Send Tier 2 public run summary to the shared telemetry experiment via ``ApiClient``. Args: tracker: Active emissions tracker instance. @@ -109,11 +109,11 @@ def send_tier2_at_stop( external_conf: Merged config for API URL, key, and experiment resolution. Returns: - True if Tier 2 was posted successfully, False otherwise. + True if the run summary was posted successfully, False otherwise. """ if _run_too_short_for_telemetry(emissions): logger.debug( - "Tier 2 telemetry not sent because run duration is shorter than 1 second." + "Public run summary not sent because run duration is shorter than 1 second." ) return False settings_conf = external_conf or {} @@ -128,7 +128,7 @@ def send_tier2_at_stop( ) return bool(api.add_emission(dataclasses.asdict(emissions))) except Exception as error: - logger.error(f"Tier 2 telemetry failed (non-critical): {error}") + logger.error(f"Public run summary failed (non-critical): {error}") return False @@ -141,7 +141,7 @@ def send_product_telemetry_at_stop( """Send product telemetry for the resolved tier at tracker ``stop()``. Tier 1 (``minimal``): private ``POST /telemetry`` only. - Tier 2 (``extensive``): Tier 1 plus ``ApiClient`` run summary. + Tier 2 (``extensive``): same private ``POST /telemetry`` plus public run summary. Args: tracker: Active emissions tracker instance. @@ -153,6 +153,6 @@ def send_product_telemetry_at_stop( return settings = external_conf or {} if level in (TelemetryLevel.minimal, TelemetryLevel.extensive): - send_tier1_at_stop(tracker, emissions, settings, level=level) + send_private_telemetry_at_stop(tracker, emissions, settings, level=level) if level == TelemetryLevel.extensive: - send_tier2_at_stop(tracker, emissions, settings) + send_public_run_summary_at_stop(tracker, emissions, settings) diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index 82ea56f7d..192709ea1 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -56,7 +56,7 @@ Private telemetry does **not** include project names, experiment ids, API keys, - Project name, experiment id, run id, API keys - Source code, file paths, hostnames -- Exact GPS coordinates (`longitude` / `latitude`), `python_executable_hash`, `host_machine_hash` +- Exact GPS coordinates, executable/host fingerprints (not in the telemetry schema) - Voluntary [user survey](https://docs.google.com/forms/d/e/1FAIpQLSeQ5Tu_rdrpDhBJvh5R1-_iB4Ld-kgh6iNMjgaMXa8AEVPxqA/viewform) demographics (role, industry, experience) ## Configure telemetry diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 741182d8f..3166addbf 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -7,7 +7,10 @@ from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.output_methods.emissions_data import EmissionsData -from codecarbon.telemetry import send_tier1_at_stop, send_tier2_at_stop +from codecarbon.telemetry import ( + send_private_telemetry_at_stop, + send_public_run_summary_at_stop, +) from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": @@ -65,7 +68,9 @@ def test_tier1_posts_private_telemetry(self): mock_client = MagicMock() mock_client.add_telemetry.return_value = {"id": "ok"} mock_client_cls.return_value = mock_client - result = send_tier1_at_stop(tracker, emissions, external_conf={}) + result = send_private_telemetry_at_stop( + tracker, emissions, external_conf={} + ) self.assertTrue(result) posted = mock_client_cls.call_args.kwargs["telemetry"] self.assertEqual(posted["telemetry_level"], "minimal") @@ -76,7 +81,9 @@ def test_tier1_skips_short_duration(self): emissions = self._emissions() emissions.duration = 0.5 with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - result = send_tier1_at_stop(tracker, emissions, external_conf={}) + result = send_private_telemetry_at_stop( + tracker, emissions, external_conf={} + ) self.assertFalse(result) mock_client_cls.assert_not_called() @@ -88,7 +95,9 @@ def test_tier2_uses_api_client(self): mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api - result = send_tier2_at_stop(tracker, emissions, external_conf={}) + result = send_public_run_summary_at_stop( + tracker, emissions, external_conf={} + ) self.assertTrue(result) mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 4d2b24b32..48ab743c5 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -93,7 +93,7 @@ def test_init_rejects_invalid_telemetry_without_calling_api(self): telemetry={ "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", - "longitude": 2.35, + "unknown_field": "value", }, ) diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 1b711ad66..1a0cdf8f7 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -2,9 +2,9 @@ from unittest.mock import MagicMock, patch from codecarbon.core.telemetry_collect import ( - build_tier1_payload, + build_telemetry_payload, collect_telemetry_context, - project_tier1, + project_private_telemetry, ) from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.output_methods.emissions_data import EmissionsData @@ -49,7 +49,7 @@ def _sample_emissions(**overrides): class TestTelemetryCollect(unittest.TestCase): - def test_project_tier1_includes_run_and_framework_flags(self): + def test_project_private_telemetry_includes_run_and_framework_flags(self): tracker = MagicMock() tracker._conf = { "os": "Linux", @@ -75,7 +75,7 @@ def test_project_tier1_includes_run_and_framework_flags(self): side_effect=lambda name: name == "torch", ): context = collect_telemetry_context(tracker, emissions) - payload = project_tier1(context) + payload = project_private_telemetry(context) self.assertEqual(payload["telemetry_level"], "minimal") self.assertEqual(payload["total_emissions_kg"], 0.5) @@ -83,7 +83,7 @@ def test_project_tier1_includes_run_and_framework_flags(self): self.assertTrue(payload["has_torch"]) self.assertIn("file", payload["output_methods"]) - def test_build_tier1_payload_includes_framework_versions(self): + def test_build_telemetry_payload_includes_framework_versions(self): tracker = MagicMock() tracker._conf = {"codecarbon_version": "3.0", "hardware": ["cpu"]} tracker._geo = None @@ -106,13 +106,13 @@ def test_build_tier1_payload_includes_framework_versions(self): "codecarbon.core.telemetry_collect._package_version", return_value="2.0.0", ): - payload = build_tier1_payload(tracker, emissions) + payload = build_telemetry_payload(tracker, emissions) self.assertEqual(payload["telemetry_level"], "minimal") self.assertTrue(payload["has_torch"]) self.assertEqual(payload["torch_version"], "2.0.0") - def test_build_tier1_payload_uses_resolved_level(self): + def test_build_telemetry_payload_uses_resolved_level(self): tracker = MagicMock() tracker._conf = {"codecarbon_version": "3.0"} tracker._save_to_file = False @@ -127,7 +127,7 @@ def test_build_tier1_payload_uses_resolved_level(self): tracker._resource_tracker = None emissions = _sample_emissions() - payload = build_tier1_payload( + payload = build_telemetry_payload( tracker, emissions, level=TelemetryLevel.extensive ) self.assertEqual(payload["telemetry_level"], "extensive") diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index e90b5987f..55f8e24f3 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -15,7 +15,10 @@ from codecarbon.core.telemetry_settings import get_telemetry_api_url from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.output_methods.emissions_data import EmissionsData -from codecarbon.telemetry import send_tier1_at_stop, warn_if_telemetry_not_configured +from codecarbon.telemetry import ( + send_private_telemetry_at_stop, + warn_if_telemetry_not_configured, +) from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": @@ -90,12 +93,12 @@ def test_tier1_posts_to_telemetry_endpoint(self): tracking_mode="machine", ) with patch( - "codecarbon.telemetry.build_tier1_payload", return_value=tier1_payload + "codecarbon.telemetry.build_telemetry_payload", return_value=tier1_payload ): with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: mock_post.return_value.status_code = 201 mock_post.return_value.json.return_value = "telemetry-id" - send_tier1_at_stop( + send_private_telemetry_at_stop( tracker, emissions, external_conf={"telemetry_api_url": "http://tier1.example"}, From e3dca3f129a5dd95407eb20e4339c4a7db444535 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 20 May 2026 08:17:09 +0200 Subject: [PATCH 14/18] refactor: simplify telemetry framework detection and enhance privacy - Removed version tracking for ML frameworks from telemetry payload to improve privacy. - Updated the framework packages list to exclude version fields, focusing solely on presence flags. - Adjusted telemetry context collection to reflect the changes in framework detection. - Modified tests to ensure the telemetry payload omits framework versions as intended. - Updated documentation to clarify the changes in ML stack data collection. --- codecarbon/core/telemetry_collect.py | 45 ++++++--------------------- codecarbon/core/telemetry_settings.py | 4 +-- docs/how-to/telemetry.md | 2 +- tests/test_telemetry_collect.py | 7 ++--- 4 files changed, 15 insertions(+), 43 deletions(-) diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry_collect.py index 4122e91dc..3a057ef77 100644 --- a/codecarbon/core/telemetry_collect.py +++ b/codecarbon/core/telemetry_collect.py @@ -15,13 +15,10 @@ from codecarbon.output_methods.emissions_data import EmissionsData FRAMEWORK_PACKAGES = ( - ("torch", "has_torch", "torch_version"), - ("transformers", "has_transformers", "transformers_version"), - ("tensorflow", "has_tensorflow", "tensorflow_version"), - ("keras", "has_keras", "keras_version"), - ("diffusers", "has_diffusers", "diffusers_version"), - ("pytorch_lightning", "has_pytorch_lightning", "pytorch_lightning_version"), - ("fastai", "has_fastai", "fastai_version"), + ("torch", "has_torch"), + ("transformers", "has_transformers"), + ("diffusers", "has_diffusers"), + ("sklearn", "has_sklearn"), ) def _non_empty(value: Any) -> bool: @@ -42,17 +39,6 @@ def _package_installed(name: str) -> bool: return importlib.util.find_spec(name) is not None -def _package_version(name: str) -> Optional[str]: - if not _package_installed(name): - return None - try: - from importlib.metadata import version - - return version(name) - except Exception: - return None - - def _detect_codecarbon_install_method() -> Optional[str]: try: from importlib.metadata import distribution @@ -233,19 +219,11 @@ def _raw_cloud_provider_and_region() -> tuple[Optional[str], Optional[str]]: return provider, region -def _collect_framework_fields(include_versions: bool) -> dict[str, Any]: - fields: dict[str, Any] = {} - primary: Optional[str] = None - for package, has_field, version_field in FRAMEWORK_PACKAGES: - installed = _package_installed(package) - fields[has_field] = installed - if include_versions and installed: - fields[version_field] = _package_version(package) - if installed and primary is None: - primary = package - if primary: - fields["ml_framework_primary"] = primary - return fields +def _collect_framework_fields() -> dict[str, Any]: + return { + has_field: _package_installed(package) + for package, has_field in FRAMEWORK_PACKAGES + } def _gpu_static_fields() -> dict[str, Any]: @@ -339,14 +317,11 @@ def collect_telemetry_context( "cpu_utilization_avg": emissions.cpu_utilization_percent, "gpu_utilization_avg": emissions.gpu_utilization_percent, "ram_utilization_avg": emissions.ram_utilization_percent, - **_collect_framework_fields(include_versions=True), + **_collect_framework_fields(), **_collect_hardware_diagnostics(tracker), } context.update(gpu_fields) - if context.get("ml_framework_primary"): - context["framework_detected"] = context["ml_framework_primary"] - return _strip_none(context) diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py index 1d2f65c87..c12d7f04f 100644 --- a/codecarbon/core/telemetry_settings.py +++ b/codecarbon/core/telemetry_settings.py @@ -7,8 +7,8 @@ from codecarbon.external.logger import logger DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" -DEFAULT_TELEMETRY_API_KEY = "cpt_JZhj-vJdEVG28qErZL5mh1ftiqbnDIBYjWSxwvX3rfI" -DEFAULT_TELEMETRY_EXPERIMENT_ID = "aa69b440-014a-4562-ac06-ba7eecb023f9" +DEFAULT_TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +DEFAULT_TELEMETRY_EXPERIMENT_ID = "d2d69403-1373-42b4-a2c1-09589aed4801" DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md index 192709ea1..b5bc56057 100644 --- a/docs/how-to/telemetry.md +++ b/docs/how-to/telemetry.md @@ -43,7 +43,7 @@ The payload includes: - **Environment:** OS, Python, CPU/GPU/RAM, country/region, cloud provider/region, GPU driver/CUDA/cuDNN when available - **Usage:** tracking mode, output methods, integration surface (library / CLI / offline), task tracking, CI/notebook/container/IDE hints, hardware diagnostics -- **ML stack:** framework presence flags **and** installed versions when detected +- **ML stack:** framework presence flags when detected (no installed package versions) - **Run outcome:** duration, emissions, energy (total and per component), utilization averages Private telemetry does **not** include project names, experiment ids, API keys, file paths, exact coordinates, executable/host hashes, or survey demographics (role, industry, etc.). diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 1a0cdf8f7..85f006ee4 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -83,7 +83,7 @@ def test_project_private_telemetry_includes_run_and_framework_flags(self): self.assertTrue(payload["has_torch"]) self.assertIn("file", payload["output_methods"]) - def test_build_telemetry_payload_includes_framework_versions(self): + def test_build_telemetry_payload_omits_framework_versions(self): tracker = MagicMock() tracker._conf = {"codecarbon_version": "3.0", "hardware": ["cpu"]} tracker._geo = None @@ -102,15 +102,12 @@ def test_build_telemetry_payload_includes_framework_versions(self): with patch( "codecarbon.core.telemetry_collect._package_installed", return_value=True, - ), patch( - "codecarbon.core.telemetry_collect._package_version", - return_value="2.0.0", ): payload = build_telemetry_payload(tracker, emissions) self.assertEqual(payload["telemetry_level"], "minimal") self.assertTrue(payload["has_torch"]) - self.assertEqual(payload["torch_version"], "2.0.0") + self.assertNotIn("torch_version", payload) def test_build_telemetry_payload_uses_resolved_level(self): tracker = MagicMock() From de3fff137bd30673f8f9c6dd255f8897f573689b Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 20 May 2026 08:32:43 +0200 Subject: [PATCH 15/18] refactor: simplify telemetry client and enhance telemetry handling - Replaced the TelemetryClient class with a standalone function, post_private_telemetry, for sending telemetry data. - Updated telemetry payload construction to improve clarity and maintainability. - Refactored telemetry warning handling to ensure it is shown only once during execution. - Enhanced tests to validate the new telemetry sending method and ensure proper payload structure. - Improved error handling and logging for telemetry requests, including handling of connection errors and HTTP response statuses. --- codecarbon/core/telemetry_client.py | 99 +++++++----------- codecarbon/core/telemetry_collect.py | 145 ++++++++++---------------- codecarbon/core/telemetry_settings.py | 96 ++++++++++------- codecarbon/telemetry.py | 54 +++++----- tests/test_telemetry.py | 70 ++++++------- tests/test_telemetry_client.py | 111 +++++--------------- tests/test_telemetry_collect.py | 11 +- tests/test_telemetry_config.py | 56 +++++----- 8 files changed, 266 insertions(+), 376 deletions(-) diff --git a/codecarbon/core/telemetry_client.py b/codecarbon/core/telemetry_client.py index 6cf821f28..fbfe76e3d 100644 --- a/codecarbon/core/telemetry_client.py +++ b/codecarbon/core/telemetry_client.py @@ -1,76 +1,47 @@ -import json -from typing import Optional, Union - import requests from codecarbon.core.telemetry_schemas import TelemetryCreate from codecarbon.external.logger import logger -class TelemetryClient: - """ - Client dedicated to sending CodeCarbon telemetry payloads. - """ +def post_private_telemetry(url: str, payload: dict, api_key: str | None) -> bool: + """POST a private telemetry payload to ``/telemetry``. - def __init__( - self, - endpoint_url: str = "https://api.codecarbon.io", - telemetry: Optional[Union[TelemetryCreate, dict]] = None, - api_key: Optional[str] = None, - ): - self.endpoint_url = endpoint_url.rstrip("/") - self.telemetry_url = self.endpoint_url + "/telemetry" - self.api_key = api_key - self.headers = self._build_headers(api_key) - self.telemetry = self._validate_telemetry(telemetry) if telemetry else None + Args: + url: API base URL. + payload: Telemetry fields dict. + api_key: Optional API token. - @staticmethod - def _build_headers(api_key: Optional[str]) -> dict[str, str]: - headers = {"Content-Type": "application/json"} - if api_key: - headers["x-api-token"] = api_key - return headers - - def add_telemetry(self, telemetry: Optional[Union[TelemetryCreate, dict]] = None): - telemetry_payload = ( - self._validate_telemetry(telemetry) if telemetry else self.telemetry + Returns: + True if the server accepted the payload (HTTP 201). + """ + headers = {"Content-Type": "application/json"} + if api_key: + headers["x-api-token"] = api_key + body = TelemetryCreate(**payload).model_dump(mode="json", exclude_none=True) + telemetry_url = f"{url.rstrip('/')}/telemetry" + try: + response = requests.post( + url=telemetry_url, + json=body, + headers=headers, + timeout=2, ) - if telemetry_payload is None: - logger.error("TelemetryClient.add_telemetry() needs a telemetry payload") - return None - payload = telemetry_payload.model_dump(mode="json", exclude_none=True) - - try: - response = requests.post( - url=self.telemetry_url, - json=payload, - timeout=2, - headers=self.headers, - ) - if response.status_code == 404: - logger.warning( - "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", - self.telemetry_url, - ) - return None - if response.status_code != 201: - self._log_error(payload, response) - return None - return response.json() - except Exception as e: - logger.error(e, exc_info=True) - return None - - @staticmethod - def _validate_telemetry(telemetry: Union[TelemetryCreate, dict]) -> TelemetryCreate: - if isinstance(telemetry, TelemetryCreate): - return telemetry - return TelemetryCreate(**telemetry) - - def _log_error(self, payload, response): - logger.error( - f"TelemetryClient Error when calling the API on {self.telemetry_url} with : {json.dumps(payload)}" + except Exception: + logger.error("Telemetry request failed.", exc_info=True) + return False + if response.status_code == 201: + return True + if response.status_code == 404: + logger.warning( + "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", + telemetry_url, ) + else: logger.error( - f"TelemetryClient API return http code {response.status_code} and answer : {response.text}" + "Telemetry API %s: %s", + response.status_code, + response.text, ) + logger.debug("Telemetry request body: %s", body) + return False diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry_collect.py index 3a057ef77..db788ad3c 100644 --- a/codecarbon/core/telemetry_collect.py +++ b/codecarbon/core/telemetry_collect.py @@ -1,4 +1,4 @@ -"""Collect and project private product telemetry (Tier 1 / Tier 2) from tracker state.""" +"""Collect private product telemetry (Tier 1 / Tier 2) from tracker state.""" from __future__ import annotations @@ -21,20 +21,46 @@ ("sklearn", "has_sklearn"), ) +PACKAGE_MANAGER_ENV = ( + ("UV", "uv"), + ("POETRY_ACTIVE", "poetry"), + ("PIP_RUN", "pip"), +) + +CI_ENVIRONMENTS = ( + ("GITHUB_ACTIONS", "github_actions"), + ("GITLAB_CI", "gitlab_ci"), + ("CIRCLECI", "circleci"), + ("JENKINS_URL", "jenkins"), + ("CI", "ci"), +) + +CONTAINER_RUNTIME_ENV = ( + ("KUBERNETES_SERVICE_HOST", "kubernetes"), +) + +OUTPUT_METHOD_ATTRS = ( + ("_save_to_file", "file"), + ("_save_to_api", "api"), + ("_save_to_logger", "logger"), + ("_emissions_endpoint", "http"), + ("_save_to_prometheus", "prometheus"), + ("_save_to_logfire", "logfire"), +) + + def _non_empty(value: Any) -> bool: - if value is None: - return False - if value == [] or value == {}: - return False - if isinstance(value, str) and value == "": - return False - return True + return value not in (None, "", [], {}) def _strip_none(data: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in data.items() if _non_empty(value)} +def _first_env_match(mapping: tuple[tuple[str, str], ...]) -> Optional[str]: + return next((label for var, label in mapping if os.environ.get(var)), None) + + def _package_installed(name: str) -> bool: return importlib.util.find_spec(name) is not None @@ -66,30 +92,6 @@ def _detect_python_env_type() -> Optional[str]: return "system" -def _detect_python_package_manager() -> Optional[str]: - if os.environ.get("UV"): - return "uv" - if os.environ.get("POETRY_ACTIVE"): - return "poetry" - if os.environ.get("PIP_RUN"): - return "pip" - return None - - -def _detect_ci_environment() -> Optional[str]: - if os.environ.get("GITHUB_ACTIONS"): - return "github_actions" - if os.environ.get("GITLAB_CI"): - return "gitlab_ci" - if os.environ.get("CIRCLECI"): - return "circleci" - if os.environ.get("JENKINS_URL"): - return "jenkins" - if os.environ.get("CI"): - return "ci" - return None - - def _detect_notebook_environment() -> Optional[str]: if os.environ.get("COLAB_GPU") is not None or "google.colab" in sys.modules: return "colab" @@ -113,8 +115,9 @@ def _detect_in_container() -> bool: def _detect_container_runtime() -> Optional[str]: - if os.environ.get("KUBERNETES_SERVICE_HOST"): - return "kubernetes" + runtime = _first_env_match(CONTAINER_RUNTIME_ENV) + if runtime: + return runtime if os.path.exists("/.dockerenv"): return "docker" return None @@ -186,18 +189,13 @@ def _detect_integration_surface(tracker: Any) -> str: def _collect_output_methods(tracker: Any) -> list[str]: methods: list[str] = [] - if getattr(tracker, "_save_to_file", False): - methods.append("file") - if getattr(tracker, "_save_to_api", False): - methods.append("api") - if getattr(tracker, "_save_to_logger", False): - methods.append("logger") - if getattr(tracker, "_emissions_endpoint", None): - methods.append("http") - if getattr(tracker, "_save_to_prometheus", False): - methods.append("prometheus") - if getattr(tracker, "_save_to_logfire", False): - methods.append("logfire") + for attr, name in OUTPUT_METHOD_ATTRS: + value = getattr(tracker, attr, False) + if attr == "_emissions_endpoint": + if value: + methods.append(name) + elif value: + methods.append(name) return methods @@ -247,18 +245,20 @@ def _gpu_static_fields() -> dict[str, Any]: return fields -def collect_telemetry_context( +def build_telemetry_payload( tracker: Any, emissions: EmissionsData, + level: TelemetryLevel = TelemetryLevel.minimal, ) -> dict[str, Any]: - """Build a flat telemetry context from tracker state and emissions at stop. + """Build a private telemetry payload dict for ``POST /telemetry``. Args: - tracker: Active ``BaseEmissionsTracker`` instance. - emissions: Total emissions row from ``_prepare_emissions_data()``. + tracker: Active emissions tracker. + emissions: Run emissions data. + level: Resolved ``TelemetryLevel`` (``minimal`` or ``extensive``). Returns: - Flat dictionary for ``project_private_telemetry``. + Payload dict for ``TelemetryCreate``. """ conf = getattr(tracker, "_conf", {}) raw_provider, raw_region = _raw_cloud_provider_and_region() @@ -272,13 +272,13 @@ def collect_telemetry_context( integration_surface = _detect_integration_surface(tracker) gpu_fields = _gpu_static_fields() - context: dict[str, Any] = { + raw: dict[str, Any] = { "timestamp": datetime.now(timezone.utc), "os": conf.get("os") or platform.platform(), "python_version": conf.get("python_version") or platform.python_version(), "python_implementation": platform.python_implementation(), "python_env_type": _detect_python_env_type(), - "python_package_manager": _detect_python_package_manager(), + "python_package_manager": _first_env_match(PACKAGE_MANAGER_ENV), "codecarbon_version": conf.get("codecarbon_version"), "codecarbon_install_method": _detect_codecarbon_install_method(), "country_name": emissions.country_name, @@ -303,7 +303,7 @@ def collect_telemetry_context( "measure_power_interval_secs": getattr(tracker, "_measure_power_secs", None), "in_container": _detect_in_container(), "container_runtime": _detect_container_runtime(), - "ci_environment": _detect_ci_environment(), + "ci_environment": _first_env_match(CI_ENVIRONMENTS), "notebook_environment": _detect_notebook_environment(), "ide_used": _detect_ide(), "cudnn_version": _cudnn_version(), @@ -317,42 +317,11 @@ def collect_telemetry_context( "cpu_utilization_avg": emissions.cpu_utilization_percent, "gpu_utilization_avg": emissions.gpu_utilization_percent, "ram_utilization_avg": emissions.ram_utilization_percent, + "telemetry_level": level.value, **_collect_framework_fields(), **_collect_hardware_diagnostics(tracker), + **gpu_fields, } - context.update(gpu_fields) - return _strip_none(context) - - -def project_private_telemetry( - context: dict[str, Any], - level: TelemetryLevel = TelemetryLevel.minimal, -) -> dict[str, Any]: - """Project context to private ``POST /telemetry`` fields for the resolved tier.""" - payload = { - key: context[key] - for key in PRIVATE_TELEMETRY_FIELDS - if key in context - } - payload["telemetry_level"] = level.value + payload = {key: raw[key] for key in PRIVATE_TELEMETRY_FIELDS if key in raw} return _strip_none(payload) - - -def build_telemetry_payload( - tracker: Any, - emissions: EmissionsData, - level: TelemetryLevel = TelemetryLevel.minimal, -) -> dict[str, Any]: - """Build a private telemetry payload dict for ``TelemetryCreate``. - - Args: - tracker: Active emissions tracker. - emissions: Run emissions data. - level: Resolved ``TelemetryLevel`` (``minimal`` or ``extensive``). - - Returns: - Payload dict for ``POST /telemetry``. - """ - context = collect_telemetry_context(tracker, emissions) - return project_private_telemetry(context, level=level) diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py index c12d7f04f..a0805ed00 100644 --- a/codecarbon/core/telemetry_settings.py +++ b/codecarbon/core/telemetry_settings.py @@ -1,7 +1,7 @@ """Resolve telemetry tier and API settings from config and environment.""" import os -from typing import Any +from typing import Any, Literal from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.external.logger import logger @@ -13,6 +13,8 @@ TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" +TelemetryLevelSource = Literal["override", "external", "file", "default"] + def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: """Parse a telemetry tier from a string or enum value. @@ -36,41 +38,13 @@ def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: ) from error -def is_telemetry_level_explicit( - config_file_conf: dict[str, Any], - *, - override: str | TelemetryLevel | None = None, - external_conf: dict[str, Any] | None = None, -) -> bool: - """Return whether the user explicitly chose a telemetry tier. - - Explicit sources: tracker ``telemetry_level`` argument, config file - ``telemetry_level``, or environment ``CODECARBON_TELEMETRY_LEVEL``. - - Args: - config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). - override: Value passed to ``EmissionsTracker(telemetry_level=...)``. - external_conf: Merged config from file and environment. - - Returns: - True if any explicit source is set. - """ - if override is not None: - return True - if config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: - return True - if external_conf is None: - return False - return external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None - - -def resolve_telemetry_level( +def resolve_telemetry_level_and_source( config_file_conf: dict[str, Any] | None = None, *, override: str | TelemetryLevel | None = None, external_conf: dict[str, Any] | None = None, -) -> TelemetryLevel: - """Resolve the active telemetry tier. +) -> tuple[TelemetryLevel, TelemetryLevelSource]: + """Resolve the active telemetry tier and where it came from. Precedence: @@ -85,27 +59,77 @@ def resolve_telemetry_level( external_conf: Merged settings from ``get_hierarchical_config()`` (optional). Returns: - Resolved ``TelemetryLevel``. + Resolved tier and its source label. """ if override is not None: raw = override + source: TelemetryLevelSource = "override" elif external_conf is not None and external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: raw = external_conf[TELEMETRY_LEVEL_CONFIG_KEY] + source = "external" elif config_file_conf is not None and config_file_conf.get( TELEMETRY_LEVEL_CONFIG_KEY ) is not None: raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] + source = "file" else: - return DEFAULT_TELEMETRY_LEVEL + return DEFAULT_TELEMETRY_LEVEL, "default" try: - return parse_telemetry_level(raw) + return parse_telemetry_level(raw), source except ValueError: logger.error( "Invalid telemetry_level %r; falling back to %r", raw, DEFAULT_TELEMETRY_LEVEL.value, ) - return DEFAULT_TELEMETRY_LEVEL + return DEFAULT_TELEMETRY_LEVEL, source + + +def resolve_telemetry_level( + config_file_conf: dict[str, Any] | None = None, + *, + override: str | TelemetryLevel | None = None, + external_conf: dict[str, Any] | None = None, +) -> TelemetryLevel: + """Resolve the active telemetry tier. + + Args: + config_file_conf: Settings from ``get_config_file_settings()`` (optional). + override: Optional tier from tracker or CLI. + external_conf: Merged settings from ``get_hierarchical_config()`` (optional). + + Returns: + Resolved ``TelemetryLevel``. + """ + return resolve_telemetry_level_and_source( + config_file_conf, + override=override, + external_conf=external_conf, + )[0] + + +def is_telemetry_level_explicit( + config_file_conf: dict[str, Any], + *, + override: str | TelemetryLevel | None = None, + external_conf: dict[str, Any] | None = None, +) -> bool: + """Return whether the user explicitly chose a telemetry tier. + + Args: + config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). + override: Value passed to ``EmissionsTracker(telemetry_level=...)``. + external_conf: Merged config from file and environment. + + Returns: + True if any explicit source is set. + """ + _, source = resolve_telemetry_level_and_source( + config_file_conf, + override=override, + external_conf=external_conf, + ) + return source != "default" def get_telemetry_api_url( diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py index ed73537b4..eeec82434 100644 --- a/codecarbon/telemetry.py +++ b/codecarbon/telemetry.py @@ -4,7 +4,7 @@ from typing import Any from codecarbon.core.api_client import ApiClient -from codecarbon.core.telemetry_client import TelemetryClient +from codecarbon.core.telemetry_client import post_private_telemetry from codecarbon.core.telemetry_collect import build_telemetry_payload from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.core.telemetry_settings import ( @@ -16,8 +16,6 @@ from codecarbon.external.logger import logger from codecarbon.output_methods.emissions_data import EmissionsData -_TELEMETRY_CONFIGURE_WARNED = False - TELEMETRY_NOT_CONFIGURED_MESSAGE = ( "CodeCarbon telemetry_level was not set explicitly; using default %r. " "Tier 1 private telemetry (per run at stop) will be sent. Set telemetry_level " @@ -26,6 +24,14 @@ "codecarbon telemetry set ." ) +_telemetry_default_warning_shown = False + + +def reset_telemetry_warning() -> None: + """Clear the one-shot default-tier warning (for tests).""" + global _telemetry_default_warning_shown + _telemetry_default_warning_shown = False + def warn_if_telemetry_not_configured( config_file_conf: dict[str, Any], @@ -42,22 +48,15 @@ def warn_if_telemetry_not_configured( override: Optional ``telemetry_level`` tracker argument. external_conf: Merged file/env settings for explicit-env detection. """ - global _TELEMETRY_CONFIGURE_WARNED - if _TELEMETRY_CONFIGURE_WARNED: - return + global _telemetry_default_warning_shown if is_telemetry_level_explicit( config_file_conf, override=override, external_conf=external_conf ): return - logger.warning( - TELEMETRY_NOT_CONFIGURED_MESSAGE, - active_level.value, - ) - _TELEMETRY_CONFIGURE_WARNED = True - - -def _run_too_short_for_telemetry(emissions: EmissionsData) -> bool: - return emissions.duration is not None and emissions.duration < 1 + if _telemetry_default_warning_shown: + return + logger.warning(TELEMETRY_NOT_CONFIGURED_MESSAGE, active_level.value) + _telemetry_default_warning_shown = True def send_private_telemetry_at_stop( @@ -68,6 +67,9 @@ def send_private_telemetry_at_stop( ) -> bool: """Send Tier 1 private telemetry via ``POST /telemetry``. + Runs shorter than one second are skipped by ``send_product_telemetry_at_stop``, + not here; direct callers may still post for sub-second runs. + Args: tracker: Active emissions tracker instance. emissions: Total emissions from ``_prepare_emissions_data()``. @@ -77,20 +79,14 @@ def send_private_telemetry_at_stop( Returns: True if the private telemetry POST was accepted, False otherwise. """ - if _run_too_short_for_telemetry(emissions): - logger.debug( - "Private telemetry not sent because run duration is shorter than 1 second." - ) - return False settings_conf = external_conf or {} try: payload = build_telemetry_payload(tracker, emissions, level=level) - client = TelemetryClient( - endpoint_url=get_telemetry_api_url(settings_conf), - telemetry=payload, - api_key=get_telemetry_api_key(settings_conf), + return post_private_telemetry( + get_telemetry_api_url(settings_conf), + payload, + get_telemetry_api_key(settings_conf), ) - return client.add_telemetry() is not None except Exception as error: logger.error(f"Private telemetry failed (non-critical): {error}") return False @@ -111,11 +107,6 @@ def send_public_run_summary_at_stop( Returns: True if the run summary was posted successfully, False otherwise. """ - if _run_too_short_for_telemetry(emissions): - logger.debug( - "Public run summary not sent because run duration is shorter than 1 second." - ) - return False settings_conf = external_conf or {} conf = getattr(tracker, "_conf", {}) try: @@ -151,6 +142,9 @@ def send_product_telemetry_at_stop( """ if level == TelemetryLevel.disabled: return + if emissions.duration is not None and emissions.duration < 1: + logger.debug("Telemetry not sent: run shorter than 1 second.") + return settings = external_conf or {} if level in (TelemetryLevel.minimal, TelemetryLevel.extensive): send_private_telemetry_at_stop(tracker, emissions, settings, level=level) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 3166addbf..757283d7a 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,7 +1,6 @@ import sys import tempfile import unittest -from pathlib import Path from unittest.mock import ANY, MagicMock, patch from codecarbon.core.telemetry_schemas import TelemetryLevel @@ -64,28 +63,32 @@ def _emissions(self) -> EmissionsData: def test_tier1_posts_private_telemetry(self): tracker = MagicMock() emissions = self._emissions() - with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = {"id": "ok"} - mock_client_cls.return_value = mock_client + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: result = send_private_telemetry_at_stop( tracker, emissions, external_conf={} ) self.assertTrue(result) - posted = mock_client_cls.call_args.kwargs["telemetry"] - self.assertEqual(posted["telemetry_level"], "minimal") - self.assertEqual(posted["total_emissions_kg"], 0.001) + mock_post.assert_called_once() + payload = mock_post.call_args[0][1] + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertEqual(payload["total_emissions_kg"], 0.001) + + def test_tier1_skips_short_duration_at_dispatcher(self): + from codecarbon.telemetry import send_product_telemetry_at_stop - def test_tier1_skips_short_duration(self): tracker = MagicMock() emissions = self._emissions() emissions.duration = 0.5 - with patch("codecarbon.telemetry.TelemetryClient") as mock_client_cls: - result = send_private_telemetry_at_stop( - tracker, emissions, external_conf={} + with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: + send_product_telemetry_at_stop( + tracker, + emissions, + TelemetryLevel.minimal, + external_conf={}, ) - self.assertFalse(result) - mock_client_cls.assert_not_called() + mock_post.assert_not_called() def test_tier2_uses_api_client(self): tracker = MagicMock() @@ -129,20 +132,19 @@ def _start_config_mock(self, conf: str) -> None: def test_emissions_tracker_does_not_send_telemetry_on_init(self, mock_cli_setup): self._start_config_mock(minimal_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) - mock_telemetry_cls.assert_not_called() + mock_post.assert_not_called() def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( self, mock_cli_setup ): self._start_config_mock(minimal_conf) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = {"id": "ok"} - mock_telemetry_cls.return_value = mock_client + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -151,15 +153,14 @@ def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_called_once() - mock_client.add_telemetry.assert_called_once() - payload = mock_telemetry_cls.call_args.kwargs["telemetry"] + mock_post.assert_called_once() + payload = mock_post.call_args[0][1] self.assertEqual(payload["telemetry_level"], "minimal") self.assertIn("total_emissions_kg", payload) def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): self._start_config_mock(disabled_conf) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -168,15 +169,14 @@ def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_not_called() + mock_post.assert_not_called() def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): self._start_config_mock(extensive_conf) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_telemetry = MagicMock() - mock_telemetry.add_telemetry.return_value = {"id": "ok"} - mock_telemetry_cls.return_value = mock_telemetry + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True @@ -189,18 +189,16 @@ def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_called_once() - mock_telemetry.add_telemetry.assert_called_once() + mock_post.assert_called_once() mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() def test_offline_tracker_sends_on_stop(self, mock_cli_setup): self._start_config_mock(minimal_conf) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = {"id": "ok"} - mock_telemetry_cls.return_value = mock_client + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: tracker = OfflineEmissionsTracker( country_iso_code="CAN", save_to_api=False, @@ -208,7 +206,7 @@ def test_offline_tracker_sends_on_stop(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_client.add_telemetry.assert_called_once() + mock_post.assert_called_once() if __name__ == "__main__": diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 48ab743c5..7d8b4be64 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -4,27 +4,12 @@ import requests_mock from pydantic import ValidationError -from codecarbon.core.telemetry_client import TelemetryClient +from codecarbon.core.telemetry_client import post_private_telemetry from codecarbon.core.telemetry_schemas import TelemetryCreate -class TestTelemetryClient(unittest.TestCase): - def test_init_sets_up_client_without_calling_api(self): - with requests_mock.Mocker() as m: - client = TelemetryClient( - endpoint_url="http://test.com/", - telemetry={ - "timestamp": "2026-05-03T12:00:00+00:00", - "telemetry_level": "minimal", - }, - ) - - self.assertEqual(client.endpoint_url, "http://test.com") - self.assertEqual(client.telemetry_url, "http://test.com/telemetry") - self.assertIsInstance(client.telemetry, TelemetryCreate) - self.assertEqual(m.call_count, 0) - - def test_add_telemetry_posts_configured_payload(self): +class TestPostPrivateTelemetry(unittest.TestCase): + def test_post_private_telemetry_sends_validated_payload(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -37,15 +22,9 @@ def test_add_telemetry_posts_configured_payload(self): json="f52fe339-164d-4c2b-a8c0-f562dfce066d", status_code=201, ) - client = TelemetryClient( - endpoint_url="http://test.com", telemetry=telemetry - ) + result = post_private_telemetry("http://test.com", telemetry, None) - actual_telemetry_id = client.add_telemetry() - - self.assertEqual( - actual_telemetry_id, "f52fe339-164d-4c2b-a8c0-f562dfce066d" - ) + self.assertTrue(result) self.assertEqual(m.call_count, 1) self.assertEqual( m.last_request.json(), @@ -55,56 +34,19 @@ def test_add_telemetry_posts_configured_payload(self): }, ) - def test_add_telemetry_posts_call_payload(self): - telemetry = TelemetryCreate( - timestamp="2026-05-03T12:00:00+00:00", - telemetry_level="minimal", - os="Linux-5.10.0-x86_64", - ) - - with requests_mock.Mocker() as m: - m.post( - "http://test.com/telemetry", - json="f52fe339-164d-4c2b-a8c0-f562dfce066d", - status_code=201, - ) - client = TelemetryClient(endpoint_url="http://test.com") - - actual_telemetry_id = client.add_telemetry(telemetry) - - self.assertEqual( - actual_telemetry_id, "f52fe339-164d-4c2b-a8c0-f562dfce066d" - ) - self.assertEqual(m.call_count, 1) - self.assertEqual( - m.last_request.json(), + def test_post_private_telemetry_rejects_invalid_payload(self): + with self.assertRaises(ValidationError): + post_private_telemetry( + "http://test.com", { - "timestamp": "2026-05-03T12:00:00Z", + "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", - "os": "Linux-5.10.0-x86_64", + "unknown_field": "value", }, + None, ) - def test_init_rejects_invalid_telemetry_without_calling_api(self): - with requests_mock.Mocker() as m: - with self.assertRaises(ValidationError): - TelemetryClient( - endpoint_url="http://test.com", - telemetry={ - "timestamp": "2026-05-03T12:00:00+00:00", - "telemetry_level": "minimal", - "unknown_field": "value", - }, - ) - - self.assertEqual(m.call_count, 0) - - def test_add_telemetry_returns_none_without_payload(self): - client = TelemetryClient(endpoint_url="http://test.com") - - self.assertIsNone(client.add_telemetry()) - - def test_add_telemetry_logs_warning_on_404(self): + def test_post_private_telemetry_logs_warning_on_404(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -116,14 +58,11 @@ def test_add_telemetry_logs_warning_on_404(self): status_code=404, ) with patch("codecarbon.core.telemetry_client.logger") as mock_logger: - client = TelemetryClient( - endpoint_url="http://test.com", telemetry=telemetry - ) - result = client.add_telemetry() - self.assertIsNone(result) + result = post_private_telemetry("http://test.com", telemetry, None) + self.assertFalse(result) mock_logger.warning.assert_called_once() - def test_add_telemetry_sends_api_key_header_when_configured(self): + def test_post_private_telemetry_sends_api_key_header_when_configured(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -134,10 +73,16 @@ def test_add_telemetry_sends_api_key_header_when_configured(self): json="telemetry-id", status_code=201, ) - client = TelemetryClient( - endpoint_url="http://test.com", - telemetry=telemetry, - api_key="cpt_test_key", - ) - client.add_telemetry() + post_private_telemetry("http://test.com", telemetry, "cpt_test_key") self.assertEqual(m.last_request.headers["x-api-token"], "cpt_test_key") + + def test_post_private_telemetry_returns_false_on_request_error(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + mock_post.side_effect = ConnectionError("network down") + with patch("codecarbon.core.telemetry_client.logger"): + result = post_private_telemetry("http://test.com", telemetry, None) + self.assertFalse(result) diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index 85f006ee4..be770e460 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -1,11 +1,7 @@ import unittest from unittest.mock import MagicMock, patch -from codecarbon.core.telemetry_collect import ( - build_telemetry_payload, - collect_telemetry_context, - project_private_telemetry, -) +from codecarbon.core.telemetry_collect import build_telemetry_payload from codecarbon.core.telemetry_schemas import TelemetryLevel from codecarbon.output_methods.emissions_data import EmissionsData @@ -49,7 +45,7 @@ def _sample_emissions(**overrides): class TestTelemetryCollect(unittest.TestCase): - def test_project_private_telemetry_includes_run_and_framework_flags(self): + def test_build_telemetry_payload_includes_run_and_framework_flags(self): tracker = MagicMock() tracker._conf = { "os": "Linux", @@ -74,8 +70,7 @@ def test_project_private_telemetry_includes_run_and_framework_flags(self): "codecarbon.core.telemetry_collect._package_installed", side_effect=lambda name: name == "torch", ): - context = collect_telemetry_context(tracker, emissions) - payload = project_private_telemetry(context) + payload = build_telemetry_payload(tracker, emissions) self.assertEqual(payload["telemetry_level"], "minimal") self.assertEqual(payload["total_emissions_kg"], 0.5) diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index 55f8e24f3..5627ae529 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -8,7 +8,6 @@ from pathlib import Path from unittest.mock import MagicMock, patch -import codecarbon.telemetry as telemetry_module from codecarbon.core.config import get_config_file_settings from codecarbon.core.telemetry_settings import resolve_telemetry_level from codecarbon.core.telemetry_schemas import TelemetryLevel @@ -16,6 +15,7 @@ from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.output_methods.emissions_data import EmissionsData from codecarbon.telemetry import ( + reset_telemetry_warning, send_private_telemetry_at_stop, warn_if_telemetry_not_configured, ) @@ -35,7 +35,7 @@ def _conf(level: str) -> str: class TestTelemetryConfigContract(unittest.TestCase): def setUp(self) -> None: - telemetry_module._TELEMETRY_CONFIGURE_WARNED = False + reset_telemetry_warning() def test_warns_once_when_telemetry_not_explicit(self): with patch("codecarbon.telemetry.logger.warning") as mock_warning: @@ -174,7 +174,7 @@ def test_telemetry_api_url_from_config_overrides_default(self): @mock_platform_cli_setup class TestTrackerTelemetryFromConfig(unittest.TestCase): def setUp(self) -> None: - telemetry_module._TELEMETRY_CONFIGURE_WARNED = False + reset_telemetry_warning() self._config_patcher = None def tearDown(self) -> None: @@ -189,7 +189,7 @@ def _mock_config(self, conf: str) -> None: def test_disabled_no_telemetry_on_stop(self, mock_cli_setup): self._mock_config(_conf("disabled")) - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -198,15 +198,14 @@ def test_disabled_no_telemetry_on_stop(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_not_called() + mock_post.assert_not_called() def test_minimal_posts_tier1_on_stop_not_on_init(self, mock_cli_setup): self._mock_config(_conf("minimal")) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -215,20 +214,18 @@ def test_minimal_posts_tier1_on_stop_not_on_init(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_called_once() - mock_client.add_telemetry.assert_called_once() + mock_post.assert_called_once() self.assertEqual( - mock_telemetry_cls.call_args.kwargs["telemetry"]["telemetry_level"], + mock_post.call_args[0][1]["telemetry_level"], TelemetryLevel.minimal.value, ) def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup): self._mock_config(_conf("extensive")) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_telemetry = MagicMock() - mock_telemetry.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_telemetry + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True @@ -241,18 +238,16 @@ def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_called_once() - mock_telemetry.add_telemetry.assert_called_once() + mock_post.assert_called_once() mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() def test_offline_minimal_posts_tier1_on_stop(self, mock_cli_setup): self._mock_config(_conf("minimal")) with ensure_telemetry_run_duration(): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: tracker = OfflineEmissionsTracker( country_iso_code="CAN", save_to_api=False, @@ -260,7 +255,7 @@ def test_offline_minimal_posts_tier1_on_stop(self, mock_cli_setup): ) tracker.start() tracker.stop() - mock_client.add_telemetry.assert_called_once() + mock_post.assert_called_once() def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup): self._mock_config("[codecarbon]\n") @@ -276,11 +271,8 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) with patch.dict(os.environ, env_without_telemetry, clear=True): with patch("codecarbon.telemetry.logger.warning") as mock_warning: with patch( - "codecarbon.telemetry.TelemetryClient" - ) as mock_telemetry_cls: - mock_client = MagicMock() - mock_client.add_telemetry.return_value = "telemetry-id" - mock_telemetry_cls.return_value = mock_client + "codecarbon.telemetry.post_private_telemetry", return_value=True + ): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) configure_warnings = [ @@ -293,7 +285,7 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): self._mock_config("[codecarbon]\n") with patch("codecarbon.telemetry.logger.warning") as mock_warning: - with patch("codecarbon.telemetry.TelemetryClient"): + with patch("codecarbon.telemetry.post_private_telemetry"): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker( telemetry_level="disabled", @@ -313,14 +305,16 @@ def test_env_telemetry_disabled_does_not_change_resolved_level( with patch.dict( os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False ): - with patch("codecarbon.telemetry.TelemetryClient") as mock_telemetry_cls: + with patch( + "codecarbon.telemetry.post_private_telemetry", return_value=True + ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( save_to_api=False, save_to_file=False ) tracker.start() tracker.stop() - mock_telemetry_cls.assert_called_once() + mock_post.assert_called_once() if __name__ == "__main__": From 5a65220b388db7b09c935376c1d909af8a799a97 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 20 May 2026 08:38:28 +0200 Subject: [PATCH 16/18] refactor: overhaul telemetry handling and structure - Replaced the existing telemetry client and settings management with a more modular approach, introducing a dedicated Telemetry class for dispatching telemetry data. - Consolidated telemetry payload construction and validation into a streamlined process, enhancing clarity and maintainability. - Removed deprecated telemetry client and settings files, ensuring a cleaner codebase. - Updated telemetry context collection to improve data accuracy and privacy compliance. - Enhanced tests to validate the new telemetry structure and ensure proper functionality across various scenarios. --- .../tests/api/test_telemetry_schema_drift.py | 4 +- codecarbon/cli/telemetry_cli.py | 14 +- codecarbon/core/telemetry/client.py | 84 ++++++++ .../collect.py} | 140 ++++++++----- codecarbon/core/telemetry/dispatcher.py | 89 +++++++++ .../schemas.py} | 0 codecarbon/core/telemetry/settings.py | 144 ++++++++++++++ codecarbon/core/telemetry_client.py | 47 ----- codecarbon/core/telemetry_settings.py | 185 ------------------ codecarbon/emissions_tracker.py | 42 ++-- codecarbon/telemetry.py | 152 -------------- tests/test_telemetry.py | 75 +++---- tests/test_telemetry_client.py | 54 +++-- tests/test_telemetry_collect.py | 118 ++++++----- tests/test_telemetry_config.py | 128 +++++------- tests/test_telemetry_settings.py | 135 +++++++------ 16 files changed, 685 insertions(+), 726 deletions(-) create mode 100644 codecarbon/core/telemetry/client.py rename codecarbon/core/{telemetry_collect.py => telemetry/collect.py} (71%) create mode 100644 codecarbon/core/telemetry/dispatcher.py rename codecarbon/core/{telemetry_schemas.py => telemetry/schemas.py} (100%) create mode 100644 codecarbon/core/telemetry/settings.py delete mode 100644 codecarbon/core/telemetry_client.py delete mode 100644 codecarbon/core/telemetry_settings.py delete mode 100644 codecarbon/telemetry.py diff --git a/carbonserver/tests/api/test_telemetry_schema_drift.py b/carbonserver/tests/api/test_telemetry_schema_drift.py index abc20d764..388e39963 100644 --- a/carbonserver/tests/api/test_telemetry_schema_drift.py +++ b/carbonserver/tests/api/test_telemetry_schema_drift.py @@ -7,7 +7,9 @@ from carbonserver.api.schemas_telemetry import TelemetryCreate as ServerTelemetryCreate REPO_ROOT = Path(__file__).resolve().parents[3] -CORE_TELEMETRY_SCHEMA_PATH = REPO_ROOT / "codecarbon" / "core" / "telemetry_schemas.py" +CORE_TELEMETRY_SCHEMA_PATH = ( + REPO_ROOT / "codecarbon" / "core" / "telemetry" / "schemas.py" +) def _load_core_telemetry_create(): diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py index 0cf7a8d77..df23a30a5 100644 --- a/codecarbon/cli/telemetry_cli.py +++ b/codecarbon/cli/telemetry_cli.py @@ -14,12 +14,11 @@ overwrite_local_config, ) from codecarbon.core.config import get_config_file_settings, get_hierarchical_config -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.core.telemetry_settings import ( +from codecarbon.core.telemetry import ( DEFAULT_TELEMETRY_LEVEL, - is_telemetry_level_explicit, + TelemetryLevel, + TelemetrySettings, parse_telemetry_level, - resolve_telemetry_level, ) telemetry_app = typer.Typer( @@ -142,11 +141,12 @@ def print_telemetry_status(config_path: Optional[Path] = None) -> None: external_conf = get_hierarchical_config() source_label = "merged ~/.codecarbon.config + ./.codecarbon.config" - level = resolve_telemetry_level( - file_settings, + settings = TelemetrySettings.resolve( + config_file_conf=file_settings, external_conf=external_conf or None, ) - explicit = is_telemetry_level_explicit(file_settings, external_conf=external_conf) + level = settings.level + explicit = settings.is_explicit stored = file_settings.get("telemetry_level") print(f"Config source: {source_label}") print(f"telemetry_level in file(s): {stored!r}") diff --git a/codecarbon/core/telemetry/client.py b/codecarbon/core/telemetry/client.py new file mode 100644 index 000000000..99a5a7ed2 --- /dev/null +++ b/codecarbon/core/telemetry/client.py @@ -0,0 +1,84 @@ +"""HTTP and API clients for product telemetry.""" + +from __future__ import annotations + +import dataclasses + +import requests + +from codecarbon.core.api_client import ApiClient +from codecarbon.core.telemetry.schemas import TelemetryCreate +from codecarbon.core.telemetry.settings import TelemetrySettings +from codecarbon.external.logger import logger +from codecarbon.output_methods.emissions_data import EmissionsData + + +def post_private(settings: TelemetrySettings, payload: dict) -> bool: + """POST a private telemetry payload to ``/telemetry``. + + Args: + settings: Resolved telemetry API settings. + payload: Telemetry fields dict. + + Returns: + True if the server accepted the payload (HTTP 201). + """ + headers = {"Content-Type": "application/json"} + if settings.api_key: + headers["x-api-token"] = settings.api_key + body = TelemetryCreate(**payload).model_dump(mode="json", exclude_none=True) + telemetry_url = f"{settings.api_url.rstrip('/')}/telemetry" + try: + response = requests.post( + url=telemetry_url, + json=body, + headers=headers, + timeout=2, + ) + except Exception: + logger.error("Telemetry request failed.", exc_info=True) + return False + if response.status_code == 201: + return True + if response.status_code == 404: + logger.warning( + "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", + telemetry_url, + ) + else: + logger.error( + "Telemetry API %s: %s", + response.status_code, + response.text, + ) + logger.debug("Telemetry request body: %s", body) + return False + + +def post_public_summary( + settings: TelemetrySettings, + conf: dict, + emissions: EmissionsData, +) -> bool: + """Send Tier 2 public run summary via ``ApiClient``. + + Args: + settings: Resolved telemetry API settings. + conf: Tracker configuration dict. + emissions: Run emissions data. + + Returns: + True if the run summary was posted successfully. + """ + try: + api = ApiClient( + endpoint_url=settings.api_url, + experiment_id=settings.experiment_id, + api_key=settings.api_key, + conf=conf, + create_run_automatically=True, + ) + return bool(api.add_emission(dataclasses.asdict(emissions))) + except Exception as error: + logger.error(f"Public run summary failed (non-critical): {error}") + return False diff --git a/codecarbon/core/telemetry_collect.py b/codecarbon/core/telemetry/collect.py similarity index 71% rename from codecarbon/core/telemetry_collect.py rename to codecarbon/core/telemetry/collect.py index db788ad3c..49f5263c3 100644 --- a/codecarbon/core/telemetry_collect.py +++ b/codecarbon/core/telemetry/collect.py @@ -6,12 +6,13 @@ import os import platform import sys +from dataclasses import dataclass from datetime import datetime, timezone from typing import Any, Optional from codecarbon.core.cloud import get_env_cloud_details from codecarbon.core.gpu import is_nvidia_system -from codecarbon.core.telemetry_schemas import PRIVATE_TELEMETRY_FIELDS, TelemetryLevel +from codecarbon.core.telemetry.schemas import PRIVATE_TELEMETRY_FIELDS, TelemetryLevel from codecarbon.output_methods.emissions_data import EmissionsData FRAMEWORK_PACKAGES = ( @@ -39,16 +40,64 @@ ("KUBERNETES_SERVICE_HOST", "kubernetes"), ) -OUTPUT_METHOD_ATTRS = ( - ("_save_to_file", "file"), - ("_save_to_api", "api"), - ("_save_to_logger", "logger"), - ("_emissions_endpoint", "http"), - ("_save_to_prometheus", "prometheus"), - ("_save_to_logfire", "logfire"), +OUTPUT_METHOD_FIELDS = ( + ("save_to_file", "file"), + ("save_to_api", "api"), + ("save_to_logger", "logger"), + ("emissions_endpoint", "http"), + ("save_to_prometheus", "prometheus"), + ("save_to_logfire", "logfire"), ) +@dataclass +class TelemetryContext: + """Snapshot of tracker state used to build a telemetry payload.""" + + conf: dict[str, Any] + emissions: EmissionsData + hardware: list[Any] + resource_tracker: Any + save_to_api: bool + save_to_file: bool + save_to_logger: bool + save_to_prometheus: bool + save_to_logfire: bool + emissions_endpoint: str | None + tasks: dict[str, Any] + measure_power_secs: float | None + is_offline: bool + + @classmethod + def from_tracker(cls, tracker: Any, emissions: EmissionsData) -> TelemetryContext: + """Build a context snapshot from an active emissions tracker. + + Args: + tracker: Active emissions tracker instance. + emissions: Run emissions data. + + Returns: + Context for ``build_payload``. + """ + from codecarbon.emissions_tracker import OfflineEmissionsTracker + + return cls( + conf=getattr(tracker, "_conf", {}), + emissions=emissions, + hardware=getattr(tracker, "_hardware", []) or [], + resource_tracker=getattr(tracker, "_resource_tracker", None), + save_to_api=bool(getattr(tracker, "_save_to_api", False)), + save_to_file=bool(getattr(tracker, "_save_to_file", False)), + save_to_logger=bool(getattr(tracker, "_save_to_logger", False)), + save_to_prometheus=bool(getattr(tracker, "_save_to_prometheus", False)), + save_to_logfire=bool(getattr(tracker, "_save_to_logfire", False)), + emissions_endpoint=getattr(tracker, "_emissions_endpoint", None), + tasks=getattr(tracker, "_tasks", {}) or {}, + measure_power_secs=getattr(tracker, "_measure_power_secs", None), + is_offline=isinstance(tracker, OfflineEmissionsTracker), + ) + + def _non_empty(value: Any) -> bool: return value not in (None, "", [], {}) @@ -106,21 +155,12 @@ def _detect_notebook_environment() -> Optional[str]: return None -def _detect_in_container() -> bool: - if os.path.exists("/.dockerenv"): - return True +def _container_info() -> tuple[bool, Optional[str]]: if os.environ.get("KUBERNETES_SERVICE_HOST"): - return True - return False - - -def _detect_container_runtime() -> Optional[str]: - runtime = _first_env_match(CONTAINER_RUNTIME_ENV) - if runtime: - return runtime + return True, "kubernetes" if os.path.exists("/.dockerenv"): - return "docker" - return None + return True, "docker" + return False, None def _detect_ide() -> Optional[str]: @@ -145,20 +185,19 @@ def _cudnn_version() -> Optional[str]: return None -def _collect_hardware_diagnostics(tracker: Any) -> dict[str, Any]: +def _collect_hardware_diagnostics(ctx: TelemetryContext) -> dict[str, Any]: from codecarbon.core import cpu hardware_tracked: list[str] = [] - for item in getattr(tracker, "_hardware", []) or []: + for item in ctx.hardware: try: hardware_tracked.append(item.description()) except Exception: pass - resource_tracker = getattr(tracker, "_resource_tracker", None) gpu_detection_method: Optional[str] = None - if resource_tracker is not None: - gpu_tracker = getattr(resource_tracker, "gpu_tracker", None) + if ctx.resource_tracker is not None: + gpu_tracker = getattr(ctx.resource_tracker, "gpu_tracker", None) if gpu_tracker and gpu_tracker != "Unspecified": gpu_detection_method = gpu_tracker @@ -166,20 +205,17 @@ def _collect_hardware_diagnostics(tracker: Any) -> dict[str, Any]: if platform.system() == "Linux": rapl_available = cpu.is_rapl_available() - save_to_api = bool(getattr(tracker, "_save_to_api", False)) return { "hardware_tracked": hardware_tracked or None, "hardware_detection_success": bool(hardware_tracked), "rapl_available": rapl_available, "gpu_detection_method": gpu_detection_method, - "api_mode": "online" if save_to_api else "offline", + "api_mode": "online" if ctx.save_to_api else "offline", } -def _detect_integration_surface(tracker: Any) -> str: - from codecarbon.emissions_tracker import OfflineEmissionsTracker - - if isinstance(tracker, OfflineEmissionsTracker): +def _detect_integration_surface(ctx: TelemetryContext) -> str: + if ctx.is_offline: return "offline_tracker" argv = " ".join(sys.argv) if "codecarbon" in argv and "monitor" in argv: @@ -187,15 +223,15 @@ def _detect_integration_surface(tracker: Any) -> str: return "library" -def _collect_output_methods(tracker: Any) -> list[str]: +def _collect_output_methods(ctx: TelemetryContext) -> list[str]: methods: list[str] = [] - for attr, name in OUTPUT_METHOD_ATTRS: - value = getattr(tracker, attr, False) - if attr == "_emissions_endpoint": + for field_name, label in OUTPUT_METHOD_FIELDS: + value = getattr(ctx, field_name) + if field_name == "emissions_endpoint": if value: - methods.append(name) + methods.append(label) elif value: - methods.append(name) + methods.append(label) return methods @@ -245,22 +281,21 @@ def _gpu_static_fields() -> dict[str, Any]: return fields -def build_telemetry_payload( - tracker: Any, - emissions: EmissionsData, +def build_payload( + ctx: TelemetryContext, level: TelemetryLevel = TelemetryLevel.minimal, ) -> dict[str, Any]: """Build a private telemetry payload dict for ``POST /telemetry``. Args: - tracker: Active emissions tracker. - emissions: Run emissions data. + ctx: Tracker snapshot from ``TelemetryContext.from_tracker``. level: Resolved ``TelemetryLevel`` (``minimal`` or ``extensive``). Returns: Payload dict for ``TelemetryCreate``. """ - conf = getattr(tracker, "_conf", {}) + emissions = ctx.emissions + conf = ctx.conf raw_provider, raw_region = _raw_cloud_provider_and_region() on_cloud = emissions.on_cloud == "Y" cloud_provider = emissions.cloud_provider or raw_provider @@ -269,7 +304,8 @@ def build_telemetry_payload( if on_cloud and cloud_region: region = region or cloud_region - integration_surface = _detect_integration_surface(tracker) + integration_surface = _detect_integration_surface(ctx) + in_container, container_runtime = _container_info() gpu_fields = _gpu_static_fields() raw: dict[str, Any] = { @@ -297,12 +333,12 @@ def build_telemetry_payload( "tracking_mode": conf.get("tracking_mode"), "integration_surface": integration_surface, "offline_mode": integration_surface == "offline_tracker", - "output_methods": _collect_output_methods(tracker), - "save_to_api_enabled": bool(getattr(tracker, "_save_to_api", False)), - "task_tracking_used": bool(getattr(tracker, "_tasks", {})), - "measure_power_interval_secs": getattr(tracker, "_measure_power_secs", None), - "in_container": _detect_in_container(), - "container_runtime": _detect_container_runtime(), + "output_methods": _collect_output_methods(ctx), + "save_to_api_enabled": ctx.save_to_api, + "task_tracking_used": bool(ctx.tasks), + "measure_power_interval_secs": ctx.measure_power_secs, + "in_container": in_container, + "container_runtime": container_runtime, "ci_environment": _first_env_match(CI_ENVIRONMENTS), "notebook_environment": _detect_notebook_environment(), "ide_used": _detect_ide(), @@ -319,7 +355,7 @@ def build_telemetry_payload( "ram_utilization_avg": emissions.ram_utilization_percent, "telemetry_level": level.value, **_collect_framework_fields(), - **_collect_hardware_diagnostics(tracker), + **_collect_hardware_diagnostics(ctx), **gpu_fields, } diff --git a/codecarbon/core/telemetry/dispatcher.py b/codecarbon/core/telemetry/dispatcher.py new file mode 100644 index 000000000..24174d94f --- /dev/null +++ b/codecarbon/core/telemetry/dispatcher.py @@ -0,0 +1,89 @@ +"""Per-tracker telemetry dispatcher.""" + +from __future__ import annotations + +from typing import Any, ClassVar + +from codecarbon.core.telemetry.client import post_private, post_public_summary +from codecarbon.core.telemetry.collect import TelemetryContext, build_payload +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.core.telemetry.settings import TelemetrySettings +from codecarbon.external.logger import logger +from codecarbon.output_methods.emissions_data import EmissionsData + +TELEMETRY_NOT_CONFIGURED_MESSAGE = ( + "CodeCarbon telemetry_level was not set explicitly; using default %r. " + "Tier 1 private telemetry (per run at stop) will be sent. Set telemetry_level " + "in .codecarbon.config, set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... " + "to EmissionsTracker / OfflineEmissionsTracker, or run " + "codecarbon telemetry set ." +) + + +class Telemetry: + """Per-tracker telemetry dispatcher.""" + + _default_warning_shown: ClassVar[bool] = False + + def __init__(self, settings: TelemetrySettings) -> None: + self.settings = settings + + @classmethod + def from_tracker(cls, tracker: Any) -> Telemetry: + """Build a dispatcher from tracker config state. + + Args: + tracker: Active emissions tracker with ``_config_file_conf``, + ``_external_conf``, and optional ``_telemetry_override``. + + Returns: + Configured ``Telemetry`` instance. + """ + return cls( + TelemetrySettings.resolve( + config_file_conf=tracker._config_file_conf, + external_conf=tracker._external_conf, + override=getattr(tracker, "_telemetry_override", None), + ) + ) + + def warn_if_implicit(self) -> None: + """Log a one-time warning when telemetry tier was not set explicitly.""" + if self.settings.is_explicit or Telemetry._default_warning_shown: + return + logger.warning( + TELEMETRY_NOT_CONFIGURED_MESSAGE, + self.settings.level.value, + ) + Telemetry._default_warning_shown = True + + def send_at_stop(self, tracker: Any, emissions: EmissionsData) -> None: + """Send product telemetry for the resolved tier at tracker ``stop()``. + + Tier 1 (``minimal``): private ``POST /telemetry`` only. + Tier 2 (``extensive``): Tier 1 plus public run summary. + + Args: + tracker: Active emissions tracker instance. + emissions: Total emissions from ``_prepare_emissions_data()``. + """ + if self.settings.level == TelemetryLevel.disabled: + return + if emissions.duration is not None and emissions.duration < 1: + logger.debug("Telemetry not sent: run shorter than 1 second.") + return + ctx = TelemetryContext.from_tracker(tracker, emissions) + payload = build_payload(ctx, level=self.settings.level) + try: + post_private(self.settings, payload) + except Exception as error: + logger.error(f"Private telemetry failed (non-critical): {error}") + if self.settings.level == TelemetryLevel.extensive: + try: + post_public_summary( + self.settings, + getattr(tracker, "_conf", {}), + emissions, + ) + except Exception as error: + logger.error(f"Public run summary failed (non-critical): {error}") diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry/schemas.py similarity index 100% rename from codecarbon/core/telemetry_schemas.py rename to codecarbon/core/telemetry/schemas.py diff --git a/codecarbon/core/telemetry/settings.py b/codecarbon/core/telemetry/settings.py new file mode 100644 index 000000000..8c2c28cb7 --- /dev/null +++ b/codecarbon/core/telemetry/settings.py @@ -0,0 +1,144 @@ +"""Resolve telemetry tier and API settings from config and environment.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, Literal + +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.external.logger import logger + +DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" +DEFAULT_TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +DEFAULT_TELEMETRY_EXPERIMENT_ID = "d2d69403-1373-42b4-a2c1-09589aed4801" +DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal + +TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" + +TelemetryLevelSource = Literal["override", "external", "file", "default"] + + +def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: + """Parse a telemetry tier from a string or enum value. + + Args: + raw: Tier name or ``TelemetryLevel`` member. + + Returns: + Parsed ``TelemetryLevel``. + + Raises: + ValueError: If ``raw`` is not a valid tier name. + """ + if isinstance(raw, TelemetryLevel): + return raw + try: + return TelemetryLevel(str(raw).lower()) + except ValueError as error: + raise ValueError( + f"Invalid telemetry_level {raw!r}. Choose: disabled, minimal, or extensive." + ) from error + + +@dataclass(frozen=True) +class TelemetrySettings: + """Resolved telemetry tier and API connection settings.""" + + level: TelemetryLevel + source: TelemetryLevelSource + api_url: str + api_key: str + experiment_id: str + + @property + def is_explicit(self) -> bool: + """Return whether the user explicitly chose a telemetry tier.""" + return self.source != "default" + + @classmethod + def resolve( + cls, + *, + config_file_conf: dict[str, Any] | None = None, + external_conf: dict[str, Any] | None = None, + override: str | TelemetryLevel | None = None, + ) -> TelemetrySettings: + """Resolve telemetry tier and API settings. + + Precedence for tier: + + 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or CLI + 2. ``external_conf`` — merged ``.codecarbon.config`` and env + 3. ``config_file_conf`` — file-only settings + 4. Default: ``minimal`` + + Args: + config_file_conf: Settings from ``get_config_file_settings()``. + external_conf: Merged settings from ``get_hierarchical_config()``. + override: Optional tier from tracker or CLI. + + Returns: + Resolved settings bundle. + """ + merged = external_conf or {} + if override is not None: + raw = override + source: TelemetryLevelSource = "override" + elif merged.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: + raw = merged[TELEMETRY_LEVEL_CONFIG_KEY] + source = "external" + elif config_file_conf is not None and config_file_conf.get( + TELEMETRY_LEVEL_CONFIG_KEY + ) is not None: + raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] + source = "file" + else: + return cls( + level=DEFAULT_TELEMETRY_LEVEL, + source="default", + api_url=cls._resolve_api_url(merged), + api_key=cls._resolve_api_key(merged), + experiment_id=cls._resolve_experiment_id(merged), + ) + try: + level = parse_telemetry_level(raw) + except ValueError: + logger.error( + "Invalid telemetry_level %r; falling back to %r", + raw, + DEFAULT_TELEMETRY_LEVEL.value, + ) + level = DEFAULT_TELEMETRY_LEVEL + return cls( + level=level, + source=source, + api_url=cls._resolve_api_url(merged), + api_key=cls._resolve_api_key(merged), + experiment_id=cls._resolve_experiment_id(merged), + ) + + @staticmethod + def _resolve_api_url(external_conf: dict[str, Any]) -> str: + url = ( + external_conf.get("telemetry_api_url") + or external_conf.get("api_endpoint") + or os.environ.get("CODECARBON_TELEMETRY_API_URL") + ) + return (url or DEFAULT_TELEMETRY_API_URL).rstrip("/") + + @staticmethod + def _resolve_api_key(external_conf: dict[str, Any]) -> str: + key = ( + external_conf.get("telemetry_api_key") + or external_conf.get("api_key") + or os.environ.get("CODECARBON_TELEMETRY_API_KEY") + ) + return key or DEFAULT_TELEMETRY_API_KEY + + @staticmethod + def _resolve_experiment_id(external_conf: dict[str, Any]) -> str: + experiment_id = external_conf.get("telemetry_experiment_id") or os.environ.get( + "CODECARBON_TELEMETRY_EXPERIMENT_ID" + ) + return experiment_id or DEFAULT_TELEMETRY_EXPERIMENT_ID diff --git a/codecarbon/core/telemetry_client.py b/codecarbon/core/telemetry_client.py deleted file mode 100644 index fbfe76e3d..000000000 --- a/codecarbon/core/telemetry_client.py +++ /dev/null @@ -1,47 +0,0 @@ -import requests - -from codecarbon.core.telemetry_schemas import TelemetryCreate -from codecarbon.external.logger import logger - - -def post_private_telemetry(url: str, payload: dict, api_key: str | None) -> bool: - """POST a private telemetry payload to ``/telemetry``. - - Args: - url: API base URL. - payload: Telemetry fields dict. - api_key: Optional API token. - - Returns: - True if the server accepted the payload (HTTP 201). - """ - headers = {"Content-Type": "application/json"} - if api_key: - headers["x-api-token"] = api_key - body = TelemetryCreate(**payload).model_dump(mode="json", exclude_none=True) - telemetry_url = f"{url.rstrip('/')}/telemetry" - try: - response = requests.post( - url=telemetry_url, - json=body, - headers=headers, - timeout=2, - ) - except Exception: - logger.error("Telemetry request failed.", exc_info=True) - return False - if response.status_code == 201: - return True - if response.status_code == 404: - logger.warning( - "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", - telemetry_url, - ) - else: - logger.error( - "Telemetry API %s: %s", - response.status_code, - response.text, - ) - logger.debug("Telemetry request body: %s", body) - return False diff --git a/codecarbon/core/telemetry_settings.py b/codecarbon/core/telemetry_settings.py deleted file mode 100644 index a0805ed00..000000000 --- a/codecarbon/core/telemetry_settings.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Resolve telemetry tier and API settings from config and environment.""" - -import os -from typing import Any, Literal - -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.external.logger import logger - -DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" -DEFAULT_TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" -DEFAULT_TELEMETRY_EXPERIMENT_ID = "d2d69403-1373-42b4-a2c1-09589aed4801" -DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal - -TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" - -TelemetryLevelSource = Literal["override", "external", "file", "default"] - - -def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: - """Parse a telemetry tier from a string or enum value. - - Args: - raw: Tier name or ``TelemetryLevel`` member. - - Returns: - Parsed ``TelemetryLevel``. - - Raises: - ValueError: If ``raw`` is not a valid tier name. - """ - if isinstance(raw, TelemetryLevel): - return raw - try: - return TelemetryLevel(str(raw).lower()) - except ValueError as error: - raise ValueError( - f"Invalid telemetry_level {raw!r}. Choose: disabled, minimal, or extensive." - ) from error - - -def resolve_telemetry_level_and_source( - config_file_conf: dict[str, Any] | None = None, - *, - override: str | TelemetryLevel | None = None, - external_conf: dict[str, Any] | None = None, -) -> tuple[TelemetryLevel, TelemetryLevelSource]: - """Resolve the active telemetry tier and where it came from. - - Precedence: - - 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or CLI - 2. ``external_conf`` — merged ``.codecarbon.config`` and ``CODECARBON_*`` env - 3. ``config_file_conf`` — file-only settings when ``external_conf`` is omitted - 4. Default: ``minimal`` (Tier 1) - - Args: - config_file_conf: Settings from ``get_config_file_settings()`` (optional). - override: Optional tier from tracker or CLI. - external_conf: Merged settings from ``get_hierarchical_config()`` (optional). - - Returns: - Resolved tier and its source label. - """ - if override is not None: - raw = override - source: TelemetryLevelSource = "override" - elif external_conf is not None and external_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: - raw = external_conf[TELEMETRY_LEVEL_CONFIG_KEY] - source = "external" - elif config_file_conf is not None and config_file_conf.get( - TELEMETRY_LEVEL_CONFIG_KEY - ) is not None: - raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] - source = "file" - else: - return DEFAULT_TELEMETRY_LEVEL, "default" - try: - return parse_telemetry_level(raw), source - except ValueError: - logger.error( - "Invalid telemetry_level %r; falling back to %r", - raw, - DEFAULT_TELEMETRY_LEVEL.value, - ) - return DEFAULT_TELEMETRY_LEVEL, source - - -def resolve_telemetry_level( - config_file_conf: dict[str, Any] | None = None, - *, - override: str | TelemetryLevel | None = None, - external_conf: dict[str, Any] | None = None, -) -> TelemetryLevel: - """Resolve the active telemetry tier. - - Args: - config_file_conf: Settings from ``get_config_file_settings()`` (optional). - override: Optional tier from tracker or CLI. - external_conf: Merged settings from ``get_hierarchical_config()`` (optional). - - Returns: - Resolved ``TelemetryLevel``. - """ - return resolve_telemetry_level_and_source( - config_file_conf, - override=override, - external_conf=external_conf, - )[0] - - -def is_telemetry_level_explicit( - config_file_conf: dict[str, Any], - *, - override: str | TelemetryLevel | None = None, - external_conf: dict[str, Any] | None = None, -) -> bool: - """Return whether the user explicitly chose a telemetry tier. - - Args: - config_file_conf: Settings from ``get_config_file_settings()`` (no env overlay). - override: Value passed to ``EmissionsTracker(telemetry_level=...)``. - external_conf: Merged config from file and environment. - - Returns: - True if any explicit source is set. - """ - _, source = resolve_telemetry_level_and_source( - config_file_conf, - override=override, - external_conf=external_conf, - ) - return source != "default" - - -def get_telemetry_api_url( - external_conf: dict[str, Any], - default: str = DEFAULT_TELEMETRY_API_URL, -) -> str: - """Return telemetry API base URL from config or environment. - - Args: - external_conf: Merged config from file and environment. - default: URL used when unset. - - Returns: - API base URL without trailing slash. - """ - url = ( - external_conf.get("telemetry_api_url") - or external_conf.get("api_endpoint") - or os.environ.get("CODECARBON_TELEMETRY_API_URL") - ) - return (url or default).rstrip("/") - - -def get_telemetry_api_key(external_conf: dict[str, Any]) -> str: - """Return telemetry API token from config, environment, or public default. - - Args: - external_conf: Merged config from file and environment. - - Returns: - API token string. - """ - key = ( - external_conf.get("telemetry_api_key") - or external_conf.get("api_key") - or os.environ.get("CODECARBON_TELEMETRY_API_KEY") - ) - return key or DEFAULT_TELEMETRY_API_KEY - - -def get_telemetry_experiment_id(external_conf: dict[str, Any]) -> str: - """Return telemetry experiment id from config, environment, or public default. - - Args: - external_conf: Merged config from file and environment. - - Returns: - Experiment UUID string (legacy leaderboard / API helpers). - """ - experiment_id = external_conf.get("telemetry_experiment_id") or os.environ.get( - "CODECARBON_TELEMETRY_EXPERIMENT_ID" - ) - return experiment_id or DEFAULT_TELEMETRY_EXPERIMENT_ID diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index fa4eeb166..30cb4b571 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -22,12 +22,7 @@ get_hierarchical_config, normalize_gpu_ids, ) -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.core.telemetry_settings import resolve_telemetry_level -from codecarbon.telemetry import ( - send_product_telemetry_at_stop, - warn_if_telemetry_not_configured, -) +from codecarbon.core.telemetry import Telemetry, TelemetrySettings from codecarbon.core.emissions import Emissions from codecarbon.core.resource_tracker import ResourceTracker from codecarbon.core.units import Energy, Power, Time, Water @@ -289,11 +284,15 @@ def __init__( self._external_conf = get_hierarchical_config() self._config_file_conf = get_config_file_settings() - telemetry_override = None if telemetry_level is _sentinel else telemetry_level - self._telemetry_level = resolve_telemetry_level( - self._config_file_conf, - override=telemetry_override, - external_conf=self._external_conf, + self._telemetry_override = ( + None if telemetry_level is _sentinel else telemetry_level + ) + self._telemetry = Telemetry( + TelemetrySettings.resolve( + config_file_conf=self._config_file_conf, + external_conf=self._external_conf, + override=self._telemetry_override, + ) ) self._set_from_conf(allow_multiple_runs, "allow_multiple_runs", True, bool) if self._allow_multiple_runs: @@ -468,25 +467,12 @@ def __init__( self._data_source, self._electricitymaps_api_token ) - self._apply_init_telemetry(telemetry_override) + self._telemetry.warn_if_implicit() self._init_output_methods(api_key=self._api_key) @suppress(Exception) - def _apply_init_telemetry(self, telemetry_override: str | None) -> None: - warn_if_telemetry_not_configured( - self._config_file_conf, - self._telemetry_level, - override=telemetry_override, - external_conf=self._external_conf, - ) - @suppress(Exception) - def _send_product_telemetry_at_stop(self, emissions_data: EmissionsData) -> None: - send_product_telemetry_at_stop( - self, - emissions_data, - self._telemetry_level, - external_conf=self._external_conf, - ) + def _send_telemetry_at_stop(self, emissions_data: EmissionsData) -> None: + self._telemetry.send_at_stop(self, emissions_data) def _init_output_methods(self, *, api_key: str = None): """ @@ -789,7 +775,7 @@ def stop(self) -> Optional[float]: emissions_data = self._prepare_emissions_data() emissions_data_delta = self._compute_emissions_delta(emissions_data) - self._send_product_telemetry_at_stop(emissions_data) + self._send_telemetry_at_stop(emissions_data) self._persist_data( total_emissions=emissions_data, diff --git a/codecarbon/telemetry.py b/codecarbon/telemetry.py deleted file mode 100644 index eeec82434..000000000 --- a/codecarbon/telemetry.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Product telemetry sent at tracker stop (Tier 1 / Tier 2).""" - -import dataclasses -from typing import Any - -from codecarbon.core.api_client import ApiClient -from codecarbon.core.telemetry_client import post_private_telemetry -from codecarbon.core.telemetry_collect import build_telemetry_payload -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.core.telemetry_settings import ( - get_telemetry_api_key, - get_telemetry_api_url, - get_telemetry_experiment_id, - is_telemetry_level_explicit, -) -from codecarbon.external.logger import logger -from codecarbon.output_methods.emissions_data import EmissionsData - -TELEMETRY_NOT_CONFIGURED_MESSAGE = ( - "CodeCarbon telemetry_level was not set explicitly; using default %r. " - "Tier 1 private telemetry (per run at stop) will be sent. Set telemetry_level " - "in .codecarbon.config, set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... " - "to EmissionsTracker / OfflineEmissionsTracker, or run " - "codecarbon telemetry set ." -) - -_telemetry_default_warning_shown = False - - -def reset_telemetry_warning() -> None: - """Clear the one-shot default-tier warning (for tests).""" - global _telemetry_default_warning_shown - _telemetry_default_warning_shown = False - - -def warn_if_telemetry_not_configured( - config_file_conf: dict[str, Any], - active_level: TelemetryLevel, - *, - override: str | TelemetryLevel | None = None, - external_conf: dict[str, Any] | None = None, -) -> None: - """Log a one-time warning when telemetry tier was not set explicitly. - - Args: - config_file_conf: File-only settings from ``get_config_file_settings()``. - active_level: Resolved tier in use for this tracker. - override: Optional ``telemetry_level`` tracker argument. - external_conf: Merged file/env settings for explicit-env detection. - """ - global _telemetry_default_warning_shown - if is_telemetry_level_explicit( - config_file_conf, override=override, external_conf=external_conf - ): - return - if _telemetry_default_warning_shown: - return - logger.warning(TELEMETRY_NOT_CONFIGURED_MESSAGE, active_level.value) - _telemetry_default_warning_shown = True - - -def send_private_telemetry_at_stop( - tracker: Any, - emissions: EmissionsData, - external_conf: dict[str, Any] | None = None, - level: TelemetryLevel = TelemetryLevel.minimal, -) -> bool: - """Send Tier 1 private telemetry via ``POST /telemetry``. - - Runs shorter than one second are skipped by ``send_product_telemetry_at_stop``, - not here; direct callers may still post for sub-second runs. - - Args: - tracker: Active emissions tracker instance. - emissions: Total emissions from ``_prepare_emissions_data()``. - external_conf: Merged config for telemetry API URL and key resolution. - level: Resolved ``TelemetryLevel`` for the ``telemetry_level`` field. - - Returns: - True if the private telemetry POST was accepted, False otherwise. - """ - settings_conf = external_conf or {} - try: - payload = build_telemetry_payload(tracker, emissions, level=level) - return post_private_telemetry( - get_telemetry_api_url(settings_conf), - payload, - get_telemetry_api_key(settings_conf), - ) - except Exception as error: - logger.error(f"Private telemetry failed (non-critical): {error}") - return False - - -def send_public_run_summary_at_stop( - tracker: Any, - emissions: EmissionsData, - external_conf: dict[str, Any] | None = None, -) -> bool: - """Send Tier 2 public run summary to the shared telemetry experiment via ``ApiClient``. - - Args: - tracker: Active emissions tracker instance. - emissions: Total emissions from ``_prepare_emissions_data()``. - external_conf: Merged config for API URL, key, and experiment resolution. - - Returns: - True if the run summary was posted successfully, False otherwise. - """ - settings_conf = external_conf or {} - conf = getattr(tracker, "_conf", {}) - try: - api = ApiClient( - endpoint_url=get_telemetry_api_url(settings_conf), - experiment_id=get_telemetry_experiment_id(settings_conf), - api_key=get_telemetry_api_key(settings_conf), - conf=conf, - create_run_automatically=True, - ) - return bool(api.add_emission(dataclasses.asdict(emissions))) - except Exception as error: - logger.error(f"Public run summary failed (non-critical): {error}") - return False - - -def send_product_telemetry_at_stop( - tracker: Any, - emissions: EmissionsData, - level: TelemetryLevel, - external_conf: dict[str, Any] | None = None, -) -> None: - """Send product telemetry for the resolved tier at tracker ``stop()``. - - Tier 1 (``minimal``): private ``POST /telemetry`` only. - Tier 2 (``extensive``): same private ``POST /telemetry`` plus public run summary. - - Args: - tracker: Active emissions tracker instance. - emissions: Total emissions from ``_prepare_emissions_data()``. - level: Resolved ``TelemetryLevel``. - external_conf: Merged config for API settings. - """ - if level == TelemetryLevel.disabled: - return - if emissions.duration is not None and emissions.duration < 1: - logger.debug("Telemetry not sent: run shorter than 1 second.") - return - settings = external_conf or {} - if level in (TelemetryLevel.minimal, TelemetryLevel.extensive): - send_private_telemetry_at_stop(tracker, emissions, settings, level=level) - if level == TelemetryLevel.extensive: - send_public_run_summary_at_stop(tracker, emissions, settings) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 757283d7a..dd157fb93 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -3,13 +3,9 @@ import unittest from unittest.mock import ANY, MagicMock, patch -from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry import Telemetry from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.output_methods.emissions_data import EmissionsData -from codecarbon.telemetry import ( - send_private_telemetry_at_stop, - send_public_run_summary_at_stop, -) from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": @@ -62,46 +58,57 @@ def _emissions(self) -> EmissionsData: def test_tier1_posts_private_telemetry(self): tracker = MagicMock() + tracker._config_file_conf = {} + tracker._external_conf = {} + tracker._telemetry_override = None + tracker._conf = {"os": "Linux", "tracking_mode": "machine"} + tracker._hardware = [] + tracker._resource_tracker = None + tracker._save_to_api = False + tracker._save_to_file = False + tracker._save_to_logger = False + tracker._emissions_endpoint = None + tracker._save_to_prometheus = False + tracker._save_to_logfire = False + tracker._tasks = {} + tracker._measure_power_secs = 15 emissions = self._emissions() + telemetry = Telemetry.from_tracker(tracker) with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: - result = send_private_telemetry_at_stop( - tracker, emissions, external_conf={} - ) - self.assertTrue(result) + telemetry.send_at_stop(tracker, emissions) mock_post.assert_called_once() payload = mock_post.call_args[0][1] self.assertEqual(payload["telemetry_level"], "minimal") self.assertEqual(payload["total_emissions_kg"], 0.001) def test_tier1_skips_short_duration_at_dispatcher(self): - from codecarbon.telemetry import send_product_telemetry_at_stop - tracker = MagicMock() + tracker._config_file_conf = {} + tracker._external_conf = {} + tracker._telemetry_override = None emissions = self._emissions() emissions.duration = 0.5 - with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: - send_product_telemetry_at_stop( - tracker, - emissions, - TelemetryLevel.minimal, - external_conf={}, - ) + telemetry = Telemetry.from_tracker(tracker) + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: + telemetry.send_at_stop(tracker, emissions) mock_post.assert_not_called() def test_tier2_uses_api_client(self): tracker = MagicMock() tracker._conf = {"os": "Linux"} + tracker._config_file_conf = {"telemetry_level": "extensive"} + tracker._external_conf = {} + tracker._telemetry_override = None emissions = self._emissions() - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: - mock_api = MagicMock() - mock_api.add_emission.return_value = True - mock_api_cls.return_value = mock_api - result = send_public_run_summary_at_stop( - tracker, emissions, external_conf={} - ) - self.assertTrue(result) + telemetry = Telemetry.from_tracker(tracker) + with patch("codecarbon.core.telemetry.dispatcher.post_private", return_value=True): + with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + telemetry.send_at_stop(tracker, emissions) mock_api_cls.assert_called_once() mock_api.add_emission.assert_called_once() mock_api_cls.assert_called_with( @@ -118,11 +125,13 @@ class TestTrackerTelemetry(unittest.TestCase): def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory() self.patcher = None + Telemetry._default_warning_shown = False def tearDown(self) -> None: if self.patcher: self.patcher.stop() self.temp_dir.cleanup() + Telemetry._default_warning_shown = False def _start_config_mock(self, conf: str) -> None: self.patcher = patch( @@ -132,7 +141,7 @@ def _start_config_mock(self, conf: str) -> None: def test_emissions_tracker_does_not_send_telemetry_on_init(self, mock_cli_setup): self._start_config_mock(minimal_conf) - with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) mock_post.assert_not_called() @@ -143,7 +152,7 @@ def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( self._start_config_mock(minimal_conf) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( @@ -160,7 +169,7 @@ def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): self._start_config_mock(disabled_conf) - with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -175,9 +184,9 @@ def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): self._start_config_mock(extensive_conf) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api @@ -197,7 +206,7 @@ def test_offline_tracker_sends_on_stop(self, mock_cli_setup): self._start_config_mock(minimal_conf) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: tracker = OfflineEmissionsTracker( country_iso_code="CAN", diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 7d8b4be64..8b0128d2a 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -4,12 +4,20 @@ import requests_mock from pydantic import ValidationError -from codecarbon.core.telemetry_client import post_private_telemetry -from codecarbon.core.telemetry_schemas import TelemetryCreate +from codecarbon.core.telemetry import TelemetrySettings, post_private -class TestPostPrivateTelemetry(unittest.TestCase): - def test_post_private_telemetry_sends_validated_payload(self): +class TestPostPrivate(unittest.TestCase): + def _settings(self, api_url: str = "http://test.com", api_key: str | None = None): + return TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url=api_url, + api_key=api_key or TelemetrySettings.resolve().api_key, + experiment_id=TelemetrySettings.resolve().experiment_id, + ) + + def test_post_private_sends_validated_payload(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -22,7 +30,7 @@ def test_post_private_telemetry_sends_validated_payload(self): json="f52fe339-164d-4c2b-a8c0-f562dfce066d", status_code=201, ) - result = post_private_telemetry("http://test.com", telemetry, None) + result = post_private(self._settings(), telemetry) self.assertTrue(result) self.assertEqual(m.call_count, 1) @@ -34,19 +42,18 @@ def test_post_private_telemetry_sends_validated_payload(self): }, ) - def test_post_private_telemetry_rejects_invalid_payload(self): + def test_post_private_rejects_invalid_payload(self): with self.assertRaises(ValidationError): - post_private_telemetry( - "http://test.com", + post_private( + self._settings(), { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", "unknown_field": "value", }, - None, ) - def test_post_private_telemetry_logs_warning_on_404(self): + def test_post_private_logs_warning_on_404(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -57,32 +64,43 @@ def test_post_private_telemetry_logs_warning_on_404(self): text='{"detail":"Not Found"}', status_code=404, ) - with patch("codecarbon.core.telemetry_client.logger") as mock_logger: - result = post_private_telemetry("http://test.com", telemetry, None) + with patch("codecarbon.core.telemetry.client.logger") as mock_logger: + result = post_private(self._settings(), telemetry) self.assertFalse(result) mock_logger.warning.assert_called_once() - def test_post_private_telemetry_sends_api_key_header_when_configured(self): + def test_post_private_sends_api_key_header_when_configured(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", } + settings = TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url="http://test.com", + api_key="cpt_test_key", + experiment_id=TelemetrySettings.resolve().experiment_id, + ) with requests_mock.Mocker() as m: m.post( "http://test.com/telemetry", json="telemetry-id", status_code=201, ) - post_private_telemetry("http://test.com", telemetry, "cpt_test_key") + post_private(settings, telemetry) self.assertEqual(m.last_request.headers["x-api-token"], "cpt_test_key") - def test_post_private_telemetry_returns_false_on_request_error(self): + def test_post_private_returns_false_on_request_error(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", } - with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: + with patch("codecarbon.core.telemetry.client.requests.post") as mock_post: mock_post.side_effect = ConnectionError("network down") - with patch("codecarbon.core.telemetry_client.logger"): - result = post_private_telemetry("http://test.com", telemetry, None) + with patch("codecarbon.core.telemetry.client.logger"): + result = post_private(self._settings(), telemetry) self.assertFalse(result) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py index be770e460..0a5f4f379 100644 --- a/tests/test_telemetry_collect.py +++ b/tests/test_telemetry_collect.py @@ -1,8 +1,7 @@ import unittest from unittest.mock import MagicMock, patch -from codecarbon.core.telemetry_collect import build_telemetry_payload -from codecarbon.core.telemetry_schemas import TelemetryLevel +from codecarbon.core.telemetry import TelemetryContext, TelemetryLevel, build_payload from codecarbon.output_methods.emissions_data import EmissionsData @@ -44,33 +43,55 @@ def _sample_emissions(**overrides): return EmissionsData(**base) -class TestTelemetryCollect(unittest.TestCase): - def test_build_telemetry_payload_includes_run_and_framework_flags(self): - tracker = MagicMock() - tracker._conf = { - "os": "Linux", - "codecarbon_version": "3.0", - "cpu_count": 4, - "tracking_mode": "machine", - } - tracker._geo = None - tracker._save_to_file = True - tracker._save_to_api = False - tracker._save_to_logger = False - tracker._emissions_endpoint = None - tracker._save_to_prometheus = False - tracker._save_to_logfire = False - tracker._tasks = {} - tracker._measure_power_secs = 15 - tracker._hardware = [] - tracker._resource_tracker = None +def _tracker_context(**overrides) -> TelemetryContext: + tracker = MagicMock() + tracker._conf = overrides.pop("conf", {"codecarbon_version": "3.0"}) + tracker._hardware = overrides.pop("hardware", []) + tracker._resource_tracker = overrides.pop("resource_tracker", None) + tracker._save_to_file = overrides.pop("save_to_file", False) + tracker._save_to_api = overrides.pop("save_to_api", False) + tracker._save_to_logger = overrides.pop("save_to_logger", False) + tracker._emissions_endpoint = overrides.pop("emissions_endpoint", None) + tracker._save_to_prometheus = overrides.pop("save_to_prometheus", False) + tracker._save_to_logfire = overrides.pop("save_to_logfire", False) + tracker._tasks = overrides.pop("tasks", {}) + tracker._measure_power_secs = overrides.pop("measure_power_secs", 15) + tracker._is_offline = overrides.pop("is_offline", False) + emissions = overrides.pop("emissions", _sample_emissions()) + ctx = TelemetryContext( + conf=tracker._conf, + emissions=emissions, + hardware=tracker._hardware, + resource_tracker=tracker._resource_tracker, + save_to_api=tracker._save_to_api, + save_to_file=tracker._save_to_file, + save_to_logger=tracker._save_to_logger, + save_to_prometheus=tracker._save_to_prometheus, + save_to_logfire=tracker._save_to_logfire, + emissions_endpoint=tracker._emissions_endpoint, + tasks=tracker._tasks, + measure_power_secs=tracker._measure_power_secs, + is_offline=tracker._is_offline, + ) + return ctx + - emissions = _sample_emissions() +class TestTelemetryCollect(unittest.TestCase): + def test_build_payload_includes_run_and_framework_flags(self): + ctx = _tracker_context( + conf={ + "os": "Linux", + "codecarbon_version": "3.0", + "cpu_count": 4, + "tracking_mode": "machine", + }, + save_to_file=True, + ) with patch( - "codecarbon.core.telemetry_collect._package_installed", + "codecarbon.core.telemetry.collect._package_installed", side_effect=lambda name: name == "torch", ): - payload = build_telemetry_payload(tracker, emissions) + payload = build_payload(ctx) self.assertEqual(payload["telemetry_level"], "minimal") self.assertEqual(payload["total_emissions_kg"], 0.5) @@ -78,48 +99,23 @@ def test_build_telemetry_payload_includes_run_and_framework_flags(self): self.assertTrue(payload["has_torch"]) self.assertIn("file", payload["output_methods"]) - def test_build_telemetry_payload_omits_framework_versions(self): - tracker = MagicMock() - tracker._conf = {"codecarbon_version": "3.0", "hardware": ["cpu"]} - tracker._geo = None - tracker._save_to_file = False - tracker._save_to_api = False - tracker._save_to_logger = False - tracker._emissions_endpoint = None - tracker._save_to_prometheus = False - tracker._save_to_logfire = False - tracker._tasks = {} - tracker._measure_power_secs = 15 - tracker._hardware = [] - tracker._resource_tracker = None - - emissions = _sample_emissions() + def test_build_payload_omits_framework_versions(self): + ctx = _tracker_context(conf={"codecarbon_version": "3.0", "hardware": ["cpu"]}) with patch( - "codecarbon.core.telemetry_collect._package_installed", + "codecarbon.core.telemetry.collect._package_installed", return_value=True, ): - payload = build_telemetry_payload(tracker, emissions) + payload = build_payload(ctx) self.assertEqual(payload["telemetry_level"], "minimal") self.assertTrue(payload["has_torch"]) self.assertNotIn("torch_version", payload) - def test_build_telemetry_payload_uses_resolved_level(self): - tracker = MagicMock() - tracker._conf = {"codecarbon_version": "3.0"} - tracker._save_to_file = False - tracker._save_to_api = False - tracker._save_to_logger = False - tracker._emissions_endpoint = None - tracker._save_to_prometheus = False - tracker._save_to_logfire = False - tracker._tasks = {} - tracker._measure_power_secs = 15 - tracker._hardware = [] - tracker._resource_tracker = None - - emissions = _sample_emissions() - payload = build_telemetry_payload( - tracker, emissions, level=TelemetryLevel.extensive - ) + def test_build_payload_uses_resolved_level(self): + ctx = _tracker_context(conf={"codecarbon_version": "3.0"}) + payload = build_payload(ctx, level=TelemetryLevel.extensive) self.assertEqual(payload["telemetry_level"], "extensive") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index 5627ae529..d80ee9ecf 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -9,16 +9,8 @@ from unittest.mock import MagicMock, patch from codecarbon.core.config import get_config_file_settings -from codecarbon.core.telemetry_settings import resolve_telemetry_level -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.core.telemetry_settings import get_telemetry_api_url +from codecarbon.core.telemetry import Telemetry, TelemetryLevel, TelemetrySettings, post_private from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker -from codecarbon.output_methods.emissions_data import EmissionsData -from codecarbon.telemetry import ( - reset_telemetry_warning, - send_private_telemetry_at_stop, - warn_if_telemetry_not_configured, -) from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open if sys.platform == "darwin": @@ -35,20 +27,27 @@ def _conf(level: str) -> str: class TestTelemetryConfigContract(unittest.TestCase): def setUp(self) -> None: - reset_telemetry_warning() + Telemetry._default_warning_shown = False + + def tearDown(self) -> None: + Telemetry._default_warning_shown = False def test_warns_once_when_telemetry_not_explicit(self): - with patch("codecarbon.telemetry.logger.warning") as mock_warning: - warn_if_telemetry_not_configured({}, TelemetryLevel.minimal) - warn_if_telemetry_not_configured({}, TelemetryLevel.minimal) + settings = TelemetrySettings.resolve(config_file_conf={}) + telemetry = Telemetry(settings) + with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + telemetry.warn_if_implicit() + telemetry.warn_if_implicit() self.assertEqual(mock_warning.call_count, 1) self.assertIn("Tier 1", mock_warning.call_args[0][0]) def test_no_warn_when_config_explicit(self): - with patch("codecarbon.telemetry.logger.warning") as mock_warning: - warn_if_telemetry_not_configured( - {"telemetry_level": "disabled"}, TelemetryLevel.disabled - ) + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "disabled"} + ) + telemetry = Telemetry(settings) + with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + telemetry.warn_if_implicit() mock_warning.assert_not_called() def test_tier1_posts_to_telemetry_endpoint(self): @@ -58,51 +57,13 @@ def test_tier1_posts_to_telemetry_endpoint(self): "total_emissions_kg": 0.001, "os": "Linux", } - tracker = MagicMock() - emissions = EmissionsData( - timestamp="2020-01-01T00:00:00", - project_name="test", - run_id="run-1", - experiment_id="exp-1", - duration=10.0, - emissions=0.001, - emissions_rate=0.0001, - cpu_power=0.0, - gpu_power=0.0, - ram_power=0.0, - cpu_energy=0.0, - gpu_energy=0.0, - ram_energy=0.0, - energy_consumed=0.01, - water_consumed=0.0, - country_name="France", - country_iso_code="FRA", - region="idf", - cloud_provider="", - cloud_region="", - os="Linux", - python_version="3.11", - codecarbon_version="2.0", - cpu_count=1.0, - cpu_model="cpu", - gpu_count=0.0, - gpu_model="", - longitude=0.0, - latitude=0.0, - ram_total_size=8.0, - tracking_mode="machine", + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://tier1.example"} ) - with patch( - "codecarbon.telemetry.build_telemetry_payload", return_value=tier1_payload - ): - with patch("codecarbon.core.telemetry_client.requests.post") as mock_post: - mock_post.return_value.status_code = 201 - mock_post.return_value.json.return_value = "telemetry-id" - send_private_telemetry_at_stop( - tracker, - emissions, - external_conf={"telemetry_api_url": "http://tier1.example"}, - ) + with patch("codecarbon.core.telemetry.client.requests.post") as mock_post: + mock_post.return_value.status_code = 201 + mock_post.return_value.json.return_value = "telemetry-id" + post_private(settings, tier1_payload) mock_post.assert_called_once() self.assertEqual( mock_post.call_args.kwargs["url"], "http://tier1.example/telemetry" @@ -128,11 +89,11 @@ def test_legacy_env_codecarbon_telemetry_does_not_change_tier(self): ): from codecarbon.core.config import get_hierarchical_config - level = resolve_telemetry_level( - get_config_file_settings(), + settings = TelemetrySettings.resolve( + config_file_conf=get_config_file_settings(), external_conf=get_hierarchical_config(), ) - self.assertEqual(level, TelemetryLevel.minimal) + self.assertEqual(settings.level, TelemetryLevel.minimal) def test_env_codecarbon_telemetry_level_overrides_file(self): with tempfile.TemporaryDirectory() as tmp: @@ -149,11 +110,11 @@ def test_env_codecarbon_telemetry_level_overrides_file(self): ): from codecarbon.core.config import get_hierarchical_config - level = resolve_telemetry_level( - get_config_file_settings(), + settings = TelemetrySettings.resolve( + config_file_conf=get_config_file_settings(), external_conf=get_hierarchical_config(), ) - self.assertEqual(level, TelemetryLevel.disabled) + self.assertEqual(settings.level, TelemetryLevel.disabled) def test_telemetry_api_url_env_used_for_tier2_client(self): with patch.dict( @@ -161,25 +122,26 @@ def test_telemetry_api_url_env_used_for_tier2_client(self): {"CODECARBON_TELEMETRY_API_URL": "http://env-telemetry.example"}, clear=False, ): - url = get_telemetry_api_url({}) - self.assertEqual(url, "http://env-telemetry.example") + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, "http://env-telemetry.example") def test_telemetry_api_url_from_config_overrides_default(self): - url = get_telemetry_api_url( - {"telemetry_api_url": "http://config-telemetry.example"} + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://config-telemetry.example"} ) - self.assertEqual(url, "http://config-telemetry.example") + self.assertEqual(settings.api_url, "http://config-telemetry.example") @mock_platform_cli_setup class TestTrackerTelemetryFromConfig(unittest.TestCase): def setUp(self) -> None: - reset_telemetry_warning() + Telemetry._default_warning_shown = False self._config_patcher = None def tearDown(self) -> None: if self._config_patcher: self._config_patcher.stop() + Telemetry._default_warning_shown = False def _mock_config(self, conf: str) -> None: self._config_patcher = patch( @@ -189,7 +151,7 @@ def _mock_config(self, conf: str) -> None: def test_disabled_no_telemetry_on_stop(self, mock_cli_setup): self._mock_config(_conf("disabled")) - with patch("codecarbon.telemetry.post_private_telemetry") as mock_post: + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( measure_power_secs=1, @@ -204,7 +166,7 @@ def test_minimal_posts_tier1_on_stop_not_on_init(self, mock_cli_setup): self._mock_config(_conf("minimal")) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( @@ -224,9 +186,9 @@ def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup self._mock_config(_conf("extensive")) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: - with patch("codecarbon.telemetry.ApiClient") as mock_api_cls: + with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api @@ -246,7 +208,7 @@ def test_offline_minimal_posts_tier1_on_stop(self, mock_cli_setup): self._mock_config(_conf("minimal")) with ensure_telemetry_run_duration(): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: tracker = OfflineEmissionsTracker( country_iso_code="CAN", @@ -269,9 +231,9 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) ) } with patch.dict(os.environ, env_without_telemetry, clear=True): - with patch("codecarbon.telemetry.logger.warning") as mock_warning: + with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) @@ -284,8 +246,8 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): self._mock_config("[codecarbon]\n") - with patch("codecarbon.telemetry.logger.warning") as mock_warning: - with patch("codecarbon.telemetry.post_private_telemetry"): + with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + with patch("codecarbon.core.telemetry.dispatcher.post_private"): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker( telemetry_level="disabled", @@ -306,7 +268,7 @@ def test_env_telemetry_disabled_does_not_change_resolved_level( os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False ): with patch( - "codecarbon.telemetry.post_private_telemetry", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py index c09582c62..e9ad2bb2e 100644 --- a/tests/test_telemetry_settings.py +++ b/tests/test_telemetry_settings.py @@ -2,17 +2,13 @@ import unittest from unittest.mock import patch -from codecarbon.core.telemetry_schemas import TelemetryLevel -from codecarbon.core.telemetry_settings import ( +from codecarbon.core.telemetry import ( DEFAULT_TELEMETRY_API_KEY, DEFAULT_TELEMETRY_API_URL, DEFAULT_TELEMETRY_EXPERIMENT_ID, - get_telemetry_api_key, - get_telemetry_api_url, - get_telemetry_experiment_id, - is_telemetry_level_explicit, + TelemetryLevel, + TelemetrySettings, parse_telemetry_level, - resolve_telemetry_level, ) @@ -30,49 +26,58 @@ def test_parse_rejects_invalid(self): parse_telemetry_level("bogus") -class TestResolveTelemetryLevel(unittest.TestCase): +class TestTelemetrySettingsResolve(unittest.TestCase): def test_default_is_minimal_when_unset(self): - level = resolve_telemetry_level({}) - self.assertEqual(level, TelemetryLevel.minimal) + settings = TelemetrySettings.resolve(config_file_conf={}) + self.assertEqual(settings.level, TelemetryLevel.minimal) + self.assertEqual(settings.source, "default") def test_telemetry_level_from_config_file(self): - level = resolve_telemetry_level({"telemetry_level": "disabled"}) - self.assertEqual(level, TelemetryLevel.disabled) + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "disabled"} + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + self.assertEqual(settings.source, "file") def test_telemetry_level_extensive(self): - level = resolve_telemetry_level({"telemetry_level": "extensive"}) - self.assertEqual(level, TelemetryLevel.extensive) + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "extensive"} + ) + self.assertEqual(settings.level, TelemetryLevel.extensive) def test_env_telemetry_key_ignored(self): with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): - level = resolve_telemetry_level({"telemetry": "extensive"}) - self.assertEqual(level, TelemetryLevel.minimal) + settings = TelemetrySettings.resolve(config_file_conf={"telemetry": "extensive"}) + self.assertEqual(settings.level, TelemetryLevel.minimal) def test_invalid_level_falls_back_to_minimal(self): - with patch("codecarbon.core.telemetry_settings.logger.error") as mock_error: - level = resolve_telemetry_level({"telemetry_level": "bogus"}) - self.assertEqual(level, TelemetryLevel.minimal) + with patch("codecarbon.core.telemetry.settings.logger.error") as mock_error: + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "bogus"} + ) + self.assertEqual(settings.level, TelemetryLevel.minimal) mock_error.assert_called_once() def test_override_kwarg_takes_precedence_over_config_file(self): - level = resolve_telemetry_level( - {"telemetry_level": "minimal"}, override="disabled" + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"}, + override="disabled", ) - self.assertEqual(level, TelemetryLevel.disabled) + self.assertEqual(settings.level, TelemetryLevel.disabled) def test_override_kwarg_takes_precedence_over_external_conf(self): - level = resolve_telemetry_level( + settings = TelemetrySettings.resolve( external_conf={"telemetry_level": "extensive"}, override="disabled", ) - self.assertEqual(level, TelemetryLevel.disabled) + self.assertEqual(settings.level, TelemetryLevel.disabled) def test_external_conf_env_overrides_file_when_merged(self): - level = resolve_telemetry_level( - {"telemetry_level": "minimal"}, + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"}, external_conf={"telemetry_level": "disabled"}, ) - self.assertEqual(level, TelemetryLevel.disabled) + self.assertEqual(settings.level, TelemetryLevel.disabled) def test_env_telemetry_level_via_external_conf(self): with patch.dict( @@ -80,75 +85,87 @@ def test_env_telemetry_level_via_external_conf(self): ): from codecarbon.core.config import get_hierarchical_config - level = resolve_telemetry_level(external_conf=get_hierarchical_config()) - self.assertEqual(level, TelemetryLevel.disabled) + settings = TelemetrySettings.resolve(external_conf=get_hierarchical_config()) + self.assertEqual(settings.level, TelemetryLevel.disabled) def test_is_explicit_with_config_file(self): - self.assertTrue(is_telemetry_level_explicit({"telemetry_level": "minimal"})) + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"} + ) + self.assertTrue(settings.is_explicit) def test_is_explicit_with_override(self): - self.assertTrue(is_telemetry_level_explicit({}, override="disabled")) + settings = TelemetrySettings.resolve(override="disabled") + self.assertTrue(settings.is_explicit) def test_is_explicit_with_env_telemetry_level(self): - self.assertTrue( - is_telemetry_level_explicit( - {}, external_conf={"telemetry_level": "minimal"} - ) + settings = TelemetrySettings.resolve( + external_conf={"telemetry_level": "minimal"} ) + self.assertTrue(settings.is_explicit) def test_is_not_explicit_when_unset(self): - self.assertFalse(is_telemetry_level_explicit({})) + settings = TelemetrySettings.resolve(config_file_conf={}) + self.assertFalse(settings.is_explicit) class TestTelemetryApiSettings(unittest.TestCase): - def test_get_telemetry_api_url_from_conf(self): - url = get_telemetry_api_url({"telemetry_api_url": "http://test.example"}) - self.assertEqual(url, "http://test.example") + def test_api_url_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://test.example"} + ) + self.assertEqual(settings.api_url, "http://test.example") - def test_get_telemetry_api_url_default(self): + def test_api_url_default(self): env = { k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_URL" } with patch.dict(os.environ, env, clear=True): - url = get_telemetry_api_url({}) - self.assertEqual(url, DEFAULT_TELEMETRY_API_URL) + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, DEFAULT_TELEMETRY_API_URL) - def test_get_telemetry_api_url_env_fallback(self): + def test_api_url_env_fallback(self): with patch.dict( os.environ, {"CODECARBON_TELEMETRY_API_URL": "http://env.example"}, clear=False, ): - url = get_telemetry_api_url({}) - self.assertEqual(url, "http://env.example") + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, "http://env.example") - def test_get_telemetry_api_key_from_conf(self): - key = get_telemetry_api_key({"telemetry_api_key": "cpt_test"}) - self.assertEqual(key, "cpt_test") + def test_api_key_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_key": "cpt_test"} + ) + self.assertEqual(settings.api_key, "cpt_test") - def test_get_telemetry_api_key_uses_public_default_when_unset(self): + def test_api_key_uses_public_default_when_unset(self): env = { k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_KEY" } with patch.dict(os.environ, env, clear=True): - key = get_telemetry_api_key({}) - self.assertEqual(key, DEFAULT_TELEMETRY_API_KEY) - - def test_get_telemetry_experiment_id_from_conf(self): - experiment_id = get_telemetry_experiment_id( - {"telemetry_experiment_id": "00000000-0000-0000-0000-000000000001"} + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_key, DEFAULT_TELEMETRY_API_KEY) + + def test_experiment_id_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={ + "telemetry_experiment_id": "00000000-0000-0000-0000-000000000001" + } + ) + self.assertEqual( + settings.experiment_id, "00000000-0000-0000-0000-000000000001" ) - self.assertEqual(experiment_id, "00000000-0000-0000-0000-000000000001") - def test_get_telemetry_experiment_id_uses_public_default_when_unset(self): + def test_experiment_id_uses_public_default_when_unset(self): env = { k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_EXPERIMENT_ID" } with patch.dict(os.environ, env, clear=True): - experiment_id = get_telemetry_experiment_id({}) - self.assertEqual(experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) + settings = TelemetrySettings.resolve() + self.assertEqual(settings.experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) if __name__ == "__main__": From aefea4c3d8ce0787d7d57e073e15e3cc9f959dd7 Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 20 May 2026 08:39:31 +0200 Subject: [PATCH 17/18] refactor: improve telemetry imports and code formatting - Reorganized imports in emissions_tracker.py for clarity and consistency. - Simplified the definition of CONTAINER_RUNTIME_ENV in collect.py for better readability. - Enhanced formatting in settings.py and various test files to maintain consistent code style. - Updated test cases to improve readability and ensure proper mocking of telemetry functions. --- codecarbon/core/telemetry/collect.py | 4 +--- codecarbon/core/telemetry/settings.py | 7 +++--- codecarbon/emissions_tracker.py | 2 +- tests/test_telemetry.py | 8 +++++-- tests/test_telemetry_config.py | 33 ++++++++++++++++++++------- tests/test_telemetry_settings.py | 12 ++++++---- 6 files changed, 44 insertions(+), 22 deletions(-) diff --git a/codecarbon/core/telemetry/collect.py b/codecarbon/core/telemetry/collect.py index 49f5263c3..bbd661525 100644 --- a/codecarbon/core/telemetry/collect.py +++ b/codecarbon/core/telemetry/collect.py @@ -36,9 +36,7 @@ ("CI", "ci"), ) -CONTAINER_RUNTIME_ENV = ( - ("KUBERNETES_SERVICE_HOST", "kubernetes"), -) +CONTAINER_RUNTIME_ENV = (("KUBERNETES_SERVICE_HOST", "kubernetes"),) OUTPUT_METHOD_FIELDS = ( ("save_to_file", "file"), diff --git a/codecarbon/core/telemetry/settings.py b/codecarbon/core/telemetry/settings.py index 8c2c28cb7..a296a9b03 100644 --- a/codecarbon/core/telemetry/settings.py +++ b/codecarbon/core/telemetry/settings.py @@ -88,9 +88,10 @@ def resolve( elif merged.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: raw = merged[TELEMETRY_LEVEL_CONFIG_KEY] source = "external" - elif config_file_conf is not None and config_file_conf.get( - TELEMETRY_LEVEL_CONFIG_KEY - ) is not None: + elif ( + config_file_conf is not None + and config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None + ): raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] source = "file" else: diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 30cb4b571..a26c88df8 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -22,9 +22,9 @@ get_hierarchical_config, normalize_gpu_ids, ) -from codecarbon.core.telemetry import Telemetry, TelemetrySettings from codecarbon.core.emissions import Emissions from codecarbon.core.resource_tracker import ResourceTracker +from codecarbon.core.telemetry import Telemetry, TelemetrySettings from codecarbon.core.units import Energy, Power, Time, Water from codecarbon.core.util import count_cpus, count_physical_cpus, suppress from codecarbon.external.geography import CloudMetadata, GeoMetadata diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index dd157fb93..78c64359d 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -103,7 +103,9 @@ def test_tier2_uses_api_client(self): tracker._telemetry_override = None emissions = self._emissions() telemetry = Telemetry.from_tracker(tracker) - with patch("codecarbon.core.telemetry.dispatcher.post_private", return_value=True): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ): with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True @@ -186,7 +188,9 @@ def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): with patch( "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: - with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: + with patch( + "codecarbon.core.telemetry.client.ApiClient" + ) as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py index d80ee9ecf..6b230ee83 100644 --- a/tests/test_telemetry_config.py +++ b/tests/test_telemetry_config.py @@ -9,7 +9,12 @@ from unittest.mock import MagicMock, patch from codecarbon.core.config import get_config_file_settings -from codecarbon.core.telemetry import Telemetry, TelemetryLevel, TelemetrySettings, post_private +from codecarbon.core.telemetry import ( + Telemetry, + TelemetryLevel, + TelemetrySettings, + post_private, +) from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open @@ -35,7 +40,9 @@ def tearDown(self) -> None: def test_warns_once_when_telemetry_not_explicit(self): settings = TelemetrySettings.resolve(config_file_conf={}) telemetry = Telemetry(settings) - with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: telemetry.warn_if_implicit() telemetry.warn_if_implicit() self.assertEqual(mock_warning.call_count, 1) @@ -46,7 +53,9 @@ def test_no_warn_when_config_explicit(self): config_file_conf={"telemetry_level": "disabled"} ) telemetry = Telemetry(settings) - with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: telemetry.warn_if_implicit() mock_warning.assert_not_called() @@ -188,7 +197,9 @@ def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup with patch( "codecarbon.core.telemetry.dispatcher.post_private", return_value=True ) as mock_post: - with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: + with patch( + "codecarbon.core.telemetry.client.ApiClient" + ) as mock_api_cls: mock_api = MagicMock() mock_api.add_emission.return_value = True mock_api_cls.return_value = mock_api @@ -231,9 +242,12 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) ) } with patch.dict(os.environ, env_without_telemetry, clear=True): - with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: with patch( - "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", + return_value=True, ): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker(save_to_api=False, save_to_file=False) @@ -246,7 +260,9 @@ def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup) def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): self._mock_config("[codecarbon]\n") - with patch("codecarbon.core.telemetry.dispatcher.logger.warning") as mock_warning: + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: with patch("codecarbon.core.telemetry.dispatcher.post_private"): with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): EmissionsTracker( @@ -268,7 +284,8 @@ def test_env_telemetry_disabled_does_not_change_resolved_level( os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False ): with patch( - "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + "codecarbon.core.telemetry.dispatcher.post_private", + return_value=True, ) as mock_post: with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): tracker = EmissionsTracker( diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py index e9ad2bb2e..e9a1e09f2 100644 --- a/tests/test_telemetry_settings.py +++ b/tests/test_telemetry_settings.py @@ -47,7 +47,9 @@ def test_telemetry_level_extensive(self): def test_env_telemetry_key_ignored(self): with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): - settings = TelemetrySettings.resolve(config_file_conf={"telemetry": "extensive"}) + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry": "extensive"} + ) self.assertEqual(settings.level, TelemetryLevel.minimal) def test_invalid_level_falls_back_to_minimal(self): @@ -85,7 +87,9 @@ def test_env_telemetry_level_via_external_conf(self): ): from codecarbon.core.config import get_hierarchical_config - settings = TelemetrySettings.resolve(external_conf=get_hierarchical_config()) + settings = TelemetrySettings.resolve( + external_conf=get_hierarchical_config() + ) self.assertEqual(settings.level, TelemetryLevel.disabled) def test_is_explicit_with_config_file(self): @@ -153,9 +157,7 @@ def test_experiment_id_from_conf(self): "telemetry_experiment_id": "00000000-0000-0000-0000-000000000001" } ) - self.assertEqual( - settings.experiment_id, "00000000-0000-0000-0000-000000000001" - ) + self.assertEqual(settings.experiment_id, "00000000-0000-0000-0000-000000000001") def test_experiment_id_uses_public_default_when_unset(self): env = { From adb50c3f8c26cf8167570946d8148157a34d155d Mon Sep 17 00:00:00 2001 From: davidberenstein1957 Date: Wed, 20 May 2026 08:41:10 +0200 Subject: [PATCH 18/18] feat: introduce telemetry module and enhance telemetry context management - Added a new telemetry module to handle product telemetry at tracker stop, including necessary imports and payload construction. - Updated tests to ensure proper handling of telemetry levels and error management with the Typer library. - Improved clarity and organization of telemetry-related code for better maintainability. --- codecarbon/core/telemetry/__init__.py | 39 +++++++++++++++++++++++++++ tests/cli/test_telemetry_cli.py | 3 ++- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 codecarbon/core/telemetry/__init__.py diff --git a/codecarbon/core/telemetry/__init__.py b/codecarbon/core/telemetry/__init__.py new file mode 100644 index 000000000..0afb851b9 --- /dev/null +++ b/codecarbon/core/telemetry/__init__.py @@ -0,0 +1,39 @@ +"""Product telemetry sent at tracker stop (Tier 1 / Tier 2).""" + +from codecarbon.core.telemetry.client import post_private, post_public_summary +from codecarbon.core.telemetry.collect import TelemetryContext, build_payload +from codecarbon.core.telemetry.dispatcher import Telemetry +from codecarbon.core.telemetry.schemas import ( + PRIVATE_TELEMETRY_FIELDS, + TelemetryCreate, + TelemetryLevel, +) +from codecarbon.core.telemetry.settings import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_API_URL, + DEFAULT_TELEMETRY_EXPERIMENT_ID, + DEFAULT_TELEMETRY_LEVEL, + TELEMETRY_LEVEL_CONFIG_KEY, + TelemetryLevelSource, + TelemetrySettings, + parse_telemetry_level, +) + +__all__ = [ + "DEFAULT_TELEMETRY_API_KEY", + "DEFAULT_TELEMETRY_API_URL", + "DEFAULT_TELEMETRY_EXPERIMENT_ID", + "DEFAULT_TELEMETRY_LEVEL", + "PRIVATE_TELEMETRY_FIELDS", + "TELEMETRY_LEVEL_CONFIG_KEY", + "Telemetry", + "TelemetryContext", + "TelemetryCreate", + "TelemetryLevel", + "TelemetryLevelSource", + "TelemetrySettings", + "build_payload", + "parse_telemetry_level", + "post_private", + "post_public_summary", +] diff --git a/tests/cli/test_telemetry_cli.py b/tests/cli/test_telemetry_cli.py index 871bd13c3..88cd9375b 100644 --- a/tests/cli/test_telemetry_cli.py +++ b/tests/cli/test_telemetry_cli.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest +import typer from typer.testing import CliRunner from codecarbon.cli import main as cli_main @@ -17,7 +18,7 @@ def test_normalize_telemetry_level_accepts_valid_values(): def test_normalize_telemetry_level_rejects_invalid(): - with pytest.raises(Exception): + with pytest.raises(typer.BadParameter): normalize_telemetry_level("bogus")