diff --git a/.gitignore b/.gitignore index 86958c842..b88e0109f 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,6 @@ tests/test_data/rapl/* credentials* .codecarbon.config* scripts/agent-vm.personal.config.sh + +# Added by ggshield +.cache_ggshield diff --git a/carbonserver/carbonserver/api/schemas_telemetry.py b/carbonserver/carbonserver/api/schemas_telemetry.py index 5517c2ff4..ced8c13fd 100644 --- a/carbonserver/carbonserver/api/schemas_telemetry.py +++ b/carbonserver/carbonserver/api/schemas_telemetry.py @@ -41,8 +41,7 @@ class TelemetryBase(BaseModel): region: Optional[str] = None cloud_provider: Optional[str] = None cloud_region: Optional[str] = None - longitude: Optional[float] = Field(default=None, ge=-180, le=180) - latitude: Optional[float] = Field(default=None, ge=-90, le=90) + on_cloud: Optional[bool] = None cpu_count: Optional[int] = Field(default=None, ge=0) cpu_physical_count: Optional[int] = Field(default=None, ge=0) @@ -58,12 +57,10 @@ class TelemetryBase(BaseModel): python_version: Optional[str] = None python_implementation: Optional[str] = None - python_executable_hash: Optional[str] = Field( - default=None, min_length=64, max_length=64 - ) python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None + python_package_manager: Optional[str] = None total_emissions_kg: Optional[float] = Field(default=None, ge=0) emissions_rate_kg_per_sec: Optional[float] = Field(default=None, ge=0) @@ -81,21 +78,18 @@ class TelemetryBase(BaseModel): output_methods: Optional[List[str]] = None hardware_tracked: Optional[List[str]] = None task_tracking_used: Optional[bool] = None - decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) + integration_surface: Optional[str] = None + offline_mode: Optional[bool] = None + save_to_api_enabled: Optional[bool] = None hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None gpu_detection_method: Optional[str] = None - first_measurement_time_ms: Optional[float] = Field(default=None, ge=0) - tracking_overhead_percent: Optional[float] = Field(default=None, ge=0) - errors_encountered: Optional[List[str]] = None - warning_count: Optional[int] = Field(default=None, ge=0) ide_used: Optional[str] = None notebook_environment: Optional[str] = None ci_environment: Optional[str] = None - python_package_manager: Optional[str] = None framework_detected: Optional[str] = None has_torch: Optional[bool] = None @@ -116,58 +110,15 @@ class TelemetryBase(BaseModel): container_runtime: Optional[str] = None in_container: Optional[bool] = None - host_machine_hash: Optional[str] = None @model_validator(mode="after") def validate_telemetry_level(self): if self.telemetry_level == TelemetryLevel.disabled: raise ValueError("Disabled telemetry must not be submitted") - - if self.telemetry_level == TelemetryLevel.minimal: - extensive_fields = set(type(self).model_fields) - MINIMAL_TELEMETRY_FIELDS - submitted_extensive_fields = [ - field - for field in extensive_fields - if getattr(self, field) not in (None, [], {}) - ] - if submitted_extensive_fields: - fields = ", ".join(sorted(submitted_extensive_fields)) - raise ValueError( - f"Minimal telemetry cannot include extensive fields: {fields}" - ) - return self -MINIMAL_TELEMETRY_FIELDS = { - "timestamp", - "telemetry_level", - "os", - "country_name", - "country_iso_code", - "region", - "cloud_provider", - "cloud_region", - "longitude", - "latitude", - "cpu_count", - "cpu_physical_count", - "cpu_model", - "cpu_architecture", - "gpu_count", - "gpu_model", - "gpu_driver_version", - "gpu_memory_total_gb", - "ram_total_size_gb", - "cuda_version", - "cudnn_version", - "python_version", - "python_implementation", - "python_executable_hash", - "python_env_type", - "codecarbon_version", - "codecarbon_install_method", -} +PRIVATE_TELEMETRY_FIELDS = frozenset(TelemetryBase.model_fields) class TelemetryCreate(TelemetryBase): diff --git a/carbonserver/tests/api/routers/test_telemetry.py b/carbonserver/tests/api/routers/test_telemetry.py index a405b59e8..a51a19ecd 100644 --- a/carbonserver/tests/api/routers/test_telemetry.py +++ b/carbonserver/tests/api/routers/test_telemetry.py @@ -14,7 +14,7 @@ TELEMETRY_ID = "f52fe339-164d-4c2b-a8c0-f562dfce066d" -MINIMAL_TELEMETRY_TO_CREATE = { +SAMPLE_PRIVATE_TELEMETRY = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", "os": "Linux-5.10.0-x86_64", @@ -46,21 +46,36 @@ def test_add_telemetry(client, custom_test_server): repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=MINIMAL_TELEMETRY_TO_CREATE) + response = client.post("/telemetry", json=SAMPLE_PRIVATE_TELEMETRY) assert response.status_code == status.HTTP_201_CREATED assert response.json() == TELEMETRY_ID -def test_minimal_telemetry_rejects_extensive_fields(client, custom_test_server): +def test_minimal_telemetry_accepts_framework_versions(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) - telemetry_with_extensive_field = { - **MINIMAL_TELEMETRY_TO_CREATE, - "total_emissions_kg": 0.42, + repository_mock.add_telemetry.return_value = UUID(TELEMETRY_ID) + telemetry_with_framework_version = { + **SAMPLE_PRIVATE_TELEMETRY, + "torch_version": "2.2.0", + } + + with custom_test_server.container.telemetry_repository.override(repository_mock): + response = client.post("/telemetry", json=telemetry_with_framework_version) + + assert response.status_code == status.HTTP_201_CREATED + repository_mock.add_telemetry.assert_called_once() + + +def test_telemetry_rejects_unknown_fields(client, custom_test_server): + repository_mock = mock.Mock(spec=TelemetryRepository) + telemetry_with_unknown_field = { + **SAMPLE_PRIVATE_TELEMETRY, + "unknown_field": "value", } with custom_test_server.container.telemetry_repository.override(repository_mock): - response = client.post("/telemetry", json=telemetry_with_extensive_field) + response = client.post("/telemetry", json=telemetry_with_unknown_field) assert response.status_code == 422 repository_mock.add_telemetry.assert_not_called() @@ -69,7 +84,7 @@ def test_minimal_telemetry_rejects_extensive_fields(client, custom_test_server): def test_disabled_telemetry_is_rejected(client, custom_test_server): repository_mock = mock.Mock(spec=TelemetryRepository) disabled_telemetry = { - **MINIMAL_TELEMETRY_TO_CREATE, + **SAMPLE_PRIVATE_TELEMETRY, "telemetry_level": "disabled", } diff --git a/carbonserver/tests/api/test_telemetry_schema_drift.py b/carbonserver/tests/api/test_telemetry_schema_drift.py index abc20d764..388e39963 100644 --- a/carbonserver/tests/api/test_telemetry_schema_drift.py +++ b/carbonserver/tests/api/test_telemetry_schema_drift.py @@ -7,7 +7,9 @@ from carbonserver.api.schemas_telemetry import TelemetryCreate as ServerTelemetryCreate REPO_ROOT = Path(__file__).resolve().parents[3] -CORE_TELEMETRY_SCHEMA_PATH = REPO_ROOT / "codecarbon" / "core" / "telemetry_schemas.py" +CORE_TELEMETRY_SCHEMA_PATH = ( + REPO_ROOT / "codecarbon" / "core" / "telemetry" / "schemas.py" +) def _load_core_telemetry_create(): diff --git a/codecarbon/cli/main.py b/codecarbon/cli/main.py index d5175ea9c..5174d6475 100644 --- a/codecarbon/cli/main.py +++ b/codecarbon/cli/main.py @@ -22,6 +22,7 @@ overwrite_local_config, ) from codecarbon.cli.monitor import run_and_monitor +from codecarbon.cli.telemetry_cli import normalize_telemetry_level, telemetry_app from codecarbon.core.api_client import ApiClient, get_datetime_with_timezone from codecarbon.core.schemas import ExperimentCreate, OrganizationCreate, ProjectCreate from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker @@ -32,6 +33,7 @@ DEFAULT_ORGANIzATION_ID = "e60afa92-17b7-4720-91a0-1ae91e409ba1" codecarbon = typer.Typer(no_args_is_help=True) +codecarbon.add_typer(telemetry_app, name="telemetry") def main(): @@ -342,6 +344,15 @@ def monitor( str, typer.Option(help="Region/province for offline mode"), ] = None, + telemetry_level: Annotated[ + Optional[str], + typer.Option( + help=( + "Override telemetry tier for this run only " + "(disabled, minimal, or extensive)." + ), + ), + ] = None, ): """Monitor your machine's carbon emissions.""" @@ -350,6 +361,8 @@ def monitor( "measure_power_secs": measure_power_secs, "api_call_interval": api_call_interval, } + if telemetry_level is not None: + tracker_args["telemetry_level"] = normalize_telemetry_level(telemetry_level) # Set up the tracker arguments based on mode (offline vs online) and validate required args for each mode if offline: if not country_iso_code: diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py new file mode 100644 index 000000000..df23a30a5 --- /dev/null +++ b/codecarbon/cli/telemetry_cli.py @@ -0,0 +1,252 @@ +"""CLI commands to configure CodeCarbon product telemetry tiers.""" + +from pathlib import Path +from typing import Optional + +import questionary +import typer +from rich import print +from typing_extensions import Annotated + +from codecarbon.cli.cli_utils import ( + create_new_config_file, + get_config, + overwrite_local_config, +) +from codecarbon.core.config import get_config_file_settings, get_hierarchical_config +from codecarbon.core.telemetry import ( + DEFAULT_TELEMETRY_LEVEL, + TelemetryLevel, + TelemetrySettings, + parse_telemetry_level, +) + +telemetry_app = typer.Typer( + help="Configure product telemetry (disabled, minimal, or extensive).", + no_args_is_help=False, +) + +TIER_DESCRIPTIONS = { + "disabled": "No product telemetry.", + "minimal": "Tier 1: private telemetry per run at stop (environment, usage, frameworks, emissions).", + "extensive": "Tier 2: Tier 1 plus run emissions summary to the shared telemetry experiment (ApiClient).", +} + + +def normalize_telemetry_level(level: str) -> str: + """Validate and normalize a telemetry tier string for CLI use. + + Args: + level: User-provided tier name. + + Returns: + Canonical tier value. + + Raises: + typer.BadParameter: If the level is not a valid ``TelemetryLevel``. + """ + try: + return parse_telemetry_level(level).value + except ValueError as error: + raise typer.BadParameter(str(error)) from error + + +def resolve_config_path(config: Optional[Path], *, create: bool = False) -> Path: + """Resolve which config file to read or write. + + Args: + config: Explicit path from ``--config``, if any. + create: When True and no file exists, create ``./.codecarbon.config``. + + Returns: + Resolved config file path. + """ + if config is not None: + path = config.expanduser().resolve() + if create and not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("[codecarbon]\n", encoding="utf-8") + return path + local_path = Path.cwd().resolve() / ".codecarbon.config" + if local_path.exists(): + return local_path + global_path = (Path.home() / ".codecarbon.config").expanduser().resolve() + if global_path.exists(): + return global_path + if create: + local_path.write_text("[codecarbon]\n", encoding="utf-8") + return local_path + return local_path + + +def pick_config_path_interactive() -> Path: + """Prompt for which config file to update. + + Returns: + Path chosen by the user. + """ + home = Path.home() + global_path = (home / ".codecarbon.config").expanduser().resolve() + local_path = Path.cwd().resolve() / ".codecarbon.config" + options = [] + if global_path.exists(): + options.append(str(global_path)) + if local_path.exists() and local_path not in options: + options.append(str(local_path)) + options.append("Create new config file") + if not options: + options = ["Create new config file"] + choice = questionary.select( + "Which configuration file should store telemetry_level?", + choices=options, + ).ask() + if choice == "Create new config file": + return create_new_config_file() + return Path(choice).expanduser().resolve() + + +def write_telemetry_level(path: Path, level: str) -> None: + """Persist ``telemetry_level`` to a config file. + + Args: + path: Target ``.codecarbon.config`` path. + level: Validated tier value. + """ + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("[codecarbon]\n", encoding="utf-8") + overwrite_local_config("telemetry_level", level, path=path) + + +def print_telemetry_status(config_path: Optional[Path] = None) -> None: + """Print resolved telemetry settings. + + Without ``config_path``, uses the same merged file settings and env overlay + as ``EmissionsTracker``. With ``config_path``, inspects that file only. + + Args: + config_path: Optional single config file to inspect. + """ + if config_path is not None: + path = config_path.expanduser().resolve() + if not path.exists(): + print(f"[yellow]Config file not found:[/yellow] {path}") + print(f"Default tier: {DEFAULT_TELEMETRY_LEVEL.value} (not explicit)") + return + file_settings = get_config(path) + external_conf: dict[str, str] = {} + source_label = str(path) + else: + file_settings = get_config_file_settings() + external_conf = get_hierarchical_config() + source_label = "merged ~/.codecarbon.config + ./.codecarbon.config" + + settings = TelemetrySettings.resolve( + config_file_conf=file_settings, + external_conf=external_conf or None, + ) + level = settings.level + explicit = settings.is_explicit + stored = file_settings.get("telemetry_level") + print(f"Config source: {source_label}") + print(f"telemetry_level in file(s): {stored!r}") + print(f"Resolved tier: {level.value}") + print(f"Explicitly configured: {explicit}") + if not explicit: + print( + "[yellow]Tier 1 minimal telemetry will be sent once per session " + "until you set telemetry_level.[/yellow]" + ) + + +def run_telemetry_interactive(config: Optional[Path] = None) -> None: + """Run the interactive telemetry configuration wizard. + + Args: + config: Optional fixed config path; otherwise prompt for file choice. + """ + print("CodeCarbon product telemetry") + print( + "Separate from your dashboard experiment (codecarbon config). " + "Controls optional usage analytics and public leaderboard data.\n" + ) + path = resolve_config_path(config) if config else None + if path is None or (config is None and not path.exists()): + path = pick_config_path_interactive() + else: + path = resolve_config_path(config, create=True) + + choices = [ + questionary.Choice("disabled — " + TIER_DESCRIPTIONS["disabled"], "disabled"), + questionary.Choice("minimal — " + TIER_DESCRIPTIONS["minimal"], "minimal"), + questionary.Choice( + "extensive — " + TIER_DESCRIPTIONS["extensive"], "extensive" + ), + ] + try: + current = get_config(path).get("telemetry_level") + except FileNotFoundError: + current = None + valid_levels = {member.value for member in TelemetryLevel} + default = current if current in valid_levels else "minimal" + level = questionary.select( + "Select telemetry_level:", + choices=choices, + default=default, + ).ask() + if level is None: + raise typer.Exit(0) + level = normalize_telemetry_level(level) + write_telemetry_level(path, level) + print(f"[green]Saved[/green] telemetry_level = {level} in {path}") + + +@telemetry_app.callback(invoke_without_command=True) +def telemetry_entry( + ctx: typer.Context, + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Path to .codecarbon.config (default: local then global).", + ), + ] = None, +) -> None: + """Configure telemetry interactively when no subcommand is given.""" + if ctx.invoked_subcommand is None: + run_telemetry_interactive(config=config) + + +@telemetry_app.command("show") +def show( + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Inspect one file only; default matches EmissionsTracker merge.", + ), + ] = None, +) -> None: + """Show the resolved telemetry tier.""" + print_telemetry_status(config_path=config) + + +@telemetry_app.command("set") +def set_level( + level: Annotated[ + str, + typer.Argument(help="Telemetry tier: disabled, minimal, or extensive."), + ], + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Path to .codecarbon.config (creates ./.codecarbon.config if missing).", + ), + ] = None, +) -> None: + """Write telemetry_level to a config file.""" + path = resolve_config_path(config, create=True) + normalized = normalize_telemetry_level(level) + write_telemetry_level(path, normalized) + print(f"[green]Saved[/green] telemetry_level = {normalized} in {path}") diff --git a/codecarbon/core/api_client.py b/codecarbon/core/api_client.py index 34067c71c..de1988d34 100644 --- a/codecarbon/core/api_client.py +++ b/codecarbon/core/api_client.py @@ -30,6 +30,13 @@ def get_datetime_with_timezone(): return timestamp +def _round_coordinate(value, decimals: int = 1) -> float: + """Round a geographic coordinate for API payloads, treating None as zero.""" + if value is None: + return round(0.0, decimals) + return round(float(value), decimals) + + class ApiClient: # (AsyncClient) """ This class call the Code Carbon API @@ -263,9 +270,8 @@ def _create_run(self, experiment_id: str): cpu_model=self.conf.get("cpu_model"), gpu_count=self.conf.get("gpu_count"), gpu_model=self.conf.get("gpu_model"), - # Reduce precision for Privacy - longitude=round(self.conf.get("longitude", 0), 1), - latitude=round(self.conf.get("latitude", 0), 1), + longitude=_round_coordinate(self.conf.get("longitude")), + latitude=_round_coordinate(self.conf.get("latitude")), region=self.conf.get("region"), provider=self.conf.get("provider"), ram_total_size=self.conf.get("ram_total_size"), diff --git a/codecarbon/core/config.py b/codecarbon/core/config.py index 7cacea41d..65330f7a0 100644 --- a/codecarbon/core/config.py +++ b/codecarbon/core/config.py @@ -110,6 +110,31 @@ def normalize_gpu_ids( return None +def _config_file_paths() -> tuple[str, str]: + """Return resolved paths for global and local CodeCarbon config files.""" + cwd = Path.cwd() + home = Path.home() + global_path = str((home / ".codecarbon.config").expanduser().resolve()) + local_path = str((cwd / ".codecarbon.config").expanduser().resolve()) + return global_path, local_path + + +def get_config_file_settings() -> dict[str, str]: + """Return the ``[codecarbon]`` section from config files without environment overlay. + + Reads ``~/.codecarbon.config`` then ``./.codecarbon.config`` (local overrides global). + + Returns: + Configuration dict from files only. Empty when no file or section exists. + """ + config = configparser.ConfigParser() + global_path, local_path = _config_file_paths() + config.read([global_path, local_path]) + if "codecarbon" not in config: + return {} + return dict(config["codecarbon"]) + + def get_hierarchical_config(): """ Get the user-defined codecarbon configuration ConfigParser dictionnary @@ -137,13 +162,7 @@ def get_hierarchical_config(): dict: The final configuration dict parsed from global, local and environment configurations. **All values are strings**. """ - - config = configparser.ConfigParser() - - cwd = Path.cwd() - home = Path.home() - global_path = str((home / ".codecarbon.config").expanduser().resolve()) - local_path = str((cwd / ".codecarbon.config").expanduser().resolve()) + global_path, local_path = _config_file_paths() if Path(global_path).exists(): logger.info( f"Codecarbon is taking the configuration from global file: {global_path}" @@ -155,7 +174,6 @@ def get_hierarchical_config(): f"Codecarbon is taking the configuration from the local file {local_path}" ) - config.read([global_path, local_path]) - config.read_dict(parse_env_config()) - - return dict(config["codecarbon"]) + conf = get_config_file_settings() + conf.update(parse_env_config().get("codecarbon", {})) + return conf diff --git a/codecarbon/core/gpu_amd.py b/codecarbon/core/gpu_amd.py index 6cff5a287..bd8eeb226 100644 --- a/codecarbon/core/gpu_amd.py +++ b/codecarbon/core/gpu_amd.py @@ -28,7 +28,7 @@ def is_rocm_system(): "Please install amdsmi to get GPU metrics." ) AMDSMI_AVAILABLE = False -except AttributeError as e: +except (AttributeError, OSError, KeyError) as e: amdsmi = None # In some environments, amdsmi may be present but not properly configured, leading to AttributeError when importing logger.warning( diff --git a/codecarbon/core/telemetry/__init__.py b/codecarbon/core/telemetry/__init__.py new file mode 100644 index 000000000..0afb851b9 --- /dev/null +++ b/codecarbon/core/telemetry/__init__.py @@ -0,0 +1,39 @@ +"""Product telemetry sent at tracker stop (Tier 1 / Tier 2).""" + +from codecarbon.core.telemetry.client import post_private, post_public_summary +from codecarbon.core.telemetry.collect import TelemetryContext, build_payload +from codecarbon.core.telemetry.dispatcher import Telemetry +from codecarbon.core.telemetry.schemas import ( + PRIVATE_TELEMETRY_FIELDS, + TelemetryCreate, + TelemetryLevel, +) +from codecarbon.core.telemetry.settings import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_API_URL, + DEFAULT_TELEMETRY_EXPERIMENT_ID, + DEFAULT_TELEMETRY_LEVEL, + TELEMETRY_LEVEL_CONFIG_KEY, + TelemetryLevelSource, + TelemetrySettings, + parse_telemetry_level, +) + +__all__ = [ + "DEFAULT_TELEMETRY_API_KEY", + "DEFAULT_TELEMETRY_API_URL", + "DEFAULT_TELEMETRY_EXPERIMENT_ID", + "DEFAULT_TELEMETRY_LEVEL", + "PRIVATE_TELEMETRY_FIELDS", + "TELEMETRY_LEVEL_CONFIG_KEY", + "Telemetry", + "TelemetryContext", + "TelemetryCreate", + "TelemetryLevel", + "TelemetryLevelSource", + "TelemetrySettings", + "build_payload", + "parse_telemetry_level", + "post_private", + "post_public_summary", +] diff --git a/codecarbon/core/telemetry/client.py b/codecarbon/core/telemetry/client.py new file mode 100644 index 000000000..99a5a7ed2 --- /dev/null +++ b/codecarbon/core/telemetry/client.py @@ -0,0 +1,84 @@ +"""HTTP and API clients for product telemetry.""" + +from __future__ import annotations + +import dataclasses + +import requests + +from codecarbon.core.api_client import ApiClient +from codecarbon.core.telemetry.schemas import TelemetryCreate +from codecarbon.core.telemetry.settings import TelemetrySettings +from codecarbon.external.logger import logger +from codecarbon.output_methods.emissions_data import EmissionsData + + +def post_private(settings: TelemetrySettings, payload: dict) -> bool: + """POST a private telemetry payload to ``/telemetry``. + + Args: + settings: Resolved telemetry API settings. + payload: Telemetry fields dict. + + Returns: + True if the server accepted the payload (HTTP 201). + """ + headers = {"Content-Type": "application/json"} + if settings.api_key: + headers["x-api-token"] = settings.api_key + body = TelemetryCreate(**payload).model_dump(mode="json", exclude_none=True) + telemetry_url = f"{settings.api_url.rstrip('/')}/telemetry" + try: + response = requests.post( + url=telemetry_url, + json=body, + headers=headers, + timeout=2, + ) + except Exception: + logger.error("Telemetry request failed.", exc_info=True) + return False + if response.status_code == 201: + return True + if response.status_code == 404: + logger.warning( + "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", + telemetry_url, + ) + else: + logger.error( + "Telemetry API %s: %s", + response.status_code, + response.text, + ) + logger.debug("Telemetry request body: %s", body) + return False + + +def post_public_summary( + settings: TelemetrySettings, + conf: dict, + emissions: EmissionsData, +) -> bool: + """Send Tier 2 public run summary via ``ApiClient``. + + Args: + settings: Resolved telemetry API settings. + conf: Tracker configuration dict. + emissions: Run emissions data. + + Returns: + True if the run summary was posted successfully. + """ + try: + api = ApiClient( + endpoint_url=settings.api_url, + experiment_id=settings.experiment_id, + api_key=settings.api_key, + conf=conf, + create_run_automatically=True, + ) + return bool(api.add_emission(dataclasses.asdict(emissions))) + except Exception as error: + logger.error(f"Public run summary failed (non-critical): {error}") + return False diff --git a/codecarbon/core/telemetry/collect.py b/codecarbon/core/telemetry/collect.py new file mode 100644 index 000000000..bbd661525 --- /dev/null +++ b/codecarbon/core/telemetry/collect.py @@ -0,0 +1,361 @@ +"""Collect private product telemetry (Tier 1 / Tier 2) from tracker state.""" + +from __future__ import annotations + +import importlib.util +import os +import platform +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Optional + +from codecarbon.core.cloud import get_env_cloud_details +from codecarbon.core.gpu import is_nvidia_system +from codecarbon.core.telemetry.schemas import PRIVATE_TELEMETRY_FIELDS, TelemetryLevel +from codecarbon.output_methods.emissions_data import EmissionsData + +FRAMEWORK_PACKAGES = ( + ("torch", "has_torch"), + ("transformers", "has_transformers"), + ("diffusers", "has_diffusers"), + ("sklearn", "has_sklearn"), +) + +PACKAGE_MANAGER_ENV = ( + ("UV", "uv"), + ("POETRY_ACTIVE", "poetry"), + ("PIP_RUN", "pip"), +) + +CI_ENVIRONMENTS = ( + ("GITHUB_ACTIONS", "github_actions"), + ("GITLAB_CI", "gitlab_ci"), + ("CIRCLECI", "circleci"), + ("JENKINS_URL", "jenkins"), + ("CI", "ci"), +) + +CONTAINER_RUNTIME_ENV = (("KUBERNETES_SERVICE_HOST", "kubernetes"),) + +OUTPUT_METHOD_FIELDS = ( + ("save_to_file", "file"), + ("save_to_api", "api"), + ("save_to_logger", "logger"), + ("emissions_endpoint", "http"), + ("save_to_prometheus", "prometheus"), + ("save_to_logfire", "logfire"), +) + + +@dataclass +class TelemetryContext: + """Snapshot of tracker state used to build a telemetry payload.""" + + conf: dict[str, Any] + emissions: EmissionsData + hardware: list[Any] + resource_tracker: Any + save_to_api: bool + save_to_file: bool + save_to_logger: bool + save_to_prometheus: bool + save_to_logfire: bool + emissions_endpoint: str | None + tasks: dict[str, Any] + measure_power_secs: float | None + is_offline: bool + + @classmethod + def from_tracker(cls, tracker: Any, emissions: EmissionsData) -> TelemetryContext: + """Build a context snapshot from an active emissions tracker. + + Args: + tracker: Active emissions tracker instance. + emissions: Run emissions data. + + Returns: + Context for ``build_payload``. + """ + from codecarbon.emissions_tracker import OfflineEmissionsTracker + + return cls( + conf=getattr(tracker, "_conf", {}), + emissions=emissions, + hardware=getattr(tracker, "_hardware", []) or [], + resource_tracker=getattr(tracker, "_resource_tracker", None), + save_to_api=bool(getattr(tracker, "_save_to_api", False)), + save_to_file=bool(getattr(tracker, "_save_to_file", False)), + save_to_logger=bool(getattr(tracker, "_save_to_logger", False)), + save_to_prometheus=bool(getattr(tracker, "_save_to_prometheus", False)), + save_to_logfire=bool(getattr(tracker, "_save_to_logfire", False)), + emissions_endpoint=getattr(tracker, "_emissions_endpoint", None), + tasks=getattr(tracker, "_tasks", {}) or {}, + measure_power_secs=getattr(tracker, "_measure_power_secs", None), + is_offline=isinstance(tracker, OfflineEmissionsTracker), + ) + + +def _non_empty(value: Any) -> bool: + return value not in (None, "", [], {}) + + +def _strip_none(data: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in data.items() if _non_empty(value)} + + +def _first_env_match(mapping: tuple[tuple[str, str], ...]) -> Optional[str]: + return next((label for var, label in mapping if os.environ.get(var)), None) + + +def _package_installed(name: str) -> bool: + return importlib.util.find_spec(name) is not None + + +def _detect_codecarbon_install_method() -> Optional[str]: + try: + from importlib.metadata import distribution + + dist = distribution("codecarbon") + if getattr(dist, "editable", False): + return "editable" + installer = (dist.metadata.get("Installer") or "").lower() + if "uv" in installer: + return "uv" + if "pip" in installer: + return "pip" + except Exception: + pass + return None + + +def _detect_python_env_type() -> Optional[str]: + if os.environ.get("CONDA_DEFAULT_ENV"): + return "conda" + if os.environ.get("VIRTUAL_ENV"): + return "venv" + if sys.prefix != getattr(sys, "base_prefix", sys.prefix): + return "venv" + return "system" + + +def _detect_notebook_environment() -> Optional[str]: + if os.environ.get("COLAB_GPU") is not None or "google.colab" in sys.modules: + return "colab" + try: + from IPython import get_ipython + + shell = get_ipython().__class__.__name__ + if "ZMQInteractiveShell" in shell: + return "jupyter" + except Exception: + pass + return None + + +def _container_info() -> tuple[bool, Optional[str]]: + if os.environ.get("KUBERNETES_SERVICE_HOST"): + return True, "kubernetes" + if os.path.exists("/.dockerenv"): + return True, "docker" + return False, None + + +def _detect_ide() -> Optional[str]: + if os.environ.get("CURSOR_TRACE_ID") or os.environ.get("CURSOR_SESSION"): + return "cursor" + if os.environ.get("VSCODE_PID") or os.environ.get("TERM_PROGRAM") == "vscode": + return "vscode" + if os.environ.get("PYCHARM_HOSTED"): + return "pycharm" + return None + + +def _cudnn_version() -> Optional[str]: + if not _package_installed("torch"): + return None + try: + import torch + + version = torch.backends.cudnn.version() + return str(version) if version is not None else None + except Exception: + return None + + +def _collect_hardware_diagnostics(ctx: TelemetryContext) -> dict[str, Any]: + from codecarbon.core import cpu + + hardware_tracked: list[str] = [] + for item in ctx.hardware: + try: + hardware_tracked.append(item.description()) + except Exception: + pass + + gpu_detection_method: Optional[str] = None + if ctx.resource_tracker is not None: + gpu_tracker = getattr(ctx.resource_tracker, "gpu_tracker", None) + if gpu_tracker and gpu_tracker != "Unspecified": + gpu_detection_method = gpu_tracker + + rapl_available: Optional[bool] = None + if platform.system() == "Linux": + rapl_available = cpu.is_rapl_available() + + return { + "hardware_tracked": hardware_tracked or None, + "hardware_detection_success": bool(hardware_tracked), + "rapl_available": rapl_available, + "gpu_detection_method": gpu_detection_method, + "api_mode": "online" if ctx.save_to_api else "offline", + } + + +def _detect_integration_surface(ctx: TelemetryContext) -> str: + if ctx.is_offline: + return "offline_tracker" + argv = " ".join(sys.argv) + if "codecarbon" in argv and "monitor" in argv: + return "cli_monitor" + return "library" + + +def _collect_output_methods(ctx: TelemetryContext) -> list[str]: + methods: list[str] = [] + for field_name, label in OUTPUT_METHOD_FIELDS: + value = getattr(ctx, field_name) + if field_name == "emissions_endpoint": + if value: + methods.append(label) + elif value: + methods.append(label) + return methods + + +def _raw_cloud_provider_and_region() -> tuple[Optional[str], Optional[str]]: + details = get_env_cloud_details() + if not details or not details.get("metadata"): + return None, None + provider = (details.get("provider") or "").lower() or None + metadata = details.get("metadata") or {} + region: Optional[str] = None + if provider == "aws": + region = metadata.get("region") + elif provider == "azure": + region = (metadata.get("compute") or {}).get("location") + elif provider == "gcp": + zone = metadata.get("zone") or "" + parts = zone.split("/") + region = parts[-1].rsplit("-", 1)[0] if parts else None + return provider, region + + +def _collect_framework_fields() -> dict[str, Any]: + return { + has_field: _package_installed(package) + for package, has_field in FRAMEWORK_PACKAGES + } + + +def _gpu_static_fields() -> dict[str, Any]: + fields: dict[str, Any] = {} + if not is_nvidia_system(): + return fields + try: + import pynvml + + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + mem = pynvml.nvmlDeviceGetMemoryInfo(handle) + fields["gpu_memory_total_gb"] = mem.total / (1024**3) + fields["gpu_driver_version"] = pynvml.nvmlSystemGetDriverVersion() + fields["cuda_version"] = pynvml.nvmlSystemGetCudaDriverVersion_v2() + if isinstance(fields["cuda_version"], int): + v = fields["cuda_version"] + fields["cuda_version"] = f"{v // 1000}.{(v % 1000) // 10}" + except Exception: + pass + return fields + + +def build_payload( + ctx: TelemetryContext, + level: TelemetryLevel = TelemetryLevel.minimal, +) -> dict[str, Any]: + """Build a private telemetry payload dict for ``POST /telemetry``. + + Args: + ctx: Tracker snapshot from ``TelemetryContext.from_tracker``. + level: Resolved ``TelemetryLevel`` (``minimal`` or ``extensive``). + + Returns: + Payload dict for ``TelemetryCreate``. + """ + emissions = ctx.emissions + conf = ctx.conf + raw_provider, raw_region = _raw_cloud_provider_and_region() + on_cloud = emissions.on_cloud == "Y" + cloud_provider = emissions.cloud_provider or raw_provider + cloud_region = emissions.cloud_region or raw_region + region = emissions.region or conf.get("region") + if on_cloud and cloud_region: + region = region or cloud_region + + integration_surface = _detect_integration_surface(ctx) + in_container, container_runtime = _container_info() + gpu_fields = _gpu_static_fields() + + raw: dict[str, Any] = { + "timestamp": datetime.now(timezone.utc), + "os": conf.get("os") or platform.platform(), + "python_version": conf.get("python_version") or platform.python_version(), + "python_implementation": platform.python_implementation(), + "python_env_type": _detect_python_env_type(), + "python_package_manager": _first_env_match(PACKAGE_MANAGER_ENV), + "codecarbon_version": conf.get("codecarbon_version"), + "codecarbon_install_method": _detect_codecarbon_install_method(), + "country_name": emissions.country_name, + "country_iso_code": emissions.country_iso_code, + "region": region, + "cloud_provider": cloud_provider, + "cloud_region": cloud_region, + "on_cloud": on_cloud, + "cpu_count": conf.get("cpu_count"), + "cpu_physical_count": conf.get("cpu_physical_count"), + "cpu_model": conf.get("cpu_model"), + "cpu_architecture": platform.machine(), + "gpu_count": conf.get("gpu_count"), + "gpu_model": conf.get("gpu_model"), + "ram_total_size_gb": conf.get("ram_total_size"), + "tracking_mode": conf.get("tracking_mode"), + "integration_surface": integration_surface, + "offline_mode": integration_surface == "offline_tracker", + "output_methods": _collect_output_methods(ctx), + "save_to_api_enabled": ctx.save_to_api, + "task_tracking_used": bool(ctx.tasks), + "measure_power_interval_secs": ctx.measure_power_secs, + "in_container": in_container, + "container_runtime": container_runtime, + "ci_environment": _first_env_match(CI_ENVIRONMENTS), + "notebook_environment": _detect_notebook_environment(), + "ide_used": _detect_ide(), + "cudnn_version": _cudnn_version(), + "duration_seconds": float(emissions.duration) if emissions.duration else None, + "total_emissions_kg": emissions.emissions, + "emissions_rate_kg_per_sec": emissions.emissions_rate, + "energy_consumed_kwh": emissions.energy_consumed, + "cpu_energy_kwh": emissions.cpu_energy, + "gpu_energy_kwh": emissions.gpu_energy, + "ram_energy_kwh": emissions.ram_energy, + "cpu_utilization_avg": emissions.cpu_utilization_percent, + "gpu_utilization_avg": emissions.gpu_utilization_percent, + "ram_utilization_avg": emissions.ram_utilization_percent, + "telemetry_level": level.value, + **_collect_framework_fields(), + **_collect_hardware_diagnostics(ctx), + **gpu_fields, + } + + payload = {key: raw[key] for key in PRIVATE_TELEMETRY_FIELDS if key in raw} + return _strip_none(payload) diff --git a/codecarbon/core/telemetry/dispatcher.py b/codecarbon/core/telemetry/dispatcher.py new file mode 100644 index 000000000..24174d94f --- /dev/null +++ b/codecarbon/core/telemetry/dispatcher.py @@ -0,0 +1,89 @@ +"""Per-tracker telemetry dispatcher.""" + +from __future__ import annotations + +from typing import Any, ClassVar + +from codecarbon.core.telemetry.client import post_private, post_public_summary +from codecarbon.core.telemetry.collect import TelemetryContext, build_payload +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.core.telemetry.settings import TelemetrySettings +from codecarbon.external.logger import logger +from codecarbon.output_methods.emissions_data import EmissionsData + +TELEMETRY_NOT_CONFIGURED_MESSAGE = ( + "CodeCarbon telemetry_level was not set explicitly; using default %r. " + "Tier 1 private telemetry (per run at stop) will be sent. Set telemetry_level " + "in .codecarbon.config, set CODECARBON_TELEMETRY_LEVEL, pass telemetry_level=... " + "to EmissionsTracker / OfflineEmissionsTracker, or run " + "codecarbon telemetry set ." +) + + +class Telemetry: + """Per-tracker telemetry dispatcher.""" + + _default_warning_shown: ClassVar[bool] = False + + def __init__(self, settings: TelemetrySettings) -> None: + self.settings = settings + + @classmethod + def from_tracker(cls, tracker: Any) -> Telemetry: + """Build a dispatcher from tracker config state. + + Args: + tracker: Active emissions tracker with ``_config_file_conf``, + ``_external_conf``, and optional ``_telemetry_override``. + + Returns: + Configured ``Telemetry`` instance. + """ + return cls( + TelemetrySettings.resolve( + config_file_conf=tracker._config_file_conf, + external_conf=tracker._external_conf, + override=getattr(tracker, "_telemetry_override", None), + ) + ) + + def warn_if_implicit(self) -> None: + """Log a one-time warning when telemetry tier was not set explicitly.""" + if self.settings.is_explicit or Telemetry._default_warning_shown: + return + logger.warning( + TELEMETRY_NOT_CONFIGURED_MESSAGE, + self.settings.level.value, + ) + Telemetry._default_warning_shown = True + + def send_at_stop(self, tracker: Any, emissions: EmissionsData) -> None: + """Send product telemetry for the resolved tier at tracker ``stop()``. + + Tier 1 (``minimal``): private ``POST /telemetry`` only. + Tier 2 (``extensive``): Tier 1 plus public run summary. + + Args: + tracker: Active emissions tracker instance. + emissions: Total emissions from ``_prepare_emissions_data()``. + """ + if self.settings.level == TelemetryLevel.disabled: + return + if emissions.duration is not None and emissions.duration < 1: + logger.debug("Telemetry not sent: run shorter than 1 second.") + return + ctx = TelemetryContext.from_tracker(tracker, emissions) + payload = build_payload(ctx, level=self.settings.level) + try: + post_private(self.settings, payload) + except Exception as error: + logger.error(f"Private telemetry failed (non-critical): {error}") + if self.settings.level == TelemetryLevel.extensive: + try: + post_public_summary( + self.settings, + getattr(tracker, "_conf", {}), + emissions, + ) + except Exception as error: + logger.error(f"Public run summary failed (non-critical): {error}") diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry/schemas.py similarity index 68% rename from codecarbon/core/telemetry_schemas.py rename to codecarbon/core/telemetry/schemas.py index ea6249b65..bc6ac2dc2 100644 --- a/codecarbon/core/telemetry_schemas.py +++ b/codecarbon/core/telemetry/schemas.py @@ -23,8 +23,7 @@ class TelemetryBase(BaseModel): region: Optional[str] = None cloud_provider: Optional[str] = None cloud_region: Optional[str] = None - longitude: Optional[float] = Field(default=None, ge=-180, le=180) - latitude: Optional[float] = Field(default=None, ge=-90, le=90) + on_cloud: Optional[bool] = None cpu_count: Optional[int] = Field(default=None, ge=0) cpu_physical_count: Optional[int] = Field(default=None, ge=0) @@ -40,12 +39,10 @@ class TelemetryBase(BaseModel): python_version: Optional[str] = None python_implementation: Optional[str] = None - python_executable_hash: Optional[str] = Field( - default=None, min_length=64, max_length=64 - ) python_env_type: Optional[str] = None codecarbon_version: Optional[str] = None codecarbon_install_method: Optional[str] = None + python_package_manager: Optional[str] = None total_emissions_kg: Optional[float] = Field(default=None, ge=0) emissions_rate_kg_per_sec: Optional[float] = Field(default=None, ge=0) @@ -63,21 +60,18 @@ class TelemetryBase(BaseModel): output_methods: Optional[List[str]] = None hardware_tracked: Optional[List[str]] = None task_tracking_used: Optional[bool] = None - decorator_vs_context: Optional[str] = None measure_power_interval_secs: Optional[float] = Field(default=None, ge=0) + integration_surface: Optional[str] = None + offline_mode: Optional[bool] = None + save_to_api_enabled: Optional[bool] = None hardware_detection_success: Optional[bool] = None rapl_available: Optional[bool] = None gpu_detection_method: Optional[str] = None - first_measurement_time_ms: Optional[float] = Field(default=None, ge=0) - tracking_overhead_percent: Optional[float] = Field(default=None, ge=0) - errors_encountered: Optional[List[str]] = None - warning_count: Optional[int] = Field(default=None, ge=0) ide_used: Optional[str] = None notebook_environment: Optional[str] = None ci_environment: Optional[str] = None - python_package_manager: Optional[str] = None framework_detected: Optional[str] = None has_torch: Optional[bool] = None @@ -98,58 +92,15 @@ class TelemetryBase(BaseModel): container_runtime: Optional[str] = None in_container: Optional[bool] = None - host_machine_hash: Optional[str] = None @model_validator(mode="after") def validate_telemetry_level(self): if self.telemetry_level == TelemetryLevel.disabled: raise ValueError("Disabled telemetry must not be submitted") - - if self.telemetry_level == TelemetryLevel.minimal: - extensive_fields = set(type(self).model_fields) - MINIMAL_TELEMETRY_FIELDS - submitted_extensive_fields = [ - field - for field in extensive_fields - if getattr(self, field) not in (None, [], {}) - ] - if submitted_extensive_fields: - fields = ", ".join(sorted(submitted_extensive_fields)) - raise ValueError( - f"Minimal telemetry cannot include extensive fields: {fields}" - ) - return self -MINIMAL_TELEMETRY_FIELDS = { - "timestamp", - "telemetry_level", - "os", - "country_name", - "country_iso_code", - "region", - "cloud_provider", - "cloud_region", - "longitude", - "latitude", - "cpu_count", - "cpu_physical_count", - "cpu_model", - "cpu_architecture", - "gpu_count", - "gpu_model", - "gpu_driver_version", - "gpu_memory_total_gb", - "ram_total_size_gb", - "cuda_version", - "cudnn_version", - "python_version", - "python_implementation", - "python_executable_hash", - "python_env_type", - "codecarbon_version", - "codecarbon_install_method", -} +PRIVATE_TELEMETRY_FIELDS = frozenset(TelemetryBase.model_fields) class TelemetryCreate(TelemetryBase): diff --git a/codecarbon/core/telemetry/settings.py b/codecarbon/core/telemetry/settings.py new file mode 100644 index 000000000..a296a9b03 --- /dev/null +++ b/codecarbon/core/telemetry/settings.py @@ -0,0 +1,145 @@ +"""Resolve telemetry tier and API settings from config and environment.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, Literal + +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.external.logger import logger + +DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" +DEFAULT_TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +DEFAULT_TELEMETRY_EXPERIMENT_ID = "d2d69403-1373-42b4-a2c1-09589aed4801" +DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal + +TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" + +TelemetryLevelSource = Literal["override", "external", "file", "default"] + + +def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: + """Parse a telemetry tier from a string or enum value. + + Args: + raw: Tier name or ``TelemetryLevel`` member. + + Returns: + Parsed ``TelemetryLevel``. + + Raises: + ValueError: If ``raw`` is not a valid tier name. + """ + if isinstance(raw, TelemetryLevel): + return raw + try: + return TelemetryLevel(str(raw).lower()) + except ValueError as error: + raise ValueError( + f"Invalid telemetry_level {raw!r}. Choose: disabled, minimal, or extensive." + ) from error + + +@dataclass(frozen=True) +class TelemetrySettings: + """Resolved telemetry tier and API connection settings.""" + + level: TelemetryLevel + source: TelemetryLevelSource + api_url: str + api_key: str + experiment_id: str + + @property + def is_explicit(self) -> bool: + """Return whether the user explicitly chose a telemetry tier.""" + return self.source != "default" + + @classmethod + def resolve( + cls, + *, + config_file_conf: dict[str, Any] | None = None, + external_conf: dict[str, Any] | None = None, + override: str | TelemetryLevel | None = None, + ) -> TelemetrySettings: + """Resolve telemetry tier and API settings. + + Precedence for tier: + + 1. ``override`` — ``EmissionsTracker(telemetry_level=...)`` or CLI + 2. ``external_conf`` — merged ``.codecarbon.config`` and env + 3. ``config_file_conf`` — file-only settings + 4. Default: ``minimal`` + + Args: + config_file_conf: Settings from ``get_config_file_settings()``. + external_conf: Merged settings from ``get_hierarchical_config()``. + override: Optional tier from tracker or CLI. + + Returns: + Resolved settings bundle. + """ + merged = external_conf or {} + if override is not None: + raw = override + source: TelemetryLevelSource = "override" + elif merged.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: + raw = merged[TELEMETRY_LEVEL_CONFIG_KEY] + source = "external" + elif ( + config_file_conf is not None + and config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None + ): + raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] + source = "file" + else: + return cls( + level=DEFAULT_TELEMETRY_LEVEL, + source="default", + api_url=cls._resolve_api_url(merged), + api_key=cls._resolve_api_key(merged), + experiment_id=cls._resolve_experiment_id(merged), + ) + try: + level = parse_telemetry_level(raw) + except ValueError: + logger.error( + "Invalid telemetry_level %r; falling back to %r", + raw, + DEFAULT_TELEMETRY_LEVEL.value, + ) + level = DEFAULT_TELEMETRY_LEVEL + return cls( + level=level, + source=source, + api_url=cls._resolve_api_url(merged), + api_key=cls._resolve_api_key(merged), + experiment_id=cls._resolve_experiment_id(merged), + ) + + @staticmethod + def _resolve_api_url(external_conf: dict[str, Any]) -> str: + url = ( + external_conf.get("telemetry_api_url") + or external_conf.get("api_endpoint") + or os.environ.get("CODECARBON_TELEMETRY_API_URL") + ) + return (url or DEFAULT_TELEMETRY_API_URL).rstrip("/") + + @staticmethod + def _resolve_api_key(external_conf: dict[str, Any]) -> str: + key = ( + external_conf.get("telemetry_api_key") + or external_conf.get("api_key") + or os.environ.get("CODECARBON_TELEMETRY_API_KEY") + ) + return key or DEFAULT_TELEMETRY_API_KEY + + @staticmethod + def _resolve_experiment_id(external_conf: dict[str, Any]) -> str: + experiment_id = external_conf.get("telemetry_experiment_id") or os.environ.get( + "CODECARBON_TELEMETRY_EXPERIMENT_ID" + ) + return experiment_id or DEFAULT_TELEMETRY_EXPERIMENT_ID diff --git a/codecarbon/core/telemetry_client.py b/codecarbon/core/telemetry_client.py deleted file mode 100644 index 8dfdad05a..000000000 --- a/codecarbon/core/telemetry_client.py +++ /dev/null @@ -1,61 +0,0 @@ -import json -from typing import Optional, Union - -import requests - -from codecarbon.core.telemetry_schemas import TelemetryCreate -from codecarbon.external.logger import logger - - -class TelemetryClient: - """ - Client dedicated to sending CodeCarbon telemetry payloads. - """ - - def __init__( - self, - endpoint_url="https://api.codecarbon.io", - telemetry: Optional[Union[TelemetryCreate, dict]] = None, - ): - self.endpoint_url = endpoint_url.rstrip("/") - self.telemetry_url = self.endpoint_url + "/telemetry" - self.headers = {"Content-Type": "application/json"} - self.telemetry = self._validate_telemetry(telemetry) if telemetry else None - - def add_telemetry(self, telemetry: Optional[Union[TelemetryCreate, dict]] = None): - telemetry_payload = ( - self._validate_telemetry(telemetry) if telemetry else self.telemetry - ) - if telemetry_payload is None: - logger.error("TelemetryClient.add_telemetry() needs a telemetry payload") - return None - payload = telemetry_payload.model_dump(mode="json", exclude_none=True) - - try: - response = requests.post( - url=self.telemetry_url, - json=payload, - timeout=2, - headers=self.headers, - ) - if response.status_code != 201: - self._log_error(payload, response) - return None - return response.json() - except Exception as e: - logger.error(e, exc_info=True) - return None - - @staticmethod - def _validate_telemetry(telemetry: Union[TelemetryCreate, dict]) -> TelemetryCreate: - if isinstance(telemetry, TelemetryCreate): - return telemetry - return TelemetryCreate(**telemetry) - - def _log_error(self, payload, response): - logger.error( - f"TelemetryClient Error when calling the API on {self.telemetry_url} with : {json.dumps(payload)}" - ) - logger.error( - f"TelemetryClient API return http code {response.status_code} and answer : {response.text}" - ) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 862eba2b4..a26c88df8 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -17,9 +17,14 @@ import psutil from codecarbon._version import __version__ -from codecarbon.core.config import get_hierarchical_config, normalize_gpu_ids +from codecarbon.core.config import ( + get_config_file_settings, + get_hierarchical_config, + normalize_gpu_ids, +) from codecarbon.core.emissions import Emissions from codecarbon.core.resource_tracker import ResourceTracker +from codecarbon.core.telemetry import Telemetry, TelemetrySettings from codecarbon.core.units import Energy, Power, Time, Water from codecarbon.core.util import count_cpus, count_physical_cpus, suppress from codecarbon.external.geography import CloudMetadata, GeoMetadata @@ -199,6 +204,7 @@ def __init__( allow_multiple_runs: Optional[bool] = _sentinel, rapl_include_dram: Optional[bool] = _sentinel, rapl_prefer_psys: Optional[bool] = _sentinel, + telemetry_level: Optional[str] = _sentinel, ): """ :param project_name: Project name for current experiment run, default name @@ -272,10 +278,22 @@ def __init__( (CPU + chipset + PCIe). When False, uses package domains which are more reliable. Note: psys can report higher values than CPU TDP and may be unreliable on older systems. + :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). + Overrides config file and ``CODECARBON_TELEMETRY_LEVEL`` when set. """ - # logger.info("base tracker init") self._external_conf = get_hierarchical_config() + self._config_file_conf = get_config_file_settings() + self._telemetry_override = ( + None if telemetry_level is _sentinel else telemetry_level + ) + self._telemetry = Telemetry( + TelemetrySettings.resolve( + config_file_conf=self._config_file_conf, + external_conf=self._external_conf, + override=self._telemetry_override, + ) + ) self._set_from_conf(allow_multiple_runs, "allow_multiple_runs", True, bool) if self._allow_multiple_runs: logger.warning( @@ -352,7 +370,6 @@ def __init__( self._set_from_conf( experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" ) - assert self._tracking_mode in ["machine", "process"] set_logger_level(self._log_level) set_logger_format(self._logger_preamble) @@ -396,6 +413,7 @@ def __init__( self._hardware = [] resource_tracker = ResourceTracker(self) resource_tracker.set_CPU_GPU_ram_tracking() + self._resource_tracker = resource_tracker self._conf["hardware"] = list(map(lambda x: x.description(), self._hardware)) @@ -439,7 +457,7 @@ def __init__( if cloud.is_on_private_infra: self._conf["longitude"] = self._geo.longitude self._conf["latitude"] = self._geo.latitude - self._conf["region"] = cloud.region + self._conf["region"] = self._geo.region self._conf["provider"] = cloud.provider else: self._conf["region"] = cloud.region @@ -448,8 +466,14 @@ def __init__( self._emissions: Emissions = Emissions( self._data_source, self._electricitymaps_api_token ) + + self._telemetry.warn_if_implicit() self._init_output_methods(api_key=self._api_key) + @suppress(Exception) + def _send_telemetry_at_stop(self, emissions_data: EmissionsData) -> None: + self._telemetry.send_at_stop(self, emissions_data) + def _init_output_methods(self, *, api_key: str = None): """ Prepare the different output methods @@ -751,6 +775,7 @@ def stop(self) -> Optional[float]: emissions_data = self._prepare_emissions_data() emissions_data_delta = self._compute_emissions_delta(emissions_data) + self._send_telemetry_at_stop(emissions_data) self._persist_data( total_emissions=emissions_data, @@ -1313,6 +1338,7 @@ def track_emissions( allow_multiple_runs: Optional[bool] = _sentinel, rapl_include_dram: Optional[bool] = _sentinel, rapl_prefer_psys: Optional[bool] = _sentinel, + telemetry_level: Optional[str] = _sentinel, ): """ Decorator that supports both `EmissionsTracker` and `OfflineEmissionsTracker` @@ -1396,6 +1422,7 @@ def track_emissions( When True, measures CPU package + DRAM. :param rapl_prefer_psys: Prefer psys over package domains for RAPL on Linux (default: False). When True, uses total platform power. + :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). :return: The decorated function """ @@ -1450,6 +1477,7 @@ def wrapped_fn(*args, **kwargs): allow_multiple_runs=allow_multiple_runs, rapl_include_dram=rapl_include_dram, rapl_prefer_psys=rapl_prefer_psys, + telemetry_level=telemetry_level, ) else: tracker = EmissionsTracker( @@ -1484,6 +1512,7 @@ def wrapped_fn(*args, **kwargs): allow_multiple_runs=allow_multiple_runs, rapl_include_dram=rapl_include_dram, rapl_prefer_psys=rapl_prefer_psys, + telemetry_level=telemetry_level, ) tracker.start() try: diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index e8b42b929..5cabf13ee 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -107,3 +107,9 @@ os.environ["HTTPS_PROXY"] = "http://0.0.0.0:0000" For more information, please read the [requests library proxy documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies) + +## Product telemetry + +Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Set it in `.codecarbon.config`, via `CODECARBON_TELEMETRY_LEVEL`, or with `EmissionsTracker(telemetry_level=...)` (argument wins). Tier 1 (`minimal`) sends private product telemetry once per run at `stop()`—see [Product telemetry](telemetry.md). + +See [Product telemetry](telemetry.md) for tiers, what is collected, and how to opt out. diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md new file mode 100644 index 000000000..b5bc56057 --- /dev/null +++ b/docs/how-to/telemetry.md @@ -0,0 +1,105 @@ +# Product telemetry + +CodeCarbon can send **optional private product telemetry** to help improve the library: hardware, environment, how the package is used, and per-run carbon/energy summaries. This is separate from sending **your** emissions to the [dashboard](cloud-api.md) with `save_to_api=True`. + +## Telemetry vs your dashboard data + +| | Product telemetry | Your emissions (`save_to_api`) | +|--|-------------------|--------------------------------| +| Purpose | Improve CodeCarbon (aggregate usage) | Your projects and experiments | +| Config | `telemetry_level`, `codecarbon telemetry` | `codecarbon config`, `experiment_id` | +| Default API target | Built-in telemetry project (private) | Your account / experiment | + +You can use one without the other. + +## Tiers + +| `telemetry_level` | Name | When | Transport | +|-------------------|------|------|-----------| +| `disabled` | — | — | Nothing | +| `minimal` | Private product telemetry | Each `stop()` | `POST /telemetry` (private) | +| `extensive` | Private telemetry + shared run summary | Each `stop()` | Same private `POST /telemetry` **and** `ApiClient` → `/emissions` | + +Tier is resolved in this order: + +1. **Tracker or CLI argument** — `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` +2. **Config + environment** — `telemetry_level` in `.codecarbon.config`, then `CODECARBON_TELEMETRY_LEVEL` when both are set +3. **Default:** `minimal` (Tier 1) + +## Lifecycle + +```text +EmissionsTracker.__init__ → collect hardware/geo (no POST) +EmissionsTracker.stop() → minimal: private POST only | extensive: private POST + /emissions +``` + +If the run lasts less than one second, telemetry is not sent. + +## Private telemetry (`minimal` and `extensive`) — per run + +Both levels send the **same private payload** to `POST /telemetry` at each `stop()`. The `telemetry_level` field records which setting was used (`minimal` or `extensive`). + +The payload includes: + +- **Environment:** OS, Python, CPU/GPU/RAM, country/region, cloud provider/region, GPU driver/CUDA/cuDNN when available +- **Usage:** tracking mode, output methods, integration surface (library / CLI / offline), task tracking, CI/notebook/container/IDE hints, hardware diagnostics +- **ML stack:** framework presence flags when detected (no installed package versions) +- **Run outcome:** duration, emissions, energy (total and per component), utilization averages + +Private telemetry does **not** include project names, experiment ids, API keys, file paths, exact coordinates, executable/host hashes, or survey demographics (role, industry, etc.). + +## `extensive` — additional public run summary + +**Also** posts a **run emissions summary** to the shared CodeCarbon telemetry experiment via `ApiClient` (`/runs` then `/emissions`). Endpoint, API key, and experiment id come from `telemetry_api_url` / `telemetry_api_key` / `telemetry_experiment_id` (or `CODECARBON_TELEMETRY_*` env vars), falling back to the built-in defaults and your `api_endpoint` / `api_key` when set. + +## Never collected + +- Project name, experiment id, run id, API keys +- Source code, file paths, hostnames +- Exact GPS coordinates, executable/host fingerprints (not in the telemetry schema) +- Voluntary [user survey](https://docs.google.com/forms/d/e/1FAIpQLSeQ5Tu_rdrpDhBJvh5R1-_iB4Ld-kgh6iNMjgaMXa8AEVPxqA/viewform) demographics (role, industry, experience) + +## Configure telemetry + +### Config file + +```ini +[codecarbon] +telemetry_level = minimal +``` + +### CLI + +```bash +codecarbon telemetry set minimal +codecarbon telemetry show +codecarbon monitor --telemetry-level disabled -- python train.py +``` + +### Python + +```python +from codecarbon import EmissionsTracker + +tracker = EmissionsTracker(telemetry_level="minimal") +tracker.start() +# ... +tracker.stop() +``` + +## Opt out + +```ini +[codecarbon] +telemetry_level = disabled +``` + +## First run without explicit configuration + +If you never set `telemetry_level`, CodeCarbon uses `minimal` (Tier 1) and logs a **one-time warning** per Python session. Set `telemetry_level` explicitly to silence it. + +## Related + +- [Configure CodeCarbon](configuration.md) +- [CLI reference](../reference/cli.md#codecarbon-telemetry) +- [Cloud API & dashboard](cloud-api.md) diff --git a/docs/reference/api.md b/docs/reference/api.md index 15117b123..8b8bea306 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -12,6 +12,20 @@ Parameters can be set via `EmissionsTracker()`, `OfflineEmissionsTracker()`, the If you use `CUDA_VISIBLE_DEVICES` or `ROCR_VISIBLE_DEVICES` to set GPUs, CodeCarbon will automatically populate `gpu_ids`. Manual `gpu_ids` overrides this. +## Product telemetry + +Optional library telemetry is controlled by **`telemetry_level`** on the tracker (same parameter on `OfflineEmissionsTracker` and `@track_emissions`): + +| Value | Behavior | +|-------|----------| +| `disabled` | No product telemetry | +| `minimal` (default) | Tier 1 private telemetry at each `stop()` | +| `extensive` | Tier 1 + Tier 2 (private telemetry and public run summary) at each `stop()` | + +**Resolution order:** tracker argument → `.codecarbon.config` → `CODECARBON_TELEMETRY_LEVEL` → default `minimal`. The tracker argument overrides config and environment. + +This is separate from `save_to_api` (your dashboard experiment). See [Product telemetry](../how-to/telemetry.md). + ## EmissionsTracker / BaseEmissionsTracker `EmissionsTracker` and `OfflineEmissionsTracker` inherit from `BaseEmissionsTracker`. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 1e93a588f..6c550f625 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -45,6 +45,7 @@ Displays real-time emissions data for all processes on your machine. Press `Ctrl | `--offline` | flag | false | Run without internet access | | `--country-iso-code` | string | - | ISO 3166-1 alpha-3 country code (required in offline mode) | | `--log-level` | choice | INFO | Log level: DEBUG, INFO, WARNING, ERROR | +| `--telemetry-level` | string | - | One-run tier: `disabled`, `minimal`, or `extensive` | **Examples:** ```bash @@ -88,6 +89,34 @@ codecarbon monitor -- node app.js --port 8080 Same options as `codecarbon monitor` apply (see above). +### `codecarbon telemetry` + +Configure **product telemetry** (library usage metadata), separate from `codecarbon config` (dashboard org/project/experiment). + +**Usage:** + +```bash +codecarbon telemetry # interactive wizard +codecarbon telemetry set # disabled | minimal | extensive +codecarbon telemetry show # resolved tier and whether it was set explicitly +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--config PATH` | Use a specific `.codecarbon.config` (default: local then global) | + +See [Product telemetry](../how-to/telemetry.md) for what Tier 1 collects and how to opt out. + +### `codecarbon monitor --telemetry-level` + +Override the telemetry tier for a single monitor run (does not update config). + +```bash +codecarbon monitor --telemetry-level disabled -- python train.py +``` + ### `codecarbon detect` Detect and print hardware information. diff --git a/mkdocs.yml b/mkdocs.yml index b82641fb7..1de7399a8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -149,6 +149,7 @@ nav: - LLMs and Agents: how-to/agents.md - How-to Guides: - Configure CodeCarbon: how-to/configuration.md + - Product telemetry: how-to/telemetry.md - Compare Model Efficiency: tutorials/comparing-model-efficiency.md - Dashboard & Visualization: - Use the Cloud API & Dashboard: how-to/cloud-api.md diff --git a/tests/cli/test_telemetry_cli.py b/tests/cli/test_telemetry_cli.py new file mode 100644 index 000000000..88cd9375b --- /dev/null +++ b/tests/cli/test_telemetry_cli.py @@ -0,0 +1,111 @@ +"""Tests for codecarbon telemetry CLI commands.""" + +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +import typer +from typer.testing import CliRunner + +from codecarbon.cli import main as cli_main +from codecarbon.cli.telemetry_cli import normalize_telemetry_level, telemetry_app + + +def test_normalize_telemetry_level_accepts_valid_values(): + assert normalize_telemetry_level("MINIMAL") == "minimal" + assert normalize_telemetry_level("disabled") == "disabled" + + +def test_normalize_telemetry_level_rejects_invalid(): + with pytest.raises(typer.BadParameter): + normalize_telemetry_level("bogus") + + +def test_telemetry_set_writes_config(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / ".codecarbon.config" + result = runner.invoke( + telemetry_app, + ["set", "disabled", "--config", str(config_path)], + ) + assert result.exit_code == 0 + assert "telemetry_level = disabled" in result.output + content = config_path.read_text() + assert "telemetry_level = disabled" in content + + +def test_telemetry_show_reports_stored_level(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / ".codecarbon.config" + config_path.write_text("[codecarbon]\ntelemetry_level = extensive\n") + result = runner.invoke( + telemetry_app, + ["show", "--config", str(config_path)], + ) + assert result.exit_code == 0 + assert "extensive" in result.output + assert "Explicitly configured: True" in result.output + + +def test_telemetry_show_merged_matches_tracker_precedence(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + global_path = Path(tmp) / "global.config" + local_path = Path(tmp) / "local.config" + global_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + local_path.write_text("[codecarbon]\ntelemetry_level = disabled\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=(str(global_path), str(local_path)), + ): + result = runner.invoke(telemetry_app, ["show"]) + assert result.exit_code == 0 + assert "Resolved tier: disabled" in result.output + assert "merged" in result.output + + +def test_monitor_passes_telemetry_level_override(monkeypatch): + from codecarbon.cli import monitor as monitor_module + + captured = {} + + class FakeTracker: + def __init__(self, **kwargs): + captured.update(kwargs) + self._conf = {"output_file": "emissions.csv"} + + def start(self): + return None + + def stop(self): + return 0.0 + + monkeypatch.setattr(monitor_module, "EmissionsTracker", FakeTracker) + + runner = CliRunner() + result = runner.invoke( + cli_main.codecarbon, + [ + "monitor", + "--no-api", + "--telemetry-level", + "disabled", + "--", + "echo", + "ok", + ], + ) + assert result.exit_code == 0 + assert captured.get("telemetry_level") == "disabled" + + +def test_monitor_rejects_invalid_telemetry_level(): + runner = CliRunner() + result = runner.invoke( + cli_main.codecarbon, + ["monitor", "--telemetry-level", "invalid"], + ) + assert result.exit_code != 0 diff --git a/tests/test_api_call.py b/tests/test_api_call.py index a4bb4cd7f..e0b61d01c 100644 --- a/tests/test_api_call.py +++ b/tests/test_api_call.py @@ -244,6 +244,22 @@ def test_add_emission_returns_false_on_unsuccessful_post(self): ) ) + def test_create_run_handles_none_coordinates(self): + conf_with_none_coords = {**conf, "longitude": None, "latitude": None} + with requests_mock.Mocker() as m: + m.post("http://test.com/runs", json={"id": "run-id"}, status_code=201) + api = ApiClient( + endpoint_url="http://test.com", + experiment_id="experiment_id", + api_key="Toto", + conf=conf_with_none_coords, + create_run_automatically=False, + ) + run_id = api._create_run("experiment_id") + self.assertEqual(run_id, "run-id") + self.assertEqual(m.last_request.json()["longitude"], 0.0) + self.assertEqual(m.last_request.json()["latitude"], 0.0) + def test_create_run_returns_none_on_unsuccessful_status(self): with requests_mock.Mocker() as m: m.post("http://test.com/runs", text="bad", status_code=400) diff --git a/tests/test_config.py b/tests/test_config.py index 181913c6c..1f431e236 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +import configparser import os import unittest from textwrap import dedent @@ -16,6 +17,19 @@ from tests.testutils import get_custom_mock_open +def _file_settings_from_ini(global_conf: str, local_conf: str) -> dict[str, str]: + """Merge mocked global and local ``[codecarbon]`` sections like config files do.""" + merged: dict[str, str] = {} + for text in (global_conf, local_conf): + if not text.strip(): + continue + parser = configparser.ConfigParser() + parser.read_string(text) + if "codecarbon" in parser: + merged.update(dict(parser["codecarbon"])) + return merged + + class TestConfig(unittest.TestCase): def setUp(self): self._original_environ = os.environ.copy() @@ -23,6 +37,12 @@ def setUp(self): "CODECARBON_API_KEY", "CODECARBON_EXPERIMENT_ID", "CODECARBON_API_ENDPOINT", + "CODECARBON_TELEMETRY", + "CODECARBON_TELEMETRY_LEVEL", + "CODECARBON_TELEMETRY_PROJECT_TOKEN", + "CODECARBON_TELEMETRY_API_URL", + "CODECARBON_TELEMETRY_API_KEY", + "CODECARBON_TELEMETRY_EXPERIMENT_ID", "codecarbon_api_key", "codecarbon_experiment_id", "codecarbon_api_endpoint", @@ -115,7 +135,8 @@ def test_read_confs(self): ) with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) + "codecarbon.core.config.get_config_file_settings", + return_value=_file_settings_from_ini(global_conf, local_conf), ): conf = dict(get_hierarchical_config()) target = { @@ -156,7 +177,8 @@ def test_read_confs_and_parse_envs(self): ) with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) + "codecarbon.core.config.get_config_file_settings", + return_value=_file_settings_from_ini(global_conf, local_conf), ): conf = dict(get_hierarchical_config()) target = { @@ -171,12 +193,7 @@ def test_read_confs_and_parse_envs(self): self.assertDictEqual(conf, target) def test_empty_conf(self): - global_conf = "" - local_conf = "" - - with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) - ): + with patch("codecarbon.core.config.get_config_file_settings", return_value={}): conf = dict(get_hierarchical_config()) # allow_multiple_runs is set in pytest.ini and not mocked, so it's visible here. target = {"allow_multiple_runs": "True"} diff --git a/tests/test_config_file_settings.py b/tests/test_config_file_settings.py new file mode 100644 index 000000000..6a155ef7e --- /dev/null +++ b/tests/test_config_file_settings.py @@ -0,0 +1,51 @@ +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from codecarbon.core.config import get_config_file_settings, get_hierarchical_config + + +class TestGetConfigFileSettings(unittest.TestCase): + def test_returns_empty_when_no_config_files(self): + with patch("codecarbon.core.config._config_file_paths") as mock_paths: + mock_paths.return_value = ("/nonexistent/global", "/nonexistent/local") + settings = get_config_file_settings() + self.assertEqual(settings, {}) + + def test_local_overrides_global_telemetry_level(self): + with tempfile.TemporaryDirectory() as tmp: + global_path = Path(tmp) / "global.config" + local_path = Path(tmp) / "local.config" + global_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + local_path.write_text("[codecarbon]\ntelemetry_level = disabled\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=(str(global_path), str(local_path)), + ): + settings = get_config_file_settings() + self.assertEqual(settings["telemetry_level"], "disabled") + + def test_hierarchical_config_includes_env_but_file_settings_do_not(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY": "disabled"}, + clear=False, + ): + file_settings = get_config_file_settings() + hierarchical = get_hierarchical_config() + self.assertEqual(file_settings.get("telemetry_level"), "minimal") + self.assertNotIn("telemetry", file_settings) + self.assertEqual(hierarchical.get("telemetry"), "disabled") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index ab4a0a275..2fe5a1199 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -35,7 +35,7 @@ def heavy_computation(run_time_secs: float = 3): pass -empty_conf = "[codecarbon]" +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" if sys.platform == "darwin": @@ -73,7 +73,8 @@ def setUp(self) -> None: # ./.codecarbon.config so that the user's local configuration does not # alter tests patcher = mock.patch( - "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + "builtins.open", + new_callable=get_custom_mock_open(disabled_conf, disabled_conf), ) self.addCleanup(patcher.stop) patcher.start() diff --git a/tests/test_offline_emissions_tracker.py b/tests/test_offline_emissions_tracker.py index 07adf403c..4b9b4b4ec 100644 --- a/tests/test_offline_emissions_tracker.py +++ b/tests/test_offline_emissions_tracker.py @@ -18,7 +18,7 @@ def heavy_computation(run_time_secs: float = 3): pass -empty_conf = "[codecarbon]" +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" class TestOfflineEmissionsTracker(unittest.TestCase): @@ -32,7 +32,8 @@ def setUp(self) -> None: # ./.codecarbon.config so that the user's local configuration does not # alter tests patcher = mock.patch( - "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + "builtins.open", + new_callable=get_custom_mock_open(disabled_conf, disabled_conf), ) self.addCleanup(patcher.stop) patcher.start() diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 000000000..78c64359d --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,226 @@ +import sys +import tempfile +import unittest +from unittest.mock import ANY, MagicMock, patch + +from codecarbon.core.telemetry import Telemetry +from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker +from codecarbon.output_methods.emissions_data import EmissionsData +from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open + +if sys.platform == "darwin": + mock_platform_cli_setup = patch( + "codecarbon.core.powermetrics.ApplePowermetrics._setup_cli" + ) +else: + mock_platform_cli_setup = patch("codecarbon.core.cpu.IntelPowerGadget._setup_cli") + +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" +minimal_conf = "[codecarbon]\ntelemetry_level = minimal\n" +extensive_conf = "[codecarbon]\ntelemetry_level = extensive\n" + + +class TestTelemetryTiersAtStop(unittest.TestCase): + def _emissions(self) -> EmissionsData: + return EmissionsData( + timestamp="2020-01-01T00:00:00", + project_name="test", + run_id="run-1", + experiment_id="exp-1", + duration=10.0, + emissions=0.001, + emissions_rate=0.0001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="2.0", + cpu_count=1.0, + cpu_model="cpu", + gpu_count=0.0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=8.0, + tracking_mode="machine", + ) + + def test_tier1_posts_private_telemetry(self): + tracker = MagicMock() + tracker._config_file_conf = {} + tracker._external_conf = {} + tracker._telemetry_override = None + tracker._conf = {"os": "Linux", "tracking_mode": "machine"} + tracker._hardware = [] + tracker._resource_tracker = None + tracker._save_to_api = False + tracker._save_to_file = False + tracker._save_to_logger = False + tracker._emissions_endpoint = None + tracker._save_to_prometheus = False + tracker._save_to_logfire = False + tracker._tasks = {} + tracker._measure_power_secs = 15 + emissions = self._emissions() + telemetry = Telemetry.from_tracker(tracker) + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + telemetry.send_at_stop(tracker, emissions) + mock_post.assert_called_once() + payload = mock_post.call_args[0][1] + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertEqual(payload["total_emissions_kg"], 0.001) + + def test_tier1_skips_short_duration_at_dispatcher(self): + tracker = MagicMock() + tracker._config_file_conf = {} + tracker._external_conf = {} + tracker._telemetry_override = None + emissions = self._emissions() + emissions.duration = 0.5 + telemetry = Telemetry.from_tracker(tracker) + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: + telemetry.send_at_stop(tracker, emissions) + mock_post.assert_not_called() + + def test_tier2_uses_api_client(self): + tracker = MagicMock() + tracker._conf = {"os": "Linux"} + tracker._config_file_conf = {"telemetry_level": "extensive"} + tracker._external_conf = {} + tracker._telemetry_override = None + emissions = self._emissions() + telemetry = Telemetry.from_tracker(tracker) + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ): + with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + telemetry.send_at_stop(tracker, emissions) + mock_api_cls.assert_called_once() + mock_api.add_emission.assert_called_once() + mock_api_cls.assert_called_with( + endpoint_url=ANY, + experiment_id=ANY, + api_key=ANY, + conf=tracker._conf, + create_run_automatically=True, + ) + + +@mock_platform_cli_setup +class TestTrackerTelemetry(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.patcher = None + Telemetry._default_warning_shown = False + + def tearDown(self) -> None: + if self.patcher: + self.patcher.stop() + self.temp_dir.cleanup() + Telemetry._default_warning_shown = False + + def _start_config_mock(self, conf: str) -> None: + self.patcher = patch( + "builtins.open", new_callable=get_custom_mock_open(conf, conf) + ) + self.patcher.start() + + def test_emissions_tracker_does_not_send_telemetry_on_init(self, mock_cli_setup): + self._start_config_mock(minimal_conf) + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker(save_to_api=False, save_to_file=False) + mock_post.assert_not_called() + + def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( + self, mock_cli_setup + ): + self._start_config_mock(minimal_conf) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + payload = mock_post.call_args[0][1] + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertIn("total_emissions_kg", payload) + + def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): + self._start_config_mock(disabled_conf) + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_not_called() + + def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): + self._start_config_mock(extensive_conf) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + with patch( + "codecarbon.core.telemetry.client.ApiClient" + ) as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + mock_api_cls.assert_called_once() + mock_api.add_emission.assert_called_once() + + def test_offline_tracker_sends_on_stop(self, mock_cli_setup): + self._start_config_mock(minimal_conf) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 14ade8b7f..8b0128d2a 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -1,29 +1,23 @@ import unittest +from unittest.mock import patch import requests_mock from pydantic import ValidationError -from codecarbon.core.telemetry_client import TelemetryClient -from codecarbon.core.telemetry_schemas import TelemetryCreate +from codecarbon.core.telemetry import TelemetrySettings, post_private -class TestTelemetryClient(unittest.TestCase): - def test_init_sets_up_client_without_calling_api(self): - with requests_mock.Mocker() as m: - client = TelemetryClient( - endpoint_url="http://test.com/", - telemetry={ - "timestamp": "2026-05-03T12:00:00+00:00", - "telemetry_level": "minimal", - }, - ) - - self.assertEqual(client.endpoint_url, "http://test.com") - self.assertEqual(client.telemetry_url, "http://test.com/telemetry") - self.assertIsInstance(client.telemetry, TelemetryCreate) - self.assertEqual(m.call_count, 0) +class TestPostPrivate(unittest.TestCase): + def _settings(self, api_url: str = "http://test.com", api_key: str | None = None): + return TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url=api_url, + api_key=api_key or TelemetrySettings.resolve().api_key, + experiment_id=TelemetrySettings.resolve().experiment_id, + ) - def test_add_telemetry_posts_configured_payload(self): + def test_post_private_sends_validated_payload(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -36,15 +30,9 @@ def test_add_telemetry_posts_configured_payload(self): json="f52fe339-164d-4c2b-a8c0-f562dfce066d", status_code=201, ) - client = TelemetryClient( - endpoint_url="http://test.com", telemetry=telemetry - ) - - actual_telemetry_id = client.add_telemetry() + result = post_private(self._settings(), telemetry) - self.assertEqual( - actual_telemetry_id, "f52fe339-164d-4c2b-a8c0-f562dfce066d" - ) + self.assertTrue(result) self.assertEqual(m.call_count, 1) self.assertEqual( m.last_request.json(), @@ -54,51 +42,65 @@ def test_add_telemetry_posts_configured_payload(self): }, ) - def test_add_telemetry_posts_call_payload(self): - telemetry = TelemetryCreate( - timestamp="2026-05-03T12:00:00+00:00", - telemetry_level="minimal", - os="Linux-5.10.0-x86_64", - ) + def test_post_private_rejects_invalid_payload(self): + with self.assertRaises(ValidationError): + post_private( + self._settings(), + { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + "unknown_field": "value", + }, + ) + def test_post_private_logs_warning_on_404(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } with requests_mock.Mocker() as m: m.post( "http://test.com/telemetry", - json="f52fe339-164d-4c2b-a8c0-f562dfce066d", - status_code=201, - ) - client = TelemetryClient(endpoint_url="http://test.com") - - actual_telemetry_id = client.add_telemetry(telemetry) - - self.assertEqual( - actual_telemetry_id, "f52fe339-164d-4c2b-a8c0-f562dfce066d" - ) - self.assertEqual(m.call_count, 1) - self.assertEqual( - m.last_request.json(), - { - "timestamp": "2026-05-03T12:00:00Z", - "telemetry_level": "minimal", - "os": "Linux-5.10.0-x86_64", - }, + text='{"detail":"Not Found"}', + status_code=404, ) + with patch("codecarbon.core.telemetry.client.logger") as mock_logger: + result = post_private(self._settings(), telemetry) + self.assertFalse(result) + mock_logger.warning.assert_called_once() - def test_init_rejects_invalid_telemetry_without_calling_api(self): + def test_post_private_sends_api_key_header_when_configured(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + settings = TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url="http://test.com", + api_key="cpt_test_key", + experiment_id=TelemetrySettings.resolve().experiment_id, + ) with requests_mock.Mocker() as m: - with self.assertRaises(ValidationError): - TelemetryClient( - endpoint_url="http://test.com", - telemetry={ - "timestamp": "2026-05-03T12:00:00+00:00", - "telemetry_level": "minimal", - "total_emissions_kg": 0.42, - }, - ) + m.post( + "http://test.com/telemetry", + json="telemetry-id", + status_code=201, + ) + post_private(settings, telemetry) + self.assertEqual(m.last_request.headers["x-api-token"], "cpt_test_key") - self.assertEqual(m.call_count, 0) + def test_post_private_returns_false_on_request_error(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + with patch("codecarbon.core.telemetry.client.requests.post") as mock_post: + mock_post.side_effect = ConnectionError("network down") + with patch("codecarbon.core.telemetry.client.logger"): + result = post_private(self._settings(), telemetry) + self.assertFalse(result) - def test_add_telemetry_returns_none_without_payload(self): - client = TelemetryClient(endpoint_url="http://test.com") - self.assertIsNone(client.add_telemetry()) +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py new file mode 100644 index 000000000..0a5f4f379 --- /dev/null +++ b/tests/test_telemetry_collect.py @@ -0,0 +1,121 @@ +import unittest +from unittest.mock import MagicMock, patch + +from codecarbon.core.telemetry import TelemetryContext, TelemetryLevel, build_payload +from codecarbon.output_methods.emissions_data import EmissionsData + + +def _sample_emissions(**overrides): + base = dict( + timestamp="2026-01-01T00:00:00", + project_name="p", + run_id="r", + experiment_id="e", + duration=10.0, + emissions=0.5, + emissions_rate=0.05, + cpu_power=1.0, + gpu_power=2.0, + ram_power=0.5, + cpu_energy=0.01, + gpu_energy=0.02, + ram_energy=0.001, + energy_consumed=0.031, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="3.0", + cpu_count=4, + cpu_model="cpu", + gpu_count=1, + gpu_model="gpu", + longitude=0.0, + latitude=0.0, + ram_total_size=16.0, + tracking_mode="machine", + ) + base.update(overrides) + return EmissionsData(**base) + + +def _tracker_context(**overrides) -> TelemetryContext: + tracker = MagicMock() + tracker._conf = overrides.pop("conf", {"codecarbon_version": "3.0"}) + tracker._hardware = overrides.pop("hardware", []) + tracker._resource_tracker = overrides.pop("resource_tracker", None) + tracker._save_to_file = overrides.pop("save_to_file", False) + tracker._save_to_api = overrides.pop("save_to_api", False) + tracker._save_to_logger = overrides.pop("save_to_logger", False) + tracker._emissions_endpoint = overrides.pop("emissions_endpoint", None) + tracker._save_to_prometheus = overrides.pop("save_to_prometheus", False) + tracker._save_to_logfire = overrides.pop("save_to_logfire", False) + tracker._tasks = overrides.pop("tasks", {}) + tracker._measure_power_secs = overrides.pop("measure_power_secs", 15) + tracker._is_offline = overrides.pop("is_offline", False) + emissions = overrides.pop("emissions", _sample_emissions()) + ctx = TelemetryContext( + conf=tracker._conf, + emissions=emissions, + hardware=tracker._hardware, + resource_tracker=tracker._resource_tracker, + save_to_api=tracker._save_to_api, + save_to_file=tracker._save_to_file, + save_to_logger=tracker._save_to_logger, + save_to_prometheus=tracker._save_to_prometheus, + save_to_logfire=tracker._save_to_logfire, + emissions_endpoint=tracker._emissions_endpoint, + tasks=tracker._tasks, + measure_power_secs=tracker._measure_power_secs, + is_offline=tracker._is_offline, + ) + return ctx + + +class TestTelemetryCollect(unittest.TestCase): + def test_build_payload_includes_run_and_framework_flags(self): + ctx = _tracker_context( + conf={ + "os": "Linux", + "codecarbon_version": "3.0", + "cpu_count": 4, + "tracking_mode": "machine", + }, + save_to_file=True, + ) + with patch( + "codecarbon.core.telemetry.collect._package_installed", + side_effect=lambda name: name == "torch", + ): + payload = build_payload(ctx) + + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertEqual(payload["total_emissions_kg"], 0.5) + self.assertEqual(payload["duration_seconds"], 10.0) + self.assertTrue(payload["has_torch"]) + self.assertIn("file", payload["output_methods"]) + + def test_build_payload_omits_framework_versions(self): + ctx = _tracker_context(conf={"codecarbon_version": "3.0", "hardware": ["cpu"]}) + with patch( + "codecarbon.core.telemetry.collect._package_installed", + return_value=True, + ): + payload = build_payload(ctx) + + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertTrue(payload["has_torch"]) + self.assertNotIn("torch_version", payload) + + def test_build_payload_uses_resolved_level(self): + ctx = _tracker_context(conf={"codecarbon_version": "3.0"}) + payload = build_payload(ctx, level=TelemetryLevel.extensive) + self.assertEqual(payload["telemetry_level"], "extensive") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py new file mode 100644 index 000000000..6b230ee83 --- /dev/null +++ b/tests/test_telemetry_config.py @@ -0,0 +1,300 @@ +"""Integration tests for telemetry tier resolution and config contract (Task 5).""" + +import os +import sys +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import MagicMock, patch + +from codecarbon.core.config import get_config_file_settings +from codecarbon.core.telemetry import ( + Telemetry, + TelemetryLevel, + TelemetrySettings, + post_private, +) +from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker +from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open + +if sys.platform == "darwin": + mock_platform_cli_setup = patch( + "codecarbon.core.powermetrics.ApplePowermetrics._setup_cli" + ) +else: + mock_platform_cli_setup = patch("codecarbon.core.cpu.IntelPowerGadget._setup_cli") + + +def _conf(level: str) -> str: + return f"[codecarbon]\ntelemetry_level = {level}\n" + + +class TestTelemetryConfigContract(unittest.TestCase): + def setUp(self) -> None: + Telemetry._default_warning_shown = False + + def tearDown(self) -> None: + Telemetry._default_warning_shown = False + + def test_warns_once_when_telemetry_not_explicit(self): + settings = TelemetrySettings.resolve(config_file_conf={}) + telemetry = Telemetry(settings) + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: + telemetry.warn_if_implicit() + telemetry.warn_if_implicit() + self.assertEqual(mock_warning.call_count, 1) + self.assertIn("Tier 1", mock_warning.call_args[0][0]) + + def test_no_warn_when_config_explicit(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "disabled"} + ) + telemetry = Telemetry(settings) + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: + telemetry.warn_if_implicit() + mock_warning.assert_not_called() + + def test_tier1_posts_to_telemetry_endpoint(self): + tier1_payload = { + "timestamp": datetime(2020, 1, 1, tzinfo=timezone.utc), + "telemetry_level": "minimal", + "total_emissions_kg": 0.001, + "os": "Linux", + } + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://tier1.example"} + ) + with patch("codecarbon.core.telemetry.client.requests.post") as mock_post: + mock_post.return_value.status_code = 201 + mock_post.return_value.json.return_value = "telemetry-id" + post_private(settings, tier1_payload) + mock_post.assert_called_once() + self.assertEqual( + mock_post.call_args.kwargs["url"], "http://tier1.example/telemetry" + ) + self.assertEqual( + mock_post.call_args.kwargs["json"]["telemetry_level"], + TelemetryLevel.minimal.value, + ) + self.assertIn("total_emissions_kg", mock_post.call_args.kwargs["json"]) + + def test_legacy_env_codecarbon_telemetry_does_not_change_tier(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text(_conf("minimal")) + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY": "disabled"}, + clear=False, + ): + from codecarbon.core.config import get_hierarchical_config + + settings = TelemetrySettings.resolve( + config_file_conf=get_config_file_settings(), + external_conf=get_hierarchical_config(), + ) + self.assertEqual(settings.level, TelemetryLevel.minimal) + + def test_env_codecarbon_telemetry_level_overrides_file(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text(_conf("minimal")) + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_LEVEL": "disabled"}, + clear=False, + ): + from codecarbon.core.config import get_hierarchical_config + + settings = TelemetrySettings.resolve( + config_file_conf=get_config_file_settings(), + external_conf=get_hierarchical_config(), + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + + def test_telemetry_api_url_env_used_for_tier2_client(self): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_API_URL": "http://env-telemetry.example"}, + clear=False, + ): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, "http://env-telemetry.example") + + def test_telemetry_api_url_from_config_overrides_default(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://config-telemetry.example"} + ) + self.assertEqual(settings.api_url, "http://config-telemetry.example") + + +@mock_platform_cli_setup +class TestTrackerTelemetryFromConfig(unittest.TestCase): + def setUp(self) -> None: + Telemetry._default_warning_shown = False + self._config_patcher = None + + def tearDown(self) -> None: + if self._config_patcher: + self._config_patcher.stop() + Telemetry._default_warning_shown = False + + def _mock_config(self, conf: str) -> None: + self._config_patcher = patch( + "builtins.open", new_callable=get_custom_mock_open(conf, conf) + ) + self._config_patcher.start() + + def test_disabled_no_telemetry_on_stop(self, mock_cli_setup): + self._mock_config(_conf("disabled")) + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_not_called() + + def test_minimal_posts_tier1_on_stop_not_on_init(self, mock_cli_setup): + self._mock_config(_conf("minimal")) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + self.assertEqual( + mock_post.call_args[0][1]["telemetry_level"], + TelemetryLevel.minimal.value, + ) + + def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup): + self._mock_config(_conf("extensive")) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + with patch( + "codecarbon.core.telemetry.client.ApiClient" + ) as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + mock_api_cls.assert_called_once() + mock_api.add_emission.assert_called_once() + + def test_offline_minimal_posts_tier1_on_stop(self, mock_cli_setup): + self._mock_config(_conf("minimal")) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + + def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup): + self._mock_config("[codecarbon]\n") + env_without_telemetry = { + key: value + for key, value in os.environ.items() + if key.lower() + not in ( + "codecarbon_telemetry", + "codecarbon_telemetry_level", + ) + } + with patch.dict(os.environ, env_without_telemetry, clear=True): + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", + return_value=True, + ): + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker(save_to_api=False, save_to_file=False) + configure_warnings = [ + c + for c in mock_warning.call_args_list + if c[0] and "telemetry" in str(c[0][0]).lower() + ] + self.assertEqual(len(configure_warnings), 1) + + def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): + self._mock_config("[codecarbon]\n") + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: + with patch("codecarbon.core.telemetry.dispatcher.post_private"): + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker( + telemetry_level="disabled", + save_to_api=False, + save_to_file=False, + ) + configure_warnings = [ + c for c in mock_warning.call_args_list if c[0] and "Tier 1" in str(c[0][0]) + ] + self.assertEqual(len(configure_warnings), 0) + + def test_env_telemetry_disabled_does_not_change_resolved_level( + self, mock_cli_setup + ): + self._mock_config(_conf("minimal")) + with ensure_telemetry_run_duration(): + with patch.dict( + os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False + ): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", + return_value=True, + ) as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + save_to_api=False, save_to_file=False + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py new file mode 100644 index 000000000..e9a1e09f2 --- /dev/null +++ b/tests/test_telemetry_settings.py @@ -0,0 +1,174 @@ +import os +import unittest +from unittest.mock import patch + +from codecarbon.core.telemetry import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_API_URL, + DEFAULT_TELEMETRY_EXPERIMENT_ID, + TelemetryLevel, + TelemetrySettings, + parse_telemetry_level, +) + + +class TestParseTelemetryLevel(unittest.TestCase): + def test_parse_accepts_enum(self): + self.assertEqual( + parse_telemetry_level(TelemetryLevel.minimal), TelemetryLevel.minimal + ) + + def test_parse_normalizes_case(self): + self.assertEqual(parse_telemetry_level("EXTENSIVE"), TelemetryLevel.extensive) + + def test_parse_rejects_invalid(self): + with self.assertRaises(ValueError): + parse_telemetry_level("bogus") + + +class TestTelemetrySettingsResolve(unittest.TestCase): + def test_default_is_minimal_when_unset(self): + settings = TelemetrySettings.resolve(config_file_conf={}) + self.assertEqual(settings.level, TelemetryLevel.minimal) + self.assertEqual(settings.source, "default") + + def test_telemetry_level_from_config_file(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "disabled"} + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + self.assertEqual(settings.source, "file") + + def test_telemetry_level_extensive(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "extensive"} + ) + self.assertEqual(settings.level, TelemetryLevel.extensive) + + def test_env_telemetry_key_ignored(self): + with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry": "extensive"} + ) + self.assertEqual(settings.level, TelemetryLevel.minimal) + + def test_invalid_level_falls_back_to_minimal(self): + with patch("codecarbon.core.telemetry.settings.logger.error") as mock_error: + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "bogus"} + ) + self.assertEqual(settings.level, TelemetryLevel.minimal) + mock_error.assert_called_once() + + def test_override_kwarg_takes_precedence_over_config_file(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"}, + override="disabled", + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + + def test_override_kwarg_takes_precedence_over_external_conf(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_level": "extensive"}, + override="disabled", + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + + def test_external_conf_env_overrides_file_when_merged(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"}, + external_conf={"telemetry_level": "disabled"}, + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + + def test_env_telemetry_level_via_external_conf(self): + with patch.dict( + os.environ, {"CODECARBON_TELEMETRY_LEVEL": "disabled"}, clear=False + ): + from codecarbon.core.config import get_hierarchical_config + + settings = TelemetrySettings.resolve( + external_conf=get_hierarchical_config() + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + + def test_is_explicit_with_config_file(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"} + ) + self.assertTrue(settings.is_explicit) + + def test_is_explicit_with_override(self): + settings = TelemetrySettings.resolve(override="disabled") + self.assertTrue(settings.is_explicit) + + def test_is_explicit_with_env_telemetry_level(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_level": "minimal"} + ) + self.assertTrue(settings.is_explicit) + + def test_is_not_explicit_when_unset(self): + settings = TelemetrySettings.resolve(config_file_conf={}) + self.assertFalse(settings.is_explicit) + + +class TestTelemetryApiSettings(unittest.TestCase): + def test_api_url_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://test.example"} + ) + self.assertEqual(settings.api_url, "http://test.example") + + def test_api_url_default(self): + env = { + k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_URL" + } + with patch.dict(os.environ, env, clear=True): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, DEFAULT_TELEMETRY_API_URL) + + def test_api_url_env_fallback(self): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_API_URL": "http://env.example"}, + clear=False, + ): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, "http://env.example") + + def test_api_key_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_key": "cpt_test"} + ) + self.assertEqual(settings.api_key, "cpt_test") + + def test_api_key_uses_public_default_when_unset(self): + env = { + k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_KEY" + } + with patch.dict(os.environ, env, clear=True): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_key, DEFAULT_TELEMETRY_API_KEY) + + def test_experiment_id_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={ + "telemetry_experiment_id": "00000000-0000-0000-0000-000000000001" + } + ) + self.assertEqual(settings.experiment_id, "00000000-0000-0000-0000-000000000001") + + def test_experiment_id_uses_public_default_when_unset(self): + env = { + k: v + for k, v in os.environ.items() + if k != "CODECARBON_TELEMETRY_EXPERIMENT_ID" + } + with patch.dict(os.environ, env, clear=True): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/testutils.py b/tests/testutils.py index e3d1dc2d1..57d94fa46 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -1,6 +1,8 @@ import builtins import unittest +from contextlib import contextmanager from pathlib import Path +from unittest.mock import patch from codecarbon.input import DataSource @@ -33,3 +35,22 @@ def conditional_open_func(path, *args, **kwargs): return conditional_open_func return mocked_open + + +@contextmanager +def ensure_telemetry_run_duration(min_seconds: float = 10.0): + """Force tracker stop emissions duration above telemetry's 1s minimum.""" + from codecarbon.emissions_tracker import BaseEmissionsTracker + + original_prepare = BaseEmissionsTracker._prepare_emissions_data + + def prepare_with_min_duration(self): + data = original_prepare(self) + if data is not None and (data.duration is None or data.duration < min_seconds): + data.duration = min_seconds + return data + + with patch.object( + BaseEmissionsTracker, "_prepare_emissions_data", prepare_with_min_duration + ): + yield