diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 00000000000..e72ef061fee --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2025-04-01 - Fast Serialization for Metrics +**Learning:** `dataclasses.asdict()` relies on recursive deep cloning internally, making it extremely slow for high-frequency operations like serializing metrics per request/token. Shallow iterating over `__dataclass_fields__` directly avoids this overhead. +**Action:** Replace `asdict()` with a custom field iteration method (falling back appropriately) in hot paths like metrics classes (`RequestMetrics`, `SpeculateMetrics`). diff --git a/fastdeploy/engine/request.py b/fastdeploy/engine/request.py index 0e95cd5e1fb..96a8c6cab68 100644 --- a/fastdeploy/engine/request.py +++ b/fastdeploy/engine/request.py @@ -16,6 +16,7 @@ from __future__ import annotations +import dataclasses import json import time import traceback @@ -897,7 +898,23 @@ def to_dict(self): """ Convert the RequestMetrics object to a dictionary. """ - return {k: v for k, v in asdict(self).items()} + res = {} + for k in self.__dataclass_fields__: + v = getattr(self, k) + if type(v) in (int, float, str, bool, type(None)): + res[k] = v + elif isinstance(v, list): + res[k] = list(v) + elif isinstance(v, dict): + res[k] = dict(v) + elif dataclasses.is_dataclass(v): + if hasattr(v, "to_dict"): + res[k] = v.to_dict() + else: + res[k] = asdict(v) + else: + res[k] = v + return res def record_recv_first_token(self): cur_time = time.time() diff --git a/fastdeploy/worker/output.py b/fastdeploy/worker/output.py index 365fec12475..0b4bf45efa4 100644 --- a/fastdeploy/worker/output.py +++ b/fastdeploy/worker/output.py @@ -14,6 +14,7 @@ # limitations under the License. """ +import dataclasses from dataclasses import dataclass, field from typing import NamedTuple, Optional @@ -164,6 +165,28 @@ class SpeculateMetrics: """ accept_ratio_per_head: list[float] + def to_dict(self): + """ + Convert the SpeculateMetrics object to a dictionary. + """ + res = {} + for k in self.__dataclass_fields__: + v = getattr(self, k) + if type(v) in (int, float, str, bool, type(None)): + res[k] = v + elif isinstance(v, list): + res[k] = list(v) + elif isinstance(v, dict): + res[k] = dict(v) + elif dataclasses.is_dataclass(v): + if hasattr(v, "to_dict"): + res[k] = v.to_dict() + else: + res[k] = dataclasses.asdict(v) + else: + res[k] = v + return res + @dataclass class SamplerOutput: