Add server observability tracing and metrics

README.md

@@ -45,6 +45,8 @@ We are very very early in this project. Expect bugs.
 
 We are not accepting contributions yet.
 
+Observability guide: [docs/observability.md](./docs/observability.md)
+
 ## If you REALLY want to contribute still.... read this first
 
 Read [CONTRIBUTING.md](./CONTRIBUTING.md) before opening an issue or PR.

apps/server/src/cli-config.test.ts

@@ -9,6 +9,18 @@ import { deriveServerPaths } from "./config";
 import { resolveServerConfig } from "./cli";
 
 it.layer(NodeServices.layer)("cli config resolution", (it) => {
+  const defaultObservabilityConfig = {
+    traceMinLevel: "Info",
+    traceTimingEnabled: true,
+    traceBatchWindowMs: 200,
+    traceMaxBytes: 10 * 1024 * 1024,
+    traceMaxFiles: 10,
+    otlpTracesUrl: undefined,
+    otlpMetricsUrl: undefined,
+    otlpExportIntervalMs: 10_000,
+    otlpServiceName: "t3-server",
+  } as const;
+
   const openBootstrapFd = Effect.fn(function* (payload: Record<string, unknown>) {
     const fs = yield* FileSystem.FileSystem;
     const filePath = yield* fs.makeTempFileScoped({ prefix: "t3-bootstrap-", suffix: ".ndjson" });
@@ -62,6 +74,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
 
       expect(resolved).toEqual({
         logLevel: "Warn",
+        ...defaultObservabilityConfig,
         mode: "desktop",
         port: 4001,
         cwd: process.cwd(),
@@ -123,6 +136,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
 
       expect(resolved).toEqual({
         logLevel: "Debug",
+        ...defaultObservabilityConfig,
         mode: "web",
         port: 8788,
         cwd: process.cwd(),
@@ -187,6 +201,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
 
       expect(resolved).toEqual({
         logLevel: "Info",
+        ...defaultObservabilityConfig,
         mode: "desktop",
         port: 4888,
         cwd: process.cwd(),
@@ -241,6 +256,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
         resolved.attachmentsDir,
         resolved.worktreesDir,
         path.dirname(resolved.serverLogPath),
+        path.dirname(resolved.serverTracePath),
       ]) {
         expect(yield* fs.exists(directory)).toBe(true);
       }
@@ -300,6 +316,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
 
       expect(resolved).toEqual({
         logLevel: "Debug",
+        ...defaultObservabilityConfig,
         mode: "web",
         port: 8788,
         cwd: process.cwd(),

apps/server/src/cli.ts

@@ -1,5 +1,5 @@
 import { NetService } from "@t3tools/shared/Net";
-import { Config, Effect, LogLevel, Option, Schema } from "effect";
+import { Config, Effect, FileSystem, LogLevel, Option, Path, Schema } from "effect";
 import { Command, Flag, GlobalFlag } from "effect/unstable/cli";
 
 import {
@@ -81,6 +81,27 @@ const logWebSocketEventsFlag = Flag.boolean("log-websocket-events").pipe(
 
 const EnvServerConfig = Config.all({
   logLevel: Config.logLevel("T3CODE_LOG_LEVEL").pipe(Config.withDefault("Info")),
+  traceMinLevel: Config.logLevel("T3CODE_TRACE_MIN_LEVEL").pipe(Config.withDefault("Info")),
+  traceTimingEnabled: Config.boolean("T3CODE_TRACE_TIMING_ENABLED").pipe(Config.withDefault(true)),
+  traceFile: Config.string("T3CODE_TRACE_FILE").pipe(
+    Config.option,
+    Config.map(Option.getOrUndefined),
+  ),
+  traceMaxBytes: Config.int("T3CODE_TRACE_MAX_BYTES").pipe(Config.withDefault(10 * 1024 * 1024)),
+  traceMaxFiles: Config.int("T3CODE_TRACE_MAX_FILES").pipe(Config.withDefault(10)),
+  traceBatchWindowMs: Config.int("T3CODE_TRACE_BATCH_WINDOW_MS").pipe(Config.withDefault(200)),
+  otlpTracesUrl: Config.string("T3CODE_OTLP_TRACES_URL").pipe(
+    Config.option,
+    Config.map(Option.getOrUndefined),
+  ),
+  otlpMetricsUrl: Config.string("T3CODE_OTLP_METRICS_URL").pipe(
+    Config.option,
+    Config.map(Option.getOrUndefined),
+  ),
+  otlpExportIntervalMs: Config.int("T3CODE_OTLP_EXPORT_INTERVAL_MS").pipe(
+    Config.withDefault(10_000),
+  ),
+  otlpServiceName: Config.string("T3CODE_OTLP_SERVICE_NAME").pipe(Config.withDefault("t3-server")),
   mode: Config.schema(RuntimeMode, "T3CODE_MODE").pipe(
     Config.option,
     Config.map(Option.getOrUndefined),
@@ -137,6 +158,8 @@ export const resolveServerConfig = (
 ) =>
   Effect.gen(function* () {
     const { findAvailablePort } = yield* NetService;
+    const path = yield* Path.Path;
+    const fs = yield* FileSystem.FileSystem;
     const env = yield* EnvServerConfig;
     const bootstrapFd = Option.getOrUndefined(flags.bootstrapFd) ?? env.bootstrapFd;
     const bootstrapEnvelope =
@@ -190,6 +213,8 @@ export const resolveServerConfig = (
     );
     const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);
     yield* ensureServerDirectories(derivedPaths);
+    const serverTracePath = env.traceFile ?? derivedPaths.serverTracePath;
+    yield* fs.makeDirectory(path.dirname(serverTracePath), { recursive: true });
     const noBrowser = resolveBooleanFlag(
       flags.noBrowser,
       Option.getOrElse(
@@ -248,11 +273,21 @@ export const resolveServerConfig = (
 
     const config: ServerConfigShape = {
       logLevel,
+      traceMinLevel: env.traceMinLevel,
+      traceTimingEnabled: env.traceTimingEnabled,
+      traceBatchWindowMs: env.traceBatchWindowMs,
+      traceMaxBytes: env.traceMaxBytes,
+      traceMaxFiles: env.traceMaxFiles,
+      otlpTracesUrl: env.otlpTracesUrl,
+      otlpMetricsUrl: env.otlpMetricsUrl,
+      otlpExportIntervalMs: env.otlpExportIntervalMs,
+      otlpServiceName: env.otlpServiceName,
       mode,
       port,
       cwd: process.cwd(),
       baseDir,
       ...derivedPaths,
+      serverTracePath,
       host,
       staticDir,
       devUrl,

apps/server/src/config.ts

@@ -25,6 +25,7 @@ export interface ServerDerivedPaths {
   readonly attachmentsDir: string;
   readonly logsDir: string;
   readonly serverLogPath: string;
+  readonly serverTracePath: string;
   readonly providerLogsDir: string;
   readonly providerEventLogPath: string;
   readonly terminalLogsDir: string;
@@ -36,6 +37,15 @@ export interface ServerDerivedPaths {
  */
 export interface ServerConfigShape extends ServerDerivedPaths {
   readonly logLevel: LogLevel.LogLevel;
+  readonly traceMinLevel: LogLevel.LogLevel;
+  readonly traceTimingEnabled: boolean;
+  readonly traceBatchWindowMs: number;
+  readonly traceMaxBytes: number;
+  readonly traceMaxFiles: number;
+  readonly otlpTracesUrl: string | undefined;
+  readonly otlpMetricsUrl: string | undefined;
+  readonly otlpExportIntervalMs: number;
+  readonly otlpServiceName: string;
   readonly mode: RuntimeMode;
   readonly port: number;
   readonly host: string | undefined;
@@ -68,6 +78,7 @@ export const deriveServerPaths = Effect.fn(function* (
     attachmentsDir,
     logsDir,
     serverLogPath: join(logsDir, "server.log"),
+    serverTracePath: join(logsDir, "server.trace.ndjson"),
     providerLogsDir,
     providerEventLogPath: join(providerLogsDir, "events.log"),
     terminalLogsDir: join(logsDir, "terminals"),
@@ -117,6 +128,15 @@ export class ServerConfig extends ServiceMap.Service<ServerConfig, ServerConfigS
 
         return {
           logLevel: "Error",
+          traceMinLevel: "Info",
+          traceTimingEnabled: true,
+          traceBatchWindowMs: 200,
+          traceMaxBytes: 10 * 1024 * 1024,
+          traceMaxFiles: 10,
+          otlpTracesUrl: undefined,
+          otlpMetricsUrl: undefined,
+          otlpExportIntervalMs: 10_000,
+          otlpServiceName: "t3-server",
           cwd,
           baseDir,
           ...derivedPaths,

apps/server/src/git/Layers/GitCore.ts

@@ -20,6 +20,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
 
 import { GitCommandError, type GitBranch } from "@t3tools/contracts";
 import { dedupeRemoteBranchesWithLocalMatches } from "@t3tools/shared/git";
+import { compactTraceAttributes } from "../../observability/Attributes.ts";
+import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../../observability/Metrics.ts";
 import {
   GitCore,
   type ExecuteGitProgress,
@@ -372,6 +374,18 @@ interface Trace2Monitor {
   readonly flush: Effect.Effect<void, never>;
 }
 
+const nowUnixNano = (): bigint => BigInt(Date.now()) * 1_000_000n;
+
+const addCurrentSpanEvent = (name: string, attributes: Record<string, unknown>) =>
+  Effect.currentSpan.pipe(
+    Effect.tap((span) =>
+      Effect.sync(() => {
+        span.event(name, nowUnixNano(), compactTraceAttributes(attributes));
+      }),
+    ),
+    Effect.catch(() => Effect.void),
+  );
+
 function trace2ChildKey(record: Record<string, unknown>): string | null {
   const childId = record.child_id;
   if (typeof childId === "number" || typeof childId === "string") {
@@ -444,6 +458,9 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (
 
     if (event === "child_start") {
       hookStartByChildKey.set(childKey, { hookName, startedAtMs: Date.now() });
+      yield* addCurrentSpanEvent("git.hook.started", {
+        hookName,
+      });
       if (progress.onHookStarted) {
         yield* progress.onHookStarted(hookName);
       }
@@ -452,12 +469,19 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (
 
     if (event === "child_exit") {
       hookStartByChildKey.delete(childKey);
+      const code = traceRecord.success.code;
+      const exitCode = typeof code === "number" && Number.isInteger(code) ? code : null;
+      const durationMs = started ? Math.max(0, Date.now() - started.startedAtMs) : null;
+      yield* addCurrentSpanEvent("git.hook.finished", {
+        hookName: started?.hookName ?? hookName,
+        exitCode,
+        durationMs,
+      });
       if (progress.onHookFinished) {
-        const code = traceRecord.success.code;
         yield* progress.onHookFinished({
           hookName: started?.hookName ?? hookName,
-          exitCode: typeof code === "number" && Number.isInteger(code) ? code : null,
-          durationMs: started ? Math.max(0, Date.now() - started.startedAtMs) : null,
+          exitCode,
+          durationMs,
         });
       }
     }
@@ -609,13 +633,13 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {
   const path = yield* Path.Path;
   const { worktreesDir } = yield* ServerConfig;
 
-  let execute: GitCoreShape["execute"];
+  let executeRaw: GitCoreShape["execute"];
 
   if (options?.executeOverride) {
-    execute = options.executeOverride;
+    executeRaw = options.executeOverride;
   } else {
     const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
-    execute = Effect.fnUntraced(function* (input) {
+    executeRaw = Effect.fnUntraced(function* (input) {
       const commandInput = {
         ...input,
         args: [...input.args],
@@ -716,6 +740,25 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {
     });
   }
 
+  const execute: GitCoreShape["execute"] = (input) =>
+    executeRaw(input).pipe(
+      withMetrics({
+        counter: gitCommandsTotal,
+        timer: gitCommandDuration,
+        attributes: {
+          operation: input.operation,
+        },
+      }),
+      Effect.withSpan(input.operation, {
+        kind: "client",
+        attributes: {
+          "git.operation": input.operation,
+          "git.cwd": input.cwd,
+          "git.args_count": input.args.length,
+        },
+      }),
+    );
+
   const executeGit = (
     operation: string,
     cwd: string,

apps/server/src/observability/Attributes.test.ts

@@ -0,0 +1,46 @@
+import { assert, describe, it } from "@effect/vitest";
+
+import { compactTraceAttributes, normalizeModelMetricLabel } from "./Attributes.ts";
+
+describe("Attributes", () => {
+  it("normalizes circular arrays, maps, and sets without recursing forever", () => {
+    const array: Array<unknown> = ["alpha"];
+    array.push(array);
+
+    const map = new Map<string, unknown>();
+    map.set("self", map);
+
+    const set = new Set<unknown>();
+    set.add(set);
+
+    assert.deepStrictEqual(
+      compactTraceAttributes({
+        array,
+        map,
+        set,
+      }),
+      {
+        array: ["alpha", "[Circular]"],
+        map: { self: "[Circular]" },
+        set: ["[Circular]"],
+      },
+    );
+  });
+
+  it("normalizes invalid dates without throwing", () => {
+    assert.deepStrictEqual(
+      compactTraceAttributes({
+        invalidDate: new Date("not-a-real-date"),
+      }),
+      {
+        invalidDate: "Invalid Date",
+      },
+    );
+  });
+
+  it("groups GPT-family models under a shared metric label", () => {
+    assert.strictEqual(normalizeModelMetricLabel("gpt-4o"), "gpt");
+    assert.strictEqual(normalizeModelMetricLabel("gpt-5.4"), "gpt");
+    assert.strictEqual(normalizeModelMetricLabel("claude-sonnet-4"), "claude");
+  });
+});