diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0a6b31d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +* text=auto + +*.js text eol=lf +*.json text eol=lf +*.py text eol=lf +*.md text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.toml text eol=lf +*.env text eol=lf +*.env.* text eol=lf + +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index ccfd1f5..db83312 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -29,7 +29,7 @@ jobs: cache: pip - name: Install bandit - run: pip install bandit[toml] + run: pip install "bandit[toml,sarif]" - name: Run Bandit run: | @@ -41,6 +41,7 @@ jobs: continue-on-error: true - name: Upload Bandit SARIF + if: always() && hashFiles('bandit-results.sarif') != '' uses: github/codeql-action/upload-sarif@v3 with: sarif_file: bandit-results.sarif diff --git a/.gitignore b/.gitignore index b028d5d..37c88b7 100644 --- a/.gitignore +++ b/.gitignore @@ -52,9 +52,6 @@ logs/ # Dev-only folders frontend/ -scripts/ -!scripts/ -!scripts/benchmark_v2_ingest.py tests/ !tests/ !tests/**/*.py @@ -65,7 +62,6 @@ rust/ # Empty root files Makefile -CHANGELOG.md ruff.toml # GitHub templates (empty) @@ -96,3 +92,8 @@ src/api/routes/search.py # Misc error.json MIGRATION_PLAN.md +node_modules/ +repos/ +exports/ +tmp/ +.xmem/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1251ec3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,6 @@ +# Changelog + +## Unreleased + +- Add local XMem setup through `npx create-xmem@latest` and `npm run dev`. +- Add local Docker storage, Chrome extension build patching, diagnostics, verification, and context export/import/sync commands. diff --git a/README.md b/README.md index e841508..2933fcf 100644 --- a/README.md +++ b/README.md @@ -309,36 +309,54 @@ The industry standard benchmark for long-term conversational memory. Tests wheth ## Quickstart -### 1. Start the XMem Server +### Local XMem ```bash -git clone https://github.com/XortexLabs/xmem.git +npx create-xmem@latest cd xmem +npm run dev +``` + +This works on Windows, macOS, and Linux. It creates a local XMem workspace, installs the backend, starts local storage, builds the Chrome extension, and launches the API at `http://localhost:8000`. + +Local prerequisites: -# Install (requires Python 3.11+) -pip install -e . +- Git +- Node.js 20+ +- Python 3.11+ +- Docker Desktop +- Ollama, unless you add a cloud LLM key to `.env` -# Configure environment -cp .env.example .env # Add your API keys +After setup, load the extension from: -# Start -uvicorn src.api.app:create_app --factory --host 0.0.0.0 --port 8000 +```text +repos/xmem-extension/dist ``` -### 2. Install the Chrome Extension +Chrome path: `chrome://extensions` -> enable Developer mode -> Load unpacked. + +### Local Commands ```bash -git clone https://github.com/XortexAI/xmem-extension.git -npm install && npm run build +npm run setup +npm run start +npm run verify +npm run doctor ``` -Load `dist/` in Chrome via `chrome://extensions` → "Load unpacked". Point it to your server URL. +If `.env` contains a real cloud LLM key, XMem uses that provider and keeps embeddings local with FastEmbed. If no cloud key is configured, XMem falls back to local Ollama and pulls the required local models during setup. +### Context Portability -https://github.com/user-attachments/assets/605985c3-ef27-4096-a28c-b0b4cc6f8b8d +```bash +npm run context:export +npm run context:import -- --file ./exports/xmem-context.json +npm run context:sync -- --file ./exports/xmem-context.json --server https://api.xmem.in --api-key +``` +`context:export` writes a local context bundle that can be imported later or synced to an XMem server. -### 3. Index a Repository (Optional) +### Index a Repository ```bash python -m src.scanner.runner \ @@ -352,8 +370,8 @@ python -m src.scanner.runner \ > For a fully local setup with no cloud dependencies: > ```ini > FALLBACK_ORDER='["ollama"]' -> EMBEDDING_PROVIDER=fastembed -> VECTOR_STORE_PROVIDER=chroma +> EMBEDDING_PROVIDER=ollama +> VECTOR_STORE_PROVIDER=pgvector > ``` > Then install local extras: `pip install -e ".[local]"` diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 0000000..6be171d --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,66 @@ +name: xmem + +services: + postgres: + image: pgvector/pgvector:pg16 + container_name: xmem-postgres + environment: + POSTGRES_DB: xmem + POSTGRES_USER: xmem + POSTGRES_PASSWORD: xmem + ports: + - "15432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U xmem -d xmem"] + interval: 10s + timeout: 5s + retries: 10 + + mongo: + image: mongo:7 + container_name: xmem-mongo + ports: + - "27018:27017" + volumes: + - mongo_data:/data/db + healthcheck: + test: + [ + "CMD-SHELL", + "mongosh --quiet --eval \"db.adminCommand('ping').ok\" || exit 1" + ] + interval: 10s + timeout: 5s + retries: 12 + + neo4j: + image: neo4j:5-community + container_name: xmem-neo4j + environment: + NEO4J_AUTH: neo4j/local-password + NEO4J_server_memory_heap_initial__size: 512m + NEO4J_server_memory_heap_max__size: 2G + NEO4J_server_memory_pagecache_size: 1G + ports: + - "17474:7474" + - "17687:7687" + volumes: + - neo4j_data:/data + - neo4j_logs:/logs + healthcheck: + test: + [ + "CMD-SHELL", + "cypher-shell -u neo4j -p local-password 'RETURN 1' || exit 1" + ] + interval: 10s + timeout: 5s + retries: 12 + +volumes: + postgres_data: + mongo_data: + neo4j_data: + neo4j_logs: diff --git a/package.json b/package.json new file mode 100644 index 0000000..b30b4a5 --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "xmem", + "version": "0.1.0", + "private": true, + "description": "XMem local-first memory layer.", + "scripts": { + "setup": "node ./scripts/xmem.js setup", + "dev": "node ./scripts/xmem.js dev", + "start": "node ./scripts/xmem.js start", + "verify": "node ./scripts/xmem.js verify", + "doctor": "node ./scripts/xmem.js doctor", + "context:export": "node ./scripts/xmem.js context:export", + "context:import": "node ./scripts/xmem.js context:import", + "context:sync": "node ./scripts/xmem.js context:sync", + "check:npm": "node ./scripts/check-npm-publish.js", + "pack:create": "npm pack ./packages/create-xmem --dry-run", + "publish:create": "npm publish ./packages/create-xmem --access public" + }, + "engines": { + "node": ">=20" + } +} diff --git a/packages/create-xmem/bin/create-xmem.js b/packages/create-xmem/bin/create-xmem.js new file mode 100644 index 0000000..94ebd5c --- /dev/null +++ b/packages/create-xmem/bin/create-xmem.js @@ -0,0 +1,143 @@ +#!/usr/bin/env node + +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +const DEFAULT_REPO = "https://github.com/XortexAI/XMem.git"; +const DEFAULT_BRANCH = "main"; + +function usage(exitCode = 0) { + console.log(`Create a local XMem workspace + +Usage: + npx create-xmem@latest + npx create-xmem@latest my-xmem + +Options: + --repo XMem git repository URL + --branch XMem branch to use + --help Show this message + +After creation: + cd xmem + npm run dev +`); + process.exit(exitCode); +} + +function parseArgs(argv) { + const options = { + target: "xmem", + repo: process.env.XMEM_REPO || DEFAULT_REPO, + branch: process.env.XMEM_BRANCH || DEFAULT_BRANCH, + }; + let targetSet = false; + + function readOptionValue(index, name) { + const value = argv[index + 1]; + if (!value || value.startsWith("-")) { + console.error(`[create-xmem] ${name} requires a value.`); + usage(1); + } + return value; + } + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + + if (arg === "--help" || arg === "-h") { + usage(0); + } + + if (arg === "--repo") { + options.repo = readOptionValue(index, arg); + index += 1; + continue; + } + + if (arg === "--branch") { + options.branch = readOptionValue(index, arg); + index += 1; + continue; + } + + if (arg.startsWith("-")) { + console.error(`[create-xmem] Unknown option: ${arg}`); + usage(1); + } + + if (!targetSet) { + options.target = arg; + targetSet = true; + continue; + } + + console.error(`[create-xmem] Unexpected extra argument: ${arg}`); + usage(1); + } + + if (!options.repo || !options.branch) { + console.error("[create-xmem] --repo and --branch require values."); + usage(1); + } + + return options; +} + +function runGit(args, cwd) { + const result = spawnSync("git", args, { + cwd, + stdio: "inherit", + shell: false, + }); + + if (result.error) { + console.error(`[create-xmem] Git is required: ${result.error.message}`); + console.error("[create-xmem] Install Git, reopen your terminal, and run the command again."); + process.exit(1); + } + + if (result.status !== 0) { + process.exit(result.status || 1); + } +} + +function assertCleanTarget(targetPath) { + if (!fs.existsSync(targetPath)) { + return; + } + + const entries = fs.readdirSync(targetPath); + if (entries.length > 0) { + console.error(`[create-xmem] Target folder is not empty: ${targetPath}`); + console.error("[create-xmem] Choose a new folder name or empty the existing folder."); + process.exit(1); + } +} + +function removeGitMetadata(targetPath) { + fs.rmSync(path.join(targetPath, ".git"), { + recursive: true, + force: true, + }); +} + +const options = parseArgs(process.argv.slice(2)); +const targetPath = path.resolve(process.cwd(), options.target); + +assertCleanTarget(targetPath); + +console.log(`[create-xmem] Creating XMem workspace in ${targetPath}`); +runGit(["clone", "--depth", "1", "--branch", options.branch, options.repo, targetPath], process.cwd()); +removeGitMetadata(targetPath); + +console.log(""); +console.log("[create-xmem] Created local XMem workspace."); +console.log(""); +console.log("Next:"); +console.log(` cd ${path.relative(process.cwd(), targetPath) || "."}`); +console.log(" npm run dev"); +console.log(""); +console.log("Chrome extension after setup:"); +console.log(" Load unpacked: repos/xmem-extension/dist"); diff --git a/packages/create-xmem/package.json b/packages/create-xmem/package.json new file mode 100644 index 0000000..3d0ca6b --- /dev/null +++ b/packages/create-xmem/package.json @@ -0,0 +1,21 @@ +{ + "name": "create-xmem", + "version": "0.1.2", + "description": "Create a local XMem workspace.", + "bin": { + "create-xmem": "bin/create-xmem.js" + }, + "files": [ + "bin" + ], + "keywords": [ + "xmem", + "xortexai", + "memory", + "local-ai" + ], + "license": "MIT", + "engines": { + "node": ">=20" + } +} diff --git a/scripts/check-npm-publish.js b/scripts/check-npm-publish.js new file mode 100644 index 0000000..ce16a4c --- /dev/null +++ b/scripts/check-npm-publish.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node + +const { spawnSync } = require("node:child_process"); + +function runNpm(args) { + return spawnSync("npm", args, { + encoding: "utf8", + shell: process.platform === "win32", + }); +} + +function printOutput(result) { + if (result.error) { + console.error(result.error.message); + } + if ((result.stdout || "").trim()) { + console.log(result.stdout.trim()); + } + if ((result.stderr || "").trim()) { + console.error(result.stderr.trim()); + } +} + +const whoami = runNpm(["whoami"]); +if (whoami.status !== 0) { + console.error("[xmem] npm is not logged in. Run: npm login"); + printOutput(whoami); + process.exit(1); +} + +console.log(`[xmem] npm user: ${whoami.stdout.trim()}`); + +const profile = runNpm(["profile", "get", "--json"]); +if (profile.status !== 0) { + const combinedOutput = `${profile.stdout}\n${profile.stderr}`; + if (combinedOutput.includes("E403")) { + console.log("[xmem] npm profile is not readable with this token."); + console.log("[xmem] That is okay for granular publish tokens; continuing package checks."); + } else { + console.error("[xmem] Could not read npm profile."); + printOutput(profile); + process.exit(1); + } +} else { + const profileJson = JSON.parse(profile.stdout); + console.log(`[xmem] npm email verified: ${profileJson.email_verified}`); + console.log(`[xmem] npm 2FA enabled: ${profileJson.tfa}`); + if (!profileJson.tfa) { + console.log("[xmem] Enable npm 2FA or use a granular publish token before publishing."); + } +} + +const packageView = runNpm(["view", "create-xmem", "name", "version", "--json"]); +if (packageView.status === 0) { + console.log("[xmem] create-xmem already exists on npm:"); + printOutput(packageView); + process.exit(0); +} + +const combinedOutput = `${packageView.stdout}\n${packageView.stderr}`; +if (combinedOutput.includes("E404")) { + console.log("[xmem] create-xmem is available on npm."); + process.exit(0); +} + +console.error("[xmem] Could not check create-xmem package availability."); +printOutput(packageView); +process.exit(packageView.status || 1); diff --git a/scripts/context.py b/scripts/context.py new file mode 100644 index 0000000..1379685 --- /dev/null +++ b/scripts/context.py @@ -0,0 +1,384 @@ +from __future__ import annotations + +import argparse +import json +import re +import sys +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +FORMAT = "xmem-context-v1" + + +def normalize_user_id(value: str) -> str: + text = str(value or "").strip() + text = re.sub(r"[^A-Za-z0-9_.@-]+", "_", text) + text = re.sub(r"_+", "_", text).strip("_") + return text[:256] + + +def read_env(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + if not path.exists(): + raise SystemExit(f"XMem .env not found at {path}. Run npm run setup first.") + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + value = value.strip().strip('"').strip("'") + values[key.strip()] = value + return values + + +def json_default(value: Any) -> str: + if isinstance(value, datetime): + return value.astimezone(timezone.utc).isoformat() + return str(value) + + +def project_paths() -> tuple[Path, Path, Path]: + root = Path(__file__).resolve().parents[1] + xmem_dir = root + return root, xmem_dir, xmem_dir / ".env" + + +def load_bundle(path: Path) -> dict[str, Any]: + bundle = json.loads(path.read_text(encoding="utf-8")) + if bundle.get("format") != FORMAT: + raise SystemExit(f"Unsupported context bundle format: {bundle.get('format')!r}") + return bundle + + +def connect_postgres(env: dict[str, str]): + import psycopg + + return psycopg.connect(env.get("PGVECTOR_URL") or "postgresql://xmem:xmem@localhost:15432/xmem") + + +def pgvector_table_identifier(env: dict[str, str]): + from psycopg import sql + + table = env.get("PGVECTOR_TABLE") or "xmem_vectors" + parts = table.split(".") + for part in parts: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", part): + raise SystemExit(f"Invalid PGVECTOR_TABLE name: {table}") + return sql.Identifier(*parts) + + +def user_filter_values(user_id: str | None) -> list[str]: + if not user_id: + return [] + values = [user_id.strip(), normalize_user_id(user_id)] + return sorted({value for value in values if value}) + + +def export_pgvector(env: dict[str, str], user_id: str | None) -> list[dict[str, Any]]: + from psycopg import sql + + filters = user_filter_values(user_id) + params: list[Any] = [] + where_sql = sql.SQL("") + if filters: + where_sql = sql.SQL("WHERE metadata->>'user_id' = ANY(%s)") + params.append(filters) + + with connect_postgres(env) as conn: + with conn.cursor() as cur: + cur.execute( + sql.SQL( + """ + SELECT namespace, id, content, embedding::text AS embedding, + metadata, created_at, updated_at + FROM {table} + {where} + ORDER BY created_at, namespace, id + """ + ).format(table=pgvector_table_identifier(env), where=where_sql), + params, + ) + rows = cur.fetchall() + + return [ + { + "namespace": row[0], + "id": row[1], + "content": row[2], + "embedding": row[3], + "metadata": row[4] or {}, + "created_at": row[5], + "updated_at": row[6], + } + for row in rows + ] + + +def export_neo4j_events(env: dict[str, str], user_id: str | None) -> list[dict[str, Any]]: + try: + from neo4j import GraphDatabase + except ImportError: + return [] + + filters = user_filter_values(user_id) + where = "WHERE size($users) = 0 OR u.user_id IN $users" + query = f""" + MATCH (u:User)-[r:HAS_EVENT]->(d:Date) + {where} + RETURN u.user_id AS user_id, d.date AS date, properties(r) AS properties + ORDER BY user_id, date, properties(r).event_name + """ + + driver = GraphDatabase.driver( + env.get("NEO4J_URI") or "bolt://localhost:17687", + auth=(env.get("NEO4J_USERNAME") or "neo4j", env.get("NEO4J_PASSWORD") or "local-password"), + ) + try: + with driver.session() as session: + rel_types = [record["relationshipType"] for record in session.run("CALL db.relationshipTypes()")] + if "HAS_EVENT" not in rel_types: + return [] + records = session.run(query, users=filters) + return [ + { + "user_id": record["user_id"], + "date": record["date"], + "properties": dict(record["properties"] or {}), + } + for record in records + ] + finally: + driver.close() + + +def export_context(args: argparse.Namespace) -> None: + root, _, env_path = project_paths() + env = read_env(env_path) + vectors = export_pgvector(env, args.user_id) + events = export_neo4j_events(env, args.user_id) + users = sorted( + { + str(row.get("metadata", {}).get("user_id") or "") + for row in vectors + if row.get("metadata", {}).get("user_id") + } + | {str(event.get("user_id") or "") for event in events if event.get("user_id")} + ) + + out = Path(args.out) if args.out else root / "exports" / f"xmem-context-{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}.json" + out.parent.mkdir(parents=True, exist_ok=True) + bundle = { + "format": FORMAT, + "exported_at": datetime.now(timezone.utc), + "source": { + "workspace": str(root), + "vector_store": "pgvector", + "graph_store": "neo4j", + }, + "filter": {"user_id": args.user_id or None}, + "users": users, + "stores": { + "pgvector": { + "table": env.get("PGVECTOR_TABLE") or "xmem_vectors", + "rows": vectors, + }, + "neo4j": { + "temporal_events": events, + }, + }, + } + out.write_text(json.dumps(bundle, indent=2, default=json_default), encoding="utf-8") + print(f"[xmem] Exported {len(vectors)} vector memories and {len(events)} temporal events.") + print(f"[xmem] Context bundle: {out}") + + +def import_pgvector(env: dict[str, str], rows: list[dict[str, Any]], user_id: str | None) -> int: + if not rows: + return 0 + from psycopg import sql + from psycopg.types.json import Jsonb + + with connect_postgres(env) as conn: + conn.autocommit = True + with conn.cursor() as cur: + for row in rows: + metadata = dict(row.get("metadata") or {}) + if user_id: + metadata["user_id"] = normalize_user_id(user_id) + cur.execute( + sql.SQL( + """ + INSERT INTO {table}(namespace, id, content, embedding, metadata, created_at, updated_at) + VALUES (%s, %s, %s, %s::vector, %s, COALESCE(%s::timestamptz, now()), COALESCE(%s::timestamptz, now())) + ON CONFLICT(namespace, id) DO UPDATE SET + content = excluded.content, + embedding = excluded.embedding, + metadata = excluded.metadata, + updated_at = now() + """ + ).format(table=pgvector_table_identifier(env)), + ( + row["namespace"], + row["id"], + row["content"], + row["embedding"], + Jsonb(metadata), + row.get("created_at"), + row.get("updated_at"), + ), + ) + return len(rows) + + +def import_neo4j_events(env: dict[str, str], events: list[dict[str, Any]], user_id: str | None) -> int: + if not events: + return 0 + from neo4j import GraphDatabase + + driver = GraphDatabase.driver( + env.get("NEO4J_URI") or "bolt://localhost:17687", + auth=(env.get("NEO4J_USERNAME") or "neo4j", env.get("NEO4J_PASSWORD") or "local-password"), + ) + query = """ + MERGE (u:User {user_id: $user_id}) + MERGE (d:Date {date: $date}) + MERGE (u)-[r:HAS_EVENT {event_name: $event_name}]->(d) + SET r += $properties + """ + try: + with driver.session() as session: + for event in events: + props = dict(event.get("properties") or {}) + target_user = normalize_user_id(user_id) if user_id else event.get("user_id") + session.run( + query, + user_id=target_user, + date=event.get("date"), + event_name=props.get("event_name") or "", + properties=props, + ) + finally: + driver.close() + return len(events) + + +def import_context(args: argparse.Namespace) -> None: + _, _, env_path = project_paths() + env = read_env(env_path) + bundle = load_bundle(Path(args.file)) + rows = bundle.get("stores", {}).get("pgvector", {}).get("rows", []) + events = bundle.get("stores", {}).get("neo4j", {}).get("temporal_events", []) + row_count = import_pgvector(env, rows, args.user_id) + event_count = import_neo4j_events(env, events, args.user_id) + print(f"[xmem] Imported {row_count} vector memories and {event_count} temporal events.") + + +def api_post_json(url: str, api_key: str, payload: dict[str, Any], timeout: int) -> dict[str, Any]: + request = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise SystemExit(f"Remote sync failed: HTTP {exc.code}\n{body}") from exc + + +def sync_context(args: argparse.Namespace) -> None: + bundle = load_bundle(Path(args.file)) + rows = bundle.get("stores", {}).get("pgvector", {}).get("rows", []) + if not rows: + print("[xmem] No vector memories found in bundle.") + return + + server = args.server.rstrip("/") + api_key = args.api_key + if not api_key: + raise SystemExit("Missing --api-key for remote sync.") + + items: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + for row in rows: + metadata = row.get("metadata") or {} + target_user = normalize_user_id(args.user_id) if args.user_id else metadata.get("user_id") or "xmem-imported-user" + content = str(row.get("content") or "").strip() + if not content: + continue + key = (target_user, content) + if key in seen: + continue + seen.add(key) + items.append( + { + "user_query": content, + "agent_response": "Imported from an XMem local context bundle.", + "user_id": target_user, + "effort_level": "low", + } + ) + + if args.dry_run: + print(f"[xmem] Dry run: would sync {len(items)} memories to {server}.") + return + + synced = 0 + for index in range(0, len(items), args.batch_size): + batch = items[index : index + args.batch_size] + response = api_post_json( + f"{server}/v1/memory/batch-ingest", + api_key, + {"items": batch}, + args.timeout, + ) + if response.get("status") != "ok": + raise SystemExit(f"Remote sync failed: {response}") + synced += len(batch) + print(f"[xmem] Synced {synced}/{len(items)} memories") + + print(f"[xmem] Remote sync complete: {synced} memories sent to {server}.") + + +def main() -> None: + parser = argparse.ArgumentParser(description="XMem local context export/import/sync") + subparsers = parser.add_subparsers(dest="command", required=True) + + export_parser = subparsers.add_parser("export", help="Export local XMem context to a JSON bundle") + export_parser.add_argument("--user-id", default="", help="Optional user id/name filter") + export_parser.add_argument("--out", default="", help="Output JSON path") + export_parser.set_defaults(func=export_context) + + import_parser = subparsers.add_parser("import", help="Import a JSON context bundle into local XMem storage") + import_parser.add_argument("--file", required=True, help="Context bundle JSON path") + import_parser.add_argument("--user-id", default="", help="Optional target user id override") + import_parser.set_defaults(func=import_context) + + sync_parser = subparsers.add_parser("sync", help="Send a context bundle to a remote XMem API") + sync_parser.add_argument("--file", required=True, help="Context bundle JSON path") + sync_parser.add_argument("--server", required=True, help="Remote XMem server URL") + sync_parser.add_argument("--api-key", required=True, help="Remote XMem API key") + sync_parser.add_argument("--user-id", default="", help="Optional target user id override") + sync_parser.add_argument("--batch-size", type=int, default=20, help="Batch size for remote ingest") + sync_parser.add_argument("--timeout", type=int, default=900, help="HTTP timeout seconds per batch") + sync_parser.add_argument("--dry-run", action="store_true", help="Show what would be synced without sending") + sync_parser.set_defaults(func=sync_context) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(130) diff --git a/scripts/patch-extension-local.js b/scripts/patch-extension-local.js new file mode 100644 index 0000000..ab36229 --- /dev/null +++ b/scripts/patch-extension-local.js @@ -0,0 +1,221 @@ +#!/usr/bin/env node + +const fs = require("node:fs"); +const path = require("node:path"); +const zlib = require("node:zlib"); + +function argValue(names, fallback = "") { + const args = process.argv.slice(2); + for (let index = 0; index < args.length; index += 1) { + if (names.includes(args[index])) { + return args[index + 1] || fallback; + } + } + return fallback; +} + +const root = path.resolve(__dirname, ".."); +const extensionDir = path.resolve( + argValue(["--extension-dir", "-ExtensionDir"], path.join(root, "repos", "xmem-extension")), +); + +function crc32(buffer) { + let crc = 0xffffffff; + for (const byte of buffer) { + crc ^= byte; + for (let index = 0; index < 8; index += 1) { + crc = (crc >>> 1) ^ (crc & 1 ? 0xedb88320 : 0); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +function pngChunk(type, data) { + const typeBuffer = Buffer.from(type); + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length, 0); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])), 0); + return Buffer.concat([length, typeBuffer, data, crc]); +} + +function distanceToSegment(px, py, ax, ay, bx, by) { + const dx = bx - ax; + const dy = by - ay; + const lengthSquared = dx * dx + dy * dy; + if (lengthSquared === 0) { + return Math.hypot(px - ax, py - ay); + } + + const t = Math.max(0, Math.min(1, ((px - ax) * dx + (py - ay) * dy) / lengthSquared)); + const x = ax + t * dx; + const y = ay + t * dy; + return Math.hypot(px - x, py - y); +} + +function writeHollowXIcon(size, outPath) { + const margin = Math.max(3, Math.round(size * 0.22)); + const outerRadius = Math.max(2, Math.round(size * 0.11)); + const innerRadius = Math.max(1, Math.round(size * 0.052)); + const rows = []; + + for (let y = 0; y < size; y += 1) { + const row = Buffer.alloc(1 + size * 4); + row[0] = 0; + for (let x = 0; x < size; x += 1) { + const px = x + 0.5; + const py = y + 0.5; + const d1 = distanceToSegment(px, py, margin, margin, size - margin, size - margin); + const d2 = distanceToSegment(px, py, size - margin, margin, margin, size - margin); + const distance = Math.min(d1, d2); + const color = distance <= outerRadius && distance > innerRadius ? 0 : 255; + const offset = 1 + x * 4; + row[offset] = color; + row[offset + 1] = color; + row[offset + 2] = color; + row[offset + 3] = 255; + } + rows.push(row); + } + + const header = Buffer.alloc(13); + header.writeUInt32BE(size, 0); + header.writeUInt32BE(size, 4); + header[8] = 8; + header[9] = 6; + header[10] = 0; + header[11] = 0; + header[12] = 0; + + const png = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", header), + pngChunk("IDAT", zlib.deflateSync(Buffer.concat(rows))), + pngChunk("IEND", Buffer.alloc(0)), + ]); + + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, png); +} + +function patchFile(relativePath, patcher) { + const file = path.join(extensionDir, relativePath); + if (!fs.existsSync(file)) { + return; + } + const previous = fs.readFileSync(file, "utf8"); + const next = patcher(previous); + if (next !== previous) { + fs.writeFileSync(file, next); + } +} + +function normalizeSource(source) { + return source + .replaceAll("https://api.xmem.in", "http://localhost:8000") + .replaceAll( + "new XMemClient(API_BASE_URL, config.apiKey, config.userId)", + "new XMemClient(API_BASE_URL, config.apiKey)", + ) + .replaceAll(".replace(/[^\\\\w.\\\\-@]+/g, '_')", ".replace(/[^A-Za-z0-9_.@-]+/g, '_')"); +} + +function ensureApiNormalizeUserId(source) { + if (!source.includes("function normalizeUserId")) { + source = source.replace( + /(const API_BASE_URL = 'http:\/\/localhost:8000';\r?\n)/, + `$1 +function normalizeUserId(userId: string): string { + const normalized = (userId || '') + .trim() + .replace(/[^A-Za-z0-9_.@-]+/g, '_') + .replace(/^_+|_+$/g, ''); + return normalized || 'xmem-local-user'; +} +`, + ); + } + + return source.replaceAll( + "userId: data.xmem_user_id || '',", + "userId: normalizeUserId(data.xmem_user_id || ''),", + ); +} + +function ensureBackgroundNormalizeUserId(source) { + if (!source.includes("function normalizeUserId")) { + source = source.replace( + /(interface XMemConfig \{\r?\n apiKey: string;\r?\n userId: string;\r?\n\}\r?\n)/, + `$1 +function normalizeUserId(userId: string): string { + const normalized = (userId || '') + .trim() + .replace(/[^A-Za-z0-9_.@-]+/g, '_') + .replace(/^_+|_+$/g, ''); + return normalized || 'xmem-local-user'; +} +`, + ); + } + + return source.replaceAll( + "userId: data.xmem_user_id || '',", + "userId: normalizeUserId(data.xmem_user_id || ''),", + ); +} + +function patchValidateCredentials(source) { + const replacement = `export async function validateCredentials(apiKey: string, username: string): Promise { + const url = \`\${API_BASE_URL}/auth/verify-key\`; + try { + const response = await fetch(url, { + headers: { + 'Authorization': \`Bearer \${apiKey}\` + } + }); + + if (!response.ok) { + console.log('[XMem] Validation failed: HTTP', response.status); + return false; + } + + const data = await response.json(); + console.log('[XMem] Validated user data:', data); + + // Local dev static keys do not always map to a real username. If the local + // API accepted the key, allow any non-empty local user id from the popup. + if (API_BASE_URL.includes('localhost') || API_BASE_URL.includes('127.0.0.1')) { + return Boolean(username && username.trim()); + } + + return Boolean(data.username && data.username.toLowerCase() === username.toLowerCase()); + } catch (err) { + console.error('[XMem] Credential validation network error:', err); + return false; + } +} + +//`; + + return source.replace(/export async function validateCredentials[\s\S]*?\r?\n}\r?\n\r?\n\/\//, replacement); +} + +const apiFile = path.join(extensionDir, "src", "api.ts"); +if (!fs.existsSync(apiFile)) { + throw new Error(`Could not find extension API file at ${apiFile}`); +} + +const iconDir = path.join(extensionDir, "icons"); +writeHollowXIcon(16, path.join(iconDir, "icon16.png")); +writeHollowXIcon(48, path.join(iconDir, "icon48.png")); +writeHollowXIcon(128, path.join(iconDir, "icon128.png")); +writeHollowXIcon(128, path.join(iconDir, "logo.png")); + +for (const file of ["src/api.ts", "src/background.ts", "src/content.ts"]) { + patchFile(file, normalizeSource); +} + +patchFile("src/api.ts", (source) => patchValidateCredentials(ensureApiNormalizeUserId(source))); +patchFile("src/background.ts", ensureBackgroundNormalizeUserId); + +console.log("[xmem] Patched extension API for http://localhost:8000"); diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 0000000..cf53825 --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import argparse +import json +import sys +import time +import urllib.error +import urllib.request +from typing import Any + + +def log(message: str) -> None: + print(f"[xmem] {message}") + + +def request_json( + url: str, + method: str = "GET", + headers: dict[str, str] | None = None, + body: dict[str, Any] | None = None, + timeout: int = 60, +) -> dict[str, Any]: + payload = json.dumps(body).encode("utf-8") if body is not None else None + req = urllib.request.Request(url, data=payload, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + text = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Request failed: {method} {url}\nHTTP {exc.code}\n{text}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"Request failed: {method} {url}\n{exc}") from exc + + +def health_ready(health: dict[str, Any]) -> bool: + data = health.get("data") or health + return bool(data.get("pipelines_ready")) + + +def health_summary(health: dict[str, Any]) -> str: + data = health.get("data") or health + return ( + f"status={data.get('status')}, " + f"pipelines_ready={data.get('pipelines_ready')}, " + f"error={data.get('error')}" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify a local XMem API") + parser.add_argument("--base-url", "-BaseUrl", default="http://localhost:8000") + parser.add_argument("--api-key", "-ApiKey", default="dev-xmem-key") + parser.add_argument("--user-id", "-UserId", default="xmem-local-user") + parser.add_argument("--timeout-seconds", "-TimeoutSeconds", type=int, default=180) + args = parser.parse_args() + + deadline = time.time() + args.timeout_seconds + health: dict[str, Any] | None = None + log(f"Waiting for API health at {args.base_url}/health") + while time.time() < deadline: + try: + health = request_json(f"{args.base_url}/health", timeout=10) + if health_ready(health): + break + except Exception: + time.sleep(3) + + if not health: + raise RuntimeError(f"XMem API did not become reachable within {args.timeout_seconds} seconds.") + + log(f"Health: {health_summary(health)}") + if not health_ready(health): + raise RuntimeError("XMem API is reachable but pipelines are not ready.") + + headers = { + "Authorization": f"Bearer {args.api_key}", + "Content-Type": "application/json", + } + + log("Ingesting a smoke-test memory") + ingest = request_json( + f"{args.base_url}/v1/memory/ingest", + method="POST", + headers=headers, + body={ + "user_query": "Remember that XMem local mode runs directly from the main XMem repository.", + "agent_response": "Got it. I will remember that XMem local mode runs from the main repository.", + "user_id": args.user_id, + "effort_level": "low", + }, + timeout=650, + ) + log(f"Ingest status: {ingest.get('status')}") + + log("Searching memory") + search = request_json( + f"{args.base_url}/v1/memory/search", + method="POST", + headers=headers, + body={ + "query": "What is XMem local mode?", + "user_id": args.user_id, + "domains": ["profile", "temporal", "summary"], + "top_k": 5, + }, + timeout=180, + ) + result_count = len((search.get("data") or {}).get("results") or []) + log(f"Search result count: {result_count}") + + log("Retrieving answer") + retrieve = request_json( + f"{args.base_url}/v1/memory/retrieve", + method="POST", + headers=headers, + body={ + "query": "Where does XMem local mode run from?", + "user_id": args.user_id, + "top_k": 5, + }, + timeout=240, + ) + print("\nAnswer:") + print((retrieve.get("data") or {}).get("answer")) + print("") + log("Verification complete") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except KeyboardInterrupt: + raise SystemExit(130) + except Exception as exc: + print(str(exc), file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/xmem.js b/scripts/xmem.js new file mode 100644 index 0000000..d9158de --- /dev/null +++ b/scripts/xmem.js @@ -0,0 +1,808 @@ +#!/usr/bin/env node + +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +const root = path.resolve(__dirname, ".."); +const scriptsDir = path.join(root, "scripts"); +const command = process.argv[2] || "help"; +const passthroughArgs = process.argv.slice(3); +const isWindows = process.platform === "win32"; + +const managedRepos = [ + { + flag: "includeMcp", + name: "xmem-mcp", + url: "https://github.com/XortexAI/xmem-mcp.git", + branch: "main", + }, + { + flag: "includeSdk", + name: "xmem-sdk", + url: "https://github.com/XortexAI/xmem-sdk.git", + branch: "master", + }, +]; + +function log(message) { + console.log(`[xmem] ${message}`); +} + +function warn(message) { + console.warn(`[xmem] ${message}`); +} + +function fail(message, exitCode = 1) { + console.error(`[xmem] ${message}`); + process.exit(exitCode); +} + +function usage(exitCode = 0) { + console.log(`XMem local workspace + +Usage: + npm run dev + npm run setup + npm run start + npm run verify + npm run doctor + npm run context:export + npm run context:import -- --file ./exports/xmem-context.json + npm run context:sync -- --file ./exports/xmem-context.json --server https://api.xmem.in --api-key + +Power-user flags can be passed after --, for example: + npm run setup -- --include-mcp + npm run setup -- --skip-model-pull + npm run start -- --skip-docker + +Windows-style flags are also accepted: + npm run setup -- -IncludeMcp + npm run start -- -SkipDocker +`); + process.exit(exitCode); +} + +function commandInvocation(commandName, args) { + if (commandName === "npm" && process.env.npm_execpath) { + return { + command: process.execPath, + args: [process.env.npm_execpath, ...args], + shell: false, + }; + } + + if (isWindows && ["npm", "npx"].includes(commandName)) { + return { + command: commandName, + args, + shell: true, + }; + } + + return { + command: commandName, + args, + shell: false, + }; +} + +function run(commandName, args = [], options = {}) { + const invocation = commandInvocation(commandName, args); + const result = spawnSync(invocation.command, invocation.args, { + cwd: options.cwd || root, + env: options.env || process.env, + encoding: options.capture ? "utf8" : undefined, + stdio: options.capture ? "pipe" : "inherit", + shell: invocation.shell, + }); + + if (result.error) { + if (options.allowFailure) { + return result; + } + throw new Error(`Could not start ${commandName}: ${result.error.message}`); + } + + if (result.status !== 0 && !options.allowFailure) { + throw new Error(`${commandName} ${args.join(" ")} failed with exit code ${result.status}`); + } + + return result; +} + +function commandExists(commandName) { + const checker = isWindows + ? ["where.exe", [commandName]] + : ["which", [commandName]]; + const result = run(checker[0], checker[1], { + capture: true, + allowFailure: true, + }); + return result.status === 0; +} + +function sleep(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function optionParser(spec) { + const aliases = new Map(); + for (const [key, config] of Object.entries(spec)) { + for (const alias of config.aliases || []) { + aliases.set(alias.toLowerCase(), { key, ...config }); + } + } + + return function parse(args) { + const values = {}; + for (const [key, config] of Object.entries(spec)) { + values[key] = config.type === "flag" ? false : config.default || ""; + } + + for (let index = 0; index < args.length; index += 1) { + let arg = args[index]; + let inlineValue = ""; + const equalsIndex = arg.indexOf("="); + if (equalsIndex > -1) { + inlineValue = arg.slice(equalsIndex + 1); + arg = arg.slice(0, equalsIndex); + } + + const match = aliases.get(arg.toLowerCase()); + if (!match) { + throw new Error(`Unknown option: ${args[index]}`); + } + + if (match.type === "flag") { + values[match.key] = true; + continue; + } + + const value = inlineValue || args[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`${arg} requires a value.`); + } + values[match.key] = value; + if (!inlineValue) { + index += 1; + } + } + + return values; + }; +} + +const parseSetupOptions = optionParser({ + reposDir: { type: "value", default: "repos", aliases: ["--repos-dir", "-ReposDir"] }, + includeMcp: { type: "flag", aliases: ["--include-mcp", "-IncludeMcp"] }, + includeSdk: { type: "flag", aliases: ["--include-sdk", "-IncludeSdk"] }, + skipModelPull: { type: "flag", aliases: ["--skip-model-pull", "-SkipModelPull"] }, + skipPythonInstall: { type: "flag", aliases: ["--skip-python-install", "-SkipPythonInstall"] }, + skipNodeInstall: { type: "flag", aliases: ["--skip-node-install", "-SkipNodeInstall"] }, + skipDocker: { type: "flag", aliases: ["--skip-docker", "-SkipDocker"] }, +}); + +const parseStartOptions = optionParser({ + reposDir: { type: "value", default: "repos", aliases: ["--repos-dir", "-ReposDir"] }, + skipDocker: { type: "flag", aliases: ["--skip-docker", "-SkipDocker"] }, +}); + +const parseDoctorOptions = optionParser({ + baseUrl: { type: "value", default: "http://localhost:8000", aliases: ["--base-url", "-BaseUrl"] }, + reposDir: { type: "value", default: "repos", aliases: ["--repos-dir", "-ReposDir"] }, +}); + +function readOption(args, names, fallback = "") { + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + const equalsIndex = arg.indexOf("="); + const name = equalsIndex > -1 ? arg.slice(0, equalsIndex) : arg; + if (!names.map((item) => item.toLowerCase()).includes(name.toLowerCase())) { + continue; + } + if (equalsIndex > -1) { + return arg.slice(equalsIndex + 1); + } + return args[index + 1] || fallback; + } + return fallback; +} + +function hasSwitch(args, names) { + const wanted = names.map((name) => name.toLowerCase()); + return args.some((arg) => wanted.includes(arg.split("=")[0].toLowerCase())); +} + +function startCompatibleArgs(args) { + const next = []; + const reposDir = readOption(args, ["--repos-dir", "-ReposDir"]); + if (reposDir) { + next.push("--repos-dir", reposDir); + } + if (hasSwitch(args, ["--skip-docker", "-SkipDocker"])) { + next.push("--skip-docker"); + } + return next; +} + +function systemPythonCommand() { + if (!isWindows && commandExists("python3")) { + return "python3"; + } + if (commandExists("python")) { + return "python"; + } + fail("Python 3.11+ is required. Install Python, reopen your terminal, and rerun this command."); +} + +function venvPythonPath() { + return path.join(root, ".venv", isWindows ? "Scripts/python.exe" : "bin/python"); +} + +function pythonForRuntime() { + const venvPython = venvPythonPath(); + if (fs.existsSync(venvPython)) { + return venvPython; + } + warn("XMem virtualenv was not found; using system Python. Run npm run setup if startup fails."); + return systemPythonCommand(); +} + +function stripQuotes(value) { + const trimmed = String(value || "").trim(); + if ( + (trimmed.startsWith("'") && trimmed.endsWith("'")) || + (trimmed.startsWith('"') && trimmed.endsWith('"')) + ) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +function readDotEnv(envPath) { + const values = {}; + if (!fs.existsSync(envPath)) { + return values; + } + + for (const rawLine of fs.readFileSync(envPath, "utf8").split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#") || !line.includes("=")) { + continue; + } + const [key, ...rest] = line.split("="); + values[key.trim()] = stripQuotes(rest.join("=")); + } + return values; +} + +function setDotEnvValues(envPath, updates) { + const original = fs.existsSync(envPath) ? fs.readFileSync(envPath, "utf8") : ""; + const lines = original ? original.split(/\r?\n/) : []; + const updatedKeys = new Set(); + const next = lines.map((line) => { + for (const [key, value] of Object.entries(updates)) { + const pattern = new RegExp(`^\\s*${key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\s*=`); + if (pattern.test(line)) { + updatedKeys.add(key); + return `${key}=${value}`; + } + } + return line; + }); + + for (const [key, value] of Object.entries(updates)) { + if (!updatedKeys.has(key)) { + next.push(`${key}=${value}`); + } + } + + fs.writeFileSync(envPath, `${next.join("\n").replace(/\n+$/g, "")}\n`); +} + +function isRealSecret(value) { + const text = stripQuotes(value).trim(); + if (!text) { + return false; + } + + return ![ + /^your[_-]/i, + /your_.*_key/i, + /example/i, + /sample/i, + /placeholder/i, + /change[-_]?me/i, + /^dummy([-_].*)?$/i, + /^fake([-_].*)?$/i, + /^test([-_].*)?$/i, + ].some((pattern) => pattern.test(text)); +} + +function configuredValue(envPath, name) { + const envValue = process.env[name]; + if (isRealSecret(envValue)) { + return envValue; + } + const fileValue = readDotEnv(envPath)[name]; + return isRealSecret(fileValue) ? fileValue : ""; +} + +function configuredProviders(envPath) { + const providers = []; + if (configuredValue(envPath, "OPENROUTER_API_KEY")) providers.push("openrouter"); + if (configuredValue(envPath, "GEMINI_API_KEY")) providers.push("gemini"); + if (configuredValue(envPath, "CLAUDE_API_KEY")) providers.push("claude"); + if (configuredValue(envPath, "OPENAI_API_KEY")) providers.push("openai"); + if ( + configuredValue(envPath, "AWS_ACCESS_KEY_ID") && + configuredValue(envPath, "AWS_SECRET_ACCESS_KEY") + ) { + providers.push("bedrock"); + } + return providers; +} + +function configureEnv(envPath, quiet = false) { + if (!fs.existsSync(envPath)) { + throw new Error(`XMem .env not found at ${envPath}. Run npm run setup first.`); + } + + const providers = configuredProviders(envPath); + + if (providers.length > 0) { + setDotEnvValues(envPath, { + FALLBACK_ORDER: `'${JSON.stringify(providers)}'`, + EMBEDDING_PROVIDER: "fastembed", + FASTEMBED_MODEL: "BAAI/bge-small-en-v1.5", + EMBEDDING_MODEL: "BAAI/bge-small-en-v1.5", + PINECONE_DIMENSION: "384", + }); + if (!quiet) { + log(`Detected cloud LLM provider(s): ${providers.join(", ")}`); + log("Configured XMem to avoid Ollama for LLM and embedding calls."); + } + return providers; + } + + setDotEnvValues(envPath, { + FALLBACK_ORDER: `'["ollama"]'`, + EMBEDDING_PROVIDER: "ollama", + OLLAMA_EMBEDDING_MODEL: "nomic-embed-text", + EMBEDDING_MODEL: "nomic-embed-text", + PINECONE_DIMENSION: "768", + }); + if (!quiet) { + log("No cloud LLM provider keys detected."); + log("Configured XMem to use local Ollama for LLM and embedding calls."); + } + return []; +} + +function dotEnvValue(envPath, name, fallback = "") { + return readDotEnv(envPath)[name] || fallback; +} + +function usesOllama(envPath) { + if (!fs.existsSync(envPath)) { + return true; + } + return /ollama/i.test(readDotEnv(envPath).FALLBACK_ORDER || ""); +} + +function syncRepo(reposDir, name, url, branch) { + const target = path.join(reposDir, name); + if (fs.existsSync(target)) { + if (!fs.existsSync(path.join(target, ".git"))) { + throw new Error(`${target} exists but is not a git checkout.`); + } + log(`Updating ${name}`); + run("git", ["-C", target, "reset", "--hard"]); + run("git", ["-C", target, "fetch", "origin"]); + run("git", ["-C", target, "checkout", branch]); + run("git", ["-C", target, "pull", "--ff-only", "origin", branch]); + return; + } + + log(`Cloning ${name}`); + run("git", ["clone", "--branch", branch, url, target]); +} + +function dockerRunning() { + return commandExists("docker") && run("docker", ["info"], { capture: true, allowFailure: true }).status === 0; +} + +function ollamaRunning() { + return commandExists("ollama") && run("ollama", ["list"], { capture: true, allowFailure: true }).status === 0; +} + +function waitForContainers(names, timeoutSeconds = 180) { + const pending = new Set(names); + const deadline = Date.now() + timeoutSeconds * 1000; + + while (Date.now() < deadline) { + for (const name of [...pending]) { + const result = run( + "docker", + [ + "inspect", + "--format", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", + name, + ], + { capture: true, allowFailure: true }, + ); + + if (result.status !== 0) { + continue; + } + + const status = String(result.stdout || "").trim(); + if (status === "healthy" || status === "running") { + pending.delete(name); + } else if (status === "unhealthy") { + throw new Error(`Container ${name} is unhealthy. Run npm run doctor or inspect it with: docker logs ${name}`); + } + } + + if (pending.size === 0) { + return; + } + + log(`Waiting for local database containers: ${[...pending].join(", ")}`); + sleep(5000); + } + + throw new Error(`Timed out waiting for local database containers: ${[...pending].join(", ")}. Run npm run doctor for details.`); +} + +function startDockerServices() { + if (!dockerRunning()) { + return false; + } + log("Starting local Docker services"); + run("docker", ["compose", "-f", path.join(root, "docker-compose.local.yml"), "up", "-d", "--remove-orphans"]); + waitForContainers(["xmem-postgres", "xmem-mongo", "xmem-neo4j"]); + return true; +} + +function installedOllamaModels() { + const result = run("ollama", ["list"], { capture: true, allowFailure: true }); + if (result.status !== 0) { + return new Set(); + } + return new Set( + String(result.stdout || "") + .split(/\r?\n/) + .slice(1) + .map((line) => line.trim().split(/\s+/)[0]) + .filter(Boolean), + ); +} + +function hasOllamaModel(model, installed) { + if (!model) { + return true; + } + return installed.has(model) || (!model.includes(":") && installed.has(`${model}:latest`)); +} + +function assertOllamaReady(envPath) { + if (!commandExists("ollama")) { + fail("Ollama was not found. Install Ollama, or add a cloud LLM key to .env and rerun.", 2); + } + if (!ollamaRunning()) { + fail("XMem is configured to use local Ollama, but Ollama is not running. Start Ollama, or add a cloud LLM key to .env and rerun.", 2); + } + + const chatModel = dotEnvValue(envPath, "OLLAMA_MODEL", "qwen2.5:1.5b"); + const embeddingModel = dotEnvValue(envPath, "OLLAMA_EMBEDDING_MODEL", "nomic-embed-text"); + const installed = installedOllamaModels(); + const missing = [chatModel, embeddingModel].filter((model) => !hasOllamaModel(model, installed)); + if (missing.length > 0) { + for (const model of missing) { + warn(`Ollama model ${model} is missing. Run: ollama pull ${model}`); + } + fail("Required Ollama model(s) are missing, or add a cloud LLM key to .env so XMem does not use Ollama.", 2); + } +} + +function ensurePrerequisites(skipPython = false) { + for (const required of ["git", "node", "npm"]) { + if (!commandExists(required)) { + fail(`${required} is required. Install it, reopen your terminal, and rerun this command.`); + } + } + if (!skipPython) { + systemPythonCommand(); + } +} + +function setupLooksComplete(reposDir) { + return ( + fs.existsSync(path.join(root, "pyproject.toml")) && + fs.existsSync(path.join(root, ".env")) && + fs.existsSync(venvPythonPath()) && + fs.existsSync(path.join(reposDir, "xmem-extension", ".git")) && + fs.existsSync(path.join(reposDir, "xmem-extension", "dist", "manifest.json")) + ); +} + +function runSetup(args) { + const options = parseSetupOptions(args); + const reposDir = path.resolve(root, options.reposDir); + const extensionDir = path.join(reposDir, "xmem-extension"); + let dockerSkipped = false; + let ollamaSkipped = false; + + ensurePrerequisites(options.skipPythonInstall); + fs.mkdirSync(reposDir, { recursive: true }); + + syncRepo(reposDir, "xmem-extension", "https://github.com/XortexAI/xmem-extension.git", "main"); + for (const repo of managedRepos) { + if (options[repo.flag]) { + syncRepo(reposDir, repo.name, repo.url, repo.branch); + } + } + + const envTemplate = path.join(root, "templates", "xmem.env.local"); + const envTarget = path.join(root, ".env"); + if (!fs.existsSync(envTarget)) { + fs.copyFileSync(envTemplate, envTarget); + log("Created .env from local template"); + } else { + log(".env already exists; leaving it unchanged"); + } + + configureEnv(envTarget); + + if (!options.skipModelPull) { + if (usesOllama(envTarget)) { + if (ollamaRunning()) { + const chatModel = dotEnvValue(envTarget, "OLLAMA_MODEL", "qwen2.5:1.5b"); + const embeddingModel = dotEnvValue(envTarget, "OLLAMA_EMBEDDING_MODEL", "nomic-embed-text"); + log("Pulling Ollama chat model"); + run("ollama", ["pull", chatModel]); + log("Pulling Ollama embedding model"); + run("ollama", ["pull", embeddingModel]); + } else { + warn("Ollama was not found or is not running."); + warn("Start Ollama, or add a cloud LLM key to .env and rerun."); + ollamaSkipped = true; + } + } else { + log("Cloud LLM provider key detected; skipping Ollama model pulls"); + } + } + + if (!options.skipDocker) { + if (!startDockerServices()) { + warn("Docker Desktop is installed but not running, or Docker was not found."); + warn("Start Docker Desktop, wait until it says Docker is running, then rerun this command."); + warn("Temporary escape hatch: rerun npm run setup -- --skip-docker to continue without local databases."); + dockerSkipped = true; + } + } + + if (!options.skipPythonInstall) { + const venvPython = venvPythonPath(); + if (!fs.existsSync(venvPython)) { + log("Creating XMem virtualenv"); + run(systemPythonCommand(), ["-m", "venv", path.join(root, ".venv")]); + } + log("Installing XMem local dependencies"); + run(venvPython, ["-m", "pip", "install", "--upgrade", "pip"]); + run(venvPython, ["-m", "pip", "install", "-e", `${root}[local,dev]`]); + } + + log("Patching extension for local API"); + run(process.execPath, [path.join(scriptsDir, "patch-extension-local.js"), "--extension-dir", extensionDir]); + + if (!options.skipNodeInstall) { + log("Installing and building Chrome extension"); + run("npm", ["--prefix", extensionDir, "install"]); + run("npm", ["--prefix", extensionDir, "run", "build"]); + } + + log("Install complete"); + console.log(""); + console.log("Next:"); + console.log(" npm run dev"); + console.log(" npm run verify"); + if (dockerSkipped) { + console.log(""); + warn("Docker services were not started. Start Docker Desktop before running npm run dev."); + } + if (ollamaSkipped) { + console.log(""); + warn("Ollama models were not pulled. Start Ollama, then rerun npm run setup or add a cloud LLM key."); + } +} + +function runStart(args) { + const options = parseStartOptions(args); + const envTarget = path.join(root, ".env"); + + if (!fs.existsSync(envTarget)) { + fail(`XMem .env not found at ${envTarget}. Run npm run setup first.`); + } + + configureEnv(envTarget); + + if (usesOllama(envTarget)) { + assertOllamaReady(envTarget); + } + + if (!options.skipDocker) { + if (!startDockerServices()) { + fail("Docker Desktop is installed but not running, or Docker was not found. Start Docker Desktop, then rerun npm run dev.", 2); + } + } + + const python = pythonForRuntime(); + log("Starting XMem API at http://localhost:8000"); + run(python, ["-m", "uvicorn", "src.api.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000"]); +} + +function runDev(args) { + const reposDir = path.resolve(root, readOption(args, ["--repos-dir", "-ReposDir"], "repos")); + if (!setupLooksComplete(reposDir)) { + log("First run detected; running setup before starting XMem."); + runSetup(args); + } + runStart(startCompatibleArgs(args)); +} + +function runVerify(args) { + const python = pythonForRuntime(); + run(python, [path.join(scriptsDir, "verify.py"), ...args]); +} + +function runContext(subcommand, args) { + const venvPython = venvPythonPath(); + if (!fs.existsSync(venvPython)) { + fail("XMem virtualenv not found. Run npm run setup first."); + } + run(venvPython, [path.join(scriptsDir, "context.py"), subcommand, ...args]); +} + +function writeCheck(name, ok, message, fix = "") { + const label = ok ? "OK" : "FIX"; + console.log(`[${label}] ${name} - ${message}`); + if (!ok && fix) { + console.log(` ${fix}`); + } +} + +async function fetchHealth(baseUrl) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + try { + const response = await fetch(`${baseUrl.replace(/\/+$/, "")}/health`, { + signal: controller.signal, + }); + if (!response.ok) { + return { ok: false, message: `HTTP ${response.status}` }; + } + const body = await response.json(); + const data = body.data || body; + return { + ok: Boolean(data.pipelines_ready), + message: `${baseUrl}/health`, + }; + } catch { + return { ok: false, message: `${baseUrl} is not reachable` }; + } finally { + clearTimeout(timeout); + } +} + +async function runDoctor(args) { + const options = parseDoctorOptions(args); + const reposDir = path.resolve(root, options.reposDir); + const envPath = path.join(root, ".env"); + const extensionDir = path.join(reposDir, "xmem-extension"); + let failures = 0; + + console.log("[xmem] Doctor report"); + console.log(""); + + for (const cmd of ["git", "node", "npm"]) { + const ok = commandExists(cmd); + if (!ok) failures += 1; + writeCheck(cmd, ok, "command lookup", `Install ${cmd} and reopen this terminal.`); + } + + const pythonOk = commandExists("python") || (!isWindows && commandExists("python3")); + if (!pythonOk) failures += 1; + writeCheck("Python", pythonOk, "Python 3.11+ lookup", "Install Python 3.11+ and reopen this terminal."); + + const dockerOk = dockerRunning(); + if (!dockerOk) failures += 1; + writeCheck("Docker", dockerOk, "local database runtime", "Start Docker Desktop, then rerun npm run dev."); + + const xmemExists = fs.existsSync(path.join(root, "pyproject.toml")); + if (!xmemExists) failures += 1; + writeCheck("XMem repo", xmemExists, root, "Run this from the XMem repository root."); + + const extensionExists = fs.existsSync(extensionDir); + if (!extensionExists) failures += 1; + writeCheck("Extension repo", extensionExists, extensionDir, "Run npm run setup."); + + const extensionBuildExists = fs.existsSync(path.join(extensionDir, "dist", "manifest.json")); + if (!extensionBuildExists) failures += 1; + writeCheck("Extension build", extensionBuildExists, "repos/xmem-extension/dist", "Run npm run setup."); + + const envExists = fs.existsSync(envPath); + if (!envExists) failures += 1; + writeCheck("XMem .env", envExists, envPath, "Run npm run setup to create it from templates/xmem.env.local."); + + if (envExists) { + const providers = configuredProviders(envPath); + if (providers.length > 0) { + writeCheck("LLM routing", true, `cloud key detected: ${providers.join(", ")}; Ollama is not required`); + } else if (usesOllama(envPath)) { + const ollamaOk = ollamaRunning(); + if (!ollamaOk) failures += 1; + writeCheck("Ollama", ollamaOk, "required because no cloud LLM key is configured", "Start Ollama, or add a cloud LLM key to .env."); + + if (ollamaOk) { + const installed = installedOllamaModels(); + for (const model of [ + dotEnvValue(envPath, "OLLAMA_MODEL", "qwen2.5:1.5b"), + dotEnvValue(envPath, "OLLAMA_EMBEDDING_MODEL", "nomic-embed-text"), + ]) { + const ok = hasOllamaModel(model, installed); + if (!ok) failures += 1; + writeCheck(`Ollama model ${model}`, ok, "local model availability", `Run: ollama pull ${model}`); + } + } + } else { + failures += 1; + writeCheck("LLM routing", false, "no cloud key detected and FALLBACK_ORDER does not include Ollama", "Run npm run setup to repair .env routing."); + } + } + + const health = await fetchHealth(options.baseUrl); + if (!health.ok) failures += 1; + writeCheck("XMem API", health.ok, health.message, "Start it with npm run dev and wait for pipelines_ready=true."); + + console.log(""); + if (failures === 0) { + log("Everything looks ready."); + } else { + warn(`Found ${failures} setup item(s) to fix.`); + } +} + +async function main() { + try { + if (command === "help" || command === "--help" || command === "-h") { + usage(0); + } else if (command === "setup") { + runSetup(passthroughArgs); + } else if (command === "dev") { + runDev(passthroughArgs); + } else if (command === "start") { + runStart(passthroughArgs); + } else if (command === "verify") { + runVerify(passthroughArgs); + } else if (command === "doctor") { + await runDoctor(passthroughArgs); + } else if (command === "context:export") { + runContext("export", passthroughArgs); + } else if (command === "context:import") { + runContext("import", passthroughArgs); + } else if (command === "context:sync") { + runContext("sync", passthroughArgs); + } else { + console.error(`[xmem] Unknown command: ${command}`); + usage(1); + } + } catch (error) { + fail(error.message || String(error)); + } +} + +main(); diff --git a/src/agents/base.py b/src/agents/base.py index f4e8c3c..d6f61f0 100644 --- a/src/agents/base.py +++ b/src/agents/base.py @@ -5,6 +5,8 @@ from langchain_core.language_models import BaseChatModel import time +from src.config import settings + @dataclass class BaseAgent(ABC): model: BaseChatModel @@ -34,7 +36,16 @@ def _build_messages(self, user_message: str) -> list: async def _call_model(self, messages: list) -> str: import asyncio start = time.perf_counter() - response = await asyncio.wait_for(self.model.ainvoke(messages), timeout=45.0) + timeout = float(getattr(settings, "llm_timeout_seconds", 45.0) or 45.0) + try: + response = await asyncio.wait_for(self.model.ainvoke(messages), timeout=timeout) + except asyncio.TimeoutError as exc: + model_name = getattr(self.model, "model", getattr(self.model, "model_name", type(self.model).__name__)) + raise TimeoutError( + f"LLM call timed out after {timeout:.0f}s in agent '{self.name}' " + f"using model '{model_name}'. For local Ollama, increase " + "LLM_TIMEOUT_SECONDS or configure a cloud LLM key." + ) from exc elapsed = time.perf_counter() - start content = response.content diff --git a/src/api/app.py b/src/api/app.py index 889151e..79bce88 100644 --- a/src/api/app.py +++ b/src/api/app.py @@ -10,7 +10,8 @@ import traceback from contextlib import asynccontextmanager -from fastapi import FastAPI, Request +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, Response @@ -63,6 +64,65 @@ def _init_pipelines_sync() -> tuple: return ingest, retrieval +def _detail_to_text(detail) -> str: + if isinstance(detail, str): + return detail + if isinstance(detail, list): + return "; ".join(str(item) for item in detail) + if isinstance(detail, dict): + return detail.get("message") or detail.get("error") or str(detail) + return str(detail) + + +def _field_name(loc) -> str: + parts = [str(part) for part in loc if part not in {"body", "query", "path"}] + return ".".join(parts) or "request" + + +def _friendly_validation_error(error: dict) -> str: + field = _field_name(error.get("loc", [])) + kind = error.get("type", "") + message = error.get("msg", "Invalid value") + + if kind == "missing": + return f"{field} is required." + if kind == "string_too_short": + return f"{field} cannot be empty." + if kind == "string_too_long": + limit = (error.get("ctx") or {}).get("max_length") + return f"{field} is too long" + (f" (max {limit} characters)." if limit else ".") + if kind in {"int_parsing", "float_parsing"}: + return f"{field} must be a number." + if kind in {"greater_than_equal", "less_than_equal"}: + return f"{field}: {message}" + if kind == "value_error": + ctx = error.get("ctx") or {} + return f"{field}: {ctx.get('error') or message}" + + return f"{field}: {message}" + + +def _public_exception_message(exc: Exception) -> str: + message = str(exc).strip() + is_local = settings.environment.lower() in {"development", "dev", "local", "test"} + + if is_local and isinstance(exc, TimeoutError): + return message or "The request timed out while waiting for an LLM response." + if is_local: + return message or type(exc).__name__ + + if isinstance(exc, TimeoutError): + return "The request timed out while waiting for an LLM response." + if isinstance(exc, ValueError): + return "Invalid request." + if isinstance(exc, ConnectionError): + return "A backend service is unavailable. Check the server logs with the request_id for details." + if isinstance(exc, RuntimeError): + return "The request could not be completed. Check the server logs with the request_id." + + return "Internal server error. Check the server logs with the request_id for details." + + async def _boot_pipelines() -> None: loop = asyncio.get_running_loop() try: @@ -204,6 +264,32 @@ async def sentry_debug(): # ── Global exception handler ────────────────────────────────────── + @app.exception_handler(RequestValidationError) + async def _validation_exception(request: Request, exc: RequestValidationError): + request_id = getattr(request.state, "request_id", None) + details = [_friendly_validation_error(error) for error in exc.errors()] + body = APIResponse( + status=StatusEnum.ERROR, + request_id=request_id, + error="Invalid request: " + " ".join(details), + data={"details": details}, + ) + return JSONResponse(content=body.model_dump(), status_code=422) + + @app.exception_handler(HTTPException) + async def _http_exception(request: Request, exc: HTTPException): + request_id = getattr(request.state, "request_id", None) + body = APIResponse( + status=StatusEnum.ERROR, + request_id=request_id, + error=_detail_to_text(exc.detail), + ) + return JSONResponse( + content=body.model_dump(), + status_code=exc.status_code, + headers=exc.headers, + ) + @app.exception_handler(Exception) async def _unhandled_exception(request: Request, exc: Exception): request_id = getattr(request.state, "request_id", None) @@ -214,7 +300,9 @@ async def _unhandled_exception(request: Request, exc: Exception): capture_exception(exc) body = APIResponse( - status=StatusEnum.ERROR, request_id=request_id, error="Internal server error.", + status=StatusEnum.ERROR, + request_id=request_id, + error=_public_exception_message(exc), ) return JSONResponse(content=body.model_dump(), status_code=500) diff --git a/src/api/routes/memory.py b/src/api/routes/memory.py index ab40c07..cab1d9a 100644 --- a/src/api/routes/memory.py +++ b/src/api/routes/memory.py @@ -48,6 +48,7 @@ import re from playwright.sync_api import sync_playwright +from src.config import settings from src.jobs.durable import ( QUEUED, get_default_job_store, @@ -58,6 +59,7 @@ logger = logging.getLogger("xmem.api.routes.memory") _ingest_semaphore = asyncio.Semaphore(5) +_LOCAL_ENVIRONMENTS = {"development", "dev", "local", "test"} router = APIRouter( prefix="/v1/memory", @@ -124,6 +126,8 @@ def _error( code: int, elapsed_ms: float = 0, ) -> JSONResponse: + if code >= 500 and settings.environment.lower() not in _LOCAL_ENVIRONMENTS: + detail = "The request could not be completed. Check the server logs with the request_id." body = APIResponse( status=StatusEnum.ERROR, request_id=getattr(request.state, "request_id", None), @@ -133,10 +137,26 @@ def _error( return JSONResponse(content=body.model_dump(), status_code=code) -def _current_user_id(user: dict) -> str: +def _is_static_key_user(user: dict) -> bool: + return user.get("email") == "static@xmem.ai" or user.get("name") == "Static Key User" + + +def _current_user_id(user: dict, requested_user_id: str = "") -> str: + if ( + requested_user_id + and settings.environment.lower() in _LOCAL_ENVIRONMENTS + and _is_static_key_user(user) + ): + return requested_user_id return user.get("username") or user.get("name") or user["id"] +def _scoped_ingest_payload(user: dict, item: IngestRequest) -> Dict[str, Any]: + payload = item.model_dump() + payload["user_id"] = _current_user_id(user, payload.get("user_id", "")) + return payload + + def _job_status_data(job: Dict[str, Any]) -> Dict[str, Any]: public = serialize_job(job) or {} return { @@ -212,11 +232,11 @@ async def _run_ingest_payload( async def _run_batch_ingest_payload( payload: Dict[str, Any], - user_id: str, ) -> Dict[str, Any]: results = [] for item in payload["items"]: - results.append(await _run_ingest_payload(item, user_id)) + item_user_id = item.get("user_id") or payload["user_id"] + results.append(await _run_ingest_payload(item, item_user_id)) return {"results": results} @@ -677,13 +697,13 @@ async def _scrape_chat_share(url: str) -> Dict[str, Any]: ) async def ingest_memory(req: IngestRequest, request: Request, user: dict = Depends(require_api_key)): start = time.perf_counter() - user_id = _current_user_id(user) + user_id = _current_user_id(user, req.user_id) payload = req.model_dump() try: data = await asyncio.wait_for( _run_ingest_payload(payload, user_id), - timeout=120.0, + timeout=float(settings.memory_ingest_timeout_seconds), ) elapsed = round((time.perf_counter() - start) * 1000, 2) return _wrap(request, data, elapsed) @@ -702,7 +722,8 @@ async def ingest_memory(req: IngestRequest, request: Request, user: dict = Depen ) async def ingest_memory_v2(req: IngestRequest, request: Request, user: dict = Depends(require_api_key)): start = time.perf_counter() - user_id = _current_user_id(user) + user_id = _current_user_id(user, req.user_id) + job_user_id = _current_user_id(user) payload = req.model_dump() payload["user_id"] = user_id @@ -720,8 +741,8 @@ async def ingest_memory_v2(req: IngestRequest, request: Request, user: dict = De "image_url": req.image_url, "effort_level": req.effort_level, }, - user_id=user_id, - timeout_seconds=120.0, + user_id=job_user_id, + timeout_seconds=float(settings.memory_ingest_timeout_seconds), max_attempts=3, ) _schedule_job( @@ -802,9 +823,10 @@ async def batch_ingest_memory(req: BatchIngestRequest, request: Request, user: d try: results = [] for item in req.items: + payload = _scoped_ingest_payload(user, item) data = await asyncio.wait_for( - _run_ingest_payload(item.model_dump(), user_id), - timeout=120.0, + _run_ingest_payload(payload, payload["user_id"]), + timeout=float(settings.memory_ingest_timeout_seconds), ) results.append(IngestResponse(**data)) @@ -829,6 +851,7 @@ async def batch_ingest_memory_v2(req: BatchIngestRequest, request: Request, user user_id = _current_user_id(user) payload = req.model_dump() payload["user_id"] = user_id + payload["items"] = [_scoped_ingest_payload(user, item) for item in req.items] try: store = get_default_job_store() @@ -841,12 +864,15 @@ async def batch_ingest_memory_v2(req: BatchIngestRequest, request: Request, user "items": payload["items"], }, user_id=user_id, - timeout_seconds=max(120.0, min(len(req.items) * 120.0, 3600.0)), + timeout_seconds=max( + float(settings.memory_ingest_timeout_seconds), + min(len(req.items) * float(settings.memory_ingest_timeout_seconds), 3600.0), + ), max_attempts=3, ) _schedule_job( job, - lambda: _run_batch_ingest_payload(payload, user_id), + lambda: _run_batch_ingest_payload(payload), ) elapsed = round((time.perf_counter() - start) * 1000, 2) return _job_accepted( @@ -875,7 +901,7 @@ async def retrieve_memory(req: RetrieveRequest, request: Request, user: dict = D pipeline = get_retrieval_pipeline() # Get username from authenticated user - user_id = user.get("username") or user.get("name") or user["id"] + user_id = _current_user_id(user, req.user_id) try: result = await pipeline.run(query=req.query, user_id=user_id, top_k=req.top_k) @@ -911,7 +937,7 @@ async def search_memory(req: SearchRequest, request: Request, user: dict = Depen pipeline = get_retrieval_pipeline() # Get username from authenticated user - user_id = user.get("username") or user.get("name") or user["id"] + user_id = _current_user_id(user, req.user_id) try: all_results: List[SourceRecord] = [] diff --git a/src/api/schemas.py b/src/api/schemas.py index b7ee122..ff10e6d 100644 --- a/src/api/schemas.py +++ b/src/api/schemas.py @@ -9,11 +9,29 @@ from datetime import datetime from enum import Enum +import re from typing import Any, Dict, List, Optional from pydantic import BaseModel, Field, field_validator +def normalize_user_id(value: Any) -> str: + """Convert friendly user input into XMem's canonical storage id.""" + text = str(value or "").strip() + text = re.sub(r"[^A-Za-z0-9_.@-]+", "_", text) + text = re.sub(r"_+", "_", text).strip("_") + return text[:256] + + +class UserScopedModel(BaseModel): + """Base model for requests that scope data to a user.""" + + @field_validator("user_id", mode="before", check_fields=False) + @classmethod + def normalize_user_id_field(cls, v: Any) -> str: + return normalize_user_id(v) + + # ── Shared envelope ──────────────────────────────────────────────────────── class StatusEnum(str, Enum): @@ -42,7 +60,7 @@ class HealthResponse(BaseModel): # ── Ingest (save memory) ────────────────────────────────────────────────── -class IngestRequest(BaseModel): +class IngestRequest(UserScopedModel): """Store a new memory from a conversation turn.""" user_query: str = Field( ..., min_length=1, max_length=10_000, @@ -53,8 +71,8 @@ class IngestRequest(BaseModel): description="The assistant's reply (used for summary extraction)", ) user_id: str = Field( - ..., min_length=1, max_length=256, pattern=r"^[\w.\-@]+$", - description="Unique user identifier (alphanumeric, dots, hyphens, underscores, @)", + ..., min_length=1, max_length=256, + description="User identifier. Friendly names are normalized internally.", ) session_datetime: str = Field( default="", @@ -74,7 +92,6 @@ class IngestRequest(BaseModel): def strip_query(cls, v: str) -> str: return v.strip() - class OperationDetail(BaseModel): type: str content: str @@ -117,14 +134,14 @@ class BatchIngestResponse(BaseModel): # ── Retrieve (answer a question from memory) ────────────────────────────── -class RetrieveRequest(BaseModel): +class RetrieveRequest(UserScopedModel): """Ask a question answered from stored memories.""" query: str = Field( ..., min_length=1, max_length=5_000, description="The question to answer from memory", ) user_id: str = Field( - ..., min_length=1, max_length=256, pattern=r"^[\w.\-@]+$", + ..., min_length=1, max_length=256, ) top_k: int = Field(default=5, ge=1, le=50) @@ -133,7 +150,6 @@ class RetrieveRequest(BaseModel): def strip_query(cls, v: str) -> str: return v.strip() - class SourceRecord(BaseModel): domain: str content: str @@ -150,13 +166,13 @@ class RetrieveResponse(BaseModel): # ── Search (raw vector / graph search without LLM answer) ───────────────── -class SearchRequest(BaseModel): +class SearchRequest(UserScopedModel): """Raw semantic search across memory domains.""" query: str = Field( ..., min_length=1, max_length=5_000, ) user_id: str = Field( - ..., min_length=1, max_length=256, pattern=r"^[\w.\-@]+$", + ..., min_length=1, max_length=256, ) domains: List[str] = Field( default=["profile", "temporal", "summary"], @@ -199,7 +215,7 @@ class ScrapeResponse(BaseModel): # ── Code retrieval (IDE mode) ───────────────────────────────────────────── -class CodeQueryRequest(BaseModel): +class CodeQueryRequest(UserScopedModel): """Query a codebase via the code retrieval pipeline.""" org_id: str = Field(..., min_length=1, max_length=256) repo: str = Field(..., min_length=1, max_length=256) @@ -219,7 +235,7 @@ class CodeQueryResponse(BaseModel): confidence: float = 0.0 -class ExecuteToolRequest(BaseModel): +class ExecuteToolRequest(UserScopedModel): """Execute a specific raw code retrieval tool natively.""" org_id: str = Field(..., min_length=1, max_length=256) repo: str = Field(..., min_length=1, max_length=256) diff --git a/src/config/settings.py b/src/config/settings.py index d846b66..b910d43 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -104,6 +104,14 @@ class Settings(BaseSettings): default=0.4, description="LLM temperature for generation" ) + llm_timeout_seconds: float = Field( + default=45.0, + description="Per-agent LLM call timeout in seconds", + ) + memory_ingest_timeout_seconds: float = Field( + default=120.0, + description="Overall memory ingest timeout in seconds", + ) fallback_order: List[str] = Field( default=["openrouter", "gemini", "claude", "openai"], description="Order of LLM providers to try on failure" diff --git a/templates/xmem.env.local b/templates/xmem.env.local new file mode 100644 index 0000000..45f2088 --- /dev/null +++ b/templates/xmem.env.local @@ -0,0 +1,92 @@ +# XMem local environment. +# Copy to .env. + +ENVIRONMENT=development +API_HOST=0.0.0.0 +API_PORT=8000 +API_KEYS='["dev-xmem-key"]' +CORS_ORIGINS='["http://localhost:3000","http://localhost:5173","http://localhost:4173","http://localhost:8000"]' +RATE_LIMIT=120 +MAX_REQUEST_BODY_BYTES=10485760 +ENABLE_PROMETHEUS=true +ENABLE_ANALYTICS=false + +# LLM routing. +# npm run setup/start rewrites FALLBACK_ORDER before local startup: +# - any real cloud LLM key below means XMem will not call Ollama +# - no cloud LLM key means XMem will use local Ollama +FALLBACK_ORDER='["ollama"]' +OLLAMA_BASE_URL=http://localhost:11434 +OLLAMA_MODEL=qwen2.5:1.5b +OLLAMA_VISION_MODEL=llava:latest +TEMPERATURE=0.3 +LLM_TIMEOUT_SECONDS=180 +MEMORY_INGEST_TIMEOUT_SECONDS=600 + +# Optional cloud providers. Add one key here to avoid Ollama calls. +GEMINI_API_KEY= +GEMINI_MODEL=gemini-2.5-flash +GEMINI_VISION_MODEL=gemini-2.5-flash-lite +CLAUDE_API_KEY= +CLAUDE_MODEL=claude-3-5-sonnet +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4.1-mini +OPENROUTER_API_KEY= +OPENROUTER_MODEL=google/gemini-2.5-flash +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +BEDROCK_REGION=us-east-1 +BEDROCK_MODEL=us.amazon.nova-lite-v1:0 + +# Local embeddings and vector storage. +# With a cloud LLM key, scripts switch embeddings to fastembed so Ollama is not used. +EMBEDDING_PROVIDER=ollama +OLLAMA_EMBEDDING_MODEL=nomic-embed-text +FASTEMBED_MODEL=BAAI/bge-small-en-v1.5 +EMBEDDING_MODEL=nomic-embed-text +VECTOR_STORE_PROVIDER=pgvector +PGVECTOR_URL=postgresql://xmem:xmem@localhost:15432/xmem +PGVECTOR_TABLE=xmem_vectors + +# Pinecone remains unset for local mode. +PINECONE_API_KEY= +PINECONE_INDEX_NAME=xmem-local +PINECONE_NAMESPACE=local +PINECONE_DIMENSION=768 +PINECONE_METRIC=cosine +PINECONE_CLOUD=aws +PINECONE_REGION=us-east-1 + +# App metadata and graph stores. +APP_STORE_PROVIDER=postgres +APP_POSTGRES_URL=postgresql://xmem:xmem@localhost:15432/xmem +MONGODB_URI=mongodb://localhost:27018 +MONGODB_DATABASE=xmem + +NEO4J_URI=bolt://localhost:17687 +NEO4J_USERNAME=neo4j +NEO4J_PASSWORD=local-password +NEO4J_DATABASE=neo4j +NEO4J_TRANSPORT=auto +NEO4J_CONNECTION_TIMEOUT=60 + +# Auth and app URLs. +GOOGLE_CLIENT_ID= +GOOGLE_CLIENT_SECRET= +GOOGLE_REDIRECT_URI=http://localhost:8000/auth/callback +JWT_SECRET_KEY=dev-only-change-me +JWT_ALGORITHM=HS256 +JWT_EXPIRATION_DAYS=7 +FRONTEND_URL=http://localhost:5173 +XMEM_SERVER_URL=http://localhost:8000 + +# Optional integrations. +SENTRY_DSN= +OPIK_API_KEY= +OPIK_WORKSPACE=xmem +OPIK_PROJECT=xmem-local +GITHUB_TOKEN= +GITHUB_REPO_OWNER=XortexAI +GITHUB_REPO_NAME=XMem +GMAIL_APP_PASSWORD= +GMAIL_SENDER_EMAIL=xmemlabs@gmail.com diff --git a/tests/api/test_app_errors.py b/tests/api/test_app_errors.py new file mode 100644 index 0000000..0dea010 --- /dev/null +++ b/tests/api/test_app_errors.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from src.api import app as api_app + + +def test_public_exception_message_redacts_connection_details_in_production(monkeypatch): + monkeypatch.setattr(api_app.settings, "environment", "production", raising=False) + + message = api_app._public_exception_message( + ConnectionError("postgresql://user:password@internal-db:5432/xmem") + ) + + assert "password" not in message + assert "internal-db" not in message + assert "backend service is unavailable" in message + + +def test_public_exception_message_redacts_value_error_in_production(monkeypatch): + monkeypatch.setattr(api_app.settings, "environment", "production", raising=False) + + message = api_app._public_exception_message( + ValueError("vector mismatch for postgresql://user:password@internal-db:5432/xmem") + ) + + assert message == "Invalid request." + assert "password" not in message + assert "internal-db" not in message + + +def test_public_exception_message_keeps_timeout_detail_in_local(monkeypatch): + monkeypatch.setattr(api_app.settings, "environment", "development", raising=False) + + message = api_app._public_exception_message(TimeoutError("LLM timed out after 180 seconds")) + + assert message == "LLM timed out after 180 seconds" diff --git a/tests/api/test_memory_versioning.py b/tests/api/test_memory_versioning.py index a7091cb..11ebbcc 100644 --- a/tests/api/test_memory_versioning.py +++ b/tests/api/test_memory_versioning.py @@ -46,14 +46,15 @@ def get(self, job_id): return self.jobs.get(job_id) -def _build_app(monkeypatch): +def _build_app(monkeypatch, user=None): ingest = FakeIngestPipeline() deps._init_error = None deps._pipelines_ready.set() deps.set_pipelines(ingest, FakeRetrievalPipeline()) + auth_user = user or {"id": "user-1", "username": "hunter"} async def fake_user(): - return {"id": "user-1", "username": "hunter"} + return auth_user async def fake_ready(): return None @@ -72,6 +73,16 @@ async def fake_rate_limit(): return app, ingest +def test_static_key_user_id_override_is_local_only(monkeypatch): + static_user = {"id": "static-key", "name": "Static Key User", "email": "static@xmem.ai"} + + monkeypatch.setattr(memory.settings, "environment", "development", raising=False) + assert memory._current_user_id(static_user, "friendly-user") == "friendly-user" + + monkeypatch.setattr(memory.settings, "environment", "production", raising=False) + assert memory._current_user_id(static_user, "friendly-user") == "Static Key User" + + def test_v1_ingest_keeps_synchronous_response_contract(monkeypatch): app, ingest = _build_app(monkeypatch) payload = { @@ -119,3 +130,49 @@ def test_v2_ingest_returns_durable_job_envelope(monkeypatch): } assert scheduled == ["memory_ingest:fake"] assert ingest.calls == [] + + +def test_v1_batch_ingest_scopes_each_item_for_local_static_key(monkeypatch): + monkeypatch.setattr(memory.settings, "environment", "development", raising=False) + static_user = {"id": "static-key", "name": "Static Key User", "email": "static@xmem.ai"} + app, ingest = _build_app(monkeypatch, user=static_user) + payload = { + "items": [ + {"user_query": "remember alpha", "agent_response": "done", "user_id": "alice"}, + {"user_query": "remember beta", "agent_response": "done", "user_id": "bob"}, + ], + } + + response = TestClient(app).post("/v1/memory/batch-ingest", json=payload) + + assert response.status_code == 200 + assert [call["user_id"] for call in ingest.calls] == ["alice", "bob"] + + +def test_v2_batch_ingest_queues_scoped_items_for_local_static_key(monkeypatch): + monkeypatch.setattr(memory.settings, "environment", "development", raising=False) + static_user = {"id": "static-key", "name": "Static Key User", "email": "static@xmem.ai"} + app, ingest = _build_app(monkeypatch, user=static_user) + store = FakeJobStore() + scheduled = [] + monkeypatch.setattr(memory, "get_default_job_store", lambda: store) + monkeypatch.setattr( + memory, + "_schedule_job", + lambda job, handler: scheduled.append(job["job_id"]), + ) + payload = { + "items": [ + {"user_query": "remember alpha", "agent_response": "done", "user_id": "alice"}, + {"user_query": "remember beta", "agent_response": "done", "user_id": "bob"}, + ], + } + + response = TestClient(app).post("/v2/memory/batch-ingest", json=payload) + + assert response.status_code == 200 + job = store.jobs["memory_batch_ingest:fake"] + assert job["user_id"] == "Static Key User" + assert [item["user_id"] for item in job["payload"]["items"]] == ["alice", "bob"] + assert scheduled == ["memory_batch_ingest:fake"] + assert ingest.calls == [] diff --git a/tests/unit/test_schemas.py b/tests/unit/test_schemas.py index 3905c53..8499977 100644 --- a/tests/unit/test_schemas.py +++ b/tests/unit/test_schemas.py @@ -3,6 +3,7 @@ import pytest from pydantic import ValidationError +from src.api.schemas import IngestRequest, normalize_user_id from src.schemas.code import ( AnnotationSeverity, AnnotationType, @@ -84,3 +85,17 @@ def test_code_schema_enums_and_namespace_helpers(): assert symbols_namespace("acme", "payments") == "acme:payments:symbols" assert annotations_namespace("acme") == "acme:annotations" assert snippets_namespace("user-1") == "user-1:snippets" + + +def test_api_user_ids_are_normalized_before_validation(): + assert normalize_user_id(" Ankit Kotnala! ") == "Ankit_Kotnala" + + request = IngestRequest( + user_query="remember this", + agent_response="done", + user_id="Ankit Kotnala!", + ) + assert request.user_id == "Ankit_Kotnala" + + with pytest.raises(ValidationError): + IngestRequest(user_query="remember this", agent_response="done", user_id=" !!! ")