Add ontology navigation to the SDK core for DQL authoring

Promote ontology introspection out of the CLI layer into a storage-agnostic
diffbot.Ontology class so library consumers (e.g. langchain) can build valid
DQL on the fly, without depending on the disk-backed CLI cache.

- New diffbot/ontology.py: Ontology with types/composites/enums/taxonomies,
  fields_for, filter_fields, taxonomy_values, enum_values, find_named,
  format_field, plus from_json/from_path. Pure; imposes no caching policy.
- Client: dql_fetch_ontology() (sync + async) downloads and returns an
  Ontology; add async DiffbotAsync.dql_parallel() to mirror the sync one.
- cli/ontology.py now delegates to Ontology, keeping its ~/.diffbot disk cache
  and existing module API unchanged.
- tests/test_ontology.py covers the core, the HTTP fetch, and async parallel.

Also includes a pre-existing, in-progress credential-resolution refactor that
was already present in the working tree (diffbot/_auth.py and its consumers in
cli/_common.py, cli/dql.py, conftest, tests, README); __init__.py carries both
sets of changes, so they commit together.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: shixish

README.md

@@ -18,12 +18,38 @@ pip install -e ".[dev]"
 ## Usage
 
 ### Authentication
-Set your Diffbot API token in your environment or .env.
+
+The CLI and the library can share a single credential. The token always has to be
+passed to the client explicitly, but `resolve_token()` gives you the same lookup the
+CLI uses, in this order:
+
+1. An explicit token passed to `resolve_token(token)`.
+2. The `DIFFBOT_API_TOKEN` environment variable.
+3. A `DIFFBOT_API_TOKEN=...` line in `~/.diffbot/credentials`.
+
+Set it once and it works for both the CLI and your scripts. Either export it:
 
 ```bash
 export DIFFBOT_API_TOKEN=<TOKEN>
 ```
 
+…or write it to the shared credentials file (handy for keeping it out of your shell environment):
+
+```bash
+mkdir -p ~/.diffbot
+printf 'DIFFBOT_API_TOKEN=%s\n' '<TOKEN>' > ~/.diffbot/credentials
+chmod 600 ~/.diffbot/credentials
+```
+
+With either in place, resolve the token and pass it to the client:
+
+```python
+from diffbot import Diffbot, resolve_token
+
+db = Diffbot(token=resolve_token())  # from env var or ~/.diffbot/credentials
+data = db.extract("https://www.example.com")
+```
+
 ### Extract structured content
 ```python
 from diffbot import Diffbot
@@ -166,7 +192,15 @@ asyncio.run(main())
 
 ## CLI
 
-This library also includes a CLI.
+This library also includes a CLI exposed as the `db` command.
+
+To make `db` available from anywhere, install it as an isolated tool with [uv](https://docs.astral.sh/uv/):
+
+```bash
+uv tool install .
+```
+
+This drops a `db` executable into `~/.local/bin` (ensure it is on your `PATH`). Use `--force` to reinstall or upgrade after changes, or `--editable` to have source edits take effect immediately. Alternatively, a plain `pip install .` (or `pip install -e .`) also installs the `db` entry point into the active environment.
 
 ```bash
 export DIFFBOT_API_TOKEN=your-token-here
@@ -189,7 +223,9 @@ Run the mock test suite:
 python -m pytest
 ```
 
-Run live integration tests against the real API (requires a valid token):
+Run live integration tests against the real API (requires a valid token).
+The token is resolved the same way as everywhere else — the `DIFFBOT_API_TOKEN`
+environment variable or `~/.diffbot/credentials`:
 ```bash
-DIFFBOT_TOKEN=your_token python -m pytest -m live
+DIFFBOT_API_TOKEN=your_token python -m pytest -m live
 ```

pyproject.toml

@@ -71,5 +71,5 @@ include = [
 ]
 
 [tool.pytest.ini_options]
-markers = ["live: marks tests as live integration tests requiring a real DIFFBOT_TOKEN"]
+markers = ["live: marks tests as live integration tests requiring a real DIFFBOT_API_TOKEN"]
 addopts = "-m 'not live'"

src/diffbot/__init__.py

@@ -4,6 +4,7 @@
 
 __version__ = "0.1.0"
 
+from ._auth import resolve_token
 from .client import Diffbot, DiffbotAsync
 from .crawl import CrawlEvent, CrawlEventType
 from .errors import (
@@ -14,12 +15,15 @@
     RateLimitError,
     ValidationError,
 )
+from .ontology import Ontology
 
 __all__ = [
     "Diffbot",
     "DiffbotAsync",
+    "resolve_token",
     "CrawlEvent",
     "CrawlEventType",
+    "Ontology",
     "DiffbotError",
     "AuthError",
     "ExtractionError",

src/diffbot/_auth.py

@@ -0,0 +1,41 @@
+"""Shared Diffbot credential resolution for both the library and the CLI.
+
+The same lookup chain is used everywhere so a single credential works for the
+``db`` CLI and any Python script that constructs a client:
+
+1. An explicit token passed to the client / function.
+2. The ``DIFFBOT_API_TOKEN`` environment variable.
+3. A ``DIFFBOT_API_TOKEN=...`` line in ``~/.diffbot/credentials``.
+"""
+
+import os
+import pathlib
+from typing import Optional
+
+TOKEN_ENV_VAR = "DIFFBOT_API_TOKEN"
+CREDENTIALS_PATH = pathlib.Path.home() / ".diffbot" / "credentials"
+
+
+def _read_credentials_file() -> str:
+    if not CREDENTIALS_PATH.exists():
+        return ""
+    for line in CREDENTIALS_PATH.read_text().splitlines():
+        line = line.strip()
+        if line.startswith(f"{TOKEN_ENV_VAR}="):
+            return line[len(TOKEN_ENV_VAR) + 1:].strip()
+    return ""
+
+
+def resolve_token(token: Optional[str] = None) -> str:
+    """Resolve a Diffbot API token from the explicit argument, env var, or file.
+
+    Returns an empty string if no token can be found.
+    """
+    if token and token.strip():
+        return token.strip()
+
+    env_token = os.environ.get(TOKEN_ENV_VAR, "").strip()
+    if env_token:
+        return env_token
+
+    return _read_credentials_file()

src/diffbot/cli/_common.py

@@ -1,35 +1,20 @@
-import os
-import pathlib
-
 import click
 
-from diffbot import Diffbot
-
-CREDENTIALS_PATH = pathlib.Path.home() / ".diffbot" / "credentials"
-
-
-def resolve_token() -> str:
-    """Return the Diffbot API token from the env var, falling back to ~/.diffbot/credentials."""
-    token = os.environ.get("DIFFBOT_API_TOKEN", "").strip()
-    if token:
-        return token
-
-    if CREDENTIALS_PATH.exists():
-        for line in CREDENTIALS_PATH.read_text().splitlines():
-            line = line.strip()
-            if line.startswith("DIFFBOT_API_TOKEN="):
-                return line[len("DIFFBOT_API_TOKEN="):].strip()
-
-    return ""
+from diffbot import Diffbot, resolve_token
+from diffbot._auth import CREDENTIALS_PATH, TOKEN_ENV_VAR
 
 
 def get_client() -> Diffbot:
+    """Build a Diffbot client using the shared credential resolution chain.
+
+    Looks at the DIFFBOT_API_TOKEN env var, then ~/.diffbot/credentials.
+    """
     token = resolve_token()
     if not token:
         click.echo(
             "Error: no Diffbot API token found.\n"
-            "  Set a DIFFBOT_API_TOKEN environment variable, or\n"
-            f"  write 'DIFFBOT_API_TOKEN=YOUR_TOKEN' to {CREDENTIALS_PATH}",
+            f"  Set a {TOKEN_ENV_VAR} environment variable, or\n"
+            f"  write '{TOKEN_ENV_VAR}=YOUR_TOKEN' to {CREDENTIALS_PATH}",
             err=True,
         )
         raise click.Abort()

src/diffbot/cli/dql.py

@@ -15,7 +15,9 @@
 from diffbot import DiffbotError
 
 from . import ontology
-from ._common import get_client, resolve_token
+from diffbot import resolve_token
+
+from ._common import get_client
 
 
 class _DqlGroup(click.Group):

src/diffbot/cli/ontology.py

@@ -1,14 +1,24 @@
+"""CLI-side ontology access: a disk cache over the storage-agnostic core.
+
+The navigation logic lives in :mod:`diffbot.ontology` (the `Ontology` class).
+This module adds the CLI's caching policy on top: the ontology is read once from
+``~/.diffbot/ontology.json`` (populated by `db dql init`) and held in
+``_CACHE``. The module-level functions preserve the historical CLI surface and
+simply delegate to an `Ontology` built from the cached document.
+"""
+
 import json
 import pathlib
-import re
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
+
+from diffbot.ontology import Ontology
 
 ONTOLOGY_PATH = pathlib.Path.home() / ".diffbot" / "ontology.json"
 
 _CACHE: Dict[str, Any] = {}
 
 
-def load() -> Dict[str, Any]:
+def _data() -> Dict[str, Any]:
     if "data" not in _CACHE:
         if not ONTOLOGY_PATH.exists():
             raise FileNotFoundError(
@@ -18,113 +28,47 @@ def load() -> Dict[str, Any]:
     return _CACHE["data"]
 
 
+def _ontology() -> Ontology:
+    return Ontology(_data())
+
+
 def list_types() -> List[str]:
-    return sorted(load().get("types", {}).keys())
+    return _ontology().types()
 
 
 def list_composites() -> List[str]:
-    return sorted(load().get("composites", {}).keys())
+    return _ontology().composites()
 
 
 def list_enums() -> List[str]:
-    return sorted(load().get("enums", {}).keys())
+    return _ontology().enums()
 
 
 def list_taxonomies() -> List[str]:
-    return sorted(load().get("taxonomies", {}).keys())
-
-
-def _fields_of(container: Dict[str, Any], type_name: str) -> Dict[str, Any]:
-    entry = container.get(type_name)
-    if entry is None:
-        raise KeyError(f"Unknown name: {type_name}")
-    return entry.get("fields", {})
+    return _ontology().taxonomies()
 
 
 def fields_for(type_name: str) -> Dict[str, Any]:
-    data = load()
-    types = data.get("types", {})
-    composites = data.get("composites", {})
-    if type_name in types:
-        return _fields_of(types, type_name)
-    if type_name in composites:
-        return _fields_of(composites, type_name)
-    raise KeyError(f"{type_name} is not a known entity type or composite")
+    return _ontology().fields_for(type_name)
 
 
 def format_field(name: str, meta: Dict[str, Any]) -> str:
-    t = meta.get("type", "?")
-    if t == "LinkedEntity":
-        le = meta.get("leType") or []
-        if le:
-            t = f"LinkedEntity ({le[0]})"
-    flags = []
-    if meta.get("isList"):
-        flags.append("isList")
-    if meta.get("isComposite"):
-        flags.append("isComposite")
-    if meta.get("isEnum"):
-        flags.append("isEnum")
-    if meta.get("isDeprecated"):
-        flags.append("DEPRECATED")
-    suffix = "".join(f" [{f}]" for f in flags)
-    return f"{name}: [{t}]{suffix}"
-
-
-def filter_fields(fields: Dict[str, Any], search: Optional[str], include_deprecated: bool = False) -> List[tuple]:
-    pattern = re.compile(search, re.IGNORECASE) if search else None
-    out = []
-    for name, meta in fields.items():
-        if not include_deprecated and meta.get("isDeprecated"):
-            continue
-        if pattern and not pattern.search(name):
-            continue
-        out.append((name, meta))
-    return out
+    return Ontology.format_field(name, meta)
 
 
-def taxonomy_values(name: str, search: Optional[str] = None) -> List[str]:
-    data = load()
-    tax = data.get("taxonomies", {}).get(name)
-    if tax is None:
-        raise KeyError(f"Unknown taxonomy: {name}")
-    pattern = re.compile(search, re.IGNORECASE) if search else None
-    out: List[str] = []
+def filter_fields(
+    fields: Dict[str, Any], search: Optional[str], include_deprecated: bool = False
+) -> List[Tuple[str, Dict[str, Any]]]:
+    return Ontology.filter_fields(fields, search, include_deprecated=include_deprecated)
 
-    def walk(node: Dict[str, Any]) -> None:
-        n = node.get("name")
-        if n and (pattern is None or pattern.search(n)):
-            out.append(n)
-        for child in node.get("children", []) or []:
-            walk(child)
 
-    for cat in tax.get("categories", []) or []:
-        walk(cat)
-    return out
+def taxonomy_values(name: str, search: Optional[str] = None) -> List[str]:
+    return _ontology().taxonomy_values(name, search)
 
 
 def enum_values(name: str) -> List[str]:
-    data = load()
-    enum = data.get("enums", {}).get(name)
-    if enum is None:
-        raise KeyError(f"Unknown enum: {name}")
-    return list(enum.get("values", []))
+    return _ontology().enum_values(name)
 
 
 def find_named(search: str) -> List[str]:
-    pattern = re.compile(search, re.IGNORECASE)
-    found = set()
-
-    def walk(node: Any) -> None:
-        if isinstance(node, dict):
-            n = node.get("name")
-            if isinstance(n, str) and pattern.search(n):
-                found.add(n)
-            for v in node.values():
-                walk(v)
-        elif isinstance(node, list):
-            for v in node:
-                walk(v)
-
-    walk(load())
-    return sorted(found)
+    return _ontology().find_named(search)