Merge branch 'release-tooling'

# Conflicts:
#	.gitignore
#	pyproject.toml

Author: shixish

.gitignore

@@ -3,7 +3,10 @@
 __pycache__
 .pytest_cache
 .env
+uv.lock
 dist/
+build/
+*.egg-info/
 
 # Claude
 .claude/settings.local.json

Makefile

@@ -0,0 +1,94 @@
+.PHONY: test test-live build clean bump-patch bump-minor bump-major set-token-pypi set-token-testpypi release-test release verify-release-test verify-release
+
+test:
+	uv run --extra dev pytest
+
+test-live:
+	uv run --extra dev pytest -m live
+
+clean:
+	rm -rf dist build *.egg-info
+
+build: clean
+	uv build
+
+# Version bumps: edits pyproject.toml in place and prints old => new.
+bump-patch:
+	uv version --bump patch
+
+bump-minor:
+	uv version --bump minor
+
+bump-major:
+	uv version --bump major
+
+# Store a PyPI / TestPyPI token in macOS Keychain. Prompts with hidden input
+# (bash `read -rsp`), so the token never appears on screen, in shell history,
+# or in `make` output. Re-running overwrites the existing entry.
+set-token-pypi:
+	@bash -c 'read -rsp "Paste PyPI token: " TOKEN && echo && \
+	  security delete-generic-password -s pypi-token-pypi >/dev/null 2>&1; \
+	  security add-generic-password -a "$$USER" -s pypi-token-pypi -w "$$TOKEN" && \
+	  echo "stored in Keychain under service: pypi-token-pypi"'
+
+set-token-testpypi:
+	@bash -c 'read -rsp "Paste TestPyPI token: " TOKEN && echo && \
+	  security delete-generic-password -s pypi-token-testpypi >/dev/null 2>&1; \
+	  security add-generic-password -a "$$USER" -s pypi-token-testpypi -w "$$TOKEN" && \
+	  echo "stored in Keychain under service: pypi-token-testpypi"'
+
+# Publish to TestPyPI. Token comes from macOS Keychain (service: pypi-token-testpypi).
+# The `@` on the recipe lines hides the actual command so the token never appears in output.
+release-test: build
+	@VERSION=$$(grep '^version' pyproject.toml | head -1 | cut -d'"' -f2) && \
+	  STATUS=$$(curl -s -o /dev/null -w "%{http_code}" "https://test.pypi.org/pypi/diffbot-python/$$VERSION/json") && \
+	  if [ "$$STATUS" = "200" ]; then \
+	    echo "ERROR: diffbot-python $$VERSION is already on TestPyPI. Bump the version in pyproject.toml."; \
+	    exit 1; \
+	  fi
+	@TOKEN=$$(security find-generic-password -s pypi-token-testpypi -w 2>/dev/null) && \
+	  if [ -z "$$TOKEN" ]; then \
+	    echo "ERROR: no Keychain entry for pypi-token-testpypi. Run 'make set-token-testpypi' first."; \
+	    exit 1; \
+	  fi && \
+	  UV_PUBLISH_TOKEN="$$TOKEN" uv publish --publish-url https://test.pypi.org/legacy/
+
+# Publish to real PyPI. Confirmation gate before upload (PyPI does not allow re-uploads).
+release: build
+	@VERSION=$$(grep '^version' pyproject.toml | head -1 | cut -d'"' -f2) && \
+	  STATUS=$$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/diffbot-python/$$VERSION/json") && \
+	  if [ "$$STATUS" = "200" ]; then \
+	    echo "ERROR: diffbot-python $$VERSION is already on PyPI. Bump the version in pyproject.toml."; \
+	    exit 1; \
+	  fi && \
+	  echo "About to publish diffbot-python $$VERSION to PyPI. This cannot be undone." && \
+	  read -p "Type the version to confirm: " CONFIRM && \
+	  [ "$$CONFIRM" = "$$VERSION" ] || { echo "Aborted."; exit 1; }
+	@TOKEN=$$(security find-generic-password -s pypi-token-pypi -w 2>/dev/null) && \
+	  if [ -z "$$TOKEN" ]; then \
+	    echo "ERROR: no Keychain entry for pypi-token-pypi. Run 'make set-token-pypi' first."; \
+	    exit 1; \
+	  fi && \
+	  UV_PUBLISH_TOKEN="$$TOKEN" uv publish
+
+# Smoke-test installs from each index in a throwaway venv.
+# `cd $$TMP` before running python so CWD doesn't shadow the venv install with this repo's source.
+# Deps live on prod PyPI, so TestPyPI install needs --extra-index-url.
+verify-release-test:
+	@VERSION=$$(grep '^version' pyproject.toml | head -1 | cut -d'"' -f2) && \
+	  TMP=$$(mktemp -d) && \
+	  uv venv --python 3.12 $$TMP/.venv >/dev/null 2>&1 && \
+	  uv pip install --quiet --python $$TMP/.venv/bin/python \
+	    --index-url https://test.pypi.org/simple/ \
+	    --extra-index-url https://pypi.org/simple/ \
+	    "diffbot-python==$$VERSION" && \
+	  (cd $$TMP && $$TMP/.venv/bin/python -c "import diffbot; print('TestPyPI install OK:', diffbot.__version__)") && \
+	  rm -rf $$TMP
+
+verify-release:
+	@VERSION=$$(grep '^version' pyproject.toml | head -1 | cut -d'"' -f2) && \
+	  TMP=$$(mktemp -d) && \
+	  uv venv --python 3.12 $$TMP/.venv >/dev/null 2>&1 && \
+	  uv pip install --quiet --python $$TMP/.venv/bin/python "diffbot-python==$$VERSION" && \
+	  (cd $$TMP && $$TMP/.venv/bin/python -c "import diffbot; print('PyPI install OK:', diffbot.__version__)") && \
+	  rm -rf $$TMP

README.md

@@ -18,12 +18,38 @@ pip install -e ".[dev]"
 ## Usage
 
 ### Authentication
-Set your Diffbot API token in your environment or .env.
+
+The CLI and the library can share a single credential. The token always has to be
+passed to the client explicitly, but `resolve_token()` gives you the same lookup the
+CLI uses, in this order:
+
+1. An explicit token passed to `resolve_token(token)`.
+2. The `DIFFBOT_API_TOKEN` environment variable.
+3. A `DIFFBOT_API_TOKEN=...` line in `~/.diffbot/credentials`.
+
+Set it once and it works for both the CLI and your scripts. Either export it:
 
 ```bash
 export DIFFBOT_API_TOKEN=<TOKEN>
 ```
 
+…or write it to the shared credentials file (handy for keeping it out of your shell environment):
+
+```bash
+mkdir -p ~/.diffbot
+printf 'DIFFBOT_API_TOKEN=%s\n' '<TOKEN>' > ~/.diffbot/credentials
+chmod 600 ~/.diffbot/credentials
+```
+
+With either in place, resolve the token and pass it to the client:
+
+```python
+from diffbot import Diffbot, resolve_token
+
+db = Diffbot(token=resolve_token())  # from env var or ~/.diffbot/credentials
+data = db.extract("https://www.example.com")
+```
+
 ### Extract structured content
 ```python
 from diffbot import Diffbot
@@ -166,7 +192,15 @@ asyncio.run(main())
 
 ## CLI
 
-This library also includes a CLI.
+This library also includes a CLI exposed as the `db` command.
+
+To make `db` available from anywhere, install it as an isolated tool with [uv](https://docs.astral.sh/uv/):
+
+```bash
+uv tool install .
+```
+
+This drops a `db` executable into `~/.local/bin` (ensure it is on your `PATH`). Use `--force` to reinstall or upgrade after changes, or `--editable` to have source edits take effect immediately. Alternatively, a plain `pip install .` (or `pip install -e .`) also installs the `db` entry point into the active environment.
 
 ```bash
 export DIFFBOT_API_TOKEN=your-token-here
@@ -189,7 +223,9 @@ Run the mock test suite:
 python -m pytest
 ```
 
-Run live integration tests against the real API (requires a valid token):
+Run live integration tests against the real API (requires a valid token).
+The token is resolved the same way as everywhere else — the `DIFFBOT_API_TOKEN`
+environment variable or `~/.diffbot/credentials`:
 ```bash
-DIFFBOT_TOKEN=your_token python -m pytest -m live
+DIFFBOT_API_TOKEN=your_token python -m pytest -m live
 ```

pyproject.toml

@@ -8,19 +8,39 @@ version = "0.1.0"
 description = "Python client library for Diffbot APIs"
 readme = "README.md"
 requires-python = ">=3.10"
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "Operating System :: OS Independent",
-    "Topic :: Software Development :: Libraries",
-    "Topic :: Scientific/Engineering :: Artificial Intelligence",
-    "Topic :: Internet :: WWW/HTTP :: Indexing/Search"
-]
 license = "MIT"
 license-files = ["LICEN[CS]E*"]
 authors = [
     { name = "Jerome Choo", email = "jerome@diffbot.com" },
     { name = "Mike Tung", email = "miket@diffbot.com" }
 ]
+keywords = [
+    "diffbot",
+    "knowledge-graph",
+    "web-scraping",
+    "extract",
+    "crawler",
+    "nlp",
+    "llm",
+    "api-client",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Typing :: Typed",
+]
 dependencies = [
     "httpx>=0.27.0",
     "click>=8.1.0",
@@ -34,6 +54,7 @@ dev = [
 
 [project.urls]
 Homepage = "https://github.com/diffbot/diffbot-python"
+Documentation = "https://github.com/diffbot/diffbot-python#readme"
 Repository = "https://github.com/diffbot/diffbot-python"
 Issues = "https://github.com/diffbot/diffbot-python/issues"
 
@@ -43,6 +64,15 @@ db = "diffbot.cli:main"
 [tool.hatch.build.targets.wheel]
 packages = ["src/diffbot"]
 
+[tool.hatch.build.targets.sdist]
+include = [
+    "/src",
+    "/tests",
+    "/README.md",
+    "/LICENSE",
+    "/pyproject.toml",
+]
+
 [tool.pytest.ini_options]
-markers = ["live: marks tests as live integration tests requiring a real DIFFBOT_TOKEN"]
-addopts = "-m 'not live'"
+markers = ["live: marks tests as live integration tests requiring a real DIFFBOT_API_TOKEN"]
+addopts = "-m 'not live'"

src/diffbot/__init__.py

@@ -4,6 +4,7 @@
 
 __version__ = "0.1.0"
 
+from ._auth import resolve_token
 from .client import Diffbot, DiffbotAsync
 from .crawl import CrawlEvent, CrawlEventType
 from .errors import (
@@ -14,12 +15,15 @@
     RateLimitError,
     ValidationError,
 )
+from .ontology import Ontology
 
 __all__ = [
     "Diffbot",
     "DiffbotAsync",
+    "resolve_token",
     "CrawlEvent",
     "CrawlEventType",
+    "Ontology",
     "DiffbotError",
     "AuthError",
     "ExtractionError",

src/diffbot/_auth.py

@@ -0,0 +1,41 @@
+"""Shared Diffbot credential resolution for both the library and the CLI.
+
+The same lookup chain is used everywhere so a single credential works for the
+``db`` CLI and any Python script that constructs a client:
+
+1. An explicit token passed to the client / function.
+2. The ``DIFFBOT_API_TOKEN`` environment variable.
+3. A ``DIFFBOT_API_TOKEN=...`` line in ``~/.diffbot/credentials``.
+"""
+
+import os
+import pathlib
+from typing import Optional
+
+TOKEN_ENV_VAR = "DIFFBOT_API_TOKEN"
+CREDENTIALS_PATH = pathlib.Path.home() / ".diffbot" / "credentials"
+
+
+def _read_credentials_file() -> str:
+    if not CREDENTIALS_PATH.exists():
+        return ""
+    for line in CREDENTIALS_PATH.read_text().splitlines():
+        line = line.strip()
+        if line.startswith(f"{TOKEN_ENV_VAR}="):
+            return line[len(TOKEN_ENV_VAR) + 1:].strip()
+    return ""
+
+
+def resolve_token(token: Optional[str] = None) -> str:
+    """Resolve a Diffbot API token from the explicit argument, env var, or file.
+
+    Returns an empty string if no token can be found.
+    """
+    if token and token.strip():
+        return token.strip()
+
+    env_token = os.environ.get(TOKEN_ENV_VAR, "").strip()
+    if env_token:
+        return env_token
+
+    return _read_credentials_file()

src/diffbot/cli/_common.py

@@ -1,35 +1,20 @@
-import os
-import pathlib
-
 import click
 
-from diffbot import Diffbot
-
-CREDENTIALS_PATH = pathlib.Path.home() / ".diffbot" / "credentials"
-
-
-def resolve_token() -> str:
-    """Return the Diffbot API token from the env var, falling back to ~/.diffbot/credentials."""
-    token = os.environ.get("DIFFBOT_API_TOKEN", "").strip()
-    if token:
-        return token
-
-    if CREDENTIALS_PATH.exists():
-        for line in CREDENTIALS_PATH.read_text().splitlines():
-            line = line.strip()
-            if line.startswith("DIFFBOT_API_TOKEN="):
-                return line[len("DIFFBOT_API_TOKEN="):].strip()
-
-    return ""
+from diffbot import Diffbot, resolve_token
+from diffbot._auth import CREDENTIALS_PATH, TOKEN_ENV_VAR
 
 
 def get_client() -> Diffbot:
+    """Build a Diffbot client using the shared credential resolution chain.
+
+    Looks at the DIFFBOT_API_TOKEN env var, then ~/.diffbot/credentials.
+    """
     token = resolve_token()
     if not token:
         click.echo(
             "Error: no Diffbot API token found.\n"
-            "  Set a DIFFBOT_API_TOKEN environment variable, or\n"
-            f"  write 'DIFFBOT_API_TOKEN=YOUR_TOKEN' to {CREDENTIALS_PATH}",
+            f"  Set a {TOKEN_ENV_VAR} environment variable, or\n"
+            f"  write '{TOKEN_ENV_VAR}=YOUR_TOKEN' to {CREDENTIALS_PATH}",
             err=True,
         )
         raise click.Abort()

src/diffbot/cli/dql.py

@@ -15,7 +15,9 @@
 from diffbot import DiffbotError
 
 from . import ontology
-from ._common import get_client, resolve_token
+from diffbot import resolve_token
+
+from ._common import get_client
 
 
 class _DqlGroup(click.Group):