Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 105 additions & 0 deletions scripts/dispatch-runner.sh
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,88 @@ OUTCOME_FILE="${HYPATIA_DATA}/outcomes/$(date -u +%Y-%m).jsonl"
OUTCOME_FILE_CENTRAL="${VERISIMDB_DATA}/outcomes/$(date -u +%Y-%m).jsonl"
mkdir -p "$(dirname "$OUTCOME_FILE")"

# --- Hypatia closed-loop contract ---
#
# Per hypatia/docs/operations/dispatch-runner-contract.adoc, after each
# fix the runner MUST invoke `mix hypatia.record_outcome` so Bayesian
# confidence updating, auto-quarantine, and re-scan verification all see
# real evidence. Without this call, every recipe stays at
# :insufficient_data and the closed-loop verification metric (PR #309)
# never engages. Closes estate-audit blocker C15.
#
# Configuration via env:
# HYPATIA_HOME path to the hypatia checkout (defaults to
# $REPOS_BASE/hypatia)
# HYPATIA_OUTCOME_REPORT set to "off" to disable mix calls (e.g. a
# CI runner without Elixir/mix on PATH);
# defaults to "on"
# HYPATIA_BATCH_ID when set, exit-2 hints suggest
# `mix hypatia.rollback_batch $HYPATIA_BATCH_ID`
#
# Exit codes from the mix task (consumed below):
# 0 outcome recorded; re-scan verified clean OR scan_unavailable
# 2 outcome recorded BUT re-scan still flags the weak point
# 1 bad arguments / unrecoverable error
HYPATIA_HOME="${HYPATIA_HOME:-${REPOS_BASE}/hypatia}"
HYPATIA_OUTCOME_REPORT="${HYPATIA_OUTCOME_REPORT:-on}"
HYPATIA_BATCH_ID="${HYPATIA_BATCH_ID:-}"

hypatia_record_outcome() {
local recipe_id="$1"
local repo="$2"
local file="$3"
local outcome="$4"

# Opt-out short-circuit.
if [[ "$HYPATIA_OUTCOME_REPORT" != "on" ]]; then
return 0
fi
# Precondition: hypatia checkout reachable and mix on PATH.
if [[ ! -d "$HYPATIA_HOME" ]]; then
echo " WARN: hypatia outcome skipped — \$HYPATIA_HOME=$HYPATIA_HOME not a directory" >&2
return 0
fi
if ! command -v mix &>/dev/null; then
echo " WARN: hypatia outcome skipped — mix not on PATH" >&2
return 0
fi
# Empty / placeholder recipe_id → skip silently rather than poison
# hypatia's recipe-health metric with anonymous entries.
if [[ -z "$recipe_id" || "$recipe_id" == "none" || "$recipe_id" == "null" ]]; then
return 0
fi

# Repo-grain fixes pass "." for --file (contract permits any
# path-inside-repo string; "." is the explicit repo-root marker).
local file_arg="${file:-.}"
[[ -z "$file_arg" ]] && file_arg="."

local rc=0
(
cd "$HYPATIA_HOME"
mix hypatia.record_outcome \
--recipe "$recipe_id" \
--repo "$repo" \
--file "$file_arg" \
--outcome "$outcome"
) || rc=$?

case "$rc" in
0)
;;
2)
echo " ::warning::Recipe $recipe_id still flags $repo/$file_arg after fix" >&2
if [[ -n "$HYPATIA_BATCH_ID" ]]; then
echo " ::warning::consider 'mix hypatia.rollback_batch $HYPATIA_BATCH_ID'" >&2
fi
;;
*)
echo " ::error::record_outcome failed for $repo/$file_arg: exit $rc" >&2
;;
esac
return "$rc"
}

record_outcome() {
local pattern_id="$1"
local recipe_id="$2"
Expand Down Expand Up @@ -162,6 +244,29 @@ record_outcome() {
if [[ -d "$(dirname "$OUTCOME_FILE_CENTRAL")" ]] || mkdir -p "$(dirname "$OUTCOME_FILE_CENTRAL")" 2>/dev/null; then
echo "$json" >> "$OUTCOME_FILE_CENTRAL"
fi

# Hypatia closed-loop contract — best-effort, never fails the dispatch
# loop. Map shell outcomes to the contract's outcome vocabulary:
# success/failure → pass through
# refused_* → false_positive (the recipe is misfiring, not
# the code; surfaces this to auto-quarantine)
# anything else → skip (don't poison the metric).
local hypatia_outcome
case "$outcome" in
success)
hypatia_outcome="success"
;;
failure)
hypatia_outcome="failure"
;;
refused_license_policy|refused_*)
hypatia_outcome="false_positive"
;;
*)
return 0
;;
esac
hypatia_record_outcome "$recipe_id" "$repo" "$file" "$hypatia_outcome" || true
}

# --- Execute a single manifest entry ---
Expand Down
128 changes: 128 additions & 0 deletions tests/hypatia-record-outcome-smoke.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: MPL-2.0
#
# Smoke test for hypatia_record_outcome() in scripts/dispatch-runner.sh.
#
# Covers the no-op short-circuit paths that don't require an actual
# hypatia checkout or `mix` on PATH:
# 1. HYPATIA_OUTCOME_REPORT=off → silent no-op, exit 0.
# 2. HYPATIA_HOME not a directory → warn to stderr, exit 0.
# 3. Empty / "none" / "null" recipe_id → silent no-op, exit 0.
#
# End-to-end paths that *do* talk to hypatia (exit 0/2/1) require a
# running mix env and are covered by the integration suite on the
# hypatia side. This file just guarantees the dispatch loop never
# crashes on the optional-call ladder.

set -euo pipefail

SCRIPT_DIR=$(dirname "$(realpath "$0")")
DISPATCH_RUNNER="${SCRIPT_DIR}/../scripts/dispatch-runner.sh"

if [[ ! -f "$DISPATCH_RUNNER" ]]; then
echo "FAIL: dispatch-runner.sh not found at $DISPATCH_RUNNER"
exit 1
fi

# We source the script in a guarded mode so we get function definitions
# without triggering the main loop. The script exits early when its
# manifest doesn't exist, so we point it at a non-existent path and
# capture the early exit.
TMP=$(mktemp -d /tmp/c15-smoke-XXXXXX)
trap 'rm -rf "$TMP"' EXIT

# Extract just the function definitions we need (between # --- Hypatia
# closed-loop contract --- and the next "# --- " section). Cheap and
# avoids running the main loop.
awk '
/^# --- Hypatia closed-loop contract ---/ { capture=1 }
capture { print }
/^record_outcome\(\)/ && capture { record_seen=1 }
record_seen && /^}/ { print "# end-extract"; exit }
' "$DISPATCH_RUNNER" > "$TMP/extracted.sh"

# Provide a stub REPOS_BASE so HYPATIA_HOME default resolves to
# something the test can control.
export REPOS_BASE="$TMP/repos"

# Source the extracted block into this shell.
# shellcheck disable=SC1091
source "$TMP/extracted.sh"

pass=0
fail=0

assert_silent_ok() {
local label="$1"
shift
local out
if out=$("$@" 2>&1); then
if [[ -z "$out" ]]; then
pass=$((pass + 1))
echo "PASS: $label"
else
fail=$((fail + 1))
echo "FAIL: $label — expected silent, got: $out"
fi
else
fail=$((fail + 1))
echo "FAIL: $label — expected exit 0, got non-zero"
fi
}

assert_warn_ok() {
local label="$1"
local expect="$2"
shift 2
local out
if out=$("$@" 2>&1); then
if [[ "$out" == *"$expect"* ]]; then
pass=$((pass + 1))
echo "PASS: $label"
else
fail=$((fail + 1))
echo "FAIL: $label — expected warning containing '$expect', got: $out"
fi
else
fail=$((fail + 1))
echo "FAIL: $label — expected exit 0, got non-zero"
fi
}

# Case 1: opt-out.
HYPATIA_OUTCOME_REPORT=off \
assert_silent_ok "opt-out silently no-ops" \
hypatia_record_outcome "rid-1" "owner/repo" "src/file.py" "success"

# Case 2: HYPATIA_HOME not a directory.
HYPATIA_OUTCOME_REPORT=on \
HYPATIA_HOME="$TMP/does-not-exist" \
assert_warn_ok "missing HYPATIA_HOME warns and no-ops" \
"not a directory" \
hypatia_record_outcome "rid-2" "owner/repo" "src/file.py" "success"

# Cases 3a-c need HYPATIA_HOME to exist so the recipe-id guard fires
# rather than the missing-dir guard.
mkdir -p "$TMP/repos/hypatia"

# Case 3a: empty recipe_id.
HYPATIA_OUTCOME_REPORT=on \
HYPATIA_HOME="$TMP/repos/hypatia" \
assert_silent_ok "empty recipe_id silently no-ops" \
hypatia_record_outcome "" "owner/repo" "src/file.py" "success"

# Case 3b: "none" recipe_id.
HYPATIA_OUTCOME_REPORT=on \
HYPATIA_HOME="$TMP/repos/hypatia" \
assert_silent_ok "'none' recipe_id silently no-ops" \
hypatia_record_outcome "none" "owner/repo" "src/file.py" "success"

# Case 3c: "null" recipe_id.
HYPATIA_OUTCOME_REPORT=on \
HYPATIA_HOME="$TMP/repos/hypatia" \
assert_silent_ok "'null' recipe_id silently no-ops" \
hypatia_record_outcome "null" "owner/repo" "src/file.py" "success"

echo ""
echo "Results: $pass passed, $fail failed"
[[ "$fail" -eq 0 ]]
Loading