diff --git a/scripts/dispatch-runner.sh b/scripts/dispatch-runner.sh index 74d066c..d6f617e 100755 --- a/scripts/dispatch-runner.sh +++ b/scripts/dispatch-runner.sh @@ -135,6 +135,88 @@ OUTCOME_FILE="${HYPATIA_DATA}/outcomes/$(date -u +%Y-%m).jsonl" OUTCOME_FILE_CENTRAL="${VERISIMDB_DATA}/outcomes/$(date -u +%Y-%m).jsonl" mkdir -p "$(dirname "$OUTCOME_FILE")" +# --- Hypatia closed-loop contract --- +# +# Per hypatia/docs/operations/dispatch-runner-contract.adoc, after each +# fix the runner MUST invoke `mix hypatia.record_outcome` so Bayesian +# confidence updating, auto-quarantine, and re-scan verification all see +# real evidence. Without this call, every recipe stays at +# :insufficient_data and the closed-loop verification metric (PR #309) +# never engages. Closes estate-audit blocker C15. +# +# Configuration via env: +# HYPATIA_HOME path to the hypatia checkout (defaults to +# $REPOS_BASE/hypatia) +# HYPATIA_OUTCOME_REPORT set to "off" to disable mix calls (e.g. a +# CI runner without Elixir/mix on PATH); +# defaults to "on" +# HYPATIA_BATCH_ID when set, exit-2 hints suggest +# `mix hypatia.rollback_batch $HYPATIA_BATCH_ID` +# +# Exit codes from the mix task (consumed below): +# 0 outcome recorded; re-scan verified clean OR scan_unavailable +# 2 outcome recorded BUT re-scan still flags the weak point +# 1 bad arguments / unrecoverable error +HYPATIA_HOME="${HYPATIA_HOME:-${REPOS_BASE}/hypatia}" +HYPATIA_OUTCOME_REPORT="${HYPATIA_OUTCOME_REPORT:-on}" +HYPATIA_BATCH_ID="${HYPATIA_BATCH_ID:-}" + +hypatia_record_outcome() { + local recipe_id="$1" + local repo="$2" + local file="$3" + local outcome="$4" + + # Opt-out short-circuit. + if [[ "$HYPATIA_OUTCOME_REPORT" != "on" ]]; then + return 0 + fi + # Precondition: hypatia checkout reachable and mix on PATH. + if [[ ! -d "$HYPATIA_HOME" ]]; then + echo " WARN: hypatia outcome skipped — \$HYPATIA_HOME=$HYPATIA_HOME not a directory" >&2 + return 0 + fi + if ! command -v mix &>/dev/null; then + echo " WARN: hypatia outcome skipped — mix not on PATH" >&2 + return 0 + fi + # Empty / placeholder recipe_id → skip silently rather than poison + # hypatia's recipe-health metric with anonymous entries. + if [[ -z "$recipe_id" || "$recipe_id" == "none" || "$recipe_id" == "null" ]]; then + return 0 + fi + + # Repo-grain fixes pass "." for --file (contract permits any + # path-inside-repo string; "." is the explicit repo-root marker). + local file_arg="${file:-.}" + [[ -z "$file_arg" ]] && file_arg="." + + local rc=0 + ( + cd "$HYPATIA_HOME" + mix hypatia.record_outcome \ + --recipe "$recipe_id" \ + --repo "$repo" \ + --file "$file_arg" \ + --outcome "$outcome" + ) || rc=$? + + case "$rc" in + 0) + ;; + 2) + echo " ::warning::Recipe $recipe_id still flags $repo/$file_arg after fix" >&2 + if [[ -n "$HYPATIA_BATCH_ID" ]]; then + echo " ::warning::consider 'mix hypatia.rollback_batch $HYPATIA_BATCH_ID'" >&2 + fi + ;; + *) + echo " ::error::record_outcome failed for $repo/$file_arg: exit $rc" >&2 + ;; + esac + return "$rc" +} + record_outcome() { local pattern_id="$1" local recipe_id="$2" @@ -162,6 +244,29 @@ record_outcome() { if [[ -d "$(dirname "$OUTCOME_FILE_CENTRAL")" ]] || mkdir -p "$(dirname "$OUTCOME_FILE_CENTRAL")" 2>/dev/null; then echo "$json" >> "$OUTCOME_FILE_CENTRAL" fi + + # Hypatia closed-loop contract — best-effort, never fails the dispatch + # loop. Map shell outcomes to the contract's outcome vocabulary: + # success/failure → pass through + # refused_* → false_positive (the recipe is misfiring, not + # the code; surfaces this to auto-quarantine) + # anything else → skip (don't poison the metric). + local hypatia_outcome + case "$outcome" in + success) + hypatia_outcome="success" + ;; + failure) + hypatia_outcome="failure" + ;; + refused_license_policy|refused_*) + hypatia_outcome="false_positive" + ;; + *) + return 0 + ;; + esac + hypatia_record_outcome "$recipe_id" "$repo" "$file" "$hypatia_outcome" || true } # --- Execute a single manifest entry --- diff --git a/tests/hypatia-record-outcome-smoke.sh b/tests/hypatia-record-outcome-smoke.sh new file mode 100755 index 0000000..7c28c46 --- /dev/null +++ b/tests/hypatia-record-outcome-smoke.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# +# Smoke test for hypatia_record_outcome() in scripts/dispatch-runner.sh. +# +# Covers the no-op short-circuit paths that don't require an actual +# hypatia checkout or `mix` on PATH: +# 1. HYPATIA_OUTCOME_REPORT=off → silent no-op, exit 0. +# 2. HYPATIA_HOME not a directory → warn to stderr, exit 0. +# 3. Empty / "none" / "null" recipe_id → silent no-op, exit 0. +# +# End-to-end paths that *do* talk to hypatia (exit 0/2/1) require a +# running mix env and are covered by the integration suite on the +# hypatia side. This file just guarantees the dispatch loop never +# crashes on the optional-call ladder. + +set -euo pipefail + +SCRIPT_DIR=$(dirname "$(realpath "$0")") +DISPATCH_RUNNER="${SCRIPT_DIR}/../scripts/dispatch-runner.sh" + +if [[ ! -f "$DISPATCH_RUNNER" ]]; then + echo "FAIL: dispatch-runner.sh not found at $DISPATCH_RUNNER" + exit 1 +fi + +# We source the script in a guarded mode so we get function definitions +# without triggering the main loop. The script exits early when its +# manifest doesn't exist, so we point it at a non-existent path and +# capture the early exit. +TMP=$(mktemp -d /tmp/c15-smoke-XXXXXX) +trap 'rm -rf "$TMP"' EXIT + +# Extract just the function definitions we need (between # --- Hypatia +# closed-loop contract --- and the next "# --- " section). Cheap and +# avoids running the main loop. +awk ' + /^# --- Hypatia closed-loop contract ---/ { capture=1 } + capture { print } + /^record_outcome\(\)/ && capture { record_seen=1 } + record_seen && /^}/ { print "# end-extract"; exit } +' "$DISPATCH_RUNNER" > "$TMP/extracted.sh" + +# Provide a stub REPOS_BASE so HYPATIA_HOME default resolves to +# something the test can control. +export REPOS_BASE="$TMP/repos" + +# Source the extracted block into this shell. +# shellcheck disable=SC1091 +source "$TMP/extracted.sh" + +pass=0 +fail=0 + +assert_silent_ok() { + local label="$1" + shift + local out + if out=$("$@" 2>&1); then + if [[ -z "$out" ]]; then + pass=$((pass + 1)) + echo "PASS: $label" + else + fail=$((fail + 1)) + echo "FAIL: $label — expected silent, got: $out" + fi + else + fail=$((fail + 1)) + echo "FAIL: $label — expected exit 0, got non-zero" + fi +} + +assert_warn_ok() { + local label="$1" + local expect="$2" + shift 2 + local out + if out=$("$@" 2>&1); then + if [[ "$out" == *"$expect"* ]]; then + pass=$((pass + 1)) + echo "PASS: $label" + else + fail=$((fail + 1)) + echo "FAIL: $label — expected warning containing '$expect', got: $out" + fi + else + fail=$((fail + 1)) + echo "FAIL: $label — expected exit 0, got non-zero" + fi +} + +# Case 1: opt-out. +HYPATIA_OUTCOME_REPORT=off \ + assert_silent_ok "opt-out silently no-ops" \ + hypatia_record_outcome "rid-1" "owner/repo" "src/file.py" "success" + +# Case 2: HYPATIA_HOME not a directory. +HYPATIA_OUTCOME_REPORT=on \ +HYPATIA_HOME="$TMP/does-not-exist" \ + assert_warn_ok "missing HYPATIA_HOME warns and no-ops" \ + "not a directory" \ + hypatia_record_outcome "rid-2" "owner/repo" "src/file.py" "success" + +# Cases 3a-c need HYPATIA_HOME to exist so the recipe-id guard fires +# rather than the missing-dir guard. +mkdir -p "$TMP/repos/hypatia" + +# Case 3a: empty recipe_id. +HYPATIA_OUTCOME_REPORT=on \ +HYPATIA_HOME="$TMP/repos/hypatia" \ + assert_silent_ok "empty recipe_id silently no-ops" \ + hypatia_record_outcome "" "owner/repo" "src/file.py" "success" + +# Case 3b: "none" recipe_id. +HYPATIA_OUTCOME_REPORT=on \ +HYPATIA_HOME="$TMP/repos/hypatia" \ + assert_silent_ok "'none' recipe_id silently no-ops" \ + hypatia_record_outcome "none" "owner/repo" "src/file.py" "success" + +# Case 3c: "null" recipe_id. +HYPATIA_OUTCOME_REPORT=on \ +HYPATIA_HOME="$TMP/repos/hypatia" \ + assert_silent_ok "'null' recipe_id silently no-ops" \ + hypatia_record_outcome "null" "owner/repo" "src/file.py" "success" + +echo "" +echo "Results: $pass passed, $fail failed" +[[ "$fail" -eq 0 ]]