fix: harden review and verifier governance

This commit is contained in:
phamnazage-jpg
2026-05-29 18:48:48 +08:00
parent 88833fac8b
commit e999d31b25
133 changed files with 2538 additions and 159 deletions

View File

@@ -7,6 +7,10 @@ DB_URL="${DATABASE_URL:-}"
INPUT_PATH=""
THRESHOLD=""
FIELD_SEP=$'\x1f'
NOW_RAW="${LLM_NOW:-}"
AGED_PRECONDITION_COUNT=0
AGED_PRECONDITION_MINUTES=1440
usage() {
cat <<'EOF'
@@ -30,10 +34,10 @@ classify_failure() {
fi
case "$normalized" in
*"api key"*|*"strict real mode"*|*"database_url"*|*"password authentication failed"*|*"permission denied"*|*"role does not exist"*|*"relation does not exist"*|*"must provide"*)
*"api key"*|*"openrouter_api_key"*|*"database_url"*|*"strict real mode"*|*"password authentication failed"*|*"permission denied"*|*"role does not exist"*|*"relation does not exist"*|*"must provide"*|*"未设置"*)
printf '%s\n' "precondition_missing"
;;
*"429"*|*"rate limit"*|*"too many requests"*|*"timeout"*|*"temporarily unavailable"*|*"transport closed"*|*"connection reset"*|*"connection refused"*|*"eof"*|*"tls handshake timeout"*|*"no such host"*|*"i/o timeout"*)
*"429"*|*"rate limit"*|*"too many requests"*|*"timeout"*|*"temporarily unavailable"*|*"transport closed"*|*"connection reset"*|*"connection refused"*|*"eof"*|*"tls handshake timeout"*|*"no such host"*|*"i/o timeout"*|*"unexpected status 403"*|*"unexpected status 502"*|*"unexpected status 503"*|*"unexpected status 504"*|*"signature drift"*|*"no pricing cards found"*|*"no model rows parsed"*|*"no model overview cards parsed"*|*"unexpected * pricing content"*)
printf '%s\n' "external_provider_failure"
;;
*)
@@ -42,6 +46,19 @@ classify_failure() {
esac
}
minutes_since_created() {
local created_at="$1"
python3 - <<'PY' "$created_at" "$NOW_RAW"
from datetime import datetime
import sys
created = datetime.strptime(sys.argv[1], '%Y-%m-%d %H:%M:%S')
raw_now = sys.argv[2].strip()
now = datetime.strptime(raw_now, '%Y-%m-%d %H:%M') if raw_now else datetime.now()
print(int((now - created).total_seconds() // 60))
PY
}
fetch_rows_from_db() {
if [[ -z "${DB_URL:-}" ]]; then
echo "missing --db / DATABASE_URL" >&2
@@ -125,10 +142,19 @@ while IFS= read -r raw_line; do
FAILURE_COUNT=$((FAILURE_COUNT + 1))
category="$(classify_failure "$error_message")"
rendered_error="${error_message:-unknown}"
if [[ "$category" == "precondition_missing" ]]; then
age_minutes="$(minutes_since_created "${created_at:-1970-01-01 00:00:00}")"
if [[ "$age_minutes" -gt "$AGED_PRECONDITION_MINUTES" ]]; then
category="aged_precondition_missing"
AGED_PRECONDITION_COUNT=$((AGED_PRECONDITION_COUNT + 1))
fi
fi
case "$category" in
precondition_missing)
PRECONDITION_COUNT=$((PRECONDITION_COUNT + 1))
;;
aged_precondition_missing)
;;
external_provider_failure)
EXTERNAL_COUNT=$((EXTERNAL_COUNT + 1))
;;
@@ -140,11 +166,13 @@ while IFS= read -r raw_line; do
;;
esac
fi
DETAIL_LINES+=$'sample_'"${ROW_COUNT}"$' created_at='"${created_at:-unknown}"$' source='"${source:-unknown}"$' outcome='"$([[ "$category" == "success" ]] && printf '%s' "success" || printf '%s' "failure")"$' category='"${category}"$' error='"${rendered_error}"$'\n'
done <<< "$ROWS"
if [[ "$ROW_COUNT" -eq 0 ]]; then
echo "window_size=0 success_count=0 failure_count=0 success_rate=0.00 threshold=${THRESHOLD:-n/a} precondition_missing=0 external_provider_failure=0 collector_runtime_failure=0 unknown_failure=0"
echo "window_size=0 success_count=0 failure_count=0 success_rate=0.00 threshold=${THRESHOLD:-n/a} precondition_missing=0 aged_precondition_missing=0 external_provider_failure=0 collector_runtime_failure=0 unknown_failure=0"
echo "sample_window=empty"
if [[ -n "$THRESHOLD" ]]; then
exit 1
@@ -152,8 +180,8 @@ if [[ "$ROW_COUNT" -eq 0 ]]; then
exit 0
fi
SUCCESS_RATE="$(awk -v success="$SUCCESS_COUNT" -v total="$ROW_COUNT" 'BEGIN { printf "%.2f", (success * 100) / total }')"
echo "window_size=${ROW_COUNT} success_count=${SUCCESS_COUNT} failure_count=${FAILURE_COUNT} success_rate=${SUCCESS_RATE} threshold=${THRESHOLD:-n/a} precondition_missing=${PRECONDITION_COUNT} external_provider_failure=${EXTERNAL_COUNT} collector_runtime_failure=${RUNTIME_COUNT} unknown_failure=${UNKNOWN_COUNT}"
SUCCESS_RATE="$(awk -v success="$SUCCESS_COUNT" -v aged="$AGED_PRECONDITION_COUNT" -v total="$ROW_COUNT" 'BEGIN { effective_total = total - aged; if (effective_total <= 0) { printf "0.00" } else { printf "%.2f", (success * 100) / effective_total } }')"
echo "window_size=${ROW_COUNT} success_count=${SUCCESS_COUNT} failure_count=${FAILURE_COUNT} success_rate=${SUCCESS_RATE} threshold=${THRESHOLD:-n/a} precondition_missing=${PRECONDITION_COUNT} aged_precondition_missing=${AGED_PRECONDITION_COUNT} external_provider_failure=${EXTERNAL_COUNT} collector_runtime_failure=${RUNTIME_COUNT} unknown_failure=${UNKNOWN_COUNT}"
printf '%s' "$DETAIL_LINES"
if [[ -n "$THRESHOLD" ]]; then