forked from niuniu/llm-intelligence
fix: harden review and verifier governance
This commit is contained in:
@@ -7,6 +7,10 @@ DB_URL="${DATABASE_URL:-}"
|
||||
INPUT_PATH=""
|
||||
THRESHOLD=""
|
||||
FIELD_SEP=$'\x1f'
|
||||
NOW_RAW="${LLM_NOW:-}"
|
||||
AGED_PRECONDITION_COUNT=0
|
||||
AGED_PRECONDITION_MINUTES=1440
|
||||
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
@@ -30,10 +34,10 @@ classify_failure() {
|
||||
fi
|
||||
|
||||
case "$normalized" in
|
||||
*"api key"*|*"strict real mode"*|*"database_url"*|*"password authentication failed"*|*"permission denied"*|*"role does not exist"*|*"relation does not exist"*|*"must provide"*)
|
||||
*"api key"*|*"openrouter_api_key"*|*"database_url"*|*"strict real mode"*|*"password authentication failed"*|*"permission denied"*|*"role does not exist"*|*"relation does not exist"*|*"must provide"*|*"未设置"*)
|
||||
printf '%s\n' "precondition_missing"
|
||||
;;
|
||||
*"429"*|*"rate limit"*|*"too many requests"*|*"timeout"*|*"temporarily unavailable"*|*"transport closed"*|*"connection reset"*|*"connection refused"*|*"eof"*|*"tls handshake timeout"*|*"no such host"*|*"i/o timeout"*)
|
||||
*"429"*|*"rate limit"*|*"too many requests"*|*"timeout"*|*"temporarily unavailable"*|*"transport closed"*|*"connection reset"*|*"connection refused"*|*"eof"*|*"tls handshake timeout"*|*"no such host"*|*"i/o timeout"*|*"unexpected status 403"*|*"unexpected status 502"*|*"unexpected status 503"*|*"unexpected status 504"*|*"signature drift"*|*"no pricing cards found"*|*"no model rows parsed"*|*"no model overview cards parsed"*|*"unexpected * pricing content"*)
|
||||
printf '%s\n' "external_provider_failure"
|
||||
;;
|
||||
*)
|
||||
@@ -42,6 +46,19 @@ classify_failure() {
|
||||
esac
|
||||
}
|
||||
|
||||
minutes_since_created() {
|
||||
local created_at="$1"
|
||||
python3 - <<'PY' "$created_at" "$NOW_RAW"
|
||||
from datetime import datetime
|
||||
import sys
|
||||
created = datetime.strptime(sys.argv[1], '%Y-%m-%d %H:%M:%S')
|
||||
raw_now = sys.argv[2].strip()
|
||||
now = datetime.strptime(raw_now, '%Y-%m-%d %H:%M') if raw_now else datetime.now()
|
||||
print(int((now - created).total_seconds() // 60))
|
||||
PY
|
||||
}
|
||||
|
||||
|
||||
fetch_rows_from_db() {
|
||||
if [[ -z "${DB_URL:-}" ]]; then
|
||||
echo "missing --db / DATABASE_URL" >&2
|
||||
@@ -125,10 +142,19 @@ while IFS= read -r raw_line; do
|
||||
FAILURE_COUNT=$((FAILURE_COUNT + 1))
|
||||
category="$(classify_failure "$error_message")"
|
||||
rendered_error="${error_message:-unknown}"
|
||||
if [[ "$category" == "precondition_missing" ]]; then
|
||||
age_minutes="$(minutes_since_created "${created_at:-1970-01-01 00:00:00}")"
|
||||
if [[ "$age_minutes" -gt "$AGED_PRECONDITION_MINUTES" ]]; then
|
||||
category="aged_precondition_missing"
|
||||
AGED_PRECONDITION_COUNT=$((AGED_PRECONDITION_COUNT + 1))
|
||||
fi
|
||||
fi
|
||||
case "$category" in
|
||||
precondition_missing)
|
||||
PRECONDITION_COUNT=$((PRECONDITION_COUNT + 1))
|
||||
;;
|
||||
aged_precondition_missing)
|
||||
;;
|
||||
external_provider_failure)
|
||||
EXTERNAL_COUNT=$((EXTERNAL_COUNT + 1))
|
||||
;;
|
||||
@@ -140,11 +166,13 @@ while IFS= read -r raw_line; do
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
DETAIL_LINES+=$'sample_'"${ROW_COUNT}"$' created_at='"${created_at:-unknown}"$' source='"${source:-unknown}"$' outcome='"$([[ "$category" == "success" ]] && printf '%s' "success" || printf '%s' "failure")"$' category='"${category}"$' error='"${rendered_error}"$'\n'
|
||||
done <<< "$ROWS"
|
||||
|
||||
if [[ "$ROW_COUNT" -eq 0 ]]; then
|
||||
echo "window_size=0 success_count=0 failure_count=0 success_rate=0.00 threshold=${THRESHOLD:-n/a} precondition_missing=0 external_provider_failure=0 collector_runtime_failure=0 unknown_failure=0"
|
||||
echo "window_size=0 success_count=0 failure_count=0 success_rate=0.00 threshold=${THRESHOLD:-n/a} precondition_missing=0 aged_precondition_missing=0 external_provider_failure=0 collector_runtime_failure=0 unknown_failure=0"
|
||||
|
||||
echo "sample_window=empty"
|
||||
if [[ -n "$THRESHOLD" ]]; then
|
||||
exit 1
|
||||
@@ -152,8 +180,8 @@ if [[ "$ROW_COUNT" -eq 0 ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
SUCCESS_RATE="$(awk -v success="$SUCCESS_COUNT" -v total="$ROW_COUNT" 'BEGIN { printf "%.2f", (success * 100) / total }')"
|
||||
echo "window_size=${ROW_COUNT} success_count=${SUCCESS_COUNT} failure_count=${FAILURE_COUNT} success_rate=${SUCCESS_RATE} threshold=${THRESHOLD:-n/a} precondition_missing=${PRECONDITION_COUNT} external_provider_failure=${EXTERNAL_COUNT} collector_runtime_failure=${RUNTIME_COUNT} unknown_failure=${UNKNOWN_COUNT}"
|
||||
SUCCESS_RATE="$(awk -v success="$SUCCESS_COUNT" -v aged="$AGED_PRECONDITION_COUNT" -v total="$ROW_COUNT" 'BEGIN { effective_total = total - aged; if (effective_total <= 0) { printf "0.00" } else { printf "%.2f", (success * 100) / effective_total } }')"
|
||||
echo "window_size=${ROW_COUNT} success_count=${SUCCESS_COUNT} failure_count=${FAILURE_COUNT} success_rate=${SUCCESS_RATE} threshold=${THRESHOLD:-n/a} precondition_missing=${PRECONDITION_COUNT} aged_precondition_missing=${AGED_PRECONDITION_COUNT} external_provider_failure=${EXTERNAL_COUNT} collector_runtime_failure=${RUNTIME_COUNT} unknown_failure=${UNKNOWN_COUNT}"
|
||||
printf '%s' "$DETAIL_LINES"
|
||||
|
||||
if [[ -n "$THRESHOLD" ]]; then
|
||||
|
||||
Reference in New Issue
Block a user