sub2api-cn-relay-manager/scripts/acceptance/verify_route_health_ui.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
# shellcheck disable=SC1091
source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"

CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
ROUTE_HEALTH_PAGE_URL="${ROUTE_HEALTH_PAGE_URL:-https://sub.tksea.top/portal/admin/route-health.html}"
TS="${TS:-$(timestamp_token)}"
ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_health_ui}"

GROUP_ID="${GROUP_ID:-p2t4-health-${TS}}"
PRIMARY_ROUTE="${PRIMARY_ROUTE:-primary-${TS}}"
FALLBACK_ROUTE="${FALLBACK_ROUTE:-fallback-${TS}}"
FAILING_ROUTE="${FAILING_ROUTE:-failing-${TS}}"
PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}"
REQUEST_ID="${REQUEST_ID:-req-p2t4-health-${TS}}"
SUBJECT_ID="${SUBJECT_ID:-conv-p2t4-health-${TS}}"

crm_auth_init
ensure_artifact_dir

curl_status_to_file "$ROUTE_HEALTH_PAGE_URL" "$ARTIFACT_DIR/00-route-health.html"

create_group_payload="$(python3 - "$GROUP_ID" <<'PY'
import json, sys
group_id = sys.argv[1]
print(json.dumps({
    "logical_group_id": group_id,
    "display_name": f"P2T4 Health {group_id}",
    "status": "active",
    "description": "P2-T4 health verification group",
    "route_policy": "priority",
    "sticky_mode": "conversation_preferred",
    "conversation_ttl_seconds": 1200,
    "user_model_ttl_seconds": 600,
    "failover_threshold": 2,
    "cooldown_seconds": 300,
}, ensure_ascii=False))
PY
)"
save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")"
save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")"

create_route() {
  local route_id="$1"
  local route_name="$2"
  local priority="$3"
  local shadow_group_id="$4"
  local shadow_host_id="$5"
  crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" \
    "{\"route_id\":\"$route_id\",\"name\":\"$route_name\",\"status\":\"active\",\"priority\":$priority,\"weight\":100,\"shadow_group_id\":\"$shadow_group_id\",\"shadow_host_id\":\"$shadow_host_id\",\"upstream_base_url_hint\":\"https://$route_id.example/v1\"}"
}

save_json 03-create-primary-route "$(create_route "$PRIMARY_ROUTE" "Primary Route" 10 "shadow-primary-$TS" "shadow-host-primary-$TS")"
save_json 04-create-fallback-route "$(create_route "$FALLBACK_ROUTE" "Fallback Route" 20 "shadow-fallback-$TS" "shadow-host-fallback-$TS")"
save_json 05-create-failing-route "$(create_route "$FAILING_ROUTE" "Failing Route" 30 "shadow-failing-$TS" "shadow-host-failing-$TS")"

for route_id in "$PRIMARY_ROUTE" "$FALLBACK_ROUTE" "$FAILING_ROUTE"; do
  save_json "route-model-${route_id}" "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$route_id/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"shadow_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")"
done

save_json 06-set-cooldown "$(crm_curl_json POST "/api/routing/sticky/cooldowns" "{\"route_id\":\"$PRIMARY_ROUTE\",\"reason\":\"degraded\",\"ttl_seconds\":600}")"
save_json 07-set-failure "$(crm_curl_json POST "/api/routing/sticky/route-failures" "{\"route_id\":\"$FAILING_ROUTE\",\"failure_count\":2,\"last_error_class\":\"timeout\",\"ttl_seconds\":600}")"
save_json 08-health-before "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")"
save_json 09-resolve "$(crm_curl_json POST "/api/routing/resolve" "{\"request_id\":\"$REQUEST_ID\",\"logical_group_id\":\"$GROUP_ID\",\"public_model\":\"$PUBLIC_MODEL\",\"scope\":\"conversation\",\"subject_id\":\"$SUBJECT_ID\",\"sync\":true}")"
save_json 10-health-after "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")"
save_json 11-failovers "$(crm_curl_json GET "/api/routing/logs/failovers?request_id=$REQUEST_ID&limit=5")"

python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$PRIMARY_ROUTE" "$FALLBACK_ROUTE" "$FAILING_ROUTE" "$REQUEST_ID" >"$ARTIFACT_DIR/12-summary.json" <<'PY'
import json
import sys
from pathlib import Path

art_dir, group_id, primary_route, fallback_route, failing_route, request_id = sys.argv[1:7]
art = Path(art_dir)
page = (art / "00-route-health.html").read_text()
before = json.loads((art / "08-health-before.json").read_text())["route_health"]
resolve = json.loads((art / "09-resolve.json").read_text())["resolve"]
after = json.loads((art / "10-health-after.json").read_text())["route_health"]
failovers = json.loads((art / "11-failovers.json").read_text())["failover_events"]

def by_id(items):
    return {item["route_id"]: item for item in items}

before_map = by_id(before)
after_map = by_id(after)

assert "Route Health Admin" in page
assert before_map[primary_route]["runtime_status"] == "cooldown"
assert before_map[failing_route]["runtime_status"] == "failing"
assert resolve["route_id"] == fallback_route
assert resolve["fallback_used"] is True
assert after_map[primary_route]["runtime_status"] == "cooldown"
assert after_map[fallback_route]["runtime_status"] == "healthy"
assert after_map[fallback_route]["recent_failover_count"] >= 1
assert failovers and failovers[0]["from_route_id"] == primary_route
assert failovers[0]["to_route_id"] == fallback_route
assert failovers[0]["reason"] == "active_cooldown:degraded"

summary = {
    "group_id": group_id,
    "request_id": request_id,
    "resolve_route_id": resolve["route_id"],
    "resolve_fallback_used": resolve["fallback_used"],
    "before_statuses": {k: v["runtime_status"] for k, v in before_map.items()},
    "after_statuses": {k: v["runtime_status"] for k, v in after_map.items()},
    "fallback_recent_failover_count": after_map[fallback_route]["recent_failover_count"],
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY

cat "$ARTIFACT_DIR/12-summary.json"