#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" SCRIPT="$ROOT_DIR/scripts/acceptance/verify_host_protocol_matrix.sh" fail() { echo "FAIL: $*" >&2 exit 1 } assert_contains() { local haystack="$1" local needle="$2" if [[ "$haystack" != *"$needle"* ]]; then fail "expected to find [$needle] in [$haystack]" fi } assert_file_contains() { local file="$1" local needle="$2" [[ -f "$file" ]] || fail "missing file: $file" local text text="$(cat "$file")" assert_contains "$text" "$needle" } [[ -f "$SCRIPT" ]] || fail "missing $SCRIPT" help_output="$(bash "$SCRIPT" --help)" assert_contains "$help_output" "Usage: verify_host_protocol_matrix.sh" assert_contains "$help_output" "PROTOCOL_MATRIX_TARGETS_JSON" assert_contains "$help_output" "DRY_RUN=1" tmpdir="$(mktemp -d)" trap 'rm -rf "$tmpdir"' EXIT set +e missing_env_output="$(ARTIFACT_DIR="$tmpdir" bash "$SCRIPT" 2>&1)" missing_env_status=$? set -e if [[ $missing_env_status -eq 0 ]]; then fail "expected missing env invocation to fail" fi assert_contains "$missing_env_output" "missing required env: PROTOCOL_MATRIX_TARGETS_JSON" dry_run_output="$(ARTIFACT_DIR="$tmpdir" DRY_RUN=1 PROTOCOL_MATRIX_TARGETS_JSON='[{"provider_id":"kimi-a7m","base_url":"https://kimi.example.com/v1","api_key_env":"KIMI_API_KEY","models":["kimi-k2.6"]}]' bash "$SCRIPT")" assert_contains "$dry_run_output" "protocol matrix summary" assert_contains "$dry_run_output" "$tmpdir" summary_file="$(find "$tmpdir" -name protocol-matrix-summary.json | head -n 1)" [[ -n "$summary_file" ]] || fail "missing protocol-matrix-summary.json" summary_text="$(cat "$summary_file")" assert_contains "$summary_text" '"provider_id": "kimi-a7m"' assert_contains "$summary_text" '"mode": "dry_run"' assert_contains "$summary_text" '"support_level": "dry_run"' assert_contains "$summary_text" '"probe_layer": "upstream"' fakebin="$tmpdir/bin" mkdir -p "$fakebin" cat > "$fakebin/curl" <<'EOF' #!/usr/bin/env bash set -euo pipefail headers_file="" body_file="" url="" request_headers_file="" request_body="" prev="" log_file="${FAKE_CURL_LOG:-}" for arg in "$@"; do case "$prev" in -D) headers_file="$arg" prev="" continue ;; -o) body_file="$arg" prev="" continue ;; -d) request_body="$arg" prev="" continue ;; -H) if [[ "$arg" == X-Hermes-Debug-Request-Headers:* ]]; then request_headers_file="${arg#X-Hermes-Debug-Request-Headers: }" fi prev="" continue ;; esac case "$arg" in -D|-o|-d|-H) prev="$arg" continue ;; http://*|https://*) url="$arg" ;; esac done [[ -n "$headers_file" && -n "$body_file" && -n "$url" ]] || { echo "missing curl capture args: $*" >&2 exit 1 } if [[ -n "$log_file" ]]; then printf '%s\n' "$*" >> "$log_file" fi if [[ -n "$request_headers_file" ]]; then printf 'Authorization: Bearer ***\n' > "$request_headers_file" printf 'Content-Type: application/json\n' >> "$request_headers_file" fi case "$url" in https://kimi.example.com/v1/models) printf 'HTTP/1.1 200 OK\nContent-Type: application/json\n' > "$headers_file" printf '{"data":[{"id":"kimi-k2.6"}]}' > "$body_file" ;; https://kimi.example.com/v1/chat/completions) printf 'HTTP/1.1 200 OK\nContent-Type: application/json\n' > "$headers_file" printf '{"choices":[{"message":{"content":"pong"}}]}' > "$body_file" ;; https://kimi.example.com/v1/responses) printf 'HTTP/1.1 403 Forbidden\nContent-Type: application/json\n' > "$headers_file" printf '{"error":{"message":"unsupported"}}' > "$body_file" ;; https://timeout.example.com/v1/models) printf 'HTTP/1.1 200 OK\nContent-Type: application/json\n' > "$headers_file" printf '{"data":[{"id":"timeout-model"}]}' > "$body_file" ;; https://timeout.example.com/v1/chat/completions) : > "$headers_file" : > "$body_file" exit 28 ;; https://timeout.example.com/v1/responses) : > "$headers_file" : > "$body_file" exit 28 ;; *) echo "unexpected curl url: $url" >&2 exit 1 ;; esac EOF chmod +x "$fakebin/curl" live_dir="$tmpdir/live" curl_log="$tmpdir/fake-curl.log" set +e live_output="$(PATH="$fakebin:$PATH" FAKE_CURL_LOG="$curl_log" ARTIFACT_DIR="$live_dir" KIMI_API_KEY='kimi-key' TIMEOUT_API_KEY='timeout-key' PROTOCOL_MATRIX_TARGETS_JSON='[ {"provider_id":"kimi-a7m","base_url":"https://kimi.example.com/v1","api_key_env":"KIMI_API_KEY","models":["kimi-k2.6"]}, {"provider_id":"timeout-provider","base_url":"https://timeout.example.com/v1","api_key_env":"TIMEOUT_API_KEY","models":["timeout-model"],"probe_layer":"host"} ]' bash "$SCRIPT")" live_status=$? set -e if [[ $live_status -ne 0 ]]; then fail "expected partial failure run to exit 0, got $live_status" fi assert_contains "$live_output" "protocol matrix summary" live_summary="$live_dir/protocol-matrix-summary.json" [[ -f "$live_summary" ]] || fail "missing live protocol-matrix-summary.json" live_summary_text="$(cat "$live_summary")" assert_contains "$live_summary_text" '"provider_id": "kimi-a7m"' assert_contains "$live_summary_text" '"models_status": 200' assert_contains "$live_summary_text" '"chat_status": 200' assert_contains "$live_summary_text" '"responses_status": 403' assert_contains "$live_summary_text" '"support_level": "supported-with-plugin-adapter"' assert_contains "$live_summary_text" '"error_code": "responses_unsupported"' assert_contains "$live_summary_text" '"probe_layer": "upstream"' assert_contains "$live_summary_text" '"provider_id": "timeout-provider"' assert_contains "$live_summary_text" '"status": "failed"' assert_contains "$live_summary_text" '"error_code": "network_timeout"' assert_contains "$live_summary_text" '"probe_layer": "host"' first_target_dir="$live_dir/targets/01-kimi-a7m" second_target_dir="$live_dir/targets/02-timeout-provider" [[ -d "$first_target_dir" ]] || fail "missing first target artifact dir" [[ -d "$second_target_dir" ]] || fail "missing second target artifact dir" assert_file_contains "$first_target_dir/01-models.request_headers.txt" 'Authorization: Bearer ***' assert_file_contains "$first_target_dir/01-models.request_headers.txt" 'Content-Type: application/json' assert_file_contains "$first_target_dir/01-models.response_headers.txt" 'HTTP/1.1 200 OK' assert_file_contains "$first_target_dir/03-responses.response_body.json" 'unsupported' assert_file_contains "$second_target_dir/02-chat.response_body.json" '' curl_log_text="$(cat "$curl_log")" assert_contains "$curl_log_text" '--connect-timeout 10' assert_contains "$curl_log_text" '--max-time 30' assert_contains "$curl_log_text" '--retry 1' assert_contains "$curl_log_text" '--retry-delay 2' if grep -R --line-number 'kimi-key\|timeout-key' "$live_dir"; then fail "artifact contains unsanitized secrets" fi echo "PASS: host protocol matrix script regression checks"