353 lines
17 KiB
Bash
Executable File
353 lines
17 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
. "$ROOT_DIR/scripts/report_utils.sh"
|
|
cd "$ROOT_DIR"
|
|
|
|
if [[ -f ".env.local" ]]; then
|
|
# shellcheck disable=SC1091
|
|
source ".env.local"
|
|
fi
|
|
if [[ -f ".env" ]]; then
|
|
# shellcheck disable=SC1091
|
|
source ".env"
|
|
fi
|
|
|
|
if [[ -z "${DATABASE_URL:-}" ]]; then
|
|
echo "DATABASE_URL 未设置" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${OPENROUTER_API_KEY:-}" ]]; then
|
|
echo "OPENROUTER_API_KEY 未设置，无法执行真实采集" >&2
|
|
exit 1
|
|
fi
|
|
|
|
REPORT_DATE="$(report_date_value)"
|
|
FETCH_OUT="$ROOT_DIR/models.json"
|
|
FETCH_TOTAL="0"
|
|
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report"
|
|
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,cucloud_pricing,mobile_cloud_pricing,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,baichuan_pricing,lingyiwanwu_pricing,sensenova_pricing,xfyun_pricing,bytedance_pricing,catalog_seed_verification"
|
|
PIPELINE_FAILED_SOURCE_SET="none"
|
|
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
|
|
PIPELINE_AUDIT_SUMMARY=""
|
|
|
|
normalize_summary_file() {
|
|
local path="$1"
|
|
if [[ ! -f "$path" ]]; then
|
|
return
|
|
fi
|
|
tr '\n' ' ' < "$path" | sed 's/[[:space:]]\+/ /g; s/^ //; s/ $//'
|
|
}
|
|
|
|
extract_failed_source_keys() {
|
|
local summary="$1"
|
|
printf '%s\n' "$summary" | sed -n 's/.*failed_source_keys=\([^ ]*\).*/\1/p'
|
|
}
|
|
|
|
merge_failed_source_keys() {
|
|
local keys="$1"
|
|
if [[ -z "$keys" || "$keys" == "none" ]]; then
|
|
return
|
|
fi
|
|
if [[ "$PIPELINE_FAILED_SOURCE_SET" == "none" ]]; then
|
|
PIPELINE_FAILED_SOURCE_SET="$keys"
|
|
return
|
|
fi
|
|
PIPELINE_FAILED_SOURCE_SET="${PIPELINE_FAILED_SOURCE_SET},${keys}"
|
|
}
|
|
|
|
refresh_pipeline_audit() {
|
|
PIPELINE_AUDIT_SUMMARY="runtime_audit stage_set=${PIPELINE_STAGE_SET} selected_source_keys=${PIPELINE_SOURCE_SET} failed_source_keys=${PIPELINE_FAILED_SOURCE_SET} openrouter_total=${FETCH_TOTAL:-0} ${MULTI_SOURCE_AUDIT}"
|
|
}
|
|
|
|
record_failure() {
|
|
local error_message output_path
|
|
error_message="$1"
|
|
output_path=""
|
|
refresh_pipeline_audit
|
|
|
|
if [[ -f "$(report_markdown_path "$REPORT_DATE")" ]]; then
|
|
output_path="$(report_markdown_path "$REPORT_DATE")"
|
|
fi
|
|
|
|
track_report_state "$DATABASE_URL" "$REPORT_DATE" "failed" "" "$PIPELINE_AUDIT_SUMMARY" "$output_path" "$error_message" "manual" "pipeline" "false" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
refresh_pipeline_audit
|
|
|
|
"$ROOT_DIR/scripts/apply_migration.sh"
|
|
|
|
if ! go run "./scripts/fetch_openrouter.go" \
|
|
-api-key "$OPENROUTER_API_KEY" \
|
|
-db "$DATABASE_URL" \
|
|
-out "$FETCH_OUT" \
|
|
-strict-real; then
|
|
merge_failed_source_keys "openrouter"
|
|
record_failure "真实采集失败"
|
|
exit 1
|
|
fi
|
|
|
|
FETCH_TOTAL=$(python3 - <<'PY' "$FETCH_OUT"
|
|
import json, sys
|
|
path = sys.argv[1]
|
|
with open(path, 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
print(int(data.get("total", 0)))
|
|
PY
|
|
)
|
|
if [[ "${FETCH_TOTAL:-0}" -lt 10 ]]; then
|
|
merge_failed_source_keys "openrouter"
|
|
record_failure "本次采集结果异常: total=${FETCH_TOTAL:-0} < 10"
|
|
exit 1
|
|
fi
|
|
refresh_pipeline_audit
|
|
|
|
MULTI_SOURCE_OUTPUT="$(mktemp)"
|
|
if ! go run "./scripts/fetch_multi_source.go" --sources moonshot,deepseek,openai > "$MULTI_SOURCE_OUTPUT"; then
|
|
MULTI_SOURCE_SUMMARY="$(normalize_summary_file "$MULTI_SOURCE_OUTPUT")"
|
|
if [[ -n "$MULTI_SOURCE_SUMMARY" ]]; then
|
|
MULTI_SOURCE_AUDIT="multi_source_audit=${MULTI_SOURCE_SUMMARY}"
|
|
merge_failed_source_keys "$(extract_failed_source_keys "$MULTI_SOURCE_SUMMARY")"
|
|
else
|
|
MULTI_SOURCE_AUDIT="multi_source_audit=stage_failed"
|
|
merge_failed_source_keys "moonshot,deepseek,openai"
|
|
fi
|
|
cat "$MULTI_SOURCE_OUTPUT"
|
|
rm -f "$MULTI_SOURCE_OUTPUT"
|
|
record_failure "多源补充同步失败"
|
|
exit 1
|
|
fi
|
|
MULTI_SOURCE_SUMMARY="$(normalize_summary_file "$MULTI_SOURCE_OUTPUT")"
|
|
MULTI_SOURCE_AUDIT="multi_source_audit=${MULTI_SOURCE_SUMMARY:-none}"
|
|
merge_failed_source_keys "$(extract_failed_source_keys "$MULTI_SOURCE_SUMMARY")"
|
|
refresh_pipeline_audit
|
|
cat "$MULTI_SOURCE_OUTPUT"
|
|
rm -f "$MULTI_SOURCE_OUTPUT"
|
|
|
|
if ! go run -tags llm_script "./scripts/import_zhipu_data.go"; then
|
|
merge_failed_source_keys "zhipu"
|
|
record_failure "智谱官方导入失败"
|
|
exit 1
|
|
fi
|
|
|
|
if ! go run -tags llm_script "./scripts/export_official_seed_json.go"; then
|
|
merge_failed_source_keys "official_seed_export"
|
|
record_failure "官方种子导出失败"
|
|
exit 1
|
|
fi
|
|
|
|
if ! go run -tags llm_script "./scripts/import_phase2_data.go"; then
|
|
merge_failed_source_keys "baidu"
|
|
record_failure "百度官方导入失败"
|
|
exit 1
|
|
fi
|
|
|
|
if ! go run -tags llm_script "./scripts/import_bytedance_data.go"; then
|
|
merge_failed_source_keys "bytedance"
|
|
record_failure "字节官方导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/aliyun_subscription_lib.go" "./scripts/import_aliyun_subscription.go"; then
|
|
merge_failed_source_keys "aliyun_subscription"
|
|
record_failure "阿里云套餐导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/baidu_subscription_lib.go" "./scripts/import_baidu_subscription.go"; then
|
|
merge_failed_source_keys "baidu_subscription"
|
|
record_failure "百度套餐导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/ctyun_subscription_lib.go" "./scripts/import_ctyun_subscription.go"; then
|
|
merge_failed_source_keys "ctyun_subscription"
|
|
record_failure "天翼云套餐导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/bytedance_subscription_lib.go" "./scripts/import_bytedance_subscription.go"; then
|
|
merge_failed_source_keys "bytedance_subscription"
|
|
record_failure "火山方舟套餐导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/huawei_package_lib.go" "./scripts/import_huawei_package.go"; then
|
|
merge_failed_source_keys "huawei_package"
|
|
record_failure "华为云套餐包导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/zhipu_coding_plan_lib.go" "./scripts/import_zhipu_coding_plan.go"; then
|
|
merge_failed_source_keys "zhipu_coding_plan"
|
|
record_failure "智谱 Coding Plan 导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/minimax_subscription_lib.go" "./scripts/import_minimax_subscription.go"; then
|
|
merge_failed_source_keys "minimax_subscription"
|
|
record_failure "MiniMax Token Plan 导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/catalog_verification_common.go" "./scripts/import_cucloud_catalog.go"; then
|
|
merge_failed_source_keys "cucloud_catalog"
|
|
record_failure "联通云目录校验失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_cucloud_pricing.go"; then
|
|
merge_failed_source_keys "cucloud_pricing"
|
|
record_failure "联通云 Token Plan 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_mobile_cloud_pricing.go"; then
|
|
merge_failed_source_keys "mobile_cloud_pricing"
|
|
record_failure "移动云 MoMA 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/tencent_catalog_lib.go" "./scripts/import_tencent_subscription.go"; then
|
|
merge_failed_source_keys "tencent_subscription"
|
|
record_failure "腾讯云套餐导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/youdao_pricing_lib.go" "./scripts/import_youdao_pricing.go"; then
|
|
merge_failed_source_keys "youdao_pricing"
|
|
record_failure "网易有道价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/platform360_pricing_lib.go" "./scripts/import_360_pricing.go"; then
|
|
merge_failed_source_keys "platform360_pricing"
|
|
record_failure "360 智脑价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/siliconflow_pricing_lib.go" "./scripts/import_siliconflow_pricing.go"; then
|
|
merge_failed_source_keys "siliconflow_pricing"
|
|
record_failure "硅基流动价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/ppio_pricing_lib.go" "./scripts/import_ppio_pricing.go"; then
|
|
merge_failed_source_keys "ppio_pricing"
|
|
record_failure "PPIO 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/ucloud_pricing_lib.go" "./scripts/import_ucloud_pricing.go"; then
|
|
merge_failed_source_keys "ucloud_pricing"
|
|
record_failure "UCloud 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/coreshub_pricing_lib.go" "./scripts/import_coreshub_pricing.go"; then
|
|
merge_failed_source_keys "coreshub_pricing"
|
|
record_failure "CoresHub 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/cloudflare_pricing_snapshot_lib.go" "./scripts/signature_guard_common.go" "./scripts/official_import_signature_audit_lib.go" "./scripts/cloudflare_pricing_signature_guard_lib.go" "./scripts/cloudflare_pricing_import_runner.go" "./scripts/cloudflare_pricing_lib.go" "./scripts/cloudflare_pricing_signature_guard.go"; then
|
|
merge_failed_source_keys "cloudflare_pricing_signature"
|
|
record_failure "Cloudflare Workers AI 价格页结构签名漂移"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/cloudflare_pricing_snapshot_lib.go" "./scripts/cloudflare_pricing_import_runner.go" "./scripts/cloudflare_pricing_lib.go" "./scripts/import_cloudflare_pricing.go"; then
|
|
merge_failed_source_keys "cloudflare_pricing"
|
|
record_failure "Cloudflare Workers AI 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/perplexity_pricing_snapshot_lib.go" "./scripts/signature_guard_common.go" "./scripts/official_import_signature_audit_lib.go" "./scripts/perplexity_pricing_signature_guard_lib.go" "./scripts/perplexity_pricing_import_runner.go" "./scripts/perplexity_pricing_lib.go" "./scripts/perplexity_pricing_signature_guard.go"; then
|
|
merge_failed_source_keys "perplexity_pricing_signature"
|
|
record_failure "Perplexity API 价格页结构签名漂移"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/perplexity_pricing_snapshot_lib.go" "./scripts/perplexity_pricing_import_runner.go" "./scripts/perplexity_pricing_lib.go" "./scripts/import_perplexity_pricing.go"; then
|
|
merge_failed_source_keys "perplexity_pricing"
|
|
record_failure "Perplexity API 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/signature_guard_common.go" "./scripts/official_import_signature_audit_lib.go" "./scripts/vertex_pricing_snapshot_lib.go" "./scripts/vertex_pricing_signature_guard_lib.go" "./scripts/vertex_pricing_import_runner.go" "./scripts/vertex_pricing_lib.go" "./scripts/vertex_pricing_signature_guard.go"; then
|
|
merge_failed_source_keys "vertex_pricing_signature"
|
|
record_failure "Vertex AI 价格页结构签名漂移"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/vertex_pricing_snapshot_lib.go" "./scripts/vertex_pricing_import_runner.go" "./scripts/vertex_pricing_lib.go" "./scripts/import_vertex_pricing.go"; then
|
|
merge_failed_source_keys "vertex_pricing"
|
|
record_failure "Vertex AI 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/bedrock_pricing_lib.go" "./scripts/import_bedrock_pricing.go"; then
|
|
merge_failed_source_keys "bedrock_pricing"
|
|
record_failure "Amazon Bedrock 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/azure_openai_pricing_lib.go" "./scripts/import_azure_openai_pricing.go"; then
|
|
merge_failed_source_keys "azure_openai_pricing"
|
|
record_failure "Azure OpenAI 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_qwen_pricing.go"; then
|
|
merge_failed_source_keys "qwen_pricing"
|
|
record_failure "通义千问价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_hunyuan_pricing.go"; then
|
|
merge_failed_source_keys "hunyuan_pricing"
|
|
record_failure "腾讯混元价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_huawei_maas_pricing.go"; then
|
|
merge_failed_source_keys "huawei_maas_pricing"
|
|
record_failure "华为云 MaaS 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_baichuan_pricing.go"; then
|
|
merge_failed_source_keys "baichuan_pricing"
|
|
record_failure "百川价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_lingyiwanwu_pricing.go"; then
|
|
merge_failed_source_keys "lingyiwanwu_pricing"
|
|
record_failure "零一万物价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_sensenova_pricing.go"; then
|
|
merge_failed_source_keys "sensenova_pricing"
|
|
record_failure "商汤 SenseNova 价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_xfyun_pricing.go"; then
|
|
merge_failed_source_keys "xfyun_pricing"
|
|
record_failure "讯飞价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_bytedance_pricing.go"; then
|
|
merge_failed_source_keys "bytedance_pricing"
|
|
record_failure "火山方舟价格导入失败"
|
|
exit 1
|
|
fi
|
|
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/import_catalog_seed_verification.go"; then
|
|
merge_failed_source_keys "catalog_seed_verification"
|
|
record_failure "目录级官方入口核验失败"
|
|
exit 1
|
|
fi
|
|
if ! SIGNAL_SOURCE_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run -tags llm_script "./scripts/materialize_daily_signals.go"; then
|
|
merge_failed_source_keys "daily_signal_snapshot"
|
|
record_failure "每日关键信号物化失败"
|
|
exit 1
|
|
fi
|
|
refresh_pipeline_audit
|
|
|
|
if ! REPORT_RUN_KIND="manual" REPORT_TRIGGER_SOURCE="pipeline" REPORT_IS_OFFICIAL_DAILY="false" REPORT_RUNTIME_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run "./scripts/generate_daily_report.go" "./scripts/official_import_signature_audit_query_lib.go"; then
|
|
record_failure "日报生成失败"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -f "$(report_archive_markdown_path "$REPORT_DATE")" || ! -f "$(report_archive_html_path "$REPORT_DATE")" ]]; then
|
|
record_failure "日报归档缺失"
|
|
exit 1
|
|
fi
|
|
|
|
if ! psql "$DATABASE_URL" -Atqc "select count(*) from report_runs where report_date = current_date and status = 'generated' and run_kind = 'manual' and trigger_source = 'pipeline';" | awk '{ exit !($1 >= 1) }'; then
|
|
record_failure "report_runs 未写入 pipeline generated 记录"
|
|
exit 1
|
|
fi
|
|
|
|
psql "$DATABASE_URL" -Atqc \
|
|
"select 'daily_report', count(*) from daily_report where report_date = current_date
|
|
union all
|
|
select 'models', count(*) from models
|
|
union all
|
|
select 'region_pricing', count(*) from region_pricing
|
|
union all
|
|
select 'report_runs', count(*) from report_runs where report_date = current_date
|
|
order by 1;"
|