feat(pipeline): wire collectors into real pipeline gates

Wire the new subscription and official pricing collectors into the daily, real, and intel pipeline entrypoints.

This commit also upgrades Phase 6 verification with recent-window collector classification so gate failures distinguish preconditions from true runtime or provider issues.
This commit is contained in:
phamnazage-jpg
2026-05-15 22:37:06 +08:00
parent 256975e10c
commit d5d18e987e
6 changed files with 739 additions and 7 deletions

View File

@@ -27,8 +27,8 @@ fi
REPORT_DATE="$(report_date_value)"
FETCH_OUT="$ROOT_DIR/models.json"
FETCH_TOTAL="0"
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_report"
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance"
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report"
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification"
PIPELINE_FAILED_SOURCE_SET="none"
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
PIPELINE_AUDIT_SUMMARY=""
@@ -149,9 +149,129 @@ if ! go run -tags llm_script "./scripts/import_bytedance_data.go"; then
record_failure "字节官方导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/aliyun_subscription_lib.go" "./scripts/import_aliyun_subscription.go"; then
merge_failed_source_keys "aliyun_subscription"
record_failure "阿里云套餐导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/baidu_subscription_lib.go" "./scripts/import_baidu_subscription.go"; then
merge_failed_source_keys "baidu_subscription"
record_failure "百度套餐导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/ctyun_subscription_lib.go" "./scripts/import_ctyun_subscription.go"; then
merge_failed_source_keys "ctyun_subscription"
record_failure "天翼云套餐导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/bytedance_subscription_lib.go" "./scripts/import_bytedance_subscription.go"; then
merge_failed_source_keys "bytedance_subscription"
record_failure "火山方舟套餐导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/huawei_package_lib.go" "./scripts/import_huawei_package.go"; then
merge_failed_source_keys "huawei_package"
record_failure "华为云套餐包导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/zhipu_coding_plan_lib.go" "./scripts/import_zhipu_coding_plan.go"; then
merge_failed_source_keys "zhipu_coding_plan"
record_failure "智谱 Coding Plan 导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/minimax_subscription_lib.go" "./scripts/import_minimax_subscription.go"; then
merge_failed_source_keys "minimax_subscription"
record_failure "MiniMax Token Plan 导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/catalog_verification_common.go" "./scripts/import_cucloud_catalog.go"; then
merge_failed_source_keys "cucloud_catalog"
record_failure "联通云目录校验失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/catalog_verification_common.go" "./scripts/import_mobile_cloud_catalog.go"; then
merge_failed_source_keys "mobile_cloud_catalog"
record_failure "移动云目录校验失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/youdao_pricing_lib.go" "./scripts/import_youdao_pricing.go"; then
merge_failed_source_keys "youdao_pricing"
record_failure "网易有道价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/platform360_pricing_lib.go" "./scripts/import_360_pricing.go"; then
merge_failed_source_keys "platform360_pricing"
record_failure "360 智脑价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/siliconflow_pricing_lib.go" "./scripts/import_siliconflow_pricing.go"; then
merge_failed_source_keys "siliconflow_pricing"
record_failure "硅基流动价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/ppio_pricing_lib.go" "./scripts/import_ppio_pricing.go"; then
merge_failed_source_keys "ppio_pricing"
record_failure "PPIO 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/ucloud_pricing_lib.go" "./scripts/import_ucloud_pricing.go"; then
merge_failed_source_keys "ucloud_pricing"
record_failure "UCloud 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/cloudflare_pricing_snapshot_lib.go" "./scripts/signature_guard_common.go" "./scripts/official_import_signature_audit_lib.go" "./scripts/cloudflare_pricing_signature_guard_lib.go" "./scripts/cloudflare_pricing_import_runner.go" "./scripts/cloudflare_pricing_lib.go" "./scripts/cloudflare_pricing_signature_guard.go"; then
merge_failed_source_keys "cloudflare_pricing_signature"
record_failure "Cloudflare Workers AI 价格页结构签名漂移"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/cloudflare_pricing_snapshot_lib.go" "./scripts/cloudflare_pricing_import_runner.go" "./scripts/cloudflare_pricing_lib.go" "./scripts/import_cloudflare_pricing.go"; then
merge_failed_source_keys "cloudflare_pricing"
record_failure "Cloudflare Workers AI 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/perplexity_pricing_snapshot_lib.go" "./scripts/signature_guard_common.go" "./scripts/official_import_signature_audit_lib.go" "./scripts/perplexity_pricing_signature_guard_lib.go" "./scripts/perplexity_pricing_import_runner.go" "./scripts/perplexity_pricing_lib.go" "./scripts/perplexity_pricing_signature_guard.go"; then
merge_failed_source_keys "perplexity_pricing_signature"
record_failure "Perplexity API 价格页结构签名漂移"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/perplexity_pricing_snapshot_lib.go" "./scripts/perplexity_pricing_import_runner.go" "./scripts/perplexity_pricing_lib.go" "./scripts/import_perplexity_pricing.go"; then
merge_failed_source_keys "perplexity_pricing"
record_failure "Perplexity API 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/signature_guard_common.go" "./scripts/official_import_signature_audit_lib.go" "./scripts/vertex_pricing_snapshot_lib.go" "./scripts/vertex_pricing_signature_guard_lib.go" "./scripts/vertex_pricing_import_runner.go" "./scripts/vertex_pricing_lib.go" "./scripts/vertex_pricing_signature_guard.go"; then
merge_failed_source_keys "vertex_pricing_signature"
record_failure "Vertex AI 价格页结构签名漂移"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/vertex_pricing_snapshot_lib.go" "./scripts/vertex_pricing_import_runner.go" "./scripts/vertex_pricing_lib.go" "./scripts/import_vertex_pricing.go"; then
merge_failed_source_keys "vertex_pricing"
record_failure "Vertex AI 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/bedrock_pricing_lib.go" "./scripts/import_bedrock_pricing.go"; then
merge_failed_source_keys "bedrock_pricing"
record_failure "Amazon Bedrock 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/azure_openai_pricing_lib.go" "./scripts/import_azure_openai_pricing.go"; then
merge_failed_source_keys "azure_openai_pricing"
record_failure "Azure OpenAI 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/import_catalog_seed_verification.go"; then
merge_failed_source_keys "catalog_seed_verification"
record_failure "目录级官方入口核验失败"
exit 1
fi
if ! SIGNAL_SOURCE_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run -tags llm_script "./scripts/materialize_daily_signals.go"; then
merge_failed_source_keys "daily_signal_snapshot"
record_failure "每日关键信号物化失败"
exit 1
fi
refresh_pipeline_audit
if ! REPORT_RUN_KIND="manual" REPORT_TRIGGER_SOURCE="pipeline" REPORT_IS_OFFICIAL_DAILY="false" REPORT_RUNTIME_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run "./scripts/generate_daily_report.go"; then
if ! REPORT_RUN_KIND="manual" REPORT_TRIGGER_SOURCE="pipeline" REPORT_IS_OFFICIAL_DAILY="false" REPORT_RUNTIME_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run "./scripts/generate_daily_report.go" "./scripts/official_import_signature_audit_query_lib.go"; then
record_failure "日报生成失败"
exit 1
fi