From 6c3569fb6500c6cac6cce9c0bb9ceafb818dc970 Mon Sep 17 00:00:00 2001 From: phamnazage-jpg Date: Fri, 22 May 2026 12:13:54 +0800 Subject: [PATCH] feat(pricing): add qwen hunyuan and huawei maas payg importers --- scripts/import_huawei_maas_pricing.go | 209 ++++++++++++++++++ scripts/import_huawei_maas_pricing_test.go | 68 ++++++ scripts/import_hunyuan_pricing.go | 168 ++++++++++++++ scripts/import_hunyuan_pricing_test.go | 61 +++++ scripts/import_plan_catalog_test.go | 10 +- scripts/import_qwen_pricing.go | 178 +++++++++++++++ scripts/import_qwen_pricing_test.go | 61 +++++ scripts/importer_smoke_gate_test.sh | 8 + scripts/official_pricing_import_common.go | 4 + scripts/pipeline_runtime_alignment_test.sh | 42 ++++ scripts/run_daily.sh | 31 ++- scripts/run_intel_pipeline.sh | 10 +- scripts/run_real_pipeline.sh | 22 +- scripts/tencent_catalog_lib.go | 19 +- .../testdata/huawei_maas_pricing_sample.json | 74 +++++++ scripts/testdata/hunyuan_pricing_sample.txt | 21 ++ scripts/testdata/qwen_pricing_sample.txt | 38 ++++ scripts/verify_importer_smoke.sh | 12 + ...og_inventory_seed_cn_relays_top20plus.json | 4 +- ...talog_inventory_seed_cn_vendors_top20.json | 8 +- 20 files changed, 1032 insertions(+), 16 deletions(-) create mode 100644 scripts/import_huawei_maas_pricing.go create mode 100644 scripts/import_huawei_maas_pricing_test.go create mode 100644 scripts/import_hunyuan_pricing.go create mode 100644 scripts/import_hunyuan_pricing_test.go create mode 100644 scripts/import_qwen_pricing.go create mode 100644 scripts/import_qwen_pricing_test.go create mode 100644 scripts/pipeline_runtime_alignment_test.sh create mode 100644 scripts/testdata/huawei_maas_pricing_sample.json create mode 100644 scripts/testdata/hunyuan_pricing_sample.txt create mode 100644 scripts/testdata/qwen_pricing_sample.txt diff --git a/scripts/import_huawei_maas_pricing.go b/scripts/import_huawei_maas_pricing.go new file mode 100644 index 0000000..9b82aac --- /dev/null +++ b/scripts/import_huawei_maas_pricing.go @@ -0,0 +1,209 @@ +//go:build llm_script + +package main + +import ( + "database/sql" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "sort" + "strings" + "time" +) + +const ( + defaultHuaweiMaaSPricingURL = "https://portal.huaweicloud.com/api/calculator/rest/cbc/portalcalculatornodeservice/v4/api/productInfo?urlPath=maas&language=zh-cn&sign=common" + defaultHuaweiMaaSPricingSourceURL = "https://support.huaweicloud.com/price-maas/price-maas-0002.html" +) + +type huaweiMaaSPricingImportConfig struct { + URL string + Fixture string + DryRun bool + Timeout time.Duration +} + +type huaweiMaaSPricingEnvelope struct { + Product map[string][]huaweiMaaSPricingRow `json:"product"` +} + +type huaweiMaaSPricingRow struct { + ResourceSpecCode string `json:"resourceSpecCode"` + ResourceSpecType string `json:"resourceSpecType"` + ModelName string `json:"Model Name"` + PlanList []huaweiMaaSPricingPlan `json:"planList"` +} + +type huaweiMaaSPricingPlan struct { + UsageFactor string `json:"usageFactor"` + Amount float64 `json:"amount"` +} + +func main() { + loadSubscriptionImportEnv() + + var url string + var fixture string + var dryRun bool + var timeoutSeconds int + + flag.StringVar(&url, "url", defaultHuaweiMaaSPricingURL, "华为云 MaaS 官方价格 JSON API") + flag.StringVar(&fixture, "fixture", "", "华为云 MaaS 价格样例文件") + flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要，不写入数据库") + flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时（秒）") + flag.Parse() + + cfg := huaweiMaaSPricingImportConfig{URL: url, Fixture: fixture, DryRun: dryRun, Timeout: time.Duration(timeoutSeconds) * time.Second} + + var db *sql.DB + var err error + if !cfg.DryRun { + db, err = subscriptionImportDB() + if err != nil { + fmt.Fprintf(os.Stderr, "open db: %v\n", err) + os.Exit(1) + } + defer db.Close() + } + + if err := runHuaweiMaaSPricingImport(cfg, db, os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "import_huawei_maas_pricing: %v\n", err) + os.Exit(1) + } +} + +func runHuaweiMaaSPricingImport(cfg huaweiMaaSPricingImportConfig, db *sql.DB, out io.Writer) error { + client := &http.Client{Timeout: cfg.Timeout} + raw, err := fetchRawPricingPage(cfg.URL, cfg.Fixture, client) + if err != nil { + return err + } + records, err := parseHuaweiMaaSPricingCatalog(raw) + if err != nil { + return err + } + records = dedupeOfficialPricingRecords(records) + if cfg.DryRun { + _, err = fmt.Fprintf(out, "source=huawei-maas-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName) + return err + } + if db == nil { + return fmt.Errorf("db is required when dry-run=false") + } + if err := upsertOfficialPricingRecords(db, records, "huawei-maas-pricing-import"); err != nil { + return err + } + var tableRows int + if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil { + return fmt.Errorf("count region_pricing: %w", err) + } + _, err = fmt.Fprintf(out, "source=huawei-maas-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows) + return err +} + +func parseHuaweiMaaSPricingCatalog(raw string) ([]officialPricingRecord, error) { + var envelope huaweiMaaSPricingEnvelope + if err := json.Unmarshal([]byte(raw), &envelope); err != nil { + return nil, fmt.Errorf("parse huawei maas pricing json: %w", err) + } + items := envelope.Product["modelarts_modelarts.tokens"] + if len(items) == 0 { + return nil, fmt.Errorf("unexpected huawei maas pricing content") + } + + type grouped struct { + providerType string + modelName string + inputs []float64 + outputs []float64 + } + byCode := map[string]*grouped{} + for _, item := range items { + entry := byCode[item.ResourceSpecCode] + if entry == nil { + entry = &grouped{providerType: item.ResourceSpecType, modelName: firstNonEmptyText(item.ModelName, item.ResourceSpecCode)} + byCode[item.ResourceSpecCode] = entry + } + for _, plan := range item.PlanList { + switch { + case strings.HasPrefix(plan.UsageFactor, "input"): + entry.inputs = append(entry.inputs, plan.Amount) + case strings.HasPrefix(plan.UsageFactor, "output"): + entry.outputs = append(entry.outputs, plan.Amount) + } + } + } + + keys := make([]string, 0, len(byCode)) + for code := range byCode { + keys = append(keys, code) + } + sort.Strings(keys) + + records := make([]officialPricingRecord, 0, len(keys)) + for _, code := range keys { + entry := byCode[code] + if len(entry.inputs) == 0 || len(entry.outputs) == 0 { + continue + } + sort.Float64s(entry.inputs) + sort.Float64s(entry.outputs) + providerName := normalizeHuaweiMaaSProvider(entry.providerType, entry.modelName) + providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName) + records = append(records, officialPricingRecord{ + ModelID: normalizeExternalID("huawei-maas", entry.modelName), + ModelName: entry.modelName, + ProviderName: providerName, + ProviderNameCn: providerNameCn, + ProviderCountry: providerCountry, + ProviderWebsite: providerWebsite, + OperatorName: "Huawei Cloud MaaS", + OperatorNameCn: "华为云 MaaS", + OperatorCountry: "CN", + OperatorWebsite: "https://www.huaweicloud.com/product/maas.html", + OperatorType: "official", + Region: "CN", + Currency: "CNY", + InputPrice: entry.inputs[0], + OutputPrice: entry.outputs[0], + SourceURL: defaultHuaweiMaaSPricingSourceURL, + ModelSourceURL: defaultHuaweiMaaSPricingSourceURL, + DateConfidence: "unknown", + DateSourceKind: "official_pricing", + Modality: detectModality(entry.modelName), + }) + } + if len(records) == 0 { + return nil, fmt.Errorf("no huawei maas input/output pricing rows found") + } + return records, nil +} + +func normalizeHuaweiMaaSProvider(providerType string, modelName string) string { + switch strings.ToLower(strings.TrimSpace(providerType)) { + case "deepseek": + return "DeepSeek" + case "qwen", "multimodalunderstanding": + return "Qwen" + case "glm": + return "Zhipu AI" + case "longcat": + return "LongCat" + default: + lower := strings.ToLower(modelName) + switch { + case strings.Contains(lower, "deepseek"): + return "DeepSeek" + case strings.Contains(lower, "qwen"): + return "Qwen" + case strings.Contains(lower, "glm"): + return "Zhipu AI" + default: + return strings.TrimSpace(providerType) + } + } +} diff --git a/scripts/import_huawei_maas_pricing_test.go b/scripts/import_huawei_maas_pricing_test.go new file mode 100644 index 0000000..1dd2f4b --- /dev/null +++ b/scripts/import_huawei_maas_pricing_test.go @@ -0,0 +1,68 @@ +//go:build llm_script + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParseHuaweiMaaSPricingCatalogBuildsRecords(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("testdata", "huawei_maas_pricing_sample.json")) + if err != nil { + t.Fatalf("读取 fixture 失败: %v", err) + } + + records, err := parseHuaweiMaaSPricingCatalog(string(raw)) + if err != nil { + t.Fatalf("parseHuaweiMaaSPricingCatalog 返回错误: %v", err) + } + if len(records) != 3 { + t.Fatalf("期望 3 条华为云 MaaS 价格记录，实际 %d", len(records)) + } + if records[0].ModelID != "huawei-maas-deepseek-v4-pro" { + t.Fatalf("首条 modelID 错误: %q", records[0].ModelID) + } + recordMap := make(map[string]officialPricingRecord, len(records)) + for _, record := range records { + recordMap[record.ModelID] = record + } + if recordMap["huawei-maas-deepseek-v4-pro"].ProviderName != "DeepSeek" { + t.Fatalf("deepseek provider 归一化错误: %q", recordMap["huawei-maas-deepseek-v4-pro"].ProviderName) + } + if recordMap["huawei-maas-qwen3-32b"].ProviderName != "Qwen" { + t.Fatalf("Qwen provider 归一化错误: %q", recordMap["huawei-maas-qwen3-32b"].ProviderName) + } + if recordMap["huawei-maas-qwen3-32b"].OutputPrice != 0.008 { + t.Fatalf("qwen3-32b 输出价格错误: %v", recordMap["huawei-maas-qwen3-32b"].OutputPrice) + } + if recordMap["huawei-maas-glm-5"].InputPrice != 0.004 || recordMap["huawei-maas-glm-5"].OutputPrice != 0.018 { + t.Fatalf("glm-5 阶梯基线价格错误: %v / %v", recordMap["huawei-maas-glm-5"].InputPrice, recordMap["huawei-maas-glm-5"].OutputPrice) + } +} + +func TestRunHuaweiMaaSPricingImportDryRunPrintsSummary(t *testing.T) { + var out bytes.Buffer + err := runHuaweiMaaSPricingImport(huaweiMaaSPricingImportConfig{ + URL: defaultHuaweiMaaSPricingURL, + Fixture: filepath.Join("testdata", "huawei_maas_pricing_sample.json"), + DryRun: true, + }, nil, &out) + if err != nil { + t.Fatalf("runHuaweiMaaSPricingImport 返回错误: %v", err) + } + output := out.String() + for _, want := range []string{ + "source=huawei-maas-pricing-import", + "models=3", + "operator=Huawei Cloud MaaS", + "dry_run=true", + } { + if !strings.Contains(output, want) { + t.Fatalf("输出缺少 %q，实际: %q", want, output) + } + } +} diff --git a/scripts/import_hunyuan_pricing.go b/scripts/import_hunyuan_pricing.go new file mode 100644 index 0000000..5869c89 --- /dev/null +++ b/scripts/import_hunyuan_pricing.go @@ -0,0 +1,168 @@ +//go:build llm_script + +package main + +import ( + "database/sql" + "flag" + "fmt" + "html" + "io" + "net/http" + "os" + "regexp" + "strings" + "time" +) + +const defaultHunyuanPricingURL = "https://cloud.tencent.com/document/product/1729/97731" + +var hunyuanModelLinePattern = regexp.MustCompile(`^[A-Za-z0-9 ._-]+$`) + +type hunyuanPricingImportConfig struct { + URL string + Fixture string + DryRun bool + Timeout time.Duration +} + +func main() { + loadSubscriptionImportEnv() + + var url string + var fixture string + var dryRun bool + var timeoutSeconds int + + flag.StringVar(&url, "url", defaultHunyuanPricingURL, "腾讯混元官方价格页") + flag.StringVar(&fixture, "fixture", "", "腾讯混元价格样例文件") + flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要，不写入数据库") + flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时（秒）") + flag.Parse() + + cfg := hunyuanPricingImportConfig{URL: url, Fixture: fixture, DryRun: dryRun, Timeout: time.Duration(timeoutSeconds) * time.Second} + + var db *sql.DB + var err error + if !cfg.DryRun { + db, err = subscriptionImportDB() + if err != nil { + fmt.Fprintf(os.Stderr, "open db: %v\n", err) + os.Exit(1) + } + defer db.Close() + } + + if err := runHunyuanPricingImport(cfg, db, os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "import_hunyuan_pricing: %v\n", err) + os.Exit(1) + } +} + +func runHunyuanPricingImport(cfg hunyuanPricingImportConfig, db *sql.DB, out io.Writer) error { + client := &http.Client{Timeout: cfg.Timeout} + raw, err := fetchRawPricingPage(cfg.URL, cfg.Fixture, client) + if err != nil { + return err + } + records, err := parseHunyuanPricingCatalog(raw) + if err != nil { + return err + } + records = dedupeOfficialPricingRecords(records) + if cfg.DryRun { + _, err = fmt.Fprintf(out, "source=hunyuan-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName) + return err + } + if db == nil { + return fmt.Errorf("db is required when dry-run=false") + } + if err := upsertOfficialPricingRecords(db, records, "hunyuan-pricing-import"); err != nil { + return err + } + var tableRows int + if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil { + return fmt.Errorf("count region_pricing: %w", err) + } + _, err = fmt.Fprintf(out, "source=hunyuan-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows) + return err +} + +func parseHunyuanPricingCatalog(raw string) ([]officialPricingRecord, error) { + lines := hunyuanPricingLines(raw) + records := make([]officialPricingRecord, 0) + currentModel := "" + currentInput := 0.0 + for _, line := range lines { + trimmed := strings.TrimSpace(line) + switch { + case trimmed == "" || strings.Contains(trimmed, "混元生文价格说明") || strings.Contains(trimmed, "token 后付费") || + strings.Contains(trimmed, "产品名") || strings.Contains(trimmed, "输入长度") || strings.Contains(trimmed, "免费额度"): + continue + case strings.HasPrefix(trimmed, "输入："): + currentInput = mustParseSubscriptionPrice(strings.TrimSuffix(strings.TrimPrefix(trimmed, "输入："), "元")) + case strings.HasPrefix(trimmed, "输出："): + if currentModel == "" || currentInput == 0 { + continue + } + outputPrice := mustParseSubscriptionPrice(strings.TrimSuffix(strings.TrimPrefix(trimmed, "输出："), "元")) + providerNameCn, providerCountry, providerWebsite := providerMetadata("Tencent") + records = append(records, officialPricingRecord{ + ModelID: normalizeExternalID("hunyuan", currentModel), + ModelName: currentModel, + ProviderName: "Tencent", + ProviderNameCn: providerNameCn, + ProviderCountry: providerCountry, + ProviderWebsite: providerWebsite, + OperatorName: "Tencent Hunyuan", + OperatorNameCn: "腾讯混元", + OperatorCountry: "CN", + OperatorWebsite: "https://cloud.tencent.com/product/hunyuan", + OperatorType: "official", + Region: "CN", + Currency: "CNY", + InputPrice: currentInput, + OutputPrice: outputPrice, + SourceURL: defaultHunyuanPricingURL, + ModelSourceURL: defaultHunyuanPricingURL, + DateConfidence: "unknown", + DateSourceKind: "official_pricing", + Modality: detectModality(currentModel), + }) + currentModel = "" + currentInput = 0 + case hunyuanModelLinePattern.MatchString(trimmed) && !strings.Contains(trimmed, "元") && !strings.Contains(trimmed, "tokens") && trimmed != "-": + currentModel = trimmed + currentInput = 0 + } + } + if len(records) == 0 { + return nil, fmt.Errorf("unexpected hunyuan pricing content") + } + return records, nil +} + +func hunyuanPricingLines(raw string) []string { + raw = strings.ReplaceAll(raw, `\u003c`, "<") + raw = strings.ReplaceAll(raw, `\u003e`, ">") + raw = strings.ReplaceAll(raw, `\n`, "\n") + raw = strings.ReplaceAll(raw, `\t`, " ") + raw = html.UnescapeString(raw) + replacer := strings.NewReplacer( + "
", "\n", "
", "\n", "
", "\n", + "

", "\n", "", "\n", "", "\n", "", "\n", + "", "\n", "", "\n", "", "\n", "", "\n", + "", "\n", "", "\n", "", "\n", "", "\n", "", "\n", + ) + withBreaks := replacer.Replace(raw) + withBreaks = regexp.MustCompile(`(?is)<[^>]+>`).ReplaceAllString(withBreaks, " ") + parts := strings.Split(withBreaks, "\n") + lines := make([]string, 0, len(parts)) + for _, part := range parts { + line := strings.TrimSpace(regexp.MustCompile(`\s+`).ReplaceAllString(part, " ")) + if line != "" { + lines = append(lines, line) + } + } + return lines +} diff --git a/scripts/import_hunyuan_pricing_test.go b/scripts/import_hunyuan_pricing_test.go new file mode 100644 index 0000000..dd2e5ac --- /dev/null +++ b/scripts/import_hunyuan_pricing_test.go @@ -0,0 +1,61 @@ +//go:build llm_script + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParseHunyuanPricingCatalogBuildsRecords(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("testdata", "hunyuan_pricing_sample.txt")) + if err != nil { + t.Fatalf("读取 fixture 失败: %v", err) + } + + records, err := parseHunyuanPricingCatalog(string(raw)) + if err != nil { + t.Fatalf("parseHunyuanPricingCatalog 返回错误: %v", err) + } + if len(records) != 5 { + t.Fatalf("期望 5 条混元价格记录，实际 %d", len(records)) + } + if records[0].ModelID != "hunyuan-tencent-hy-2-0-think" { + t.Fatalf("首条 modelID 错误: %q", records[0].ModelID) + } + if records[2].ModelName != "Hunyuan-T1" { + t.Fatalf("第三条模型名错误: %q", records[2].ModelName) + } + if records[3].InputPrice != 0.8 || records[3].OutputPrice != 2 { + t.Fatalf("Hunyuan-TurboS 定价错误: %v / %v", records[3].InputPrice, records[3].OutputPrice) + } + if records[4].ProviderName != "Tencent" { + t.Fatalf("provider 错误: %q", records[4].ProviderName) + } +} + +func TestRunHunyuanPricingImportDryRunPrintsSummary(t *testing.T) { + var out bytes.Buffer + err := runHunyuanPricingImport(hunyuanPricingImportConfig{ + URL: defaultHunyuanPricingURL, + Fixture: filepath.Join("testdata", "hunyuan_pricing_sample.txt"), + DryRun: true, + }, nil, &out) + if err != nil { + t.Fatalf("runHunyuanPricingImport 返回错误: %v", err) + } + output := out.String() + for _, want := range []string{ + "source=hunyuan-pricing-import", + "models=5", + "operator=Tencent Hunyuan", + "dry_run=true", + } { + if !strings.Contains(output, want) { + t.Fatalf("输出缺少 %q，实际: %q", want, output) + } + } +} diff --git a/scripts/import_plan_catalog_test.go b/scripts/import_plan_catalog_test.go index 7bad250..198d3d6 100644 --- a/scripts/import_plan_catalog_test.go +++ b/scripts/import_plan_catalog_test.go @@ -47,16 +47,18 @@ func TestBuildPlanCatalogRows(t *testing.T) { "cucloud-aicp-platform": "import_cucloud_catalog.go", "cucloud-ai-app-platform": "import_cucloud_catalog.go", "mobile-cloud-ai-market": "import_mobile_cloud_catalog.go", + "aliyun-modelscope-api-inference": "import_catalog_seed_verification.go", "youdao-zhiyun-maas": "import_youdao_pricing.go", + "ctyun-model-inference-payg": "import_catalog_seed_verification.go", "360-open-platform": "import_360_pricing.go", "siliconflow-siliconcloud": "import_siliconflow_pricing.go", "ppio-model-api": "import_ppio_pricing.go", "ucloud-umodelverse": "import_ucloud_pricing.go", "anthropic-api-payg": "import_catalog_seed_verification.go", "xai-api-payg": "import_catalog_seed_verification.go", - "alibaba-qwen-api-payg": "import_catalog_seed_verification.go", - "tencent-hunyuan-api-payg": "import_catalog_seed_verification.go", - "huawei-pangu-api-payg": "import_catalog_seed_verification.go", + "alibaba-qwen-api-payg": "import_qwen_pricing.go", + "tencent-hunyuan-api-payg": "import_hunyuan_pricing.go", + "huawei-pangu-api-payg": "import_huawei_maas_pricing.go", "baichuan-api-payg": "import_catalog_seed_verification.go", "01ai-api-payg": "import_catalog_seed_verification.go", "sensenova-api-payg": "import_catalog_seed_verification.go", @@ -67,7 +69,7 @@ func TestBuildPlanCatalogRows(t *testing.T) { "baai-flagopen-api-payg": "import_catalog_seed_verification.go", "skywork-api-payg": "import_catalog_seed_verification.go", "infinigence-api-payg": "import_catalog_seed_verification.go", - "qingcloud-coreshub": "import_catalog_seed_verification.go", + "qingcloud-coreshub": "import_coreshub_pricing.go", "ksyun-xingliu-platform": "import_catalog_seed_verification.go", "google-gemini-api-payg": "import_catalog_seed_verification.go", "mistral-api-payg": "import_catalog_seed_verification.go", diff --git a/scripts/import_qwen_pricing.go b/scripts/import_qwen_pricing.go new file mode 100644 index 0000000..04f7730 --- /dev/null +++ b/scripts/import_qwen_pricing.go @@ -0,0 +1,178 @@ +//go:build llm_script + +package main + +import ( + "database/sql" + "flag" + "fmt" + "html" + "io" + "net/http" + "os" + "regexp" + "strings" + "time" +) + +const defaultQwenPricingURL = "https://help.aliyun.com/zh/model-studio/model-pricing" + +var qwenModelLinePattern = regexp.MustCompile(`^(qwen[0-9a-z.-]+|qwq[0-9a-z.-]+|qvq[0-9a-z.-]+)$`) + +type qwenPricingImportConfig struct { + URL string + Fixture string + DryRun bool + Timeout time.Duration +} + +func main() { + loadSubscriptionImportEnv() + + var url string + var fixture string + var dryRun bool + var timeoutSeconds int + + flag.StringVar(&url, "url", defaultQwenPricingURL, "通义千问官方模型价格页") + flag.StringVar(&fixture, "fixture", "", "通义千问价格样例文件") + flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要，不写入数据库") + flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时（秒）") + flag.Parse() + + cfg := qwenPricingImportConfig{URL: url, Fixture: fixture, DryRun: dryRun, Timeout: time.Duration(timeoutSeconds) * time.Second} + + var db *sql.DB + var err error + if !cfg.DryRun { + db, err = subscriptionImportDB() + if err != nil { + fmt.Fprintf(os.Stderr, "open db: %v\n", err) + os.Exit(1) + } + defer db.Close() + } + + if err := runQwenPricingImport(cfg, db, os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "import_qwen_pricing: %v\n", err) + os.Exit(1) + } +} + +func runQwenPricingImport(cfg qwenPricingImportConfig, db *sql.DB, out io.Writer) error { + client := &http.Client{Timeout: cfg.Timeout} + raw, err := fetchRawPricingPage(cfg.URL, cfg.Fixture, client) + if err != nil { + return err + } + records, err := parseQwenPricingCatalog(raw) + if err != nil { + return err + } + records = dedupeOfficialPricingRecords(records) + if cfg.DryRun { + _, err = fmt.Fprintf(out, "source=qwen-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName) + return err + } + if db == nil { + return fmt.Errorf("db is required when dry-run=false") + } + if err := upsertOfficialPricingRecords(db, records, "qwen-pricing-import"); err != nil { + return err + } + var tableRows int + if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil { + return fmt.Errorf("count region_pricing: %w", err) + } + _, err = fmt.Fprintf(out, "source=qwen-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows) + return err +} + +func parseQwenPricingCatalog(raw string) ([]officialPricingRecord, error) { + lines := qwenPricingLines(raw) + records := make([]officialPricingRecord, 0) + for i := 0; i < len(lines); i++ { + modelName := strings.ToLower(strings.TrimSpace(lines[i])) + if !qwenModelLinePattern.MatchString(modelName) { + continue + } + block := make([]string, 0, 12) + for j := i + 1; j < len(lines) && j < i+14; j++ { + next := strings.ToLower(strings.TrimSpace(lines[j])) + if qwenModelLinePattern.MatchString(next) { + break + } + block = append(block, lines[j]) + } + prices := qwenBlockPrices(block) + if len(prices) < 2 { + continue + } + providerNameCn, providerCountry, providerWebsite := providerMetadata("Qwen") + record := officialPricingRecord{ + ModelID: normalizeExternalID("qwen", modelName), + ModelName: modelName, + ProviderName: "Qwen", + ProviderNameCn: providerNameCn, + ProviderCountry: providerCountry, + ProviderWebsite: providerWebsite, + OperatorName: "DashScope", + OperatorNameCn: "通义千问 API", + OperatorCountry: "CN", + OperatorWebsite: "https://help.aliyun.com/zh/model-studio/model-pricing", + OperatorType: "official", + Region: "CN", + Currency: "CNY", + InputPrice: prices[0], + OutputPrice: prices[1], + SourceURL: defaultQwenPricingURL, + ModelSourceURL: defaultQwenPricingURL, + DateConfidence: "unknown", + DateSourceKind: "official_pricing", + Modality: detectModality(modelName), + } + records = append(records, record) + } + if len(records) == 0 { + return nil, fmt.Errorf("unexpected qwen pricing content") + } + return records, nil +} + +func qwenPricingLines(raw string) []string { + raw = strings.ReplaceAll(raw, `\u003c`, "<") + raw = strings.ReplaceAll(raw, `\u003e`, ">") + raw = strings.ReplaceAll(raw, `\n`, "\n") + raw = strings.ReplaceAll(raw, `\t`, " ") + raw = html.UnescapeString(raw) + replacer := strings.NewReplacer( + "
", "\n", "
", "\n", "
", "\n", + "

", "\n", "", "\n", "", "\n", "", "\n", + "", "\n", "", "\n", "", "\n", "", "\n", + "", "\n", "", "\n", "", "\n", "", "\n", "", "\n", + ) + withBreaks := replacer.Replace(raw) + tagPattern := regexp.MustCompile(`(?is)<[^>]+>`) + withBreaks = tagPattern.ReplaceAllString(withBreaks, " ") + parts := strings.Split(withBreaks, "\n") + lines := make([]string, 0, len(parts)) + for _, part := range parts { + line := strings.TrimSpace(regexp.MustCompile(`\s+`).ReplaceAllString(part, " ")) + if line != "" { + lines = append(lines, line) + } + } + return lines +} + +func qwenBlockPrices(lines []string) []float64 { + pricePattern := regexp.MustCompile(`^([0-9]+(?:\.[0-9]+)?) 元$`) + prices := make([]float64, 0, 4) + for _, line := range lines { + match := pricePattern.FindStringSubmatch(strings.TrimSpace(line)) + if len(match) == 2 { + prices = append(prices, mustParseSubscriptionPrice(match[1])) + } + } + return prices +} diff --git a/scripts/import_qwen_pricing_test.go b/scripts/import_qwen_pricing_test.go new file mode 100644 index 0000000..07f0d40 --- /dev/null +++ b/scripts/import_qwen_pricing_test.go @@ -0,0 +1,61 @@ +//go:build llm_script + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParseQwenPricingCatalogBuildsRecords(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("testdata", "qwen_pricing_sample.txt")) + if err != nil { + t.Fatalf("读取 fixture 失败: %v", err) + } + + records, err := parseQwenPricingCatalog(string(raw)) + if err != nil { + t.Fatalf("parseQwenPricingCatalog 返回错误: %v", err) + } + if len(records) != 4 { + t.Fatalf("期望 4 条通义千问价格记录，实际 %d", len(records)) + } + if records[0].ModelID != "qwen-qwen-max" { + t.Fatalf("首条 modelID 错误: %q", records[0].ModelID) + } + if records[1].InputPrice != 0.8 || records[1].OutputPrice != 2 { + t.Fatalf("qwen-plus 定价错误: %v / %v", records[1].InputPrice, records[1].OutputPrice) + } + if records[2].Modality != "multimodal" { + t.Fatalf("qwen-vl-max modality 错误: %q", records[2].Modality) + } + if records[3].ProviderName != "Qwen" { + t.Fatalf("provider 错误: %q", records[3].ProviderName) + } +} + +func TestRunQwenPricingImportDryRunPrintsSummary(t *testing.T) { + var out bytes.Buffer + err := runQwenPricingImport(qwenPricingImportConfig{ + URL: defaultQwenPricingURL, + Fixture: filepath.Join("testdata", "qwen_pricing_sample.txt"), + DryRun: true, + }, nil, &out) + if err != nil { + t.Fatalf("runQwenPricingImport 返回错误: %v", err) + } + output := out.String() + for _, want := range []string{ + "source=qwen-pricing-import", + "models=4", + "operator=DashScope", + "dry_run=true", + } { + if !strings.Contains(output, want) { + t.Fatalf("输出缺少 %q，实际: %q", want, output) + } + } +} diff --git a/scripts/importer_smoke_gate_test.sh b/scripts/importer_smoke_gate_test.sh index fa13238..e49b10a 100755 --- a/scripts/importer_smoke_gate_test.sh +++ b/scripts/importer_smoke_gate_test.sh @@ -24,5 +24,13 @@ printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=coreshub-fixture' printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=coreshub-live' printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=ctyun-fixture' printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=ctyun-live' +printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=tencent-fixture' +printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=tencent-live' +printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=qwen-fixture' +printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=qwen-live' +printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=hunyuan-fixture' +printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=hunyuan-live' +printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=huawei-maas-fixture' +printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=huawei-maas-live' echo "importer_smoke_gate_test: PASS" diff --git a/scripts/official_pricing_import_common.go b/scripts/official_pricing_import_common.go index 4de99ce..d9da41f 100644 --- a/scripts/official_pricing_import_common.go +++ b/scripts/official_pricing_import_common.go @@ -438,6 +438,10 @@ func providerMetadata(providerName string) (string, string, string) { return "OpenAI", "US", "https://openai.com" case "Perplexity": return "Perplexity", "US", "https://www.perplexity.ai" + case "Tencent": + return "腾讯", "CN", "https://cloud.tencent.com" + case "Huawei": + return "华为", "CN", "https://www.huaweicloud.com" case "xAI": return "xAI", "US", "https://x.ai" case "Zhipu AI": diff --git a/scripts/pipeline_runtime_alignment_test.sh b/scripts/pipeline_runtime_alignment_test.sh new file mode 100644 index 0000000..2ffc8fd --- /dev/null +++ b/scripts/pipeline_runtime_alignment_test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +check_contains() { + local file="$1" + local needle="$2" + grep -Fq "$needle" "$file" || { + echo "missing in ${file}: ${needle}" + exit 1 + } +} + +check_contains "scripts/run_intel_pipeline.sh" 'PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription' +check_contains "scripts/run_real_pipeline.sh" 'PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription' +check_contains "scripts/run_daily.sh" 'PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription' + +check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "tencent_subscription" "腾讯云套餐导入失败"' +check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "tencent_subscription"' +check_contains "scripts/run_real_pipeline.sh" 'record_failure "腾讯云套餐导入失败"' +check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "tencent_subscription"' +check_contains "scripts/run_daily.sh" 'error_exit "腾讯云套餐导入失败"' +check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "qwen_pricing" "通义千问价格导入失败"' +check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "hunyuan_pricing" "腾讯混元价格导入失败"' +check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "huawei_maas_pricing" "华为云 MaaS 价格导入失败"' +check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "qwen_pricing"' +check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "hunyuan_pricing"' +check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "huawei_maas_pricing"' +check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "qwen_pricing"' +check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "hunyuan_pricing"' +check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "huawei_maas_pricing"' + + +check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "tencent-live"' +check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "qwen-fixture"' +check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "hunyuan-fixture"' +check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "huawei-maas-fixture"' + +echo "pipeline_runtime_alignment_test: PASS" diff --git a/scripts/run_daily.sh b/scripts/run_daily.sh index 7c48aad..fce5aed 100755 --- a/scripts/run_daily.sh +++ b/scripts/run_daily.sh @@ -22,7 +22,7 @@ MODEL_COUNT="" FETCH_OUT="${PROJECT_DIR}/models.json" FETCH_TOTAL="0" PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report" -PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification" +PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,catalog_seed_verification" PIPELINE_FAILED_SOURCE_SET="none" MULTI_SOURCE_AUDIT="multi_source_audit=unavailable" PIPELINE_AUDIT_SUMMARY="" @@ -245,6 +245,14 @@ if ! go run -tags llm_script \ merge_failed_source_keys "mobile_cloud_catalog" error_exit "移动云目录校验失败" fi +if ! go run -tags llm_script \ + scripts/subscription_import_common.go \ + scripts/tencent_catalog_lib.go \ + scripts/import_tencent_subscription.go >> "$LOG_FILE" 2>&1; then + merge_failed_source_keys "tencent_subscription" + error_exit "腾讯云套餐导入失败" +fi + if ! go run -tags llm_script \ scripts/subscription_import_common.go \ scripts/official_pricing_import_common.go \ @@ -383,6 +391,27 @@ if ! go run -tags llm_script \ merge_failed_source_keys "azure_openai_pricing" error_exit "Azure OpenAI 价格导入失败" fi +if ! go run -tags llm_script \ + scripts/subscription_import_common.go \ + scripts/official_pricing_import_common.go \ + scripts/import_qwen_pricing.go >> "$LOG_FILE" 2>&1; then + merge_failed_source_keys "qwen_pricing" + error_exit "通义千问价格导入失败" +fi +if ! go run -tags llm_script \ + scripts/subscription_import_common.go \ + scripts/official_pricing_import_common.go \ + scripts/import_hunyuan_pricing.go >> "$LOG_FILE" 2>&1; then + merge_failed_source_keys "hunyuan_pricing" + error_exit "腾讯混元价格导入失败" +fi +if ! go run -tags llm_script \ + scripts/subscription_import_common.go \ + scripts/official_pricing_import_common.go \ + scripts/import_huawei_maas_pricing.go >> "$LOG_FILE" 2>&1; then + merge_failed_source_keys "huawei_maas_pricing" + error_exit "华为云 MaaS 价格导入失败" +fi if ! go run -tags llm_script \ scripts/subscription_import_common.go \ scripts/import_catalog_seed_verification.go >> "$LOG_FILE" 2>&1; then diff --git a/scripts/run_intel_pipeline.sh b/scripts/run_intel_pipeline.sh index 96a3aaa..d9b9b94 100755 --- a/scripts/run_intel_pipeline.sh +++ b/scripts/run_intel_pipeline.sh @@ -27,7 +27,7 @@ REPORT_DATE="${REPORT_DATE:-$(date +%F)}" FETCH_OUT="$ROOT_DIR/models.json" FETCH_TOTAL="0" PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot" -PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification" +PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,catalog_seed_verification" PIPELINE_FAILED_SOURCE_SET="none" MULTI_SOURCE_AUDIT="multi_source_audit=unavailable" PIPELINE_AUDIT_SUMMARY="" @@ -142,6 +142,8 @@ run_or_fail "cucloud_catalog" "联通云目录校验失败" \ go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/catalog_verification_common.go ./scripts/import_cucloud_catalog.go run_or_fail "mobile_cloud_catalog" "移动云目录校验失败" \ go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/catalog_verification_common.go ./scripts/import_mobile_cloud_catalog.go +run_or_fail "tencent_subscription" "腾讯云套餐导入失败" \ + go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/tencent_catalog_lib.go ./scripts/import_tencent_subscription.go run_or_fail "youdao_pricing" "网易有道价格导入失败" \ go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/youdao_pricing_lib.go ./scripts/import_youdao_pricing.go run_or_fail "platform360_pricing" "360 智脑价格导入失败" \ @@ -171,6 +173,12 @@ run_or_fail "bedrock_pricing" "Amazon Bedrock 价格导入失败" \ go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/bedrock_pricing_lib.go ./scripts/import_bedrock_pricing.go run_or_fail "azure_openai_pricing" "Azure OpenAI 价格导入失败" \ go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/azure_openai_pricing_lib.go ./scripts/import_azure_openai_pricing.go +run_or_fail "qwen_pricing" "通义千问价格导入失败" \ + go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_qwen_pricing.go +run_or_fail "hunyuan_pricing" "腾讯混元价格导入失败" \ + go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_hunyuan_pricing.go +run_or_fail "huawei_maas_pricing" "华为云 MaaS 价格导入失败" \ + go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_huawei_maas_pricing.go refresh_pipeline_audit run_or_fail "catalog_seed_verification" "目录级官方入口核验失败" \ diff --git a/scripts/run_real_pipeline.sh b/scripts/run_real_pipeline.sh index d1eb075..8846d79 100755 --- a/scripts/run_real_pipeline.sh +++ b/scripts/run_real_pipeline.sh @@ -28,7 +28,7 @@ REPORT_DATE="$(report_date_value)" FETCH_OUT="$ROOT_DIR/models.json" FETCH_TOTAL="0" PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report" -PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification" +PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,catalog_seed_verification" PIPELINE_FAILED_SOURCE_SET="none" MULTI_SOURCE_AUDIT="multi_source_audit=unavailable" PIPELINE_AUDIT_SUMMARY="" @@ -194,6 +194,11 @@ if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./script record_failure "移动云目录校验失败" exit 1 fi +if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/tencent_catalog_lib.go" "./scripts/import_tencent_subscription.go"; then + merge_failed_source_keys "tencent_subscription" + record_failure "腾讯云套餐导入失败" + exit 1 +fi if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/youdao_pricing_lib.go" "./scripts/import_youdao_pricing.go"; then merge_failed_source_keys "youdao_pricing" record_failure "网易有道价格导入失败" @@ -264,6 +269,21 @@ if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./script record_failure "Azure OpenAI 价格导入失败" exit 1 fi +if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_qwen_pricing.go"; then + merge_failed_source_keys "qwen_pricing" + record_failure "通义千问价格导入失败" + exit 1 +fi +if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_hunyuan_pricing.go"; then + merge_failed_source_keys "hunyuan_pricing" + record_failure "腾讯混元价格导入失败" + exit 1 +fi +if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_huawei_maas_pricing.go"; then + merge_failed_source_keys "huawei_maas_pricing" + record_failure "华为云 MaaS 价格导入失败" + exit 1 +fi if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/import_catalog_seed_verification.go"; then merge_failed_source_keys "catalog_seed_verification" record_failure "目录级官方入口核验失败" diff --git a/scripts/tencent_catalog_lib.go b/scripts/tencent_catalog_lib.go index cab0f27..cc40f49 100644 --- a/scripts/tencent_catalog_lib.go +++ b/scripts/tencent_catalog_lib.go @@ -106,10 +106,16 @@ func parseTencentCatalog(raw string) (tencentCatalog, error) { } switch line { - case "### 套餐详情": + case "### 套餐详情", "套餐详情": + if currentSeries == "" { + continue + } currentMode = "plans" continue - case "### 可用模型": + case "### 可用模型", "可用模型": + if currentSeries == "" { + continue + } currentMode = "models" continue } @@ -159,7 +165,7 @@ func normalizeTencentCatalogLines(raw string) []string { rawLines := strings.Split(text, "\n") lines := make([]string, 0, len(rawLines)) for _, rawLine := range rawLines { - line := strings.TrimSpace(rawLine) + line := strings.Trim(strings.TrimSpace(rawLine), "\uFEFF") if line == "" { continue } @@ -178,6 +184,13 @@ func extractUpdatedAt(line string) string { func extractSeriesHeading(line string) string { if !strings.HasPrefix(line, "## ") { + trimmed := strings.Trim(line, "\uFEFF ") + switch trimmed { + case "通用 Token Plan 套餐": + return "通用 Token Plan" + case "Hy Token Plan 套餐": + return "Hy Token Plan" + } return "" } series := strings.TrimSpace(strings.TrimPrefix(line, "## ")) diff --git a/scripts/testdata/huawei_maas_pricing_sample.json b/scripts/testdata/huawei_maas_pricing_sample.json new file mode 100644 index 0000000..16f2077 --- /dev/null +++ b/scripts/testdata/huawei_maas_pricing_sample.json @@ -0,0 +1,74 @@ +{ + "product": { + "modelarts_modelarts.tokens": [ + { + "resourceSpecCode": "modelarts.tokens.deepseek.v4.pro", + "resourceSpecType": "DeepSeek", + "Model Name": "deepseek-v4-pro", + "planList": [{"usageFactor": "input", "amount": 0.012}], + "tableUnit": "detail_6_" + }, + { + "resourceSpecCode": "modelarts.tokens.deepseek.v4.pro", + "resourceSpecType": "DeepSeek", + "Model Name": "deepseek-v4-pro", + "planList": [{"usageFactor": "output", "amount": 0.024}], + "tableUnit": "detail_6_" + }, + { + "resourceSpecCode": "modelarts.tokens.qwen3-32b", + "resourceSpecType": "Qwen", + "Model Name": "qwen3-32b", + "planList": [{"usageFactor": "input", "amount": 0.002}], + "tableUnit": "detail_6_" + }, + { + "resourceSpecCode": "modelarts.tokens.qwen3-32b", + "resourceSpecType": "Qwen", + "Model Name": "qwen3-32b", + "planList": [{"usageFactor": "output", "amount": 0.008}], + "tableUnit": "detail_6_" + }, + { + "resourceSpecCode": "modelarts.tokens.qwen3-32b", + "resourceSpecType": "Qwen", + "Model Name": "qwen3-32b", + "planList": [{"usageFactor": "output_with_think", "amount": 0.02}], + "tableUnit": "detail_6_" + }, + { + "resourceSpecCode": "modelarts.tokens.glm.5", + "resourceSpecType": "GLM", + "Model Name": "glm-5", + "planList": [{"usageFactor": "input_token_interval_1", "amount": 0.004}], + "tableUnit": "detail_6_" + }, + { + "resourceSpecCode": "modelarts.tokens.glm.5", + "resourceSpecType": "GLM", + "Model Name": "glm-5", + "planList": [{"usageFactor": "input_token_interval_2", "amount": 0.006}], + "tableUnit": "detail_6_" + }, + { + "resourceSpecCode": "modelarts.tokens.glm.5", + "resourceSpecType": "GLM", + "Model Name": "glm-5", + "planList": [{"usageFactor": "output_token_interval_1", "amount": 0.018}], + "tableUnit": "detail_6_" + }, + { + "resourceSpecCode": "modelarts.tokens.glm.5", + "resourceSpecType": "GLM", + "Model Name": "glm-5", + "planList": [{"usageFactor": "output_token_interval_2", "amount": 0.022}], + "tableUnit": "detail_6_" + } + ] + }, + "period": {}, + "region": {}, + "tag": {}, + "urlPath": "maas", + "tab": {} +} diff --git a/scripts/testdata/hunyuan_pricing_sample.txt b/scripts/testdata/hunyuan_pricing_sample.txt new file mode 100644 index 0000000..a502145 --- /dev/null +++ b/scripts/testdata/hunyuan_pricing_sample.txt @@ -0,0 +1,21 @@ +混元生文价格说明 +token 后付费 +在免费额度用完后，按如下价格进行后付费计费。 +产品名 +输入长度 ≤32k tokens +输入长度 （32k, 128k] tokens +Tencent HY 2.0 Think +输入：4.13元 +输出：16.58元 +Tencent HY 2.0 Instruct +输入：0.57元 +输出：2.27元 +Hunyuan-T1 +输入：1元 +输出：4元 +Hunyuan-TurboS +输入：0.8元 +输出：2元 +hunyuan-large-role +输入：0.24元 +输出：0.96元 diff --git a/scripts/testdata/qwen_pricing_sample.txt b/scripts/testdata/qwen_pricing_sample.txt new file mode 100644 index 0000000..4e047b3 --- /dev/null +++ b/scripts/testdata/qwen_pricing_sample.txt @@ -0,0 +1,38 @@ +更新时间：2026-05-22 +模型调用计费 +文本生成-千问 +模型名称 +输入单价（每百万 Token） +输出单价（每百万 Token） +免费额度 （注） +qwen-max +当前能力等同于 qwen-max-2024-09-19 Batch 调用 半价 +仅非思考模式 +无阶梯计价 +2.4 元 +9.6 元 +各 100 万 Token +qwen-plus +当前能力等同于 qwen-plus-2025-12-01 Batch 调用 半价 +0