feat(pricing): add qwen hunyuan and huawei maas payg importers
This commit is contained in:
209
scripts/import_huawei_maas_pricing.go
Normal file
209
scripts/import_huawei_maas_pricing.go
Normal file
@@ -0,0 +1,209 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultHuaweiMaaSPricingURL = "https://portal.huaweicloud.com/api/calculator/rest/cbc/portalcalculatornodeservice/v4/api/productInfo?urlPath=maas&language=zh-cn&sign=common"
|
||||
defaultHuaweiMaaSPricingSourceURL = "https://support.huaweicloud.com/price-maas/price-maas-0002.html"
|
||||
)
|
||||
|
||||
type huaweiMaaSPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type huaweiMaaSPricingEnvelope struct {
|
||||
Product map[string][]huaweiMaaSPricingRow `json:"product"`
|
||||
}
|
||||
|
||||
type huaweiMaaSPricingRow struct {
|
||||
ResourceSpecCode string `json:"resourceSpecCode"`
|
||||
ResourceSpecType string `json:"resourceSpecType"`
|
||||
ModelName string `json:"Model Name"`
|
||||
PlanList []huaweiMaaSPricingPlan `json:"planList"`
|
||||
}
|
||||
|
||||
type huaweiMaaSPricingPlan struct {
|
||||
UsageFactor string `json:"usageFactor"`
|
||||
Amount float64 `json:"amount"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultHuaweiMaaSPricingURL, "华为云 MaaS 官方价格 JSON API")
|
||||
flag.StringVar(&fixture, "fixture", "", "华为云 MaaS 价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := huaweiMaaSPricingImportConfig{URL: url, Fixture: fixture, DryRun: dryRun, Timeout: time.Duration(timeoutSeconds) * time.Second}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runHuaweiMaaSPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_huawei_maas_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runHuaweiMaaSPricingImport(cfg huaweiMaaSPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchRawPricingPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseHuaweiMaaSPricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=huawei-maas-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "huawei-maas-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=huawei-maas-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
|
||||
func parseHuaweiMaaSPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
var envelope huaweiMaaSPricingEnvelope
|
||||
if err := json.Unmarshal([]byte(raw), &envelope); err != nil {
|
||||
return nil, fmt.Errorf("parse huawei maas pricing json: %w", err)
|
||||
}
|
||||
items := envelope.Product["modelarts_modelarts.tokens"]
|
||||
if len(items) == 0 {
|
||||
return nil, fmt.Errorf("unexpected huawei maas pricing content")
|
||||
}
|
||||
|
||||
type grouped struct {
|
||||
providerType string
|
||||
modelName string
|
||||
inputs []float64
|
||||
outputs []float64
|
||||
}
|
||||
byCode := map[string]*grouped{}
|
||||
for _, item := range items {
|
||||
entry := byCode[item.ResourceSpecCode]
|
||||
if entry == nil {
|
||||
entry = &grouped{providerType: item.ResourceSpecType, modelName: firstNonEmptyText(item.ModelName, item.ResourceSpecCode)}
|
||||
byCode[item.ResourceSpecCode] = entry
|
||||
}
|
||||
for _, plan := range item.PlanList {
|
||||
switch {
|
||||
case strings.HasPrefix(plan.UsageFactor, "input"):
|
||||
entry.inputs = append(entry.inputs, plan.Amount)
|
||||
case strings.HasPrefix(plan.UsageFactor, "output"):
|
||||
entry.outputs = append(entry.outputs, plan.Amount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(byCode))
|
||||
for code := range byCode {
|
||||
keys = append(keys, code)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
records := make([]officialPricingRecord, 0, len(keys))
|
||||
for _, code := range keys {
|
||||
entry := byCode[code]
|
||||
if len(entry.inputs) == 0 || len(entry.outputs) == 0 {
|
||||
continue
|
||||
}
|
||||
sort.Float64s(entry.inputs)
|
||||
sort.Float64s(entry.outputs)
|
||||
providerName := normalizeHuaweiMaaSProvider(entry.providerType, entry.modelName)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
records = append(records, officialPricingRecord{
|
||||
ModelID: normalizeExternalID("huawei-maas", entry.modelName),
|
||||
ModelName: entry.modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Huawei Cloud MaaS",
|
||||
OperatorNameCn: "华为云 MaaS",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.huaweicloud.com/product/maas.html",
|
||||
OperatorType: "official",
|
||||
Region: "CN",
|
||||
Currency: "CNY",
|
||||
InputPrice: entry.inputs[0],
|
||||
OutputPrice: entry.outputs[0],
|
||||
SourceURL: defaultHuaweiMaaSPricingSourceURL,
|
||||
ModelSourceURL: defaultHuaweiMaaSPricingSourceURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(entry.modelName),
|
||||
})
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no huawei maas input/output pricing rows found")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func normalizeHuaweiMaaSProvider(providerType string, modelName string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(providerType)) {
|
||||
case "deepseek":
|
||||
return "DeepSeek"
|
||||
case "qwen", "multimodalunderstanding":
|
||||
return "Qwen"
|
||||
case "glm":
|
||||
return "Zhipu AI"
|
||||
case "longcat":
|
||||
return "LongCat"
|
||||
default:
|
||||
lower := strings.ToLower(modelName)
|
||||
switch {
|
||||
case strings.Contains(lower, "deepseek"):
|
||||
return "DeepSeek"
|
||||
case strings.Contains(lower, "qwen"):
|
||||
return "Qwen"
|
||||
case strings.Contains(lower, "glm"):
|
||||
return "Zhipu AI"
|
||||
default:
|
||||
return strings.TrimSpace(providerType)
|
||||
}
|
||||
}
|
||||
}
|
||||
68
scripts/import_huawei_maas_pricing_test.go
Normal file
68
scripts/import_huawei_maas_pricing_test.go
Normal file
@@ -0,0 +1,68 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseHuaweiMaaSPricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "huawei_maas_pricing_sample.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseHuaweiMaaSPricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseHuaweiMaaSPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 3 {
|
||||
t.Fatalf("期望 3 条华为云 MaaS 价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].ModelID != "huawei-maas-deepseek-v4-pro" {
|
||||
t.Fatalf("首条 modelID 错误: %q", records[0].ModelID)
|
||||
}
|
||||
recordMap := make(map[string]officialPricingRecord, len(records))
|
||||
for _, record := range records {
|
||||
recordMap[record.ModelID] = record
|
||||
}
|
||||
if recordMap["huawei-maas-deepseek-v4-pro"].ProviderName != "DeepSeek" {
|
||||
t.Fatalf("deepseek provider 归一化错误: %q", recordMap["huawei-maas-deepseek-v4-pro"].ProviderName)
|
||||
}
|
||||
if recordMap["huawei-maas-qwen3-32b"].ProviderName != "Qwen" {
|
||||
t.Fatalf("Qwen provider 归一化错误: %q", recordMap["huawei-maas-qwen3-32b"].ProviderName)
|
||||
}
|
||||
if recordMap["huawei-maas-qwen3-32b"].OutputPrice != 0.008 {
|
||||
t.Fatalf("qwen3-32b 输出价格错误: %v", recordMap["huawei-maas-qwen3-32b"].OutputPrice)
|
||||
}
|
||||
if recordMap["huawei-maas-glm-5"].InputPrice != 0.004 || recordMap["huawei-maas-glm-5"].OutputPrice != 0.018 {
|
||||
t.Fatalf("glm-5 阶梯基线价格错误: %v / %v", recordMap["huawei-maas-glm-5"].InputPrice, recordMap["huawei-maas-glm-5"].OutputPrice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHuaweiMaaSPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runHuaweiMaaSPricingImport(huaweiMaaSPricingImportConfig{
|
||||
URL: defaultHuaweiMaaSPricingURL,
|
||||
Fixture: filepath.Join("testdata", "huawei_maas_pricing_sample.json"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runHuaweiMaaSPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=huawei-maas-pricing-import",
|
||||
"models=3",
|
||||
"operator=Huawei Cloud MaaS",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
168
scripts/import_hunyuan_pricing.go
Normal file
168
scripts/import_hunyuan_pricing.go
Normal file
@@ -0,0 +1,168 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultHunyuanPricingURL = "https://cloud.tencent.com/document/product/1729/97731"
|
||||
|
||||
var hunyuanModelLinePattern = regexp.MustCompile(`^[A-Za-z0-9 ._-]+$`)
|
||||
|
||||
type hunyuanPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultHunyuanPricingURL, "腾讯混元官方价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "腾讯混元价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := hunyuanPricingImportConfig{URL: url, Fixture: fixture, DryRun: dryRun, Timeout: time.Duration(timeoutSeconds) * time.Second}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runHunyuanPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_hunyuan_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runHunyuanPricingImport(cfg hunyuanPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchRawPricingPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseHunyuanPricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=hunyuan-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "hunyuan-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=hunyuan-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
|
||||
func parseHunyuanPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
lines := hunyuanPricingLines(raw)
|
||||
records := make([]officialPricingRecord, 0)
|
||||
currentModel := ""
|
||||
currentInput := 0.0
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
switch {
|
||||
case trimmed == "" || strings.Contains(trimmed, "混元生文价格说明") || strings.Contains(trimmed, "token 后付费") ||
|
||||
strings.Contains(trimmed, "产品名") || strings.Contains(trimmed, "输入长度") || strings.Contains(trimmed, "免费额度"):
|
||||
continue
|
||||
case strings.HasPrefix(trimmed, "输入:"):
|
||||
currentInput = mustParseSubscriptionPrice(strings.TrimSuffix(strings.TrimPrefix(trimmed, "输入:"), "元"))
|
||||
case strings.HasPrefix(trimmed, "输出:"):
|
||||
if currentModel == "" || currentInput == 0 {
|
||||
continue
|
||||
}
|
||||
outputPrice := mustParseSubscriptionPrice(strings.TrimSuffix(strings.TrimPrefix(trimmed, "输出:"), "元"))
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata("Tencent")
|
||||
records = append(records, officialPricingRecord{
|
||||
ModelID: normalizeExternalID("hunyuan", currentModel),
|
||||
ModelName: currentModel,
|
||||
ProviderName: "Tencent",
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Tencent Hunyuan",
|
||||
OperatorNameCn: "腾讯混元",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://cloud.tencent.com/product/hunyuan",
|
||||
OperatorType: "official",
|
||||
Region: "CN",
|
||||
Currency: "CNY",
|
||||
InputPrice: currentInput,
|
||||
OutputPrice: outputPrice,
|
||||
SourceURL: defaultHunyuanPricingURL,
|
||||
ModelSourceURL: defaultHunyuanPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(currentModel),
|
||||
})
|
||||
currentModel = ""
|
||||
currentInput = 0
|
||||
case hunyuanModelLinePattern.MatchString(trimmed) && !strings.Contains(trimmed, "元") && !strings.Contains(trimmed, "tokens") && trimmed != "-":
|
||||
currentModel = trimmed
|
||||
currentInput = 0
|
||||
}
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("unexpected hunyuan pricing content")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func hunyuanPricingLines(raw string) []string {
|
||||
raw = strings.ReplaceAll(raw, `\u003c`, "<")
|
||||
raw = strings.ReplaceAll(raw, `\u003e`, ">")
|
||||
raw = strings.ReplaceAll(raw, `\n`, "\n")
|
||||
raw = strings.ReplaceAll(raw, `\t`, " ")
|
||||
raw = html.UnescapeString(raw)
|
||||
replacer := strings.NewReplacer(
|
||||
"<br>", "\n", "<br/>", "\n", "<br />", "\n",
|
||||
"</p>", "\n", "</div>", "\n", "</section>", "\n", "</tr>", "\n",
|
||||
"</td>", "\n", "</th>", "\n", "</li>", "\n", "</h1>", "\n",
|
||||
"</h2>", "\n", "</h3>", "\n", "</h4>", "\n", "</h5>", "\n", "</h6>", "\n",
|
||||
)
|
||||
withBreaks := replacer.Replace(raw)
|
||||
withBreaks = regexp.MustCompile(`(?is)<[^>]+>`).ReplaceAllString(withBreaks, " ")
|
||||
parts := strings.Split(withBreaks, "\n")
|
||||
lines := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
line := strings.TrimSpace(regexp.MustCompile(`\s+`).ReplaceAllString(part, " "))
|
||||
if line != "" {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
}
|
||||
return lines
|
||||
}
|
||||
61
scripts/import_hunyuan_pricing_test.go
Normal file
61
scripts/import_hunyuan_pricing_test.go
Normal file
@@ -0,0 +1,61 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseHunyuanPricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "hunyuan_pricing_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseHunyuanPricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseHunyuanPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 5 {
|
||||
t.Fatalf("期望 5 条混元价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].ModelID != "hunyuan-tencent-hy-2-0-think" {
|
||||
t.Fatalf("首条 modelID 错误: %q", records[0].ModelID)
|
||||
}
|
||||
if records[2].ModelName != "Hunyuan-T1" {
|
||||
t.Fatalf("第三条模型名错误: %q", records[2].ModelName)
|
||||
}
|
||||
if records[3].InputPrice != 0.8 || records[3].OutputPrice != 2 {
|
||||
t.Fatalf("Hunyuan-TurboS 定价错误: %v / %v", records[3].InputPrice, records[3].OutputPrice)
|
||||
}
|
||||
if records[4].ProviderName != "Tencent" {
|
||||
t.Fatalf("provider 错误: %q", records[4].ProviderName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHunyuanPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runHunyuanPricingImport(hunyuanPricingImportConfig{
|
||||
URL: defaultHunyuanPricingURL,
|
||||
Fixture: filepath.Join("testdata", "hunyuan_pricing_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runHunyuanPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=hunyuan-pricing-import",
|
||||
"models=5",
|
||||
"operator=Tencent Hunyuan",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -47,16 +47,18 @@ func TestBuildPlanCatalogRows(t *testing.T) {
|
||||
"cucloud-aicp-platform": "import_cucloud_catalog.go",
|
||||
"cucloud-ai-app-platform": "import_cucloud_catalog.go",
|
||||
"mobile-cloud-ai-market": "import_mobile_cloud_catalog.go",
|
||||
"aliyun-modelscope-api-inference": "import_catalog_seed_verification.go",
|
||||
"youdao-zhiyun-maas": "import_youdao_pricing.go",
|
||||
"ctyun-model-inference-payg": "import_catalog_seed_verification.go",
|
||||
"360-open-platform": "import_360_pricing.go",
|
||||
"siliconflow-siliconcloud": "import_siliconflow_pricing.go",
|
||||
"ppio-model-api": "import_ppio_pricing.go",
|
||||
"ucloud-umodelverse": "import_ucloud_pricing.go",
|
||||
"anthropic-api-payg": "import_catalog_seed_verification.go",
|
||||
"xai-api-payg": "import_catalog_seed_verification.go",
|
||||
"alibaba-qwen-api-payg": "import_catalog_seed_verification.go",
|
||||
"tencent-hunyuan-api-payg": "import_catalog_seed_verification.go",
|
||||
"huawei-pangu-api-payg": "import_catalog_seed_verification.go",
|
||||
"alibaba-qwen-api-payg": "import_qwen_pricing.go",
|
||||
"tencent-hunyuan-api-payg": "import_hunyuan_pricing.go",
|
||||
"huawei-pangu-api-payg": "import_huawei_maas_pricing.go",
|
||||
"baichuan-api-payg": "import_catalog_seed_verification.go",
|
||||
"01ai-api-payg": "import_catalog_seed_verification.go",
|
||||
"sensenova-api-payg": "import_catalog_seed_verification.go",
|
||||
@@ -67,7 +69,7 @@ func TestBuildPlanCatalogRows(t *testing.T) {
|
||||
"baai-flagopen-api-payg": "import_catalog_seed_verification.go",
|
||||
"skywork-api-payg": "import_catalog_seed_verification.go",
|
||||
"infinigence-api-payg": "import_catalog_seed_verification.go",
|
||||
"qingcloud-coreshub": "import_catalog_seed_verification.go",
|
||||
"qingcloud-coreshub": "import_coreshub_pricing.go",
|
||||
"ksyun-xingliu-platform": "import_catalog_seed_verification.go",
|
||||
"google-gemini-api-payg": "import_catalog_seed_verification.go",
|
||||
"mistral-api-payg": "import_catalog_seed_verification.go",
|
||||
|
||||
178
scripts/import_qwen_pricing.go
Normal file
178
scripts/import_qwen_pricing.go
Normal file
@@ -0,0 +1,178 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultQwenPricingURL = "https://help.aliyun.com/zh/model-studio/model-pricing"
|
||||
|
||||
var qwenModelLinePattern = regexp.MustCompile(`^(qwen[0-9a-z.-]+|qwq[0-9a-z.-]+|qvq[0-9a-z.-]+)$`)
|
||||
|
||||
type qwenPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultQwenPricingURL, "通义千问官方模型价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "通义千问价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := qwenPricingImportConfig{URL: url, Fixture: fixture, DryRun: dryRun, Timeout: time.Duration(timeoutSeconds) * time.Second}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runQwenPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_qwen_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runQwenPricingImport(cfg qwenPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchRawPricingPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseQwenPricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=qwen-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "qwen-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=qwen-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
|
||||
func parseQwenPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
lines := qwenPricingLines(raw)
|
||||
records := make([]officialPricingRecord, 0)
|
||||
for i := 0; i < len(lines); i++ {
|
||||
modelName := strings.ToLower(strings.TrimSpace(lines[i]))
|
||||
if !qwenModelLinePattern.MatchString(modelName) {
|
||||
continue
|
||||
}
|
||||
block := make([]string, 0, 12)
|
||||
for j := i + 1; j < len(lines) && j < i+14; j++ {
|
||||
next := strings.ToLower(strings.TrimSpace(lines[j]))
|
||||
if qwenModelLinePattern.MatchString(next) {
|
||||
break
|
||||
}
|
||||
block = append(block, lines[j])
|
||||
}
|
||||
prices := qwenBlockPrices(block)
|
||||
if len(prices) < 2 {
|
||||
continue
|
||||
}
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata("Qwen")
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("qwen", modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: "Qwen",
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "DashScope",
|
||||
OperatorNameCn: "通义千问 API",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://help.aliyun.com/zh/model-studio/model-pricing",
|
||||
OperatorType: "official",
|
||||
Region: "CN",
|
||||
Currency: "CNY",
|
||||
InputPrice: prices[0],
|
||||
OutputPrice: prices[1],
|
||||
SourceURL: defaultQwenPricingURL,
|
||||
ModelSourceURL: defaultQwenPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(modelName),
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("unexpected qwen pricing content")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func qwenPricingLines(raw string) []string {
|
||||
raw = strings.ReplaceAll(raw, `\u003c`, "<")
|
||||
raw = strings.ReplaceAll(raw, `\u003e`, ">")
|
||||
raw = strings.ReplaceAll(raw, `\n`, "\n")
|
||||
raw = strings.ReplaceAll(raw, `\t`, " ")
|
||||
raw = html.UnescapeString(raw)
|
||||
replacer := strings.NewReplacer(
|
||||
"<br>", "\n", "<br/>", "\n", "<br />", "\n",
|
||||
"</p>", "\n", "</div>", "\n", "</section>", "\n", "</tr>", "\n",
|
||||
"</td>", "\n", "</th>", "\n", "</li>", "\n", "</h1>", "\n",
|
||||
"</h2>", "\n", "</h3>", "\n", "</h4>", "\n", "</h5>", "\n", "</h6>", "\n",
|
||||
)
|
||||
withBreaks := replacer.Replace(raw)
|
||||
tagPattern := regexp.MustCompile(`(?is)<[^>]+>`)
|
||||
withBreaks = tagPattern.ReplaceAllString(withBreaks, " ")
|
||||
parts := strings.Split(withBreaks, "\n")
|
||||
lines := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
line := strings.TrimSpace(regexp.MustCompile(`\s+`).ReplaceAllString(part, " "))
|
||||
if line != "" {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func qwenBlockPrices(lines []string) []float64 {
|
||||
pricePattern := regexp.MustCompile(`^([0-9]+(?:\.[0-9]+)?) 元$`)
|
||||
prices := make([]float64, 0, 4)
|
||||
for _, line := range lines {
|
||||
match := pricePattern.FindStringSubmatch(strings.TrimSpace(line))
|
||||
if len(match) == 2 {
|
||||
prices = append(prices, mustParseSubscriptionPrice(match[1]))
|
||||
}
|
||||
}
|
||||
return prices
|
||||
}
|
||||
61
scripts/import_qwen_pricing_test.go
Normal file
61
scripts/import_qwen_pricing_test.go
Normal file
@@ -0,0 +1,61 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseQwenPricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "qwen_pricing_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseQwenPricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseQwenPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 4 {
|
||||
t.Fatalf("期望 4 条通义千问价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].ModelID != "qwen-qwen-max" {
|
||||
t.Fatalf("首条 modelID 错误: %q", records[0].ModelID)
|
||||
}
|
||||
if records[1].InputPrice != 0.8 || records[1].OutputPrice != 2 {
|
||||
t.Fatalf("qwen-plus 定价错误: %v / %v", records[1].InputPrice, records[1].OutputPrice)
|
||||
}
|
||||
if records[2].Modality != "multimodal" {
|
||||
t.Fatalf("qwen-vl-max modality 错误: %q", records[2].Modality)
|
||||
}
|
||||
if records[3].ProviderName != "Qwen" {
|
||||
t.Fatalf("provider 错误: %q", records[3].ProviderName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunQwenPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runQwenPricingImport(qwenPricingImportConfig{
|
||||
URL: defaultQwenPricingURL,
|
||||
Fixture: filepath.Join("testdata", "qwen_pricing_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runQwenPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=qwen-pricing-import",
|
||||
"models=4",
|
||||
"operator=DashScope",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,5 +24,13 @@ printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=coreshub-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=coreshub-live'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=ctyun-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=ctyun-live'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=tencent-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=tencent-live'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=qwen-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=qwen-live'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=hunyuan-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=hunyuan-live'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=huawei-maas-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=huawei-maas-live'
|
||||
|
||||
echo "importer_smoke_gate_test: PASS"
|
||||
|
||||
@@ -438,6 +438,10 @@ func providerMetadata(providerName string) (string, string, string) {
|
||||
return "OpenAI", "US", "https://openai.com"
|
||||
case "Perplexity":
|
||||
return "Perplexity", "US", "https://www.perplexity.ai"
|
||||
case "Tencent":
|
||||
return "腾讯", "CN", "https://cloud.tencent.com"
|
||||
case "Huawei":
|
||||
return "华为", "CN", "https://www.huaweicloud.com"
|
||||
case "xAI":
|
||||
return "xAI", "US", "https://x.ai"
|
||||
case "Zhipu AI":
|
||||
|
||||
42
scripts/pipeline_runtime_alignment_test.sh
Normal file
42
scripts/pipeline_runtime_alignment_test.sh
Normal file
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
check_contains() {
|
||||
local file="$1"
|
||||
local needle="$2"
|
||||
grep -Fq "$needle" "$file" || {
|
||||
echo "missing in ${file}: ${needle}"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription'
|
||||
check_contains "scripts/run_daily.sh" 'PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription'
|
||||
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "tencent_subscription" "腾讯云套餐导入失败"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "tencent_subscription"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'record_failure "腾讯云套餐导入失败"'
|
||||
check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "tencent_subscription"'
|
||||
check_contains "scripts/run_daily.sh" 'error_exit "腾讯云套餐导入失败"'
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "qwen_pricing" "通义千问价格导入失败"'
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "hunyuan_pricing" "腾讯混元价格导入失败"'
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "huawei_maas_pricing" "华为云 MaaS 价格导入失败"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "qwen_pricing"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "hunyuan_pricing"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "huawei_maas_pricing"'
|
||||
check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "qwen_pricing"'
|
||||
check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "hunyuan_pricing"'
|
||||
check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "huawei_maas_pricing"'
|
||||
|
||||
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "tencent-live"'
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "qwen-fixture"'
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "hunyuan-fixture"'
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "huawei-maas-fixture"'
|
||||
|
||||
echo "pipeline_runtime_alignment_test: PASS"
|
||||
@@ -22,7 +22,7 @@ MODEL_COUNT=""
|
||||
FETCH_OUT="${PROJECT_DIR}/models.json"
|
||||
FETCH_TOTAL="0"
|
||||
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,catalog_seed_verification"
|
||||
PIPELINE_FAILED_SOURCE_SET="none"
|
||||
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
|
||||
PIPELINE_AUDIT_SUMMARY=""
|
||||
@@ -245,6 +245,14 @@ if ! go run -tags llm_script \
|
||||
merge_failed_source_keys "mobile_cloud_catalog"
|
||||
error_exit "移动云目录校验失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/tencent_catalog_lib.go \
|
||||
scripts/import_tencent_subscription.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "tencent_subscription"
|
||||
error_exit "腾讯云套餐导入失败"
|
||||
fi
|
||||
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
@@ -383,6 +391,27 @@ if ! go run -tags llm_script \
|
||||
merge_failed_source_keys "azure_openai_pricing"
|
||||
error_exit "Azure OpenAI 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/import_qwen_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "qwen_pricing"
|
||||
error_exit "通义千问价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/import_hunyuan_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "hunyuan_pricing"
|
||||
error_exit "腾讯混元价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/import_huawei_maas_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "huawei_maas_pricing"
|
||||
error_exit "华为云 MaaS 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/import_catalog_seed_verification.go >> "$LOG_FILE" 2>&1; then
|
||||
|
||||
@@ -27,7 +27,7 @@ REPORT_DATE="${REPORT_DATE:-$(date +%F)}"
|
||||
FETCH_OUT="$ROOT_DIR/models.json"
|
||||
FETCH_TOTAL="0"
|
||||
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,catalog_seed_verification"
|
||||
PIPELINE_FAILED_SOURCE_SET="none"
|
||||
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
|
||||
PIPELINE_AUDIT_SUMMARY=""
|
||||
@@ -142,6 +142,8 @@ run_or_fail "cucloud_catalog" "联通云目录校验失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/catalog_verification_common.go ./scripts/import_cucloud_catalog.go
|
||||
run_or_fail "mobile_cloud_catalog" "移动云目录校验失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/catalog_verification_common.go ./scripts/import_mobile_cloud_catalog.go
|
||||
run_or_fail "tencent_subscription" "腾讯云套餐导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/tencent_catalog_lib.go ./scripts/import_tencent_subscription.go
|
||||
run_or_fail "youdao_pricing" "网易有道价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/youdao_pricing_lib.go ./scripts/import_youdao_pricing.go
|
||||
run_or_fail "platform360_pricing" "360 智脑价格导入失败" \
|
||||
@@ -171,6 +173,12 @@ run_or_fail "bedrock_pricing" "Amazon Bedrock 价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/bedrock_pricing_lib.go ./scripts/import_bedrock_pricing.go
|
||||
run_or_fail "azure_openai_pricing" "Azure OpenAI 价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/azure_openai_pricing_lib.go ./scripts/import_azure_openai_pricing.go
|
||||
run_or_fail "qwen_pricing" "通义千问价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_qwen_pricing.go
|
||||
run_or_fail "hunyuan_pricing" "腾讯混元价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_hunyuan_pricing.go
|
||||
run_or_fail "huawei_maas_pricing" "华为云 MaaS 价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_huawei_maas_pricing.go
|
||||
|
||||
refresh_pipeline_audit
|
||||
run_or_fail "catalog_seed_verification" "目录级官方入口核验失败" \
|
||||
|
||||
@@ -28,7 +28,7 @@ REPORT_DATE="$(report_date_value)"
|
||||
FETCH_OUT="$ROOT_DIR/models.json"
|
||||
FETCH_TOTAL="0"
|
||||
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,catalog_seed_verification"
|
||||
PIPELINE_FAILED_SOURCE_SET="none"
|
||||
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
|
||||
PIPELINE_AUDIT_SUMMARY=""
|
||||
@@ -194,6 +194,11 @@ if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./script
|
||||
record_failure "移动云目录校验失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/tencent_catalog_lib.go" "./scripts/import_tencent_subscription.go"; then
|
||||
merge_failed_source_keys "tencent_subscription"
|
||||
record_failure "腾讯云套餐导入失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/youdao_pricing_lib.go" "./scripts/import_youdao_pricing.go"; then
|
||||
merge_failed_source_keys "youdao_pricing"
|
||||
record_failure "网易有道价格导入失败"
|
||||
@@ -264,6 +269,21 @@ if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./script
|
||||
record_failure "Azure OpenAI 价格导入失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_qwen_pricing.go"; then
|
||||
merge_failed_source_keys "qwen_pricing"
|
||||
record_failure "通义千问价格导入失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_hunyuan_pricing.go"; then
|
||||
merge_failed_source_keys "hunyuan_pricing"
|
||||
record_failure "腾讯混元价格导入失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_huawei_maas_pricing.go"; then
|
||||
merge_failed_source_keys "huawei_maas_pricing"
|
||||
record_failure "华为云 MaaS 价格导入失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/import_catalog_seed_verification.go"; then
|
||||
merge_failed_source_keys "catalog_seed_verification"
|
||||
record_failure "目录级官方入口核验失败"
|
||||
|
||||
@@ -106,10 +106,16 @@ func parseTencentCatalog(raw string) (tencentCatalog, error) {
|
||||
}
|
||||
|
||||
switch line {
|
||||
case "### 套餐详情":
|
||||
case "### 套餐详情", "套餐详情":
|
||||
if currentSeries == "" {
|
||||
continue
|
||||
}
|
||||
currentMode = "plans"
|
||||
continue
|
||||
case "### 可用模型":
|
||||
case "### 可用模型", "可用模型":
|
||||
if currentSeries == "" {
|
||||
continue
|
||||
}
|
||||
currentMode = "models"
|
||||
continue
|
||||
}
|
||||
@@ -159,7 +165,7 @@ func normalizeTencentCatalogLines(raw string) []string {
|
||||
rawLines := strings.Split(text, "\n")
|
||||
lines := make([]string, 0, len(rawLines))
|
||||
for _, rawLine := range rawLines {
|
||||
line := strings.TrimSpace(rawLine)
|
||||
line := strings.Trim(strings.TrimSpace(rawLine), "\uFEFF")
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
@@ -178,6 +184,13 @@ func extractUpdatedAt(line string) string {
|
||||
|
||||
func extractSeriesHeading(line string) string {
|
||||
if !strings.HasPrefix(line, "## ") {
|
||||
trimmed := strings.Trim(line, "\uFEFF ")
|
||||
switch trimmed {
|
||||
case "通用 Token Plan 套餐":
|
||||
return "通用 Token Plan"
|
||||
case "Hy Token Plan 套餐":
|
||||
return "Hy Token Plan"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
series := strings.TrimSpace(strings.TrimPrefix(line, "## "))
|
||||
|
||||
74
scripts/testdata/huawei_maas_pricing_sample.json
vendored
Normal file
74
scripts/testdata/huawei_maas_pricing_sample.json
vendored
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"product": {
|
||||
"modelarts_modelarts.tokens": [
|
||||
{
|
||||
"resourceSpecCode": "modelarts.tokens.deepseek.v4.pro",
|
||||
"resourceSpecType": "DeepSeek",
|
||||
"Model Name": "deepseek-v4-pro",
|
||||
"planList": [{"usageFactor": "input", "amount": 0.012}],
|
||||
"tableUnit": "detail_6_"
|
||||
},
|
||||
{
|
||||
"resourceSpecCode": "modelarts.tokens.deepseek.v4.pro",
|
||||
"resourceSpecType": "DeepSeek",
|
||||
"Model Name": "deepseek-v4-pro",
|
||||
"planList": [{"usageFactor": "output", "amount": 0.024}],
|
||||
"tableUnit": "detail_6_"
|
||||
},
|
||||
{
|
||||
"resourceSpecCode": "modelarts.tokens.qwen3-32b",
|
||||
"resourceSpecType": "Qwen",
|
||||
"Model Name": "qwen3-32b",
|
||||
"planList": [{"usageFactor": "input", "amount": 0.002}],
|
||||
"tableUnit": "detail_6_"
|
||||
},
|
||||
{
|
||||
"resourceSpecCode": "modelarts.tokens.qwen3-32b",
|
||||
"resourceSpecType": "Qwen",
|
||||
"Model Name": "qwen3-32b",
|
||||
"planList": [{"usageFactor": "output", "amount": 0.008}],
|
||||
"tableUnit": "detail_6_"
|
||||
},
|
||||
{
|
||||
"resourceSpecCode": "modelarts.tokens.qwen3-32b",
|
||||
"resourceSpecType": "Qwen",
|
||||
"Model Name": "qwen3-32b",
|
||||
"planList": [{"usageFactor": "output_with_think", "amount": 0.02}],
|
||||
"tableUnit": "detail_6_"
|
||||
},
|
||||
{
|
||||
"resourceSpecCode": "modelarts.tokens.glm.5",
|
||||
"resourceSpecType": "GLM",
|
||||
"Model Name": "glm-5",
|
||||
"planList": [{"usageFactor": "input_token_interval_1", "amount": 0.004}],
|
||||
"tableUnit": "detail_6_"
|
||||
},
|
||||
{
|
||||
"resourceSpecCode": "modelarts.tokens.glm.5",
|
||||
"resourceSpecType": "GLM",
|
||||
"Model Name": "glm-5",
|
||||
"planList": [{"usageFactor": "input_token_interval_2", "amount": 0.006}],
|
||||
"tableUnit": "detail_6_"
|
||||
},
|
||||
{
|
||||
"resourceSpecCode": "modelarts.tokens.glm.5",
|
||||
"resourceSpecType": "GLM",
|
||||
"Model Name": "glm-5",
|
||||
"planList": [{"usageFactor": "output_token_interval_1", "amount": 0.018}],
|
||||
"tableUnit": "detail_6_"
|
||||
},
|
||||
{
|
||||
"resourceSpecCode": "modelarts.tokens.glm.5",
|
||||
"resourceSpecType": "GLM",
|
||||
"Model Name": "glm-5",
|
||||
"planList": [{"usageFactor": "output_token_interval_2", "amount": 0.022}],
|
||||
"tableUnit": "detail_6_"
|
||||
}
|
||||
]
|
||||
},
|
||||
"period": {},
|
||||
"region": {},
|
||||
"tag": {},
|
||||
"urlPath": "maas",
|
||||
"tab": {}
|
||||
}
|
||||
21
scripts/testdata/hunyuan_pricing_sample.txt
vendored
Normal file
21
scripts/testdata/hunyuan_pricing_sample.txt
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
混元生文价格说明
|
||||
token 后付费
|
||||
在免费额度用完后,按如下价格进行后付费计费。
|
||||
产品名
|
||||
输入长度 ≤32k tokens
|
||||
输入长度 (32k, 128k] tokens
|
||||
Tencent HY 2.0 Think
|
||||
输入:4.13元
|
||||
输出:16.58元
|
||||
Tencent HY 2.0 Instruct
|
||||
输入:0.57元
|
||||
输出:2.27元
|
||||
Hunyuan-T1
|
||||
输入:1元
|
||||
输出:4元
|
||||
Hunyuan-TurboS
|
||||
输入:0.8元
|
||||
输出:2元
|
||||
hunyuan-large-role
|
||||
输入:0.24元
|
||||
输出:0.96元
|
||||
38
scripts/testdata/qwen_pricing_sample.txt
vendored
Normal file
38
scripts/testdata/qwen_pricing_sample.txt
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
更新时间:2026-05-22
|
||||
模型调用计费
|
||||
文本生成-千问
|
||||
模型名称
|
||||
输入单价(每百万 Token)
|
||||
输出单价(每百万 Token)
|
||||
免费额度 (注)
|
||||
qwen-max
|
||||
当前能力等同于 qwen-max-2024-09-19 Batch 调用 半价
|
||||
仅非思考模式
|
||||
无阶梯计价
|
||||
2.4 元
|
||||
9.6 元
|
||||
各 100 万 Token
|
||||
qwen-plus
|
||||
当前能力等同于 qwen-plus-2025-12-01 Batch 调用 半价
|
||||
0<Token≤128K
|
||||
0.8 元
|
||||
2 元
|
||||
8 元
|
||||
128K<Token≤256K
|
||||
2.4 元
|
||||
20 元
|
||||
qwen-vl-max
|
||||
当前能力等同于 qwen-vl-max-2025-08-13 Batch 调用 半价 上下文缓存 享有折扣
|
||||
无阶梯计价
|
||||
1.6 元
|
||||
4 元
|
||||
各 100 万 Token
|
||||
qwen3-coder-plus
|
||||
当前能力等同于 qwen3-coder-plus-2025-09-23 上下文缓存 享有折扣
|
||||
0<Token≤32K
|
||||
4 元
|
||||
16 元
|
||||
各 100 万 Token
|
||||
32K<Token≤128K
|
||||
6 元
|
||||
24 元
|
||||
@@ -8,6 +8,10 @@ cd "$ROOT_DIR"
|
||||
CORESHUB_FIXTURE_PATH="${CORESHUB_FIXTURE_PATH:-./scripts/testdata/coreshub_pricing_sample.txt}"
|
||||
CTYUN_CODING_FIXTURE_PATH="${CTYUN_CODING_FIXTURE_PATH:-./scripts/testdata/ctyun_coding_plan_sample.txt}"
|
||||
CTYUN_TOKEN_FIXTURE_PATH="${CTYUN_TOKEN_FIXTURE_PATH:-./scripts/testdata/ctyun_token_plan_sample.txt}"
|
||||
TENCENT_FIXTURE_PATH="${TENCENT_FIXTURE_PATH:-./scripts/testdata/tencent_token_plan_sample.txt}"
|
||||
QWEN_FIXTURE_PATH="${QWEN_FIXTURE_PATH:-./scripts/testdata/qwen_pricing_sample.txt}"
|
||||
HUNYUAN_FIXTURE_PATH="${HUNYUAN_FIXTURE_PATH:-./scripts/testdata/hunyuan_pricing_sample.txt}"
|
||||
HUAWEI_MAAS_FIXTURE_PATH="${HUAWEI_MAAS_FIXTURE_PATH:-./scripts/testdata/huawei_maas_pricing_sample.json}"
|
||||
|
||||
last_meaningful_line() {
|
||||
awk 'NF && $0 !~ /^exit status [0-9]+$/ { line=$0 } END { print line }'
|
||||
@@ -38,5 +42,13 @@ run_smoke "coreshub-fixture" "go run -tags llm_script ./scripts/subscription_imp
|
||||
run_smoke "coreshub-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/coreshub_pricing_lib.go ./scripts/import_coreshub_pricing.go -dry-run"
|
||||
run_smoke "ctyun-fixture" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/ctyun_subscription_lib.go ./scripts/import_ctyun_subscription.go -coding-fixture ${CTYUN_CODING_FIXTURE_PATH@Q} -token-fixture ${CTYUN_TOKEN_FIXTURE_PATH@Q} -dry-run"
|
||||
run_smoke "ctyun-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/ctyun_subscription_lib.go ./scripts/import_ctyun_subscription.go -dry-run"
|
||||
run_smoke "tencent-fixture" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/tencent_catalog_lib.go ./scripts/import_tencent_subscription.go -fixture ${TENCENT_FIXTURE_PATH@Q} -dry-run"
|
||||
run_smoke "tencent-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/tencent_catalog_lib.go ./scripts/import_tencent_subscription.go -dry-run"
|
||||
run_smoke "qwen-fixture" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_qwen_pricing.go -fixture ${QWEN_FIXTURE_PATH@Q} -dry-run"
|
||||
run_smoke "qwen-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_qwen_pricing.go -dry-run"
|
||||
run_smoke "hunyuan-fixture" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_hunyuan_pricing.go -fixture ${HUNYUAN_FIXTURE_PATH@Q} -dry-run"
|
||||
run_smoke "hunyuan-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_hunyuan_pricing.go -dry-run"
|
||||
run_smoke "huawei-maas-fixture" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_huawei_maas_pricing.go -fixture ${HUAWEI_MAAS_FIXTURE_PATH@Q} -dry-run"
|
||||
run_smoke "huawei-maas-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_huawei_maas_pricing.go -dry-run"
|
||||
|
||||
echo "IMPORTER_SMOKE_RESULT: PASS"
|
||||
|
||||
@@ -347,7 +347,7 @@
|
||||
"region": "CN",
|
||||
"currency": "CNY",
|
||||
"billingCycle": "usage",
|
||||
"importerKey": "import_youdao_pricing.go",
|
||||
"importerKey": "import_catalog_seed_verification.go",
|
||||
"notes": "魔搭社区提供统一模型推理网关。",
|
||||
"catalogSegment": "relay_top20plus",
|
||||
"marketRank": 8
|
||||
@@ -401,7 +401,7 @@
|
||||
"region": "CN",
|
||||
"currency": "CNY",
|
||||
"billingCycle": "usage",
|
||||
"importerKey": "import_360_pricing.go",
|
||||
"importerKey": "import_catalog_seed_verification.go",
|
||||
"notes": "与 Token Plan 并行提供按量计费。",
|
||||
"catalogSegment": "relay_top20plus",
|
||||
"marketRank": 9
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"region": "CN",
|
||||
"currency": "CNY",
|
||||
"billingCycle": "usage",
|
||||
"importerKey": "import_catalog_seed_verification.go",
|
||||
"importerKey": "import_qwen_pricing.go",
|
||||
"notes": "按量计费为主,套餐型信息单独由百炼 Token Plan/Coding Plan 行承载。",
|
||||
"catalogSegment": "vendor_top20",
|
||||
"marketRank": 1
|
||||
@@ -50,7 +50,7 @@
|
||||
"region": "CN",
|
||||
"currency": "CNY",
|
||||
"billingCycle": "usage",
|
||||
"importerKey": "import_catalog_seed_verification.go",
|
||||
"importerKey": "import_hunyuan_pricing.go",
|
||||
"notes": "腾讯官方模型能力入口,订阅套餐另走腾讯云 TokenHub。",
|
||||
"catalogSegment": "vendor_top20",
|
||||
"marketRank": 2
|
||||
@@ -158,8 +158,8 @@
|
||||
"region": "CN",
|
||||
"currency": "CNY",
|
||||
"billingCycle": "usage",
|
||||
"importerKey": "import_catalog_seed_verification.go",
|
||||
"notes": "当前公开能力同时存在按量计费与资源包计费两类入口。",
|
||||
"importerKey": "import_huawei_maas_pricing.go",
|
||||
"notes": "当前公开按量价格页实际覆盖华为云 MaaS 文本生成模型集合,未见独立盘古 SKU 公开单价。",
|
||||
"catalogSegment": "vendor_top20",
|
||||
"marketRank": 6
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user