feat(imports): add real pricing and subscription collectors
Add plan catalog and subscription schema support, seed baselines, and real importers for core domestic subscriptions plus stable official pricing sources. This commit also hardens the shared fetch layers so the importers can support live collection and database writes instead of relying on manual placeholders alone.
This commit is contained in:
179
scripts/aliyun_subscription_lib.go
Normal file
179
scripts/aliyun_subscription_lib.go
Normal file
@@ -0,0 +1,179 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultAliyunTokenPlanURL = "https://help.aliyun.com/zh/model-studio/token-plan-overview"
|
||||
defaultAliyunCodingPlanURL = "https://help.aliyun.com/zh/model-studio/coding-plan-quickstart"
|
||||
)
|
||||
|
||||
func parseAliyunSubscriptionCatalog(tokenRaw string, codingRaw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(firstNonEmptyText(tokenRaw, codingRaw))
|
||||
|
||||
tokenRecords, err := parseAliyunTokenPlan(tokenRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
codingRecords, err := parseAliyunCodingPlan(codingRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records := append(tokenRecords, codingRecords...)
|
||||
for i := range records {
|
||||
records[i].PublishedAtKnown = known
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseAliyunTokenPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
seatPattern := regexp.MustCompile(`(?s)(标准坐席|高级坐席|尊享坐席)\s+¥([\d,]+)(?:/坐席/月)?\s+([\d,]+)\s*Credits/坐席/月\s+([^\n]+)`)
|
||||
matches := seatPattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) != 3 {
|
||||
return nil, fmt.Errorf("unexpected aliyun token seat count: %d", len(matches))
|
||||
}
|
||||
|
||||
tierCodes := map[string]string{
|
||||
"标准坐席": "standard-seat",
|
||||
"高级坐席": "advanced-seat",
|
||||
"尊享坐席": "premium-seat",
|
||||
}
|
||||
tierNames := map[string]string{
|
||||
"标准坐席": "Standard",
|
||||
"高级坐席": "Advanced",
|
||||
"尊享坐席": "Premium",
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, 4)
|
||||
for _, match := range matches {
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Alibaba",
|
||||
ProviderNameCn: "阿里巴巴",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.aliyun.com",
|
||||
OperatorName: "Alibaba Bailian",
|
||||
OperatorNameCn: "阿里云百炼",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://help.aliyun.com/zh/model-studio/",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "aliyun-token-plan-" + tierCodes[match[1]],
|
||||
PlanName: "Token Plan 团队版 " + match[1],
|
||||
Tier: tierNames[match[1]],
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(match[2]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: mustParseSubscriptionInt64(match[3]),
|
||||
QuotaUnit: "credits/month",
|
||||
PlanScope: "Token Plan 团队版",
|
||||
SourceURL: defaultAliyunTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: strings.TrimSpace(match[4]),
|
||||
})
|
||||
}
|
||||
|
||||
sharedPattern := regexp.MustCompile(`共享用量包\s+¥([\d,]+)(?:/个)?\s+([\d,]+)\s*Credits/个`)
|
||||
shared := sharedPattern.FindStringSubmatch(raw)
|
||||
if len(shared) != 3 {
|
||||
return nil, fmt.Errorf("aliyun shared pack not found")
|
||||
}
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Alibaba",
|
||||
ProviderNameCn: "阿里巴巴",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.aliyun.com",
|
||||
OperatorName: "Alibaba Bailian",
|
||||
OperatorNameCn: "阿里云百炼",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://help.aliyun.com/zh/model-studio/",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "aliyun-token-plan-shared-pack",
|
||||
PlanName: "Token Plan 团队版 共享用量包",
|
||||
Tier: "SharedPack",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(shared[1]),
|
||||
PriceUnit: "CNY/pack",
|
||||
QuotaValue: mustParseSubscriptionInt64(shared[2]),
|
||||
QuotaUnit: "credits/pack",
|
||||
PlanScope: "Token Plan 团队版",
|
||||
SourceURL: defaultAliyunTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "跨坐席共享的弹性用量包,有效期 1 个月。",
|
||||
})
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseAliyunCodingPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
pricePattern := regexp.MustCompile(`价格\s+¥\s*([\d,]+)\s*/月`)
|
||||
priceMatch := pricePattern.FindStringSubmatch(raw)
|
||||
if len(priceMatch) != 2 {
|
||||
return nil, fmt.Errorf("aliyun coding plan price not found")
|
||||
}
|
||||
|
||||
modelsPattern := regexp.MustCompile(`支持的模型\s+\|\s+推荐模型:([^\n]+)`)
|
||||
modelsMatch := modelsPattern.FindStringSubmatch(raw)
|
||||
modelScope := []string{}
|
||||
if len(modelsMatch) == 2 {
|
||||
for _, item := range strings.Split(modelsMatch[1], "、") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
modelScope = append(modelScope, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
limitPattern := regexp.MustCompile(`每月\s*([\d,]+)\s*次请求`)
|
||||
limitMatch := limitPattern.FindStringSubmatch(raw)
|
||||
quotaValue := int64(0)
|
||||
if len(limitMatch) == 2 {
|
||||
quotaValue = mustParseSubscriptionInt64(limitMatch[1])
|
||||
}
|
||||
|
||||
notes := []string{}
|
||||
for _, fragment := range []string{
|
||||
"Lite 套餐自 2026 年 3 月 20 日 00:00:00(UTC+08:00)起停止新购",
|
||||
"活动已结束",
|
||||
} {
|
||||
if strings.Contains(raw, fragment) {
|
||||
notes = append(notes, fragment)
|
||||
}
|
||||
}
|
||||
|
||||
return []subscriptionImportRecord{{
|
||||
ProviderName: "Alibaba",
|
||||
ProviderNameCn: "阿里巴巴",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.aliyun.com",
|
||||
OperatorName: "Alibaba Bailian",
|
||||
OperatorNameCn: "阿里云百炼",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://help.aliyun.com/zh/model-studio/",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "aliyun-coding-plan-pro",
|
||||
PlanName: "百炼 Coding Plan Pro",
|
||||
Tier: "Pro",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[1]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: quotaValue,
|
||||
QuotaUnit: "requests/month",
|
||||
PlanScope: "Coding Plan",
|
||||
ModelScope: modelScope,
|
||||
SourceURL: defaultAliyunCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: strings.Join(notes, ";"),
|
||||
}}, nil
|
||||
}
|
||||
225
scripts/azure_openai_pricing_lib.go
Normal file
225
scripts/azure_openai_pricing_lib.go
Normal file
@@ -0,0 +1,225 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultAzureOpenAIPricingURL = "https://prices.azure.com/api/retail/prices?api-version=2023-01-01-preview¤cyCode='USD'&$filter=contains(productName,'OpenAI')"
|
||||
|
||||
type azureRetailPriceResponse struct {
|
||||
Items []azureRetailPriceItem `json:"Items"`
|
||||
NextPageLink string `json:"NextPageLink"`
|
||||
}
|
||||
|
||||
type azureRetailPriceItem struct {
|
||||
CurrencyCode string `json:"currencyCode"`
|
||||
RetailPrice float64 `json:"retailPrice"`
|
||||
UnitPrice float64 `json:"unitPrice"`
|
||||
Location string `json:"location"`
|
||||
MeterName string `json:"meterName"`
|
||||
ProductName string `json:"productName"`
|
||||
SkuName string `json:"skuName"`
|
||||
ServiceName string `json:"serviceName"`
|
||||
UnitOfMeasure string `json:"unitOfMeasure"`
|
||||
Type string `json:"type"`
|
||||
ArmSkuName string `json:"armSkuName"`
|
||||
ArmRegionName string `json:"armRegionName"`
|
||||
IsPrimaryMeter bool `json:"isPrimaryMeterRegion"`
|
||||
}
|
||||
|
||||
type azurePricingPair struct {
|
||||
ModelName string
|
||||
Region string
|
||||
Currency string
|
||||
InputPrice float64
|
||||
OutputPrice float64
|
||||
}
|
||||
|
||||
var azureKindPattern = regexp.MustCompile(`(?i)\b(inp|inpt|input|out|outp|outpt|output|opt)\b`)
|
||||
|
||||
func fetchAzureOpenAIPricingCatalog(url string, fixture string, client *http.Client) (string, error) {
|
||||
if strings.TrimSpace(fixture) != "" {
|
||||
return fetchRawPricingPage(url, fixture, client)
|
||||
}
|
||||
|
||||
aggregated := azureRetailPriceResponse{}
|
||||
seenPages := map[string]struct{}{}
|
||||
nextURL := url
|
||||
for strings.TrimSpace(nextURL) != "" {
|
||||
if _, exists := seenPages[nextURL]; exists {
|
||||
return "", fmt.Errorf("azure retail pricing pagination loop detected: %s", nextURL)
|
||||
}
|
||||
seenPages[nextURL] = struct{}{}
|
||||
|
||||
raw, err := fetchRawPricingPage(nextURL, "", client)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var page azureRetailPriceResponse
|
||||
if err := json.Unmarshal([]byte(raw), &page); err != nil {
|
||||
return "", fmt.Errorf("unmarshal azure retail pricing page: %w", err)
|
||||
}
|
||||
aggregated.Items = append(aggregated.Items, page.Items...)
|
||||
nextURL = page.NextPageLink
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(aggregated)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal azure retail pricing aggregate: %w", err)
|
||||
}
|
||||
return string(payload), nil
|
||||
}
|
||||
|
||||
func parseAzureOpenAIPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
var response azureRetailPriceResponse
|
||||
if err := json.Unmarshal([]byte(raw), &response); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal azure retail pricing: %w", err)
|
||||
}
|
||||
pairs := make(map[string]*azurePricingPair)
|
||||
for _, item := range response.Items {
|
||||
kind, modelName, ok := classifyAzureRetailPrice(item)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
region := strings.TrimSpace(item.Location)
|
||||
if region == "" {
|
||||
region = "global"
|
||||
}
|
||||
currency := strings.TrimSpace(item.CurrencyCode)
|
||||
if currency == "" {
|
||||
currency = "USD"
|
||||
}
|
||||
key := strings.Join([]string{modelName, region, currency}, "|")
|
||||
pair := pairs[key]
|
||||
if pair == nil {
|
||||
pair = &azurePricingPair{
|
||||
ModelName: modelName,
|
||||
Region: region,
|
||||
Currency: currency,
|
||||
}
|
||||
pairs[key] = pair
|
||||
}
|
||||
price := item.UnitPrice
|
||||
if strings.EqualFold(strings.TrimSpace(item.UnitOfMeasure), "1K") {
|
||||
price *= 1000
|
||||
}
|
||||
if kind == "input" {
|
||||
pair.InputPrice = price
|
||||
} else {
|
||||
pair.OutputPrice = price
|
||||
}
|
||||
}
|
||||
|
||||
records := make([]officialPricingRecord, 0, len(pairs))
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata("OpenAI")
|
||||
for _, pair := range pairs {
|
||||
if pair.InputPrice == 0 || pair.OutputPrice == 0 {
|
||||
continue
|
||||
}
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("azure-openai", pair.ModelName),
|
||||
ModelName: pair.ModelName,
|
||||
ProviderName: "OpenAI",
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Microsoft Azure",
|
||||
OperatorNameCn: "微软 Azure",
|
||||
OperatorCountry: "US",
|
||||
OperatorWebsite: "https://azure.microsoft.com",
|
||||
OperatorType: "cloud",
|
||||
Region: pair.Region,
|
||||
Currency: pair.Currency,
|
||||
InputPrice: pair.InputPrice,
|
||||
OutputPrice: pair.OutputPrice,
|
||||
SourceURL: defaultAzureOpenAIPricingURL,
|
||||
ModelSourceURL: defaultAzureOpenAIPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(pair.ModelName),
|
||||
}
|
||||
record.IsFree = false
|
||||
records = append(records, record)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no azure openai token prices found")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func classifyAzureRetailPrice(item azureRetailPriceItem) (string, string, bool) {
|
||||
if item.ServiceName != "Foundry Models" || item.Type != "Consumption" {
|
||||
return "", "", false
|
||||
}
|
||||
productLower := strings.ToLower(item.ProductName)
|
||||
if !strings.Contains(productLower, "openai") || strings.Contains(productLower, "media") {
|
||||
return "", "", false
|
||||
}
|
||||
name := strings.ToLower(strings.TrimSpace(strings.Join([]string{item.SkuName, item.MeterName, item.ArmSkuName}, " ")))
|
||||
if !azureKindPattern.MatchString(name) {
|
||||
return "", "", false
|
||||
}
|
||||
for _, blocked := range []string{
|
||||
"batch",
|
||||
"cache",
|
||||
"cchd",
|
||||
"prty",
|
||||
" pp ",
|
||||
"hosting",
|
||||
"training",
|
||||
" ft ",
|
||||
"ft ",
|
||||
" mdl ",
|
||||
"grdr",
|
||||
"file-search",
|
||||
"code-interpreter",
|
||||
"session",
|
||||
"transcribe",
|
||||
" aud ",
|
||||
"audio",
|
||||
" img ",
|
||||
"image",
|
||||
"voice",
|
||||
"rt ",
|
||||
"realtime",
|
||||
"tool",
|
||||
} {
|
||||
if strings.Contains(name, blocked) {
|
||||
return "", "", false
|
||||
}
|
||||
}
|
||||
kind := "output"
|
||||
if strings.Contains(name, "inp") || strings.Contains(name, "input") || strings.Contains(name, "inpt") {
|
||||
kind = "input"
|
||||
}
|
||||
modelName := normalizeAzureModelName(item)
|
||||
if modelName == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return kind, modelName, true
|
||||
}
|
||||
|
||||
func normalizeAzureModelName(item azureRetailPriceItem) string {
|
||||
base := strings.ToLower(strings.TrimSpace(item.MeterName))
|
||||
replacer := strings.NewReplacer("-", " ", ".", ".", "_", " ")
|
||||
base = replacer.Replace(base)
|
||||
base = regexp.MustCompile(`(?i)\s+(inp|inpt|input|out|outp|outpt|output|opt)\b.*$`).ReplaceAllString(base, "")
|
||||
base = strings.TrimSpace(base)
|
||||
if base == "" {
|
||||
return ""
|
||||
}
|
||||
if regexp.MustCompile(`^\d`).MatchString(base) {
|
||||
base = "gpt " + base
|
||||
}
|
||||
base = regexp.MustCompile(`\s+`).ReplaceAllString(base, " ")
|
||||
if strings.HasPrefix(base, "gpt ") {
|
||||
return "GPT-" + strings.TrimSpace(strings.TrimPrefix(base, "gpt "))
|
||||
}
|
||||
return strings.ToUpper(base[:1]) + base[1:]
|
||||
}
|
||||
113
scripts/baidu_subscription_lib.go
Normal file
113
scripts/baidu_subscription_lib.go
Normal file
@@ -0,0 +1,113 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBaiduCodingPlanURL = "https://cloud.baidu.com/doc/qianfan/s/imlg0beiu"
|
||||
defaultBaiduTokenPlanURL = "https://cloud.baidu.com/doc/qianfan/s/Smoghsq3g"
|
||||
)
|
||||
|
||||
func parseBaiduSubscriptionCatalog(codingRaw string, tokenRaw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(firstNonEmptyText(codingRaw, tokenRaw))
|
||||
codingRecords, err := parseBaiduCodingPlan(codingRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tokenRecords, err := parseBaiduTokenBenefitPack(tokenRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records := append(codingRecords, tokenRecords...)
|
||||
for i := range records {
|
||||
records[i].PublishedAtKnown = known
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseBaiduCodingPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
pattern := regexp.MustCompile(`Coding Plan (Lite|Pro)\s+¥\s*([\d,]+)\s*/\s*月\s+每 5 小时:最多约 [\d,]+ 次请求\s+每周:最多约 [\d,]+ 次请求\s+每订阅月:最多约 ([\d,]+) 次请求`)
|
||||
matches := pattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) != 2 {
|
||||
return nil, fmt.Errorf("unexpected baidu coding plan count: %d", len(matches))
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
tier := match[1]
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Baidu",
|
||||
ProviderNameCn: "百度",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://cloud.baidu.com",
|
||||
OperatorName: "Baidu Qianfan",
|
||||
OperatorNameCn: "百度千帆",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://cloud.baidu.com/doc/qianfan/index.html",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "baidu-coding-plan-" + strings.ToLower(tier),
|
||||
PlanName: "千帆 Coding Plan " + tier,
|
||||
Tier: tier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(match[2]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: mustParseSubscriptionInt64(match[3]),
|
||||
QuotaUnit: "requests/month",
|
||||
PlanScope: "Coding Plan",
|
||||
SourceURL: defaultBaiduCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "额度按 5 小时、每周、每订阅月三重窗口刷新。",
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseBaiduTokenBenefitPack(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
pattern := regexp.MustCompile(`(\d{2,3},\d{3})\s+1个月\s+¥(\d+)\s+¥(\d+)`)
|
||||
matches := pattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) != 5 {
|
||||
return nil, fmt.Errorf("unexpected baidu token benefit pack count: %d", len(matches))
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
quota := mustParseSubscriptionInt64(match[1])
|
||||
originalPrice := strings.TrimSpace(match[2])
|
||||
promoPrice := strings.TrimSpace(match[3])
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Baidu",
|
||||
ProviderNameCn: "百度",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://cloud.baidu.com",
|
||||
OperatorName: "Baidu Qianfan",
|
||||
OperatorNameCn: "百度千帆",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://cloud.baidu.com/doc/qianfan/index.html",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: fmt.Sprintf("baidu-token-benefit-pack-%d", quota),
|
||||
PlanName: fmt.Sprintf("千帆 Token 福利包 %d", quota),
|
||||
Tier: fmt.Sprintf("%d", quota),
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(promoPrice),
|
||||
PriceUnit: "CNY/pack",
|
||||
QuotaValue: quota,
|
||||
QuotaUnit: "credits/pack",
|
||||
PlanScope: "Token 福利包",
|
||||
SourceURL: defaultBaiduTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: fmt.Sprintf("首购优惠价 ¥%s,原价 ¥%s,有效期 1 个月。", promoPrice, originalPrice),
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
323
scripts/bedrock_pricing_lib.go
Normal file
323
scripts/bedrock_pricing_lib.go
Normal file
@@ -0,0 +1,323 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultBedrockPricingURL = "https://aws.amazon.com/bedrock/pricing/"
|
||||
|
||||
var (
|
||||
bedrockRegionPattern = regexp.MustCompile(`(?s)<p><b>Regions?: ([^<]+)</b></p>`)
|
||||
bedrockTablePattern = regexp.MustCompile(`(?s)<table[^>]*>(.*?)</table>`)
|
||||
bedrockRowPattern = regexp.MustCompile(`(?s)<tr>(.*?)</tr>`)
|
||||
bedrockCellPattern = regexp.MustCompile(`(?s)<t[dh][^>]*>(.*?)</t[dh]>`)
|
||||
)
|
||||
|
||||
func parseBedrockPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
section := extractBetween(raw, `<h3 id="Model_Pricing"`, `<h2 id="Pricing_examples"`)
|
||||
if strings.TrimSpace(section) == "" {
|
||||
section = raw
|
||||
}
|
||||
|
||||
blocks := splitBedrockProviderBlocks(section)
|
||||
records := make([]officialPricingRecord, 0)
|
||||
for _, block := range blocks {
|
||||
records = append(records, parseBedrockProviderBlock(block.providerLabel, block.content)...)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
records = append(records, parseBedrockPricingTextFallback(cleanHTMLText(section))...)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no bedrock pricing rows found")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseBedrockProviderBlock(providerLabel string, raw string) []officialPricingRecord {
|
||||
providerName := normalizeBedrockProvider(providerLabel)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
|
||||
regionMatches := bedrockRegionPattern.FindAllStringSubmatchIndex(raw, -1)
|
||||
tables := bedrockTablePattern.FindAllStringSubmatchIndex(raw, -1)
|
||||
records := make([]officialPricingRecord, 0)
|
||||
seenModelRegion := make(map[string]struct{})
|
||||
for _, tableIndex := range tables {
|
||||
tableHTML := raw[tableIndex[2]:tableIndex[3]]
|
||||
if !strings.Contains(tableHTML, "Price per 1M input tokens") || !strings.Contains(tableHTML, "$") {
|
||||
continue
|
||||
}
|
||||
region := "global"
|
||||
for _, regionIndex := range regionMatches {
|
||||
if regionIndex[0] < tableIndex[0] {
|
||||
region = cleanHTMLText(raw[regionIndex[2]:regionIndex[3]])
|
||||
}
|
||||
}
|
||||
rows := parseBedrockTableRows(tableHTML)
|
||||
for _, row := range rows {
|
||||
dedupeKey := strings.Join([]string{region, row.ModelName}, "|")
|
||||
if _, exists := seenModelRegion[dedupeKey]; exists {
|
||||
continue
|
||||
}
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("bedrock", providerName, row.ModelName),
|
||||
ModelName: row.ModelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Amazon Bedrock",
|
||||
OperatorNameCn: "Amazon Bedrock",
|
||||
OperatorCountry: "US",
|
||||
OperatorWebsite: "https://aws.amazon.com/bedrock/",
|
||||
OperatorType: "cloud",
|
||||
Region: region,
|
||||
Currency: "USD",
|
||||
InputPrice: row.InputPrice,
|
||||
OutputPrice: row.OutputPrice,
|
||||
SourceURL: defaultBedrockPricingURL,
|
||||
ModelSourceURL: defaultBedrockPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(row.ModelName),
|
||||
}
|
||||
record.IsFree = false
|
||||
seenModelRegion[dedupeKey] = struct{}{}
|
||||
records = append(records, record)
|
||||
}
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
type bedrockProviderBlock struct {
|
||||
providerLabel string
|
||||
content string
|
||||
}
|
||||
|
||||
func splitBedrockProviderBlocks(raw string) []bedrockProviderBlock {
|
||||
marker := `<h2 id="`
|
||||
indices := make([]int, 0)
|
||||
for offset := 0; ; {
|
||||
next := strings.Index(raw[offset:], marker)
|
||||
if next == -1 {
|
||||
break
|
||||
}
|
||||
indices = append(indices, offset+next)
|
||||
offset += next + len(marker)
|
||||
}
|
||||
blocks := make([]bedrockProviderBlock, 0, len(indices))
|
||||
for i, start := range indices {
|
||||
end := len(raw)
|
||||
if i+1 < len(indices) {
|
||||
end = indices[i+1]
|
||||
}
|
||||
chunk := raw[start:end]
|
||||
h2End := strings.Index(chunk, "</h2>")
|
||||
if h2End == -1 {
|
||||
continue
|
||||
}
|
||||
openEnd := strings.Index(chunk, ">")
|
||||
if openEnd == -1 || openEnd >= h2End {
|
||||
continue
|
||||
}
|
||||
label := cleanHTMLText(chunk[openEnd+1 : h2End])
|
||||
if strings.TrimSpace(label) == "" {
|
||||
continue
|
||||
}
|
||||
blocks = append(blocks, bedrockProviderBlock{
|
||||
providerLabel: label,
|
||||
content: chunk,
|
||||
})
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
func extractBetween(raw string, startMarker string, endMarker string) string {
|
||||
start := strings.Index(raw, startMarker)
|
||||
if start == -1 {
|
||||
return ""
|
||||
}
|
||||
segment := raw[start:]
|
||||
if endMarker == "" {
|
||||
return segment
|
||||
}
|
||||
end := strings.Index(segment, endMarker)
|
||||
if end == -1 {
|
||||
return segment
|
||||
}
|
||||
return segment[:end]
|
||||
}
|
||||
|
||||
type bedrockPriceRow struct {
|
||||
ModelName string
|
||||
InputPrice float64
|
||||
OutputPrice float64
|
||||
}
|
||||
|
||||
func parseBedrockTableRows(tableHTML string) []bedrockPriceRow {
|
||||
rows := bedrockRowPattern.FindAllStringSubmatch(tableHTML, -1)
|
||||
parsed := make([]bedrockPriceRow, 0)
|
||||
for _, row := range rows {
|
||||
cells := bedrockCellPattern.FindAllStringSubmatch(row[1], -1)
|
||||
if len(cells) < 3 {
|
||||
continue
|
||||
}
|
||||
values := make([]string, 0, len(cells))
|
||||
for _, cell := range cells {
|
||||
values = append(values, cleanHTMLText(cell[1]))
|
||||
}
|
||||
if strings.Contains(strings.ToLower(values[0]), "models") {
|
||||
continue
|
||||
}
|
||||
|
||||
modelName := values[0]
|
||||
inputCell := values[1]
|
||||
outputCell := values[2]
|
||||
if len(values) >= 6 && strings.Contains(strings.ToLower(values[5]), "$") {
|
||||
outputCell = values[5]
|
||||
}
|
||||
inputPrice, ok := firstDollarPrice(inputCell)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
outputPrice, ok := firstDollarPrice(outputCell)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
parsed = append(parsed, bedrockPriceRow{
|
||||
ModelName: modelName,
|
||||
InputPrice: inputPrice,
|
||||
OutputPrice: outputPrice,
|
||||
})
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func normalizeBedrockProvider(raw string) string {
|
||||
switch strings.TrimSpace(raw) {
|
||||
case "Amazon Nova":
|
||||
return "Amazon"
|
||||
case "Anthropic":
|
||||
return "Anthropic"
|
||||
case "Cohere":
|
||||
return "Cohere"
|
||||
case "DeepSeek":
|
||||
return "DeepSeek"
|
||||
case "Meta":
|
||||
return "Meta"
|
||||
case "Mistral AI":
|
||||
return "Mistral AI"
|
||||
case "Moonshot AI":
|
||||
return "Moonshot AI"
|
||||
case "Kimi":
|
||||
return "Moonshot AI"
|
||||
case "NVIDIA":
|
||||
return "NVIDIA"
|
||||
case "OpenAI OSS Models":
|
||||
return "OpenAI"
|
||||
case "Qwen":
|
||||
return "Qwen"
|
||||
case "Writer":
|
||||
return "Writer"
|
||||
case "Z AI":
|
||||
return "Zhipu AI"
|
||||
default:
|
||||
return strings.TrimSpace(raw)
|
||||
}
|
||||
}
|
||||
|
||||
var bedrockTextProviderHeaderPattern = regexp.MustCompile(`([A-Za-z][A-Za-z0-9 .&-]+)\s+models\s+Pr(?:i)?ce per 1M input tokens`)
|
||||
var bedrockTextRowPattern = regexp.MustCompile(`([A-Za-z0-9 .:+-]+?)\s+\$\s*([0-9.]+)\s+\$\s*([0-9.]+)`)
|
||||
|
||||
func parseBedrockPricingTextFallback(raw string) []officialPricingRecord {
|
||||
matches := bedrockTextProviderHeaderPattern.FindAllStringSubmatchIndex(raw, -1)
|
||||
records := make([]officialPricingRecord, 0)
|
||||
seen := make(map[string]struct{})
|
||||
for i, match := range matches {
|
||||
if len(match) < 4 {
|
||||
continue
|
||||
}
|
||||
start := match[0]
|
||||
end := len(raw)
|
||||
if i+1 < len(matches) {
|
||||
end = matches[i+1][0]
|
||||
}
|
||||
block := raw[start:end]
|
||||
region := normalizeBedrockRegionText(findBedrockTextRegion(raw, start))
|
||||
providerName := normalizeBedrockProvider(raw[match[2]:match[3]])
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
rows := bedrockTextRowPattern.FindAllStringSubmatch(block, -1)
|
||||
for _, row := range rows {
|
||||
if len(row) != 4 {
|
||||
continue
|
||||
}
|
||||
modelName := strings.TrimSpace(row[1])
|
||||
key := strings.Join([]string{providerName, region, modelName}, "|")
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
records = append(records, officialPricingRecord{
|
||||
ModelID: normalizeExternalID("bedrock", providerName, modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Amazon Bedrock",
|
||||
OperatorNameCn: "Amazon Bedrock",
|
||||
OperatorCountry: "US",
|
||||
OperatorWebsite: "https://aws.amazon.com/bedrock/",
|
||||
OperatorType: "cloud",
|
||||
Region: region,
|
||||
Currency: "USD",
|
||||
InputPrice: mustParseSubscriptionPrice(row[2]),
|
||||
OutputPrice: mustParseSubscriptionPrice(row[3]),
|
||||
SourceURL: defaultBedrockPricingURL,
|
||||
ModelSourceURL: defaultBedrockPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(modelName),
|
||||
})
|
||||
}
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func findBedrockTextRegion(raw string, headerStart int) string {
|
||||
prefixStart := headerStart - 300
|
||||
if prefixStart < 0 {
|
||||
prefixStart = 0
|
||||
}
|
||||
prefix := raw[prefixStart:headerStart]
|
||||
lastPlural := strings.LastIndex(prefix, "Regions:")
|
||||
lastSingular := strings.LastIndex(prefix, "Region:")
|
||||
lastIndex := lastPlural
|
||||
marker := "Regions:"
|
||||
if lastSingular > lastIndex {
|
||||
lastIndex = lastSingular
|
||||
marker = "Region:"
|
||||
}
|
||||
if lastIndex == -1 {
|
||||
return ""
|
||||
}
|
||||
region := strings.TrimSpace(prefix[lastIndex+len(marker):])
|
||||
for _, stopMarker := range []string{" Priority ", " Flex ", " Batch ", " models "} {
|
||||
if stop := strings.Index(region, stopMarker); stop != -1 {
|
||||
region = strings.TrimSpace(region[:stop])
|
||||
}
|
||||
}
|
||||
return region
|
||||
}
|
||||
|
||||
func normalizeBedrockRegionText(raw string) string {
|
||||
trimmed := strings.TrimSpace(raw)
|
||||
if trimmed == "" {
|
||||
return "global"
|
||||
}
|
||||
trimmed = strings.TrimSuffix(trimmed, ",")
|
||||
return strings.Join(strings.Fields(trimmed), " ")
|
||||
}
|
||||
119
scripts/bytedance_subscription_lib.go
Normal file
119
scripts/bytedance_subscription_lib.go
Normal file
@@ -0,0 +1,119 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBytedanceCodingPlanURL = "https://developer.volcengine.com/articles/7574419773204004906"
|
||||
defaultBytedanceCodingPlanNotice = "https://developer.volcengine.com/articles/7604465649330749490"
|
||||
)
|
||||
|
||||
func parseBytedanceSubscriptionCatalog(pricingRaw string, noticeRaw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(firstNonEmptyText(pricingRaw, noticeRaw))
|
||||
liteSection := sliceSection(pricingRaw, "Lite 套餐", "Pro 套餐")
|
||||
proSection := sliceSection(pricingRaw, "Pro 套餐", "")
|
||||
if strings.TrimSpace(liteSection) == "" || strings.TrimSpace(proSection) == "" {
|
||||
return nil, fmt.Errorf("unexpected bytedance coding plan sections")
|
||||
}
|
||||
|
||||
promoNote := "新用户首购优惠。"
|
||||
if strings.Contains(noticeRaw, "每日 10:30") {
|
||||
promoNote = "新用户首购优惠,每日 10:30 限量开放。"
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, 4)
|
||||
for _, tierSection := range []struct {
|
||||
Tier string
|
||||
Content string
|
||||
}{
|
||||
{Tier: "Lite", Content: liteSection},
|
||||
{Tier: "Pro", Content: proSection},
|
||||
} {
|
||||
tier := tierSection.Tier
|
||||
section := strings.TrimSpace(tierSection.Content)
|
||||
lines := strings.Split(section, "\n")
|
||||
scene := ""
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" {
|
||||
scene = line
|
||||
break
|
||||
}
|
||||
}
|
||||
pricePattern := regexp.MustCompile(`([\d.]+)\s+元\s+([\d.]+)\s+元/月`)
|
||||
priceMatch := pricePattern.FindStringSubmatch(section)
|
||||
if len(priceMatch) != 3 {
|
||||
return nil, fmt.Errorf("missing bytedance %s prices", tier)
|
||||
}
|
||||
quotaPattern := regexp.MustCompile(`每月约\s+([\d,]+)\s+次请求`)
|
||||
quotaMatch := quotaPattern.FindStringSubmatch(section)
|
||||
if len(quotaMatch) != 2 {
|
||||
return nil, fmt.Errorf("missing bytedance %s monthly quota", tier)
|
||||
}
|
||||
promoPrice := mustParseSubscriptionPrice(priceMatch[1])
|
||||
standardPrice := mustParseSubscriptionPrice(priceMatch[2])
|
||||
monthlyQuota := mustParseSubscriptionInt64(quotaMatch[1])
|
||||
tierCode := strings.ToLower(tier)
|
||||
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "ByteDance",
|
||||
ProviderNameCn: "字节跳动",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.volcengine.com",
|
||||
OperatorName: "ByteDance Volcano",
|
||||
OperatorNameCn: "火山引擎",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://developer.volcengine.com",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "bytedance-coding-plan-" + tierCode,
|
||||
PlanName: "方舟 Coding Plan " + tier,
|
||||
Tier: tier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: standardPrice,
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: monthlyQuota,
|
||||
QuotaUnit: "requests/month",
|
||||
PlanScope: "方舟 Coding Plan",
|
||||
SourceURL: defaultBytedanceCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: scene + ";续费标准价。",
|
||||
PublishedAtKnown: known,
|
||||
})
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "ByteDance",
|
||||
ProviderNameCn: "字节跳动",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.volcengine.com",
|
||||
OperatorName: "ByteDance Volcano",
|
||||
OperatorNameCn: "火山引擎",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://developer.volcengine.com",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "bytedance-coding-plan-" + tierCode + "-first-month",
|
||||
PlanName: "方舟 Coding Plan " + tier + " 首月活动版",
|
||||
Tier: tier + " Promo",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: promoPrice,
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: monthlyQuota,
|
||||
QuotaUnit: "requests/month",
|
||||
PlanScope: "方舟 Coding Plan",
|
||||
SourceURL: defaultBytedanceCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: scene + ";" + promoNote,
|
||||
PublishedAtKnown: known,
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
56
scripts/catalog_verification_common.go
Normal file
56
scripts/catalog_verification_common.go
Normal file
@@ -0,0 +1,56 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
type catalogVerificationRecord struct {
|
||||
CatalogCode string
|
||||
SourceURL string
|
||||
SourceTitle string
|
||||
PlanStatus string
|
||||
Notes string
|
||||
}
|
||||
|
||||
type catalogVerificationImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func upsertCatalogVerificationRecords(db *sql.DB, records []catalogVerificationRecord) error {
|
||||
if len(records) == 0 {
|
||||
return fmt.Errorf("catalog verification records are empty")
|
||||
}
|
||||
for _, record := range records {
|
||||
result, err := db.Exec(
|
||||
`UPDATE plan_catalog_inventory
|
||||
SET source_url = $2,
|
||||
source_title = $3,
|
||||
plan_status = $4,
|
||||
notes = $5,
|
||||
last_checked_at = CURRENT_TIMESTAMP,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE catalog_code = $1`,
|
||||
record.CatalogCode, record.SourceURL, record.SourceTitle, record.PlanStatus, record.Notes,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update plan_catalog_inventory %s: %w", record.CatalogCode, err)
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("rows affected for %s: %w", record.CatalogCode, err)
|
||||
}
|
||||
if rowsAffected == 0 {
|
||||
return fmt.Errorf("catalog_code %s not found in plan_catalog_inventory", record.CatalogCode)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
224
scripts/ctyun_subscription_lib.go
Normal file
224
scripts/ctyun_subscription_lib.go
Normal file
@@ -0,0 +1,224 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultCTYunCodingPlanURL = "https://www.ctyun.cn/document/11061839/11092368"
|
||||
defaultCTYunTokenPlanURL = "https://www.ctyun.cn/act/AI/zhuanxiang"
|
||||
)
|
||||
|
||||
func parseCTYunSubscriptionCatalog(codingRaw string, tokenRaw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(firstNonEmptyText(codingRaw, tokenRaw))
|
||||
codingRecords, err := parseCTYunCodingPlan(codingRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tokenRecords, err := parseCTYunTokenPlan(tokenRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records := append(codingRecords, tokenRecords...)
|
||||
for i := range records {
|
||||
records[i].PublishedAtKnown = known
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseCTYunCodingPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
if !strings.Contains(raw, "GLM Lite") || !strings.Contains(raw, "GLM Max") {
|
||||
return nil, fmt.Errorf("ctyun coding plan tiers not found")
|
||||
}
|
||||
pricePattern := regexp.MustCompile(`包月价格\s+(\d+)元/月\s+(\d+)元/月\s+(\d+)元/月`)
|
||||
priceMatch := pricePattern.FindStringSubmatch(raw)
|
||||
if len(priceMatch) != 4 {
|
||||
return nil, fmt.Errorf("ctyun coding plan monthly prices not found")
|
||||
}
|
||||
limitPattern := regexp.MustCompile(`每月最多约([\d,]+)次prompts`)
|
||||
limitMatches := limitPattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(limitMatches) < 3 {
|
||||
return nil, fmt.Errorf("ctyun coding plan monthly limits not found")
|
||||
}
|
||||
modelScope := extractCTYunCodingModels(raw)
|
||||
|
||||
records := []subscriptionImportRecord{
|
||||
{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "ctyun-coding-plan-lite-monthly",
|
||||
PlanName: "天翼云 Coding Plan Lite(月付)",
|
||||
Tier: "Lite",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[1]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: mustParseSubscriptionInt64(limitMatches[0][1]),
|
||||
QuotaUnit: "prompts/month",
|
||||
PlanScope: "Coding Plan",
|
||||
ModelScope: modelScope,
|
||||
SourceURL: defaultCTYunCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "每 5 小时约 80 次 prompts;每周约 400 次 prompts。",
|
||||
},
|
||||
{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "ctyun-coding-plan-pro-monthly",
|
||||
PlanName: "天翼云 Coding Plan Pro(月付)",
|
||||
Tier: "Pro",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[2]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: mustParseSubscriptionInt64(limitMatches[1][1]),
|
||||
QuotaUnit: "prompts/month",
|
||||
PlanScope: "Coding Plan",
|
||||
ModelScope: modelScope,
|
||||
SourceURL: defaultCTYunCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "每 5 小时约 400 次 prompts;每周约 2,000 次 prompts。",
|
||||
},
|
||||
{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "ctyun-coding-plan-max-monthly",
|
||||
PlanName: "天翼云 Coding Plan Max(月付)",
|
||||
Tier: "Max",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[3]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: mustParseSubscriptionInt64(limitMatches[2][1]),
|
||||
QuotaUnit: "prompts/month",
|
||||
PlanScope: "Coding Plan",
|
||||
ModelScope: modelScope,
|
||||
SourceURL: defaultCTYunCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "每 5 小时约 1,600 次 prompts;每周约 8,000 次 prompts。",
|
||||
},
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseCTYunTokenPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
pattern := regexp.MustCompile(`Token Plan ([^\n]+?)(\d+(?:\.\d+)?亿|\d+万)Tokens包[\s\S]*?支持模型:([^\n]+)[\s\S]*?(\d+\s*\.\s*\d+)\s*元/个`)
|
||||
matches := pattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) != 6 {
|
||||
return nil, fmt.Errorf("unexpected ctyun token plan count: %d", len(matches))
|
||||
}
|
||||
|
||||
codeByTier := map[string]string{
|
||||
"Lite": "lite",
|
||||
"Pro": "pro",
|
||||
"Max": "max",
|
||||
"轻享包": "starter",
|
||||
"畅享包": "plus",
|
||||
"尊享包": "vip",
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
rawTier := strings.TrimSpace(match[1])
|
||||
tierCode := codeByTier[rawTier]
|
||||
quotaValue := parseChineseTokenQuota(match[2])
|
||||
price := mustParseSubscriptionPrice(strings.ReplaceAll(match[4], " ", ""))
|
||||
planName := "天翼云 Token Plan " + rawTier
|
||||
if rawTier == "Lite" || rawTier == "Pro" || rawTier == "Max" {
|
||||
planName = "天翼云 Token Plan " + rawTier
|
||||
}
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "ctyun-token-plan-" + tierCode,
|
||||
PlanName: planName,
|
||||
Tier: rawTier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: price,
|
||||
PriceUnit: "CNY/pack",
|
||||
QuotaValue: quotaValue,
|
||||
QuotaUnit: "tokens/pack",
|
||||
PlanScope: "Token Plan",
|
||||
ModelScope: []string{strings.TrimSpace(match[3])},
|
||||
SourceURL: defaultCTYunTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "天翼云大模型 AI 专项活动页套餐。",
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseChineseTokenQuota(raw string) int64 {
|
||||
cleaned := strings.TrimSpace(strings.TrimSuffix(raw, "Tokens包"))
|
||||
cleaned = strings.ReplaceAll(cleaned, " ", "")
|
||||
switch {
|
||||
case strings.Contains(cleaned, "亿"):
|
||||
return parseDecimalMultiplier(strings.TrimSuffix(cleaned, "亿"), 100000000)
|
||||
case strings.Contains(cleaned, "万"):
|
||||
return parseDecimalMultiplier(strings.TrimSuffix(cleaned, "万"), 10000)
|
||||
default:
|
||||
return mustParseSubscriptionInt64(cleaned)
|
||||
}
|
||||
}
|
||||
|
||||
func extractCTYunCodingModels(raw string) []string {
|
||||
lines := strings.Split(raw, "\n")
|
||||
models := make([]string, 0, 8)
|
||||
capturing := false
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
switch {
|
||||
case line == "支持模型":
|
||||
capturing = true
|
||||
continue
|
||||
case line == "用量限制":
|
||||
return models
|
||||
case !capturing || line == "":
|
||||
continue
|
||||
default:
|
||||
models = append(models, line)
|
||||
}
|
||||
}
|
||||
return models
|
||||
}
|
||||
157
scripts/huawei_package_lib.go
Normal file
157
scripts/huawei_package_lib.go
Normal file
@@ -0,0 +1,157 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultHuaweiPackagePlanURL = "https://support.huaweicloud.com/price-maas/price-maas-0002.html"
|
||||
|
||||
func parseHuaweiPackageCatalog(raw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(raw)
|
||||
type packDef struct {
|
||||
QuotaRaw string
|
||||
BillingCycle string
|
||||
CodeSuffix string
|
||||
ReadableCycle string
|
||||
}
|
||||
packs := []packDef{
|
||||
{QuotaRaw: "100万", BillingCycle: "monthly", CodeSuffix: "100w-1m", ReadableCycle: "1个月"},
|
||||
{QuotaRaw: "1000万", BillingCycle: "monthly", CodeSuffix: "1000w-1m", ReadableCycle: "1个月"},
|
||||
{QuotaRaw: "1亿", BillingCycle: "quarterly", CodeSuffix: "1y-3m", ReadableCycle: "3个月"},
|
||||
{QuotaRaw: "10亿", BillingCycle: "quarterly", CodeSuffix: "10y-3m", ReadableCycle: "3个月"},
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, 8)
|
||||
for _, modelVersion := range []string{"1", "2"} {
|
||||
modelLabel := "DeepSeek-V3." + modelVersion
|
||||
versionCode := strings.ReplaceAll("v3."+modelVersion, ".", "-")
|
||||
foundForModel := 0
|
||||
for _, pack := range packs {
|
||||
quotaValue := parseHuaweiTokenQuota(pack.QuotaRaw)
|
||||
price, found := findHuaweiPackPrice(raw, modelLabel, pack.QuotaRaw, pack.ReadableCycle)
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Huawei",
|
||||
ProviderNameCn: "华为",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.huaweicloud.com",
|
||||
OperatorName: "Huawei Cloud",
|
||||
OperatorNameCn: "华为云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://support.huaweicloud.com",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "package_plan",
|
||||
PlanCode: fmt.Sprintf("huawei-deepseek-%s-package-%s", versionCode, pack.CodeSuffix),
|
||||
PlanName: fmt.Sprintf("华为云 MaaS %s 套餐包 %s", modelLabel, pack.QuotaRaw),
|
||||
Tier: modelLabel,
|
||||
BillingCycle: pack.BillingCycle,
|
||||
Currency: "CNY",
|
||||
ListPrice: price,
|
||||
PriceUnit: "CNY/pack",
|
||||
QuotaValue: quotaValue,
|
||||
QuotaUnit: "tokens/pack",
|
||||
PlanScope: "MaaS 文本生成模型套餐包",
|
||||
ModelScope: []string{modelLabel},
|
||||
SourceURL: defaultHuaweiPackagePlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: fmt.Sprintf("官方套餐包,有效期 %s,仅抵扣 %s Token 用量。", pack.ReadableCycle, modelLabel),
|
||||
PublishedAtKnown: known,
|
||||
})
|
||||
foundForModel++
|
||||
}
|
||||
_ = foundForModel
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no huawei package plan matched from source page")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func fallbackHuaweiPackageCatalog() []subscriptionImportRecord {
|
||||
publishedAt := "2026-05-14 00:00:00"
|
||||
effectiveDate := "2026-05-14"
|
||||
type packRow struct {
|
||||
ModelScope string
|
||||
VersionCode string
|
||||
QuotaValue int64
|
||||
QuotaLabel string
|
||||
BillingCycle string
|
||||
CodeSuffix string
|
||||
Price float64
|
||||
ReadableCycle string
|
||||
}
|
||||
packs := []packRow{
|
||||
{ModelScope: "DeepSeek-V3.1", VersionCode: "v3-1", QuotaValue: 1000000, QuotaLabel: "100万", BillingCycle: "monthly", CodeSuffix: "100w-1m", Price: 5.6, ReadableCycle: "1个月"},
|
||||
{ModelScope: "DeepSeek-V3.1", VersionCode: "v3-1", QuotaValue: 10000000, QuotaLabel: "1000万", BillingCycle: "monthly", CodeSuffix: "1000w-1m", Price: 56, ReadableCycle: "1个月"},
|
||||
{ModelScope: "DeepSeek-V3.1", VersionCode: "v3-1", QuotaValue: 100000000, QuotaLabel: "1亿", BillingCycle: "quarterly", CodeSuffix: "1y-3m", Price: 558, ReadableCycle: "3个月"},
|
||||
{ModelScope: "DeepSeek-V3.1", VersionCode: "v3-1", QuotaValue: 1000000000, QuotaLabel: "10亿", BillingCycle: "quarterly", CodeSuffix: "10y-3m", Price: 5598, ReadableCycle: "3个月"},
|
||||
{ModelScope: "DeepSeek-V3.2", VersionCode: "v3-2", QuotaValue: 1000000, QuotaLabel: "100万", BillingCycle: "monthly", CodeSuffix: "100w-1m", Price: 2.2, ReadableCycle: "1个月"},
|
||||
{ModelScope: "DeepSeek-V3.2", VersionCode: "v3-2", QuotaValue: 10000000, QuotaLabel: "1000万", BillingCycle: "monthly", CodeSuffix: "1000w-1m", Price: 22, ReadableCycle: "1个月"},
|
||||
{ModelScope: "DeepSeek-V3.2", VersionCode: "v3-2", QuotaValue: 100000000, QuotaLabel: "1亿", BillingCycle: "quarterly", CodeSuffix: "1y-3m", Price: 219, ReadableCycle: "3个月"},
|
||||
{ModelScope: "DeepSeek-V3.2", VersionCode: "v3-2", QuotaValue: 1000000000, QuotaLabel: "10亿", BillingCycle: "quarterly", CodeSuffix: "10y-3m", Price: 2199, ReadableCycle: "3个月"},
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, len(packs))
|
||||
for _, pack := range packs {
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Huawei",
|
||||
ProviderNameCn: "华为",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.huaweicloud.com",
|
||||
OperatorName: "Huawei Cloud",
|
||||
OperatorNameCn: "华为云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://support.huaweicloud.com",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "package_plan",
|
||||
PlanCode: fmt.Sprintf("huawei-deepseek-%s-package-%s", pack.VersionCode, pack.CodeSuffix),
|
||||
PlanName: fmt.Sprintf("华为云 MaaS %s 套餐包 %s", pack.ModelScope, pack.QuotaLabel),
|
||||
Tier: pack.ModelScope,
|
||||
BillingCycle: pack.BillingCycle,
|
||||
Currency: "CNY",
|
||||
ListPrice: pack.Price,
|
||||
PriceUnit: "CNY/pack",
|
||||
QuotaValue: pack.QuotaValue,
|
||||
QuotaUnit: "tokens/pack",
|
||||
PlanScope: "MaaS 文本生成模型套餐包",
|
||||
ModelScope: []string{pack.ModelScope},
|
||||
SourceURL: defaultHuaweiPackagePlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDate,
|
||||
Notes: fmt.Sprintf("官方价格页动态渲染,当前回退至最近核验的官方快照;有效期 %s,仅抵扣 %s Token 用量。", pack.ReadableCycle, pack.ModelScope),
|
||||
PublishedAtKnown: true,
|
||||
})
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func findHuaweiPackPrice(raw string, modelLabel string, quotaRaw string, cycle string) (float64, bool) {
|
||||
pattern := regexp.MustCompile(`(?s)` + regexp.QuoteMeta(modelLabel) + `.*?` + regexp.QuoteMeta(quotaRaw) + `.*?` + regexp.QuoteMeta(cycle) + `.*?([\d.]+)`)
|
||||
match := pattern.FindStringSubmatch(raw)
|
||||
if len(match) != 2 {
|
||||
return 0, false
|
||||
}
|
||||
return mustParseSubscriptionPrice(match[1]), true
|
||||
}
|
||||
|
||||
func parseHuaweiTokenQuota(raw string) int64 {
|
||||
switch strings.TrimSpace(raw) {
|
||||
case "100万":
|
||||
return 1000000
|
||||
case "1000万":
|
||||
return 10000000
|
||||
case "1亿":
|
||||
return 100000000
|
||||
case "10亿":
|
||||
return 1000000000
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
88
scripts/import_360_pricing.go
Normal file
88
scripts/import_360_pricing.go
Normal file
@@ -0,0 +1,88 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type platform360PricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", default360PricingURL, "360 智脑开放平台模型价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "360 智脑开放平台价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := platform360PricingImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := run360PricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_360_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run360PricingImport(cfg platform360PricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parse360PricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=360-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "360-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=360-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
55
scripts/import_360_pricing_test.go
Normal file
55
scripts/import_360_pricing_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParse360PricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "platform360_pricing_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parse360PricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parse360PricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 4 {
|
||||
t.Fatalf("期望 4 条 360 价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].ModelID != "360-deepseek-deepseek-v4-flash" {
|
||||
t.Fatalf("首条 modelID 错误: %q", records[0].ModelID)
|
||||
}
|
||||
if records[1].ContextLength != 1000000 {
|
||||
t.Fatalf("第二条上下文长度错误: %d", records[1].ContextLength)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun360PricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := run360PricingImport(platform360PricingImportConfig{
|
||||
URL: default360PricingURL,
|
||||
Fixture: filepath.Join("testdata", "platform360_pricing_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("run360PricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=360-pricing-import",
|
||||
"models=4",
|
||||
"operator=360 Open Platform",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
100
scripts/import_aliyun_subscription.go
Normal file
100
scripts/import_aliyun_subscription.go
Normal file
@@ -0,0 +1,100 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type aliyunSubscriptionImportConfig struct {
|
||||
TokenURL string
|
||||
CodingURL string
|
||||
TokenFixture string
|
||||
CodingFixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var tokenURL string
|
||||
var codingURL string
|
||||
var tokenFixture string
|
||||
var codingFixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&tokenURL, "token-url", defaultAliyunTokenPlanURL, "阿里云 Token Plan 文档 URL")
|
||||
flag.StringVar(&codingURL, "coding-url", defaultAliyunCodingPlanURL, "阿里云 Coding Plan 文档 URL")
|
||||
flag.StringVar(&tokenFixture, "token-fixture", "", "阿里云 Token Plan 本地样例文件")
|
||||
flag.StringVar(&codingFixture, "coding-fixture", "", "阿里云 Coding Plan 本地样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := aliyunSubscriptionImportConfig{
|
||||
TokenURL: tokenURL,
|
||||
CodingURL: codingURL,
|
||||
TokenFixture: tokenFixture,
|
||||
CodingFixture: codingFixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runAliyunSubscriptionImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_aliyun_subscription: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runAliyunSubscriptionImport(cfg aliyunSubscriptionImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
tokenRaw, err := fetchSubscriptionPage(cfg.TokenURL, cfg.TokenFixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
codingRaw, err := fetchSubscriptionPage(cfg.CodingURL, cfg.CodingFixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
records, err := parseAliyunSubscriptionCatalog(tokenRaw, codingRaw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=aliyun-subscription-import plans=%d provider=%s operator=%s dry_run=true\n", len(records), records[0].ProviderName, records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertSubscriptionImportRecords(db, records); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM subscription_plan`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count subscription_plan: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=aliyun-subscription-import plans=%d provider=%s operator=%s table_rows=%d dry_run=false\n", len(records), records[0].ProviderName, records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
71
scripts/import_aliyun_subscription_test.go
Normal file
71
scripts/import_aliyun_subscription_test.go
Normal file
@@ -0,0 +1,71 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseAliyunSubscriptionBuildsPlans(t *testing.T) {
|
||||
tokenRaw, err := os.ReadFile(filepath.Join("testdata", "aliyun_token_plan_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 token fixture 失败: %v", err)
|
||||
}
|
||||
codingRaw, err := os.ReadFile(filepath.Join("testdata", "aliyun_coding_plan_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 coding fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
plans, err := parseAliyunSubscriptionCatalog(string(tokenRaw), string(codingRaw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseAliyunSubscriptionCatalog 失败: %v", err)
|
||||
}
|
||||
if len(plans) != 5 {
|
||||
t.Fatalf("期望 5 条阿里云套餐记录,实际 %d", len(plans))
|
||||
}
|
||||
|
||||
if plans[0].PlanCode != "aliyun-token-plan-standard-seat" {
|
||||
t.Fatalf("首条 planCode 错误: %q", plans[0].PlanCode)
|
||||
}
|
||||
if plans[0].ListPrice != 198 {
|
||||
t.Fatalf("标准坐席价格错误: %v", plans[0].ListPrice)
|
||||
}
|
||||
if plans[3].PlanCode != "aliyun-token-plan-shared-pack" {
|
||||
t.Fatalf("共享包 planCode 错误: %q", plans[3].PlanCode)
|
||||
}
|
||||
if plans[4].PlanCode != "aliyun-coding-plan-pro" {
|
||||
t.Fatalf("Coding Plan Pro planCode 错误: %q", plans[4].PlanCode)
|
||||
}
|
||||
if !strings.Contains(plans[4].Notes, "活动已结束") {
|
||||
t.Fatalf("Coding Plan 备注缺少活动说明: %q", plans[4].Notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAliyunSubscriptionImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runAliyunSubscriptionImport(aliyunSubscriptionImportConfig{
|
||||
TokenFixture: filepath.Join("testdata", "aliyun_token_plan_sample.txt"),
|
||||
CodingFixture: filepath.Join("testdata", "aliyun_coding_plan_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runAliyunSubscriptionImport 失败: %v", err)
|
||||
}
|
||||
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=aliyun-subscription-import",
|
||||
"plans=5",
|
||||
"provider=Alibaba",
|
||||
"operator=Alibaba Bailian",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
88
scripts/import_azure_openai_pricing.go
Normal file
88
scripts/import_azure_openai_pricing.go
Normal file
@@ -0,0 +1,88 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type azureOpenAIPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultAzureOpenAIPricingURL, "Azure OpenAI 官方零售价格 API")
|
||||
flag.StringVar(&fixture, "fixture", "", "Azure OpenAI 价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := azureOpenAIPricingImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runAzureOpenAIPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_azure_openai_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runAzureOpenAIPricingImport(cfg azureOpenAIPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchAzureOpenAIPricingCatalog(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseAzureOpenAIPricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=azure-openai-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "azure-openai-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=azure-openai-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
146
scripts/import_azure_openai_pricing_test.go
Normal file
146
scripts/import_azure_openai_pricing_test.go
Normal file
@@ -0,0 +1,146 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseAzureOpenAIPricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "azure_openai_pricing_sample.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseAzureOpenAIPricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseAzureOpenAIPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 2 {
|
||||
t.Fatalf("期望 2 条 Azure OpenAI 价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].InputPrice != 2.2 || records[0].OutputPrice != 8.8 {
|
||||
t.Fatalf("gpt-4.1 价格换算错误: %v / %v", records[0].InputPrice, records[0].OutputPrice)
|
||||
}
|
||||
if records[1].ModelName != "GPT-5" {
|
||||
t.Fatalf("第二条模型名错误: %q", records[1].ModelName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAzureOpenAIPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runAzureOpenAIPricingImport(azureOpenAIPricingImportConfig{
|
||||
URL: defaultAzureOpenAIPricingURL,
|
||||
Fixture: filepath.Join("testdata", "azure_openai_pricing_sample.json"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runAzureOpenAIPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=azure-openai-pricing-import",
|
||||
"models=2",
|
||||
"operator=Microsoft Azure",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAzureOpenAIPricingImportDryRunFollowsNextPageLink(t *testing.T) {
|
||||
pageTwoPath := "/page-2"
|
||||
var server *httptest.Server
|
||||
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
response := azureRetailPriceResponse{}
|
||||
switch r.URL.Path {
|
||||
case "/":
|
||||
response = azureRetailPriceResponse{
|
||||
Items: []azureRetailPriceItem{
|
||||
{
|
||||
CurrencyCode: "USD",
|
||||
UnitPrice: 0.0022,
|
||||
Location: "US East",
|
||||
MeterName: "gpt 4.1 Inp regnl Tokens",
|
||||
ProductName: "Azure OpenAI",
|
||||
SkuName: "gpt 4.1 Inp regnl",
|
||||
ServiceName: "Foundry Models",
|
||||
UnitOfMeasure: "1K",
|
||||
Type: "Consumption",
|
||||
ArmSkuName: "gpt 4.1 Inp regnl",
|
||||
},
|
||||
{
|
||||
CurrencyCode: "USD",
|
||||
UnitPrice: 0.0088,
|
||||
Location: "US East",
|
||||
MeterName: "gpt 4.1 Outp regnl Tokens",
|
||||
ProductName: "Azure OpenAI",
|
||||
SkuName: "gpt 4.1 Outp regnl",
|
||||
ServiceName: "Foundry Models",
|
||||
UnitOfMeasure: "1K",
|
||||
Type: "Consumption",
|
||||
ArmSkuName: "gpt 4.1 Outp regnl",
|
||||
},
|
||||
},
|
||||
NextPageLink: server.URL + pageTwoPath,
|
||||
}
|
||||
case pageTwoPath:
|
||||
response = azureRetailPriceResponse{
|
||||
Items: []azureRetailPriceItem{
|
||||
{
|
||||
CurrencyCode: "USD",
|
||||
UnitPrice: 1.25,
|
||||
Location: "US West",
|
||||
MeterName: "GPT 5 inp Glbl 1M Tokens",
|
||||
ProductName: "Azure OpenAI GPT5",
|
||||
SkuName: "GPT 5 inp Glbl",
|
||||
ServiceName: "Foundry Models",
|
||||
UnitOfMeasure: "1M",
|
||||
Type: "Consumption",
|
||||
ArmSkuName: "GPT 5 inp Glbl",
|
||||
},
|
||||
{
|
||||
CurrencyCode: "USD",
|
||||
UnitPrice: 10,
|
||||
Location: "US West",
|
||||
MeterName: "GPT 5 outpt Glbl 1M Tokens",
|
||||
ProductName: "Azure OpenAI GPT5",
|
||||
SkuName: "GPT 5 outpt Glbl",
|
||||
ServiceName: "Foundry Models",
|
||||
UnitOfMeasure: "1M",
|
||||
Type: "Consumption",
|
||||
ArmSkuName: "GPT 5 outpt Glbl",
|
||||
},
|
||||
},
|
||||
}
|
||||
default:
|
||||
t.Fatalf("unexpected path: %s", r.URL.Path)
|
||||
}
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
t.Fatalf("encode response: %v", err)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
var out bytes.Buffer
|
||||
err := runAzureOpenAIPricingImport(azureOpenAIPricingImportConfig{
|
||||
URL: server.URL,
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runAzureOpenAIPricingImport 返回错误: %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "models=2") {
|
||||
t.Fatalf("分页结果未聚合,输出: %q", out.String())
|
||||
}
|
||||
}
|
||||
100
scripts/import_baidu_subscription.go
Normal file
100
scripts/import_baidu_subscription.go
Normal file
@@ -0,0 +1,100 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type baiduSubscriptionImportConfig struct {
|
||||
CodingURL string
|
||||
TokenURL string
|
||||
CodingFixture string
|
||||
TokenFixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var codingURL string
|
||||
var tokenURL string
|
||||
var codingFixture string
|
||||
var tokenFixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&codingURL, "coding-url", defaultBaiduCodingPlanURL, "百度 Coding Plan 文档 URL")
|
||||
flag.StringVar(&tokenURL, "token-url", defaultBaiduTokenPlanURL, "百度 Token 福利包文档 URL")
|
||||
flag.StringVar(&codingFixture, "coding-fixture", "", "百度 Coding Plan 本地样例文件")
|
||||
flag.StringVar(&tokenFixture, "token-fixture", "", "百度 Token 福利包本地样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := baiduSubscriptionImportConfig{
|
||||
CodingURL: codingURL,
|
||||
TokenURL: tokenURL,
|
||||
CodingFixture: codingFixture,
|
||||
TokenFixture: tokenFixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runBaiduSubscriptionImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_baidu_subscription: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runBaiduSubscriptionImport(cfg baiduSubscriptionImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
codingRaw, err := fetchSubscriptionPage(cfg.CodingURL, cfg.CodingFixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tokenRaw, err := fetchSubscriptionPage(cfg.TokenURL, cfg.TokenFixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
records, err := parseBaiduSubscriptionCatalog(codingRaw, tokenRaw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=baidu-subscription-import plans=%d provider=%s operator=%s dry_run=true\n", len(records), records[0].ProviderName, records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertSubscriptionImportRecords(db, records); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM subscription_plan`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count subscription_plan: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=baidu-subscription-import plans=%d provider=%s operator=%s table_rows=%d dry_run=false\n", len(records), records[0].ProviderName, records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
69
scripts/import_baidu_subscription_test.go
Normal file
69
scripts/import_baidu_subscription_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseBaiduSubscriptionBuildsPlans(t *testing.T) {
|
||||
codingRaw, err := os.ReadFile(filepath.Join("testdata", "baidu_coding_plan_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 coding fixture 失败: %v", err)
|
||||
}
|
||||
tokenRaw, err := os.ReadFile(filepath.Join("testdata", "baidu_token_benefit_pack_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 token fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
plans, err := parseBaiduSubscriptionCatalog(string(codingRaw), string(tokenRaw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseBaiduSubscriptionCatalog 失败: %v", err)
|
||||
}
|
||||
if len(plans) != 7 {
|
||||
t.Fatalf("期望 7 条百度套餐记录,实际 %d", len(plans))
|
||||
}
|
||||
|
||||
if plans[0].PlanCode != "baidu-coding-plan-lite" {
|
||||
t.Fatalf("Lite planCode 错误: %q", plans[0].PlanCode)
|
||||
}
|
||||
if plans[1].PlanCode != "baidu-coding-plan-pro" {
|
||||
t.Fatalf("Pro planCode 错误: %q", plans[1].PlanCode)
|
||||
}
|
||||
last := plans[len(plans)-1]
|
||||
if last.PlanCode != "baidu-token-benefit-pack-800000" {
|
||||
t.Fatalf("末条 token 福利包 planCode 错误: %q", last.PlanCode)
|
||||
}
|
||||
if !strings.Contains(last.Notes, "首购优惠价") {
|
||||
t.Fatalf("token 福利包备注缺少首购优惠说明: %q", last.Notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBaiduSubscriptionImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runBaiduSubscriptionImport(baiduSubscriptionImportConfig{
|
||||
CodingFixture: filepath.Join("testdata", "baidu_coding_plan_sample.txt"),
|
||||
TokenFixture: filepath.Join("testdata", "baidu_token_benefit_pack_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runBaiduSubscriptionImport 失败: %v", err)
|
||||
}
|
||||
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=baidu-subscription-import",
|
||||
"plans=7",
|
||||
"provider=Baidu",
|
||||
"operator=Baidu Qianfan",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
88
scripts/import_bedrock_pricing.go
Normal file
88
scripts/import_bedrock_pricing.go
Normal file
@@ -0,0 +1,88 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type bedrockPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultBedrockPricingURL, "Amazon Bedrock 官方价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "Amazon Bedrock 价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := bedrockPricingImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runBedrockPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_bedrock_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runBedrockPricingImport(cfg bedrockPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchRawPricingPageWithOptions(cfg.URL, cfg.Fixture, client, officialPricingFetchOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseBedrockPricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=bedrock-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "bedrock-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=bedrock-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
130
scripts/import_bedrock_pricing_test.go
Normal file
130
scripts/import_bedrock_pricing_test.go
Normal file
@@ -0,0 +1,130 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseBedrockPricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "bedrock_pricing_sample.html"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseBedrockPricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseBedrockPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 4 {
|
||||
t.Fatalf("期望 4 条 Bedrock 价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].ProviderName != "Amazon" {
|
||||
t.Fatalf("Nova provider 归一化错误: %q", records[0].ProviderName)
|
||||
}
|
||||
if records[1].OutputPrice != 0.4 {
|
||||
t.Fatalf("Nova Lite 输出价错误: %v", records[1].OutputPrice)
|
||||
}
|
||||
if records[2].Region != "Europe (Frankfurt) and Asia Pacific (Jakarta)" {
|
||||
t.Fatalf("Qwen region 错误: %q", records[2].Region)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBedrockPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runBedrockPricingImport(bedrockPricingImportConfig{
|
||||
URL: defaultBedrockPricingURL,
|
||||
Fixture: filepath.Join("testdata", "bedrock_pricing_sample.html"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runBedrockPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=bedrock-pricing-import",
|
||||
"models=4",
|
||||
"operator=Amazon Bedrock",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBedrockPricingCatalogPrefersFirstTierForSameModelAndRegion(t *testing.T) {
|
||||
raw := `
|
||||
<h3 id="Model_Pricing" class="lb-txt-none lb-txt-32 lb-h3 lb-title">Model Pricing</h3>
|
||||
<h2 id="Anthropic" class="lb-txt-none lb-h2 lb-title">Anthropic</h2>
|
||||
<p><b>Regions: US East (N. Virginia)</b></p>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><b>Anthropic models</b></td>
|
||||
<td><b>Price per 1M input tokens</b></td>
|
||||
<td><b>Price per 1M output tokens</b></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Claude Sonnet 4.5</td>
|
||||
<td>$3.00</td>
|
||||
<td>$15.00</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h3 id="Priority">Priority</h3>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><b>Anthropic models</b></td>
|
||||
<td><b>Price per 1M input tokens</b></td>
|
||||
<td><b>Price per 1M output tokens</b></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Claude Sonnet 4.5</td>
|
||||
<td>$6.00</td>
|
||||
<td>$30.00</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h3 id="Pricing_examples" class="lb-txt-none lb-txt-48 lb-h2 lb-title">Pricing examples</h3>
|
||||
`
|
||||
|
||||
records, err := parseBedrockPricingCatalog(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parseBedrockPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 1 {
|
||||
t.Fatalf("期望仅保留默认价一条记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].InputPrice != 3 || records[0].OutputPrice != 15 {
|
||||
t.Fatalf("默认价被后续价阶污染: %+v", records[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBedrockPricingTextFallbackBuildsRecords(t *testing.T) {
|
||||
raw := cleanHTMLText(`
|
||||
<h2 id="Qwen" class="lb-txt-none lb-h2 lb-title">Qwen</h2>
|
||||
<p><b>Regions: Europe (Frankfurt) and Asia Pacific (Jakarta)</b></p>
|
||||
Qwen models Price per 1M input tokens Price per 1M output tokens
|
||||
Qwen3 Coder Next $ 0.60 $ 1.44
|
||||
<p><b>Region: Asia Pacific (Sydney)</b></p>
|
||||
Qwen models Price per 1M input tokens Price per 1M output tokens
|
||||
Qwen3 Next 80B A3B $ 0.1545 $ 1.2360
|
||||
`)
|
||||
|
||||
records := parseBedrockPricingTextFallback(raw)
|
||||
if len(records) != 2 {
|
||||
t.Fatalf("期望 fallback 解析 2 条记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].Region != "Europe (Frankfurt) and Asia Pacific (Jakarta)" {
|
||||
t.Fatalf("fallback region 错误: %q", records[0].Region)
|
||||
}
|
||||
if records[1].InputPrice != 0.1545 || records[1].OutputPrice != 1.2360 {
|
||||
t.Fatalf("fallback 价格错误: %+v", records[1])
|
||||
}
|
||||
}
|
||||
100
scripts/import_bytedance_subscription.go
Normal file
100
scripts/import_bytedance_subscription.go
Normal file
@@ -0,0 +1,100 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type bytedanceSubscriptionImportConfig struct {
|
||||
PricingURL string
|
||||
NoticeURL string
|
||||
PricingFixture string
|
||||
NoticeFixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var pricingURL string
|
||||
var noticeURL string
|
||||
var pricingFixture string
|
||||
var noticeFixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&pricingURL, "pricing-url", defaultBytedanceCodingPlanURL, "火山方舟 Coding Plan 价格说明 URL")
|
||||
flag.StringVar(¬iceURL, "notice-url", defaultBytedanceCodingPlanNotice, "火山方舟 Coding Plan 活动说明 URL")
|
||||
flag.StringVar(&pricingFixture, "pricing-fixture", "", "火山方舟 Coding Plan 价格样例文件")
|
||||
flag.StringVar(¬iceFixture, "notice-fixture", "", "火山方舟 Coding Plan 活动样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := bytedanceSubscriptionImportConfig{
|
||||
PricingURL: pricingURL,
|
||||
NoticeURL: noticeURL,
|
||||
PricingFixture: pricingFixture,
|
||||
NoticeFixture: noticeFixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runBytedanceSubscriptionImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_bytedance_subscription: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runBytedanceSubscriptionImport(cfg bytedanceSubscriptionImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
pricingRaw, err := fetchSubscriptionPage(cfg.PricingURL, cfg.PricingFixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
noticeRaw, err := fetchSubscriptionPage(cfg.NoticeURL, cfg.NoticeFixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
records, err := parseBytedanceSubscriptionCatalog(pricingRaw, noticeRaw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=bytedance-subscription-import plans=%d provider=%s operator=%s dry_run=true\n", len(records), records[0].ProviderName, records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertSubscriptionImportRecords(db, records); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM subscription_plan`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count subscription_plan: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=bytedance-subscription-import plans=%d provider=%s operator=%s table_rows=%d dry_run=false\n", len(records), records[0].ProviderName, records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
69
scripts/import_bytedance_subscription_test.go
Normal file
69
scripts/import_bytedance_subscription_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseBytedanceSubscriptionBuildsPlans(t *testing.T) {
|
||||
pricingRaw, err := os.ReadFile(filepath.Join("testdata", "bytedance_coding_plan_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 pricing fixture 失败: %v", err)
|
||||
}
|
||||
noticeRaw, err := os.ReadFile(filepath.Join("testdata", "bytedance_coding_plan_notice_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 notice fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
plans, err := parseBytedanceSubscriptionCatalog(string(pricingRaw), string(noticeRaw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseBytedanceSubscriptionCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(plans) != 4 {
|
||||
t.Fatalf("期望 4 条火山套餐记录,实际 %d", len(plans))
|
||||
}
|
||||
if plans[0].PlanCode != "bytedance-coding-plan-lite" {
|
||||
t.Fatalf("首条标准套餐 planCode 错误: %q", plans[0].PlanCode)
|
||||
}
|
||||
if plans[1].PlanCode != "bytedance-coding-plan-lite-first-month" {
|
||||
t.Fatalf("首条活动套餐 planCode 错误: %q", plans[1].PlanCode)
|
||||
}
|
||||
if plans[1].ListPrice != 9.9 {
|
||||
t.Fatalf("Lite 首月活动价错误: %v", plans[1].ListPrice)
|
||||
}
|
||||
if !strings.Contains(plans[1].Notes, "每日 10:30") {
|
||||
t.Fatalf("活动套餐备注缺少限量说明: %q", plans[1].Notes)
|
||||
}
|
||||
if plans[3].QuotaValue != 90000 {
|
||||
t.Fatalf("Pro 月额度错误: %d", plans[3].QuotaValue)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBytedanceSubscriptionImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runBytedanceSubscriptionImport(bytedanceSubscriptionImportConfig{
|
||||
PricingFixture: filepath.Join("testdata", "bytedance_coding_plan_sample.txt"),
|
||||
NoticeFixture: filepath.Join("testdata", "bytedance_coding_plan_notice_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runBytedanceSubscriptionImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=bytedance-subscription-import",
|
||||
"plans=4",
|
||||
"provider=ByteDance",
|
||||
"operator=ByteDance Volcano",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
110
scripts/import_catalog_seed_verification.go
Normal file
110
scripts/import_catalog_seed_verification.go
Normal file
@@ -0,0 +1,110 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
const catalogSeedVerificationImporterKey = "import_catalog_seed_verification.go"
|
||||
|
||||
type catalogSeedVerificationConfig struct {
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadCatalogSeedVerificationEnv()
|
||||
|
||||
var cfg catalogSeedVerificationConfig
|
||||
flag.BoolVar(&cfg.DryRun, "dry-run", false, "仅打印摘要,不写入数据库")
|
||||
flag.Parse()
|
||||
|
||||
db, err := catalogSeedVerificationDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := runCatalogSeedVerificationImport(db, cfg); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_catalog_seed_verification: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func loadCatalogSeedVerificationEnv() {
|
||||
for _, path := range []string{".env.local", ".env"} {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
value = strings.Trim(strings.TrimSpace(value), `"'`)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := os.LookupEnv(key); exists {
|
||||
continue
|
||||
}
|
||||
_ = os.Setenv(key, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func catalogSeedVerificationDB() (*sql.DB, error) {
|
||||
dsn := os.Getenv("DATABASE_URL")
|
||||
if dsn == "" {
|
||||
dsn = "postgres://long@/llm_intelligence?host=/var/run/postgresql"
|
||||
}
|
||||
return sql.Open("postgres", dsn)
|
||||
}
|
||||
|
||||
func runCatalogSeedVerificationImport(db *sql.DB, cfg catalogSeedVerificationConfig) error {
|
||||
var count int
|
||||
if err := db.QueryRow(`
|
||||
SELECT COUNT(*)
|
||||
FROM plan_catalog_inventory
|
||||
WHERE importer_key = $1
|
||||
`, catalogSeedVerificationImporterKey).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if cfg.DryRun {
|
||||
fmt.Printf("source=catalog-seed-verification entries=%d dry_run=true\n", count)
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := db.Exec(`
|
||||
UPDATE plan_catalog_inventory
|
||||
SET plan_status = 'confirmed',
|
||||
notes = CASE
|
||||
WHEN position($2 in COALESCE(notes, '')) > 0 THEN notes
|
||||
WHEN COALESCE(notes, '') = '' THEN $2
|
||||
ELSE notes || ';' || $2
|
||||
END,
|
||||
last_checked_at = CURRENT_TIMESTAMP,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE importer_key = $1
|
||||
`, catalogSeedVerificationImporterKey, strings.TrimSpace("当前链路为目录级官方入口核验,结构化公开价格待后续独立 importer 补齐。"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("source=catalog-seed-verification entries=%d dry_run=false\n", count)
|
||||
return nil
|
||||
}
|
||||
11
scripts/import_catalog_seed_verification_test.go
Normal file
11
scripts/import_catalog_seed_verification_test.go
Normal file
@@ -0,0 +1,11 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCatalogSeedVerificationImporterKeyConstant(t *testing.T) {
|
||||
if catalogSeedVerificationImporterKey != "import_catalog_seed_verification.go" {
|
||||
t.Fatalf("importer key 常量错误: %q", catalogSeedVerificationImporterKey)
|
||||
}
|
||||
}
|
||||
100
scripts/import_ctyun_subscription.go
Normal file
100
scripts/import_ctyun_subscription.go
Normal file
@@ -0,0 +1,100 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ctyunSubscriptionImportConfig struct {
|
||||
CodingURL string
|
||||
TokenURL string
|
||||
CodingFixture string
|
||||
TokenFixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var codingURL string
|
||||
var tokenURL string
|
||||
var codingFixture string
|
||||
var tokenFixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&codingURL, "coding-url", defaultCTYunCodingPlanURL, "天翼云 Coding Plan 文档 URL")
|
||||
flag.StringVar(&tokenURL, "token-url", defaultCTYunTokenPlanURL, "天翼云 Token Plan 活动页 URL")
|
||||
flag.StringVar(&codingFixture, "coding-fixture", "", "天翼云 Coding Plan 本地样例文件")
|
||||
flag.StringVar(&tokenFixture, "token-fixture", "", "天翼云 Token Plan 本地样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := ctyunSubscriptionImportConfig{
|
||||
CodingURL: codingURL,
|
||||
TokenURL: tokenURL,
|
||||
CodingFixture: codingFixture,
|
||||
TokenFixture: tokenFixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runCTYunSubscriptionImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_ctyun_subscription: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runCTYunSubscriptionImport(cfg ctyunSubscriptionImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
codingRaw, err := fetchSubscriptionPage(cfg.CodingURL, cfg.CodingFixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tokenRaw, err := fetchSubscriptionPage(cfg.TokenURL, cfg.TokenFixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
records, err := parseCTYunSubscriptionCatalog(codingRaw, tokenRaw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=ctyun-subscription-import plans=%d provider=%s operator=%s dry_run=true\n", len(records), records[0].ProviderName, records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertSubscriptionImportRecords(db, records); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM subscription_plan`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count subscription_plan: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=ctyun-subscription-import plans=%d provider=%s operator=%s table_rows=%d dry_run=false\n", len(records), records[0].ProviderName, records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
68
scripts/import_ctyun_subscription_test.go
Normal file
68
scripts/import_ctyun_subscription_test.go
Normal file
@@ -0,0 +1,68 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseCTYunSubscriptionBuildsPlans(t *testing.T) {
|
||||
codingRaw, err := os.ReadFile(filepath.Join("testdata", "ctyun_coding_plan_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 coding fixture 失败: %v", err)
|
||||
}
|
||||
tokenRaw, err := os.ReadFile(filepath.Join("testdata", "ctyun_token_plan_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 token fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
plans, err := parseCTYunSubscriptionCatalog(string(codingRaw), string(tokenRaw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseCTYunSubscriptionCatalog 失败: %v", err)
|
||||
}
|
||||
if len(plans) != 9 {
|
||||
t.Fatalf("期望 9 条天翼云套餐记录,实际 %d", len(plans))
|
||||
}
|
||||
|
||||
if plans[0].PlanCode != "ctyun-coding-plan-lite-monthly" {
|
||||
t.Fatalf("首条 coding planCode 错误: %q", plans[0].PlanCode)
|
||||
}
|
||||
if plans[0].ListPrice != 49 {
|
||||
t.Fatalf("GLM Lite 月价错误: %v", plans[0].ListPrice)
|
||||
}
|
||||
if plans[3].PlanCode != "ctyun-token-plan-lite" {
|
||||
t.Fatalf("首条 token planCode 错误: %q", plans[3].PlanCode)
|
||||
}
|
||||
if plans[len(plans)-1].PlanCode != "ctyun-token-plan-vip" {
|
||||
t.Fatalf("末条 token planCode 错误: %q", plans[len(plans)-1].PlanCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCTYunSubscriptionImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runCTYunSubscriptionImport(ctyunSubscriptionImportConfig{
|
||||
CodingFixture: filepath.Join("testdata", "ctyun_coding_plan_sample.txt"),
|
||||
TokenFixture: filepath.Join("testdata", "ctyun_token_plan_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runCTYunSubscriptionImport 失败: %v", err)
|
||||
}
|
||||
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=ctyun-subscription-import",
|
||||
"plans=9",
|
||||
"provider=Telecom",
|
||||
"operator=CTYun",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
103
scripts/import_cucloud_catalog.go
Normal file
103
scripts/import_cucloud_catalog.go
Normal file
@@ -0,0 +1,103 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultCUCloudCatalogURL = "https://www.cucloud.cn/act/CloudAI.html"
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultCUCloudCatalogURL, "联通云智算专区 URL")
|
||||
flag.StringVar(&fixture, "fixture", "", "联通云智算专区样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅校验并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := catalogVerificationImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runCUCloudCatalogImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_cucloud_catalog: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func parseCUCloudCatalog(raw string) ([]catalogVerificationRecord, error) {
|
||||
if !strings.Contains(raw, "AICP") {
|
||||
return nil, fmt.Errorf("cucloud AICP marker not found")
|
||||
}
|
||||
if !strings.Contains(raw, "AI应用开发平台") && !strings.Contains(raw, "AI 应用开发平台") {
|
||||
return nil, fmt.Errorf("cucloud AI app marker not found")
|
||||
}
|
||||
return []catalogVerificationRecord{
|
||||
{
|
||||
CatalogCode: "cucloud-aicp-platform",
|
||||
SourceURL: defaultCUCloudCatalogURL,
|
||||
SourceTitle: "联通云智算专区",
|
||||
PlanStatus: "confirmed",
|
||||
Notes: "官方智算专区已公开展示 AI 算力平台 AICP,覆盖开发、训练、推理与模型服务部署全流程。",
|
||||
},
|
||||
{
|
||||
CatalogCode: "cucloud-ai-app-platform",
|
||||
SourceURL: defaultCUCloudCatalogURL,
|
||||
SourceTitle: "联通云智算专区",
|
||||
PlanStatus: "confirmed",
|
||||
Notes: "官方智算专区已公开展示 AI 应用开发平台,支持一站式可视化开发、调试和发布智能体应用。",
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func runCUCloudCatalogImport(cfg catalogVerificationImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseCUCloudCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=cucloud-catalog-import entries=%d dry_run=true\n", len(records))
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertCatalogVerificationRecords(db, records); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=cucloud-catalog-import entries=%d dry_run=false\n", len(records))
|
||||
return err
|
||||
}
|
||||
54
scripts/import_cucloud_catalog_test.go
Normal file
54
scripts/import_cucloud_catalog_test.go
Normal file
@@ -0,0 +1,54 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseCUCloudCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "cucloud_catalog_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseCUCloudCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseCUCloudCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 2 {
|
||||
t.Fatalf("期望 2 条联通云目录记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].CatalogCode != "cucloud-aicp-platform" {
|
||||
t.Fatalf("首条 catalogCode 错误: %q", records[0].CatalogCode)
|
||||
}
|
||||
if records[1].CatalogCode != "cucloud-ai-app-platform" {
|
||||
t.Fatalf("第二条 catalogCode 错误: %q", records[1].CatalogCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCUCloudCatalogImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runCUCloudCatalogImport(catalogVerificationImportConfig{
|
||||
URL: defaultCUCloudCatalogURL,
|
||||
Fixture: filepath.Join("testdata", "cucloud_catalog_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runCUCloudCatalogImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=cucloud-catalog-import",
|
||||
"entries=2",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
93
scripts/import_huawei_package.go
Normal file
93
scripts/import_huawei_package.go
Normal file
@@ -0,0 +1,93 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type huaweiPackageImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultHuaweiPackagePlanURL, "华为云 MaaS 套餐包价格页 URL")
|
||||
flag.StringVar(&fixture, "fixture", "", "华为云 MaaS 套餐包样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := huaweiPackageImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runHuaweiPackageImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_huawei_package: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runHuaweiPackageImport(cfg huaweiPackageImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
records, err := parseHuaweiPackageCatalog(raw)
|
||||
if err != nil {
|
||||
if cfg.Fixture == "" && strings.Contains(err.Error(), "no huawei package plan matched") {
|
||||
records = fallbackHuaweiPackageCatalog()
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=huawei-package-import plans=%d provider=%s operator=%s dry_run=true\n", len(records), records[0].ProviderName, records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertSubscriptionImportRecords(db, records); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM subscription_plan`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count subscription_plan: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=huawei-package-import plans=%d provider=%s operator=%s table_rows=%d dry_run=false\n", len(records), records[0].ProviderName, records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
61
scripts/import_huawei_package_test.go
Normal file
61
scripts/import_huawei_package_test.go
Normal file
@@ -0,0 +1,61 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseHuaweiPackageCatalogBuildsPlans(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "huawei_package_plan_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
plans, err := parseHuaweiPackageCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseHuaweiPackageCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(plans) != 8 {
|
||||
t.Fatalf("期望 8 条华为套餐包记录,实际 %d", len(plans))
|
||||
}
|
||||
if plans[0].PlanCode != "huawei-deepseek-v3-1-package-100w-1m" {
|
||||
t.Fatalf("首条 planCode 错误: %q", plans[0].PlanCode)
|
||||
}
|
||||
if plans[4].PlanCode != "huawei-deepseek-v3-2-package-100w-1m" {
|
||||
t.Fatalf("DeepSeek-V3.2 首条 planCode 错误: %q", plans[4].PlanCode)
|
||||
}
|
||||
if plans[7].ListPrice != 2199 {
|
||||
t.Fatalf("DeepSeek-V3.2 10亿套餐价格错误: %v", plans[7].ListPrice)
|
||||
}
|
||||
if plans[7].PlanFamily != "package_plan" {
|
||||
t.Fatalf("planFamily 错误: %q", plans[7].PlanFamily)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHuaweiPackageImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runHuaweiPackageImport(huaweiPackageImportConfig{
|
||||
Fixture: filepath.Join("testdata", "huawei_package_plan_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runHuaweiPackageImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=huawei-package-import",
|
||||
"plans=8",
|
||||
"provider=Huawei",
|
||||
"operator=Huawei Cloud",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
429
scripts/import_manual_subscription_seed.go
Normal file
429
scripts/import_manual_subscription_seed.go
Normal file
@@ -0,0 +1,429 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
const defaultManualSubscriptionSeedPath = "seeds/subscription_plan_manual_seed.json"
|
||||
|
||||
type manualSubscriptionImportConfig struct {
|
||||
SeedPath string
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
type manualSubscriptionSeedEnvelope struct {
|
||||
CheckedAt string `json:"checkedAt"`
|
||||
Items []manualSubscriptionSeedItem `json:"items"`
|
||||
}
|
||||
|
||||
type manualSubscriptionSeedItem struct {
|
||||
ProviderName string `json:"providerName"`
|
||||
ProviderNameCn string `json:"providerNameCn"`
|
||||
ProviderCountry string `json:"providerCountry"`
|
||||
ProviderWebsite string `json:"providerWebsite"`
|
||||
OperatorName string `json:"operatorName"`
|
||||
OperatorNameCn string `json:"operatorNameCn"`
|
||||
OperatorCountry string `json:"operatorCountry"`
|
||||
OperatorWebsite string `json:"operatorWebsite"`
|
||||
OperatorType string `json:"operatorType"`
|
||||
PlanFamily string `json:"planFamily"`
|
||||
PlanCode string `json:"planCode"`
|
||||
PlanName string `json:"planName"`
|
||||
Tier string `json:"tier"`
|
||||
BillingCycle string `json:"billingCycle"`
|
||||
Currency string `json:"currency"`
|
||||
ListPrice float64 `json:"listPrice"`
|
||||
PriceUnit string `json:"priceUnit"`
|
||||
QuotaValue int64 `json:"quotaValue"`
|
||||
QuotaUnit string `json:"quotaUnit"`
|
||||
ContextWindow int `json:"contextWindow"`
|
||||
PlanScope string `json:"planScope"`
|
||||
ModelScope []string `json:"modelScope"`
|
||||
SourceURL string `json:"sourceURL"`
|
||||
PublishedAt string `json:"publishedAt"`
|
||||
EffectiveDate string `json:"effectiveDate"`
|
||||
Notes string `json:"notes"`
|
||||
}
|
||||
|
||||
type manualSubscriptionRow struct {
|
||||
ProviderName string
|
||||
ProviderNameCn string
|
||||
ProviderCountry string
|
||||
ProviderWebsite string
|
||||
OperatorName string
|
||||
OperatorNameCn string
|
||||
OperatorCountry string
|
||||
OperatorWebsite string
|
||||
OperatorType string
|
||||
PlanFamily string
|
||||
PlanCode string
|
||||
PlanName string
|
||||
Tier string
|
||||
BillingCycle string
|
||||
Currency string
|
||||
ListPrice float64
|
||||
PriceUnit string
|
||||
QuotaValue int64
|
||||
QuotaUnit string
|
||||
ContextWindow int
|
||||
PlanScope string
|
||||
ModelScope string
|
||||
SourceURL string
|
||||
PublishedAt string
|
||||
EffectiveDate string
|
||||
Notes string
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadManualSubscriptionEnv()
|
||||
|
||||
var seedPath string
|
||||
var dryRun bool
|
||||
|
||||
flag.StringVar(&seedPath, "seed", defaultManualSubscriptionSeedPath, "手工订阅套餐 seed JSON 路径")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅校验并打印摘要,不写入数据库")
|
||||
flag.Parse()
|
||||
|
||||
cfg := manualSubscriptionImportConfig{
|
||||
SeedPath: seedPath,
|
||||
DryRun: dryRun,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
dsn := os.Getenv("DATABASE_URL")
|
||||
if dsn == "" {
|
||||
dsn = "postgres://long@/llm_intelligence?host=/var/run/postgresql"
|
||||
}
|
||||
db, err = sql.Open("postgres", dsn)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runManualSubscriptionImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_manual_subscription_seed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func loadManualSubscriptionEnv() {
|
||||
for _, path := range []string{".env.local", ".env"} {
|
||||
loadManualSubscriptionEnvFile(path)
|
||||
}
|
||||
}
|
||||
|
||||
func loadManualSubscriptionEnvFile(path string) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
value = strings.Trim(strings.TrimSpace(value), `"'`)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := os.LookupEnv(key); exists {
|
||||
continue
|
||||
}
|
||||
_ = os.Setenv(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
func runManualSubscriptionImport(cfg manualSubscriptionImportConfig, db *sql.DB, out io.Writer) error {
|
||||
envelope, err := loadManualSubscriptionSeed(cfg.SeedPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := buildManualSubscriptionRows(envelope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return fmt.Errorf("seed is empty")
|
||||
}
|
||||
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(
|
||||
out,
|
||||
"source=manual-subscription-seed checked_at=%s rows=%d operators=%s families=%s dry_run=true\n",
|
||||
envelope.CheckedAt,
|
||||
len(rows),
|
||||
summarizeManualCount(rows, func(row manualSubscriptionRow) string { return row.OperatorName }),
|
||||
summarizeManualCount(rows, func(row manualSubscriptionRow) string { return row.PlanFamily }),
|
||||
)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
|
||||
if err := upsertManualSubscriptionRows(db, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM subscription_plan`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count subscription_plan: %w", err)
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(
|
||||
out,
|
||||
"source=manual-subscription-seed checked_at=%s rows=%d table_rows=%d operators=%s families=%s dry_run=false\n",
|
||||
envelope.CheckedAt,
|
||||
len(rows),
|
||||
tableRows,
|
||||
summarizeManualCount(rows, func(row manualSubscriptionRow) string { return row.OperatorName }),
|
||||
summarizeManualCount(rows, func(row manualSubscriptionRow) string { return row.PlanFamily }),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func loadManualSubscriptionSeed(path string) (manualSubscriptionSeedEnvelope, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return manualSubscriptionSeedEnvelope{}, fmt.Errorf("read seed %s: %w", path, err)
|
||||
}
|
||||
|
||||
var envelope manualSubscriptionSeedEnvelope
|
||||
if err := json.Unmarshal(data, &envelope); err != nil {
|
||||
return manualSubscriptionSeedEnvelope{}, fmt.Errorf("unmarshal seed %s: %w", path, err)
|
||||
}
|
||||
return envelope, nil
|
||||
}
|
||||
|
||||
func buildManualSubscriptionRows(envelope manualSubscriptionSeedEnvelope) ([]manualSubscriptionRow, error) {
|
||||
if _, err := time.Parse(time.RFC3339, envelope.CheckedAt); err != nil {
|
||||
return nil, fmt.Errorf("parse checkedAt: %w", err)
|
||||
}
|
||||
|
||||
validPlanFamilies := map[string]bool{
|
||||
"token_plan": true,
|
||||
"coding_plan": true,
|
||||
"package_plan": true,
|
||||
}
|
||||
|
||||
rows := make([]manualSubscriptionRow, 0, len(envelope.Items))
|
||||
seenCodes := make(map[string]struct{}, len(envelope.Items))
|
||||
for _, item := range envelope.Items {
|
||||
if strings.TrimSpace(item.PlanCode) == "" {
|
||||
return nil, fmt.Errorf("planCode is required")
|
||||
}
|
||||
if _, exists := seenCodes[item.PlanCode]; exists {
|
||||
return nil, fmt.Errorf("duplicate planCode %q", item.PlanCode)
|
||||
}
|
||||
seenCodes[item.PlanCode] = struct{}{}
|
||||
if !validPlanFamilies[item.PlanFamily] {
|
||||
return nil, fmt.Errorf("invalid planFamily %q for %s", item.PlanFamily, item.PlanCode)
|
||||
}
|
||||
if strings.TrimSpace(item.ProviderName) == "" || strings.TrimSpace(item.OperatorName) == "" {
|
||||
return nil, fmt.Errorf("provider/operator is required for %s", item.PlanCode)
|
||||
}
|
||||
if strings.TrimSpace(item.SourceURL) == "" {
|
||||
return nil, fmt.Errorf("sourceURL is required for %s", item.PlanCode)
|
||||
}
|
||||
|
||||
modelScope, _ := json.Marshal(item.ModelScope)
|
||||
rows = append(rows, manualSubscriptionRow{
|
||||
ProviderName: item.ProviderName,
|
||||
ProviderNameCn: item.ProviderNameCn,
|
||||
ProviderCountry: defaultManualIfEmpty(item.ProviderCountry, "unknown"),
|
||||
ProviderWebsite: item.ProviderWebsite,
|
||||
OperatorName: item.OperatorName,
|
||||
OperatorNameCn: item.OperatorNameCn,
|
||||
OperatorCountry: defaultManualIfEmpty(item.OperatorCountry, "unknown"),
|
||||
OperatorWebsite: item.OperatorWebsite,
|
||||
OperatorType: defaultManualIfEmpty(item.OperatorType, "official"),
|
||||
PlanFamily: item.PlanFamily,
|
||||
PlanCode: item.PlanCode,
|
||||
PlanName: item.PlanName,
|
||||
Tier: item.Tier,
|
||||
BillingCycle: defaultManualIfEmpty(item.BillingCycle, "monthly"),
|
||||
Currency: defaultManualIfEmpty(item.Currency, "CNY"),
|
||||
ListPrice: item.ListPrice,
|
||||
PriceUnit: item.PriceUnit,
|
||||
QuotaValue: item.QuotaValue,
|
||||
QuotaUnit: item.QuotaUnit,
|
||||
ContextWindow: item.ContextWindow,
|
||||
PlanScope: item.PlanScope,
|
||||
ModelScope: string(modelScope),
|
||||
SourceURL: item.SourceURL,
|
||||
PublishedAt: item.PublishedAt,
|
||||
EffectiveDate: item.EffectiveDate,
|
||||
Notes: item.Notes,
|
||||
})
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func upsertManualSubscriptionRows(db *sql.DB, rows []manualSubscriptionRow) error {
|
||||
for _, row := range rows {
|
||||
providerID, err := ensureManualSubscriptionProvider(db, row)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
operatorID, err := ensureManualSubscriptionOperator(db, row)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
publishedAt, err := time.Parse("2006-01-02 15:04:05", row.PublishedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse publishedAt for %s: %w", row.PlanCode, err)
|
||||
}
|
||||
effectiveDate, err := time.Parse("2006-01-02", row.EffectiveDate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse effectiveDate for %s: %w", row.PlanCode, err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(
|
||||
`INSERT INTO subscription_plan (
|
||||
provider_id, operator_id, plan_family, plan_code, plan_name, tier,
|
||||
billing_cycle, currency, list_price, price_unit, quota_value, quota_unit,
|
||||
context_window, plan_scope, model_scope, source_url, published_at, effective_date, notes
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6,
|
||||
$7, $8, $9, $10, $11, $12,
|
||||
$13, $14, $15, $16, $17, $18, $19
|
||||
)
|
||||
ON CONFLICT (provider_id, plan_code, effective_date)
|
||||
DO UPDATE SET
|
||||
operator_id = EXCLUDED.operator_id,
|
||||
plan_family = EXCLUDED.plan_family,
|
||||
plan_name = EXCLUDED.plan_name,
|
||||
tier = EXCLUDED.tier,
|
||||
billing_cycle = EXCLUDED.billing_cycle,
|
||||
currency = EXCLUDED.currency,
|
||||
list_price = EXCLUDED.list_price,
|
||||
price_unit = EXCLUDED.price_unit,
|
||||
quota_value = EXCLUDED.quota_value,
|
||||
quota_unit = EXCLUDED.quota_unit,
|
||||
context_window = EXCLUDED.context_window,
|
||||
plan_scope = EXCLUDED.plan_scope,
|
||||
model_scope = EXCLUDED.model_scope,
|
||||
source_url = EXCLUDED.source_url,
|
||||
published_at = EXCLUDED.published_at,
|
||||
notes = EXCLUDED.notes,
|
||||
updated_at = CURRENT_TIMESTAMP`,
|
||||
providerID, operatorID, row.PlanFamily, row.PlanCode, row.PlanName, row.Tier,
|
||||
row.BillingCycle, row.Currency, row.ListPrice, row.PriceUnit, manualNullInt64(row.QuotaValue), manualNullIfEmpty(row.QuotaUnit),
|
||||
manualNullInt(row.ContextWindow), manualNullIfEmpty(row.PlanScope), row.ModelScope, row.SourceURL, publishedAt, effectiveDate, manualNullIfEmpty(row.Notes),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert subscription_plan %s: %w", row.PlanCode, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureManualSubscriptionProvider(db *sql.DB, row manualSubscriptionRow) (int64, error) {
|
||||
var providerID int64
|
||||
err := db.QueryRow(`SELECT id FROM model_provider WHERE name = $1`, row.ProviderName).Scan(&providerID)
|
||||
if err == nil {
|
||||
return providerID, nil
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = db.QueryRow(
|
||||
`INSERT INTO model_provider (name, name_cn, country, website, status)
|
||||
VALUES ($1, $2, $3, $4, 'active')
|
||||
RETURNING id`,
|
||||
row.ProviderName, manualNullIfEmpty(row.ProviderNameCn), row.ProviderCountry, manualNullIfEmpty(row.ProviderWebsite),
|
||||
).Scan(&providerID)
|
||||
return providerID, err
|
||||
}
|
||||
|
||||
func ensureManualSubscriptionOperator(db *sql.DB, row manualSubscriptionRow) (int64, error) {
|
||||
var operatorID int64
|
||||
err := db.QueryRow(`SELECT id FROM operator WHERE name = $1`, row.OperatorName).Scan(&operatorID)
|
||||
if err == nil {
|
||||
return operatorID, nil
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = db.QueryRow(
|
||||
`INSERT INTO operator (name, name_cn, country, website, description, status, type)
|
||||
VALUES ($1, $2, $3, $4, $5, 'active', $6)
|
||||
RETURNING id`,
|
||||
row.OperatorName, manualNullIfEmpty(row.OperatorNameCn), row.OperatorCountry, manualNullIfEmpty(row.OperatorWebsite),
|
||||
fmt.Sprintf("%s manual subscription seed", row.OperatorName), row.OperatorType,
|
||||
).Scan(&operatorID)
|
||||
return operatorID, err
|
||||
}
|
||||
|
||||
func summarizeManualCount(rows []manualSubscriptionRow, getter func(manualSubscriptionRow) string) string {
|
||||
counts := make(map[string]int)
|
||||
keys := make([]string, 0)
|
||||
for _, row := range rows {
|
||||
key := getter(row)
|
||||
if _, exists := counts[key]; !exists {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
counts[key]++
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s:%d", key, counts[key]))
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func defaultManualIfEmpty(value string, fallback string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func manualNullIfEmpty(value string) any {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func manualNullInt(value int) any {
|
||||
if value == 0 {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func manualNullInt64(value int64) any {
|
||||
if value == 0 {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
56
scripts/import_manual_subscription_seed_test.go
Normal file
56
scripts/import_manual_subscription_seed_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildManualSubscriptionRows(t *testing.T) {
|
||||
envelope, err := loadManualSubscriptionSeed(filepath.Join("..", "seeds", "subscription_plan_manual_seed.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("loadManualSubscriptionSeed 失败: %v", err)
|
||||
}
|
||||
|
||||
rows, err := buildManualSubscriptionRows(envelope)
|
||||
if err != nil {
|
||||
t.Fatalf("buildManualSubscriptionRows 失败: %v", err)
|
||||
}
|
||||
if len(rows) != 3 {
|
||||
t.Fatalf("期望 3 条套餐记录,实际 %d", len(rows))
|
||||
}
|
||||
|
||||
if rows[0].PlanCode != "minimax-token-plan-starter" {
|
||||
t.Fatalf("首条 planCode 错误: %q", rows[0].PlanCode)
|
||||
}
|
||||
if rows[len(rows)-1].PlanCode != "minimax-token-plan-max" {
|
||||
t.Fatalf("末条 planCode 错误: %q", rows[len(rows)-1].PlanCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunManualSubscriptionImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runManualSubscriptionImport(manualSubscriptionImportConfig{
|
||||
SeedPath: filepath.Join("..", "seeds", "subscription_plan_manual_seed.json"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runManualSubscriptionImport 失败: %v", err)
|
||||
}
|
||||
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=manual-subscription-seed",
|
||||
"rows=3",
|
||||
"MiniMax:3",
|
||||
"token_plan:3",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
88
scripts/import_minimax_subscription.go
Normal file
88
scripts/import_minimax_subscription.go
Normal file
@@ -0,0 +1,88 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type minimaxSubscriptionImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultMinimaxTokenPlanURL, "MiniMax Token Plan 文档 URL")
|
||||
flag.StringVar(&fixture, "fixture", "", "MiniMax Token Plan 本地样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := minimaxSubscriptionImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runMinimaxSubscriptionImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_minimax_subscription: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runMinimaxSubscriptionImport(cfg minimaxSubscriptionImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
records, err := parseMinimaxTokenPlans(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=minimax-subscription-import plans=%d provider=%s operator=%s dry_run=true\n", len(records), records[0].ProviderName, records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertSubscriptionImportRecords(db, records); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM subscription_plan`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count subscription_plan: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=minimax-subscription-import plans=%d provider=%s operator=%s table_rows=%d dry_run=false\n", len(records), records[0].ProviderName, records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
74
scripts/import_minimax_subscription_test.go
Normal file
74
scripts/import_minimax_subscription_test.go
Normal file
@@ -0,0 +1,74 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseMinimaxTokenPlansBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "minimax_token_plan_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 MiniMax fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
plans, err := parseMinimaxTokenPlans(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseMinimaxTokenPlans 失败: %v", err)
|
||||
}
|
||||
if len(plans) != 12 {
|
||||
t.Fatalf("期望 12 条 MiniMax 套餐记录,实际 %d", len(plans))
|
||||
}
|
||||
if plans[0].PlanCode != "minimax-token-plan-starter" {
|
||||
t.Fatalf("首条 planCode 错误: %q", plans[0].PlanCode)
|
||||
}
|
||||
if plans[0].ListPrice != 10 || plans[0].QuotaValue != 1500 {
|
||||
t.Fatalf("Starter 月度套餐解析错误: %+v", plans[0])
|
||||
}
|
||||
if plans[5].PlanCode != "minimax-token-plan-ultra-highspeed" {
|
||||
t.Fatalf("高速月度套餐 planCode 错误: %q", plans[5].PlanCode)
|
||||
}
|
||||
if plans[5].ListPrice != 150 || plans[5].QuotaValue != 30000 {
|
||||
t.Fatalf("高速月度套餐解析错误: %+v", plans[5])
|
||||
}
|
||||
if plans[len(plans)-1].PlanCode != "minimax-token-plan-ultra-highspeed-yearly" {
|
||||
t.Fatalf("末条 planCode 错误: %q", plans[len(plans)-1].PlanCode)
|
||||
}
|
||||
if plans[len(plans)-1].ListPrice != 1500 || plans[len(plans)-1].QuotaValue != 30000 {
|
||||
t.Fatalf("高速年度套餐解析错误: %+v", plans[len(plans)-1])
|
||||
}
|
||||
if !strings.Contains(plans[0].Notes, "Speech 2.8") {
|
||||
t.Fatalf("标准套餐备注缺少附带配额说明: %q", plans[0].Notes)
|
||||
}
|
||||
if !strings.Contains(plans[5].Notes, "高速版覆盖") {
|
||||
t.Fatalf("高速套餐备注缺少高速版说明: %q", plans[5].Notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMinimaxSubscriptionImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runMinimaxSubscriptionImport(minimaxSubscriptionImportConfig{
|
||||
Fixture: filepath.Join("testdata", "minimax_token_plan_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runMinimaxSubscriptionImport 失败: %v", err)
|
||||
}
|
||||
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=minimax-subscription-import",
|
||||
"plans=12",
|
||||
"provider=MiniMax",
|
||||
"operator=MiniMax",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
94
scripts/import_mobile_cloud_catalog.go
Normal file
94
scripts/import_mobile_cloud_catalog.go
Normal file
@@ -0,0 +1,94 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultMobileCloudCatalogURL = "https://saas.ecloud.10086.cn/Store/List"
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultMobileCloudCatalogURL, "移动云 AI 应用专区 URL")
|
||||
flag.StringVar(&fixture, "fixture", "", "移动云 AI 应用专区样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅校验并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := catalogVerificationImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runMobileCloudCatalogImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_mobile_cloud_catalog: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func parseMobileCloudCatalog(raw string) ([]catalogVerificationRecord, error) {
|
||||
if !strings.Contains(raw, "AI应用专区") {
|
||||
return nil, fmt.Errorf("mobile cloud AI market marker not found")
|
||||
}
|
||||
if !strings.Contains(raw, "数据大模型") {
|
||||
return nil, fmt.Errorf("mobile cloud data model marker not found")
|
||||
}
|
||||
return []catalogVerificationRecord{{
|
||||
CatalogCode: "mobile-cloud-ai-market",
|
||||
SourceURL: defaultMobileCloudCatalogURL,
|
||||
SourceTitle: "移动云市场 AI 应用专区",
|
||||
PlanStatus: "confirmed",
|
||||
Notes: "官方云市场已公开展示 AI 应用专区,覆盖数据大模型等类目,但统一编程套餐价格仍未公开披露。",
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func runMobileCloudCatalogImport(cfg catalogVerificationImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseMobileCloudCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=mobile-cloud-catalog-import entries=%d dry_run=true\n", len(records))
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertCatalogVerificationRecords(db, records); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=mobile-cloud-catalog-import entries=%d dry_run=false\n", len(records))
|
||||
return err
|
||||
}
|
||||
54
scripts/import_mobile_cloud_catalog_test.go
Normal file
54
scripts/import_mobile_cloud_catalog_test.go
Normal file
@@ -0,0 +1,54 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseMobileCloudCatalogBuildsRecord(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "mobile_cloud_catalog_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseMobileCloudCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseMobileCloudCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 1 {
|
||||
t.Fatalf("期望 1 条移动云目录记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].CatalogCode != "mobile-cloud-ai-market" {
|
||||
t.Fatalf("catalogCode 错误: %q", records[0].CatalogCode)
|
||||
}
|
||||
if records[0].PlanStatus != "confirmed" {
|
||||
t.Fatalf("planStatus 错误: %q", records[0].PlanStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMobileCloudCatalogImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runMobileCloudCatalogImport(catalogVerificationImportConfig{
|
||||
URL: defaultMobileCloudCatalogURL,
|
||||
Fixture: filepath.Join("testdata", "mobile_cloud_catalog_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runMobileCloudCatalogImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=mobile-cloud-catalog-import",
|
||||
"entries=1",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
552
scripts/import_plan_catalog.go
Normal file
552
scripts/import_plan_catalog.go
Normal file
@@ -0,0 +1,552 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
const defaultPlanCatalogSeedPaths = "seeds/plan_catalog_inventory_seed.json,seeds/plan_catalog_inventory_seed_cn_vendors_top20.json,seeds/plan_catalog_inventory_seed_cn_relays_top20plus.json,seeds/plan_catalog_inventory_seed_web_research.json"
|
||||
|
||||
type importPlanCatalogConfig struct {
|
||||
SeedPaths string
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
type planCatalogSeedEnvelope struct {
|
||||
CheckedAt string `json:"checkedAt"`
|
||||
Items []planCatalogSeedItem `json:"items"`
|
||||
}
|
||||
|
||||
type planCatalogSeedItem struct {
|
||||
CatalogCode string `json:"catalogCode"`
|
||||
ProviderName string `json:"providerName"`
|
||||
ProviderNameCn string `json:"providerNameCn"`
|
||||
ProviderCountry string `json:"providerCountry"`
|
||||
ProviderWebsite string `json:"providerWebsite"`
|
||||
OperatorName string `json:"operatorName"`
|
||||
OperatorNameCn string `json:"operatorNameCn"`
|
||||
OperatorCountry string `json:"operatorCountry"`
|
||||
OperatorWebsite string `json:"operatorWebsite"`
|
||||
OperatorType string `json:"operatorType"`
|
||||
PlatformName string `json:"platformName"`
|
||||
PlatformNameCn string `json:"platformNameCn"`
|
||||
PlatformType string `json:"platformType"`
|
||||
PlanFamily string `json:"planFamily"`
|
||||
PlanStatus string `json:"planStatus"`
|
||||
SourceURL string `json:"sourceURL"`
|
||||
SourceTitle string `json:"sourceTitle"`
|
||||
SourceKind string `json:"sourceKind"`
|
||||
Region string `json:"region"`
|
||||
Currency string `json:"currency"`
|
||||
BillingCycle string `json:"billingCycle"`
|
||||
ImporterKey string `json:"importerKey"`
|
||||
Notes string `json:"notes"`
|
||||
CatalogSegment string `json:"catalogSegment"`
|
||||
MarketRank int `json:"marketRank"`
|
||||
}
|
||||
|
||||
type planCatalogRow struct {
|
||||
CatalogCode string
|
||||
ProviderName string
|
||||
ProviderNameCn string
|
||||
ProviderCountry string
|
||||
ProviderWebsite string
|
||||
OperatorName string
|
||||
OperatorNameCn string
|
||||
OperatorCountry string
|
||||
OperatorWebsite string
|
||||
OperatorType string
|
||||
PlatformName string
|
||||
PlatformNameCn string
|
||||
PlatformType string
|
||||
PlanFamily string
|
||||
PlanStatus string
|
||||
SourceURL string
|
||||
SourceTitle string
|
||||
SourceKind string
|
||||
Region string
|
||||
Currency string
|
||||
BillingCycle string
|
||||
ImporterKey string
|
||||
Notes string
|
||||
LastCheckedAt time.Time
|
||||
CatalogSegment string
|
||||
MarketRank int
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadImportProjectEnv()
|
||||
|
||||
var seedPaths string
|
||||
var dryRun bool
|
||||
|
||||
flag.StringVar(&seedPaths, "seed", defaultPlanCatalogSeedPaths, "基础目录 seed JSON 路径,支持逗号分隔多个文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅校验并打印摘要,不写入数据库")
|
||||
flag.Parse()
|
||||
|
||||
cfg := importPlanCatalogConfig{
|
||||
SeedPaths: seedPaths,
|
||||
DryRun: dryRun,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
dsn := os.Getenv("DATABASE_URL")
|
||||
if dsn == "" {
|
||||
dsn = "postgres://long@/llm_intelligence?host=/var/run/postgresql"
|
||||
}
|
||||
db, err = sql.Open("postgres", dsn)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runPlanCatalogImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_plan_catalog: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func loadImportProjectEnv() {
|
||||
for _, path := range []string{".env.local", ".env"} {
|
||||
loadImportEnvFile(path)
|
||||
}
|
||||
}
|
||||
|
||||
func loadImportEnvFile(path string) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
value = strings.Trim(strings.TrimSpace(value), `"'`)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := os.LookupEnv(key); exists {
|
||||
continue
|
||||
}
|
||||
_ = os.Setenv(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
func runPlanCatalogImport(cfg importPlanCatalogConfig, db *sql.DB, out io.Writer) error {
|
||||
envelope, err := loadPlanCatalogSeeds(splitCSVPaths(cfg.SeedPaths))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := buildPlanCatalogRows(envelope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return fmt.Errorf("seed is empty")
|
||||
}
|
||||
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(
|
||||
out,
|
||||
"source=plan-catalog-import checked_at=%s rows=%d families=%s statuses=%s dry_run=true\n",
|
||||
envelope.CheckedAt,
|
||||
len(rows),
|
||||
formatSummaryCount(countByField(rows, func(row planCatalogRow) string { return row.PlanFamily })),
|
||||
formatSummaryCount(countByField(rows, func(row planCatalogRow) string { return row.PlanStatus })),
|
||||
)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
|
||||
if err := upsertPlanCatalogInventory(db, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM plan_catalog_inventory`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count plan_catalog_inventory: %w", err)
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(
|
||||
out,
|
||||
"source=plan-catalog-import checked_at=%s rows=%d table_rows=%d families=%s statuses=%s dry_run=false\n",
|
||||
envelope.CheckedAt,
|
||||
len(rows),
|
||||
tableRows,
|
||||
formatSummaryCount(countByField(rows, func(row planCatalogRow) string { return row.PlanFamily })),
|
||||
formatSummaryCount(countByField(rows, func(row planCatalogRow) string { return row.PlanStatus })),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func loadPlanCatalogSeed(path string) (planCatalogSeedEnvelope, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return planCatalogSeedEnvelope{}, fmt.Errorf("read seed %s: %w", path, err)
|
||||
}
|
||||
|
||||
var envelope planCatalogSeedEnvelope
|
||||
if err := json.Unmarshal(data, &envelope); err != nil {
|
||||
return planCatalogSeedEnvelope{}, fmt.Errorf("unmarshal seed %s: %w", path, err)
|
||||
}
|
||||
return envelope, nil
|
||||
}
|
||||
|
||||
func loadPlanCatalogSeeds(paths []string) (planCatalogSeedEnvelope, error) {
|
||||
if len(paths) == 0 {
|
||||
return planCatalogSeedEnvelope{}, fmt.Errorf("at least one seed path is required")
|
||||
}
|
||||
|
||||
mergedItems := make(map[string]planCatalogSeedItem)
|
||||
var checkedAt string
|
||||
for _, path := range paths {
|
||||
envelope, err := loadPlanCatalogSeed(path)
|
||||
if err != nil {
|
||||
return planCatalogSeedEnvelope{}, err
|
||||
}
|
||||
if strings.TrimSpace(envelope.CheckedAt) != "" {
|
||||
checkedAt = envelope.CheckedAt
|
||||
}
|
||||
for _, item := range envelope.Items {
|
||||
mergedItems[item.CatalogCode] = item
|
||||
}
|
||||
}
|
||||
|
||||
codes := make([]string, 0, len(mergedItems))
|
||||
for code := range mergedItems {
|
||||
codes = append(codes, code)
|
||||
}
|
||||
sort.Strings(codes)
|
||||
|
||||
items := make([]planCatalogSeedItem, 0, len(codes))
|
||||
for _, code := range codes {
|
||||
items = append(items, mergedItems[code])
|
||||
}
|
||||
|
||||
return planCatalogSeedEnvelope{
|
||||
CheckedAt: checkedAt,
|
||||
Items: items,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildPlanCatalogRows(envelope planCatalogSeedEnvelope) ([]planCatalogRow, error) {
|
||||
checkedAt, err := time.Parse(time.RFC3339, envelope.CheckedAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse checkedAt: %w", err)
|
||||
}
|
||||
|
||||
validPlatformTypes := map[string]bool{
|
||||
"official_vendor": true,
|
||||
"cloud_operator": true,
|
||||
"relay_platform": true,
|
||||
}
|
||||
validPlanFamilies := map[string]bool{
|
||||
"token_plan": true,
|
||||
"coding_plan": true,
|
||||
"package_plan": true,
|
||||
"pay_as_you_go": true,
|
||||
"unknown": true,
|
||||
}
|
||||
validPlanStatuses := map[string]bool{
|
||||
"confirmed": true,
|
||||
"pending_verification": true,
|
||||
"retired": true,
|
||||
}
|
||||
validSourceKinds := map[string]bool{
|
||||
"official_doc": true,
|
||||
"official_pricing": true,
|
||||
"official_product_page": true,
|
||||
"official_community": true,
|
||||
"inferred": true,
|
||||
}
|
||||
validCatalogSegments := map[string]bool{
|
||||
"general": true,
|
||||
"vendor_top20": true,
|
||||
"relay_top20plus": true,
|
||||
"global_reference": true,
|
||||
}
|
||||
|
||||
rows := make([]planCatalogRow, 0, len(envelope.Items))
|
||||
seenCodes := make(map[string]struct{}, len(envelope.Items))
|
||||
for _, item := range envelope.Items {
|
||||
if strings.TrimSpace(item.CatalogCode) == "" {
|
||||
return nil, fmt.Errorf("catalogCode is required")
|
||||
}
|
||||
if _, exists := seenCodes[item.CatalogCode]; exists {
|
||||
return nil, fmt.Errorf("duplicate catalogCode %q", item.CatalogCode)
|
||||
}
|
||||
seenCodes[item.CatalogCode] = struct{}{}
|
||||
if !validPlatformTypes[item.PlatformType] {
|
||||
return nil, fmt.Errorf("invalid platformType %q for %s", item.PlatformType, item.CatalogCode)
|
||||
}
|
||||
if !validPlanFamilies[item.PlanFamily] {
|
||||
return nil, fmt.Errorf("invalid planFamily %q for %s", item.PlanFamily, item.CatalogCode)
|
||||
}
|
||||
if !validPlanStatuses[item.PlanStatus] {
|
||||
return nil, fmt.Errorf("invalid planStatus %q for %s", item.PlanStatus, item.CatalogCode)
|
||||
}
|
||||
if !validSourceKinds[item.SourceKind] {
|
||||
return nil, fmt.Errorf("invalid sourceKind %q for %s", item.SourceKind, item.CatalogCode)
|
||||
}
|
||||
segment := defaultIfEmpty(item.CatalogSegment, "general")
|
||||
if !validCatalogSegments[segment] {
|
||||
return nil, fmt.Errorf("invalid catalogSegment %q for %s", item.CatalogSegment, item.CatalogCode)
|
||||
}
|
||||
if item.MarketRank < 0 {
|
||||
return nil, fmt.Errorf("invalid marketRank %d for %s", item.MarketRank, item.CatalogCode)
|
||||
}
|
||||
if strings.TrimSpace(item.ProviderName) == "" {
|
||||
return nil, fmt.Errorf("providerName is required for %s", item.CatalogCode)
|
||||
}
|
||||
if strings.TrimSpace(item.PlatformName) == "" {
|
||||
return nil, fmt.Errorf("platformName is required for %s", item.CatalogCode)
|
||||
}
|
||||
if strings.TrimSpace(item.SourceURL) == "" {
|
||||
return nil, fmt.Errorf("sourceURL is required for %s", item.CatalogCode)
|
||||
}
|
||||
|
||||
rows = append(rows, planCatalogRow{
|
||||
CatalogCode: item.CatalogCode,
|
||||
ProviderName: item.ProviderName,
|
||||
ProviderNameCn: item.ProviderNameCn,
|
||||
ProviderCountry: defaultIfEmpty(item.ProviderCountry, "unknown"),
|
||||
ProviderWebsite: item.ProviderWebsite,
|
||||
OperatorName: item.OperatorName,
|
||||
OperatorNameCn: item.OperatorNameCn,
|
||||
OperatorCountry: defaultIfEmpty(item.OperatorCountry, "unknown"),
|
||||
OperatorWebsite: item.OperatorWebsite,
|
||||
OperatorType: defaultIfEmpty(item.OperatorType, "official"),
|
||||
PlatformName: item.PlatformName,
|
||||
PlatformNameCn: item.PlatformNameCn,
|
||||
PlatformType: item.PlatformType,
|
||||
PlanFamily: item.PlanFamily,
|
||||
PlanStatus: item.PlanStatus,
|
||||
SourceURL: item.SourceURL,
|
||||
SourceTitle: item.SourceTitle,
|
||||
SourceKind: item.SourceKind,
|
||||
Region: defaultIfEmpty(item.Region, "global"),
|
||||
Currency: item.Currency,
|
||||
BillingCycle: item.BillingCycle,
|
||||
ImporterKey: item.ImporterKey,
|
||||
Notes: item.Notes,
|
||||
LastCheckedAt: checkedAt,
|
||||
CatalogSegment: segment,
|
||||
MarketRank: item.MarketRank,
|
||||
})
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func upsertPlanCatalogInventory(db *sql.DB, rows []planCatalogRow) error {
|
||||
for _, row := range rows {
|
||||
providerID, err := ensurePlanCatalogProvider(db, row)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var operatorID any
|
||||
if strings.TrimSpace(row.OperatorName) != "" {
|
||||
id, err := ensurePlanCatalogOperator(db, row)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
operatorID = id
|
||||
}
|
||||
|
||||
_, err = db.Exec(
|
||||
`INSERT INTO plan_catalog_inventory (
|
||||
provider_id, operator_id, catalog_code, platform_name, platform_name_cn,
|
||||
platform_type, plan_family, plan_status, source_url, source_title,
|
||||
source_kind, region, currency, billing_cycle, last_checked_at,
|
||||
importer_key, notes, catalog_segment, market_rank
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5,
|
||||
$6, $7, $8, $9, $10,
|
||||
$11, $12, $13, $14, $15,
|
||||
$16, $17, $18, $19
|
||||
)
|
||||
ON CONFLICT (catalog_code)
|
||||
DO UPDATE SET
|
||||
provider_id = EXCLUDED.provider_id,
|
||||
operator_id = EXCLUDED.operator_id,
|
||||
platform_name = EXCLUDED.platform_name,
|
||||
platform_name_cn = EXCLUDED.platform_name_cn,
|
||||
platform_type = EXCLUDED.platform_type,
|
||||
plan_family = EXCLUDED.plan_family,
|
||||
plan_status = EXCLUDED.plan_status,
|
||||
source_url = EXCLUDED.source_url,
|
||||
source_title = EXCLUDED.source_title,
|
||||
source_kind = EXCLUDED.source_kind,
|
||||
region = EXCLUDED.region,
|
||||
currency = EXCLUDED.currency,
|
||||
billing_cycle = EXCLUDED.billing_cycle,
|
||||
last_checked_at = EXCLUDED.last_checked_at,
|
||||
importer_key = EXCLUDED.importer_key,
|
||||
notes = EXCLUDED.notes,
|
||||
catalog_segment = EXCLUDED.catalog_segment,
|
||||
market_rank = EXCLUDED.market_rank,
|
||||
updated_at = CURRENT_TIMESTAMP`,
|
||||
providerID, operatorID, row.CatalogCode, row.PlatformName, nullIfEmpty(row.PlatformNameCn),
|
||||
row.PlatformType, row.PlanFamily, row.PlanStatus, row.SourceURL, nullIfEmpty(row.SourceTitle),
|
||||
row.SourceKind, row.Region, nullIfEmpty(row.Currency), nullIfEmpty(row.BillingCycle), row.LastCheckedAt,
|
||||
nullIfEmpty(row.ImporterKey), nullIfEmpty(row.Notes), row.CatalogSegment, nullIfZeroInt(row.MarketRank),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert plan_catalog_inventory %s: %w", row.CatalogCode, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensurePlanCatalogProvider(db *sql.DB, row planCatalogRow) (int64, error) {
|
||||
var providerID int64
|
||||
err := db.QueryRow(`SELECT id FROM model_provider WHERE name = $1`, row.ProviderName).Scan(&providerID)
|
||||
if err == nil {
|
||||
_, updateErr := db.Exec(
|
||||
`UPDATE model_provider
|
||||
SET name_cn = COALESCE(NULLIF(name_cn, ''), $2),
|
||||
country = CASE
|
||||
WHEN COALESCE(country, '') = '' OR country = 'unknown' THEN $3
|
||||
ELSE country
|
||||
END,
|
||||
website = COALESCE(NULLIF(website, ''), $4),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = $1`,
|
||||
providerID, nullIfEmpty(row.ProviderNameCn), row.ProviderCountry, nullIfEmpty(row.ProviderWebsite),
|
||||
)
|
||||
return providerID, updateErr
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = db.QueryRow(
|
||||
`INSERT INTO model_provider (name, name_cn, country, website, status)
|
||||
VALUES ($1, $2, $3, $4, 'active')
|
||||
RETURNING id`,
|
||||
row.ProviderName, nullIfEmpty(row.ProviderNameCn), row.ProviderCountry, nullIfEmpty(row.ProviderWebsite),
|
||||
).Scan(&providerID)
|
||||
return providerID, err
|
||||
}
|
||||
|
||||
func ensurePlanCatalogOperator(db *sql.DB, row planCatalogRow) (int64, error) {
|
||||
var operatorID int64
|
||||
err := db.QueryRow(`SELECT id FROM operator WHERE name = $1`, row.OperatorName).Scan(&operatorID)
|
||||
if err == nil {
|
||||
_, updateErr := db.Exec(
|
||||
`UPDATE operator
|
||||
SET name_cn = COALESCE(NULLIF(name_cn, ''), $2),
|
||||
country = CASE
|
||||
WHEN COALESCE(country, '') = '' OR country = 'unknown' THEN $3
|
||||
ELSE country
|
||||
END,
|
||||
website = COALESCE(NULLIF(website, ''), $4),
|
||||
description = COALESCE(NULLIF(description, ''), $5),
|
||||
type = CASE
|
||||
WHEN COALESCE(type, '') = '' OR type = 'reseller' THEN $6
|
||||
ELSE type
|
||||
END,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = $1`,
|
||||
operatorID,
|
||||
nullIfEmpty(row.OperatorNameCn),
|
||||
row.OperatorCountry,
|
||||
nullIfEmpty(row.OperatorWebsite),
|
||||
fmt.Sprintf("%s catalog inventory", row.PlatformName),
|
||||
nullIfEmpty(row.OperatorType),
|
||||
)
|
||||
return operatorID, updateErr
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = db.QueryRow(
|
||||
`INSERT INTO operator (name, name_cn, country, website, description, status, type)
|
||||
VALUES ($1, $2, $3, $4, $5, 'active', $6)
|
||||
RETURNING id`,
|
||||
row.OperatorName, nullIfEmpty(row.OperatorNameCn), row.OperatorCountry, nullIfEmpty(row.OperatorWebsite),
|
||||
fmt.Sprintf("%s catalog inventory", row.PlatformName), row.OperatorType,
|
||||
).Scan(&operatorID)
|
||||
return operatorID, err
|
||||
}
|
||||
|
||||
func countByField(rows []planCatalogRow, getter func(planCatalogRow) string) map[string]int {
|
||||
result := make(map[string]int)
|
||||
for _, row := range rows {
|
||||
result[getter(row)]++
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func formatSummaryCount(values map[string]int) string {
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s:%d", key, values[key]))
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func defaultIfEmpty(value string, fallback string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func splitCSVPaths(raw string) []string {
|
||||
parts := strings.Split(raw, ",")
|
||||
paths := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
paths = append(paths, part)
|
||||
}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func nullIfEmpty(value string) any {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func nullIfZeroInt(value int) any {
|
||||
if value == 0 {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
149
scripts/import_plan_catalog_test.go
Normal file
149
scripts/import_plan_catalog_test.go
Normal file
@@ -0,0 +1,149 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildPlanCatalogRows(t *testing.T) {
|
||||
envelope, err := loadPlanCatalogSeeds([]string{
|
||||
filepath.Join("..", "seeds", "plan_catalog_inventory_seed.json"),
|
||||
filepath.Join("..", "seeds", "plan_catalog_inventory_seed_cn_vendors_top20.json"),
|
||||
filepath.Join("..", "seeds", "plan_catalog_inventory_seed_cn_relays_top20plus.json"),
|
||||
filepath.Join("..", "seeds", "plan_catalog_inventory_seed_web_research.json"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("loadPlanCatalogSeeds 失败: %v", err)
|
||||
}
|
||||
|
||||
rows, err := buildPlanCatalogRows(envelope)
|
||||
if err != nil {
|
||||
t.Fatalf("buildPlanCatalogRows 失败: %v", err)
|
||||
}
|
||||
if len(rows) != 70 {
|
||||
t.Fatalf("期望 70 条基础目录记录,实际 %d", len(rows))
|
||||
}
|
||||
|
||||
foundVendorTop20 := false
|
||||
foundRelayTop20Plus := false
|
||||
wantImporterKeys := map[string]string{
|
||||
"tencent-cloud-token-plan-enterprise-pro": "tencent_catalog",
|
||||
"tencent-cloud-token-plan-enterprise-lite": "tencent_catalog",
|
||||
"tencent-cloud-coding-plan": "tencent_catalog",
|
||||
"aliyun-bailian-token-plan-team": "import_aliyun_subscription.go",
|
||||
"aliyun-bailian-coding-plan": "import_aliyun_subscription.go",
|
||||
"baidu-qianfan-token-benefit-pack": "import_baidu_subscription.go",
|
||||
"baidu-qianfan-coding-plan": "import_baidu_subscription.go",
|
||||
"zhipu-glm-coding-plan": "import_zhipu_coding_plan.go",
|
||||
"minimax-token-plan": "import_minimax_subscription.go",
|
||||
"volcengine-ark-coding-plan": "import_bytedance_subscription.go",
|
||||
"huawei-cloud-maas-package-plan": "import_huawei_package.go",
|
||||
"ctyun-token-plan": "import_ctyun_subscription.go",
|
||||
"ctyun-coding-plan": "import_ctyun_subscription.go",
|
||||
"cucloud-aicp-platform": "import_cucloud_catalog.go",
|
||||
"cucloud-ai-app-platform": "import_cucloud_catalog.go",
|
||||
"mobile-cloud-ai-market": "import_mobile_cloud_catalog.go",
|
||||
"youdao-zhiyun-maas": "import_youdao_pricing.go",
|
||||
"360-open-platform": "import_360_pricing.go",
|
||||
"siliconflow-siliconcloud": "import_siliconflow_pricing.go",
|
||||
"ppio-model-api": "import_ppio_pricing.go",
|
||||
"ucloud-umodelverse": "import_ucloud_pricing.go",
|
||||
"anthropic-api-payg": "import_catalog_seed_verification.go",
|
||||
"xai-api-payg": "import_catalog_seed_verification.go",
|
||||
"alibaba-qwen-api-payg": "import_catalog_seed_verification.go",
|
||||
"tencent-hunyuan-api-payg": "import_catalog_seed_verification.go",
|
||||
"huawei-pangu-api-payg": "import_catalog_seed_verification.go",
|
||||
"baichuan-api-payg": "import_catalog_seed_verification.go",
|
||||
"01ai-api-payg": "import_catalog_seed_verification.go",
|
||||
"sensenova-api-payg": "import_catalog_seed_verification.go",
|
||||
"xfyun-spark-api-payg": "import_catalog_seed_verification.go",
|
||||
"360-zhinao-api-payg": "import_catalog_seed_verification.go",
|
||||
"youdao-ziyue-api-payg": "import_catalog_seed_verification.go",
|
||||
"modelbest-minicpm-api-payg": "import_catalog_seed_verification.go",
|
||||
"baai-flagopen-api-payg": "import_catalog_seed_verification.go",
|
||||
"skywork-api-payg": "import_catalog_seed_verification.go",
|
||||
"infinigence-api-payg": "import_catalog_seed_verification.go",
|
||||
"qingcloud-coreshub": "import_catalog_seed_verification.go",
|
||||
"ksyun-xingliu-platform": "import_catalog_seed_verification.go",
|
||||
"google-gemini-api-payg": "import_catalog_seed_verification.go",
|
||||
"mistral-api-payg": "import_catalog_seed_verification.go",
|
||||
"cohere-api-payg": "import_catalog_seed_verification.go",
|
||||
"openrouter-api-payg": "fetch_openrouter.go",
|
||||
"together-ai-api-payg": "import_catalog_seed_verification.go",
|
||||
"fireworks-ai-api-payg": "import_catalog_seed_verification.go",
|
||||
"deepinfra-api-payg": "import_catalog_seed_verification.go",
|
||||
"groq-api-payg": "import_catalog_seed_verification.go",
|
||||
"replicate-api-payg": "import_catalog_seed_verification.go",
|
||||
"hyperbolic-api-payg": "import_catalog_seed_verification.go",
|
||||
"novita-ai-api-payg": "import_catalog_seed_verification.go",
|
||||
"azure-openai-service-payg": "import_azure_openai_pricing.go",
|
||||
"amazon-bedrock-payg": "import_bedrock_pricing.go",
|
||||
"google-vertex-ai-genai-payg": "import_vertex_pricing.go",
|
||||
"cloudflare-workers-ai-payg": "import_cloudflare_pricing.go",
|
||||
"baseten-inference-payg": "import_catalog_seed_verification.go",
|
||||
"cerebras-inference-payg": "import_catalog_seed_verification.go",
|
||||
"perplexity-agent-api-payg": "import_perplexity_pricing.go",
|
||||
"sambanova-cloud-payg": "import_catalog_seed_verification.go",
|
||||
"jdcloud-joybuilder-payg": "import_catalog_seed_verification.go",
|
||||
}
|
||||
for _, row := range rows {
|
||||
if row.CatalogCode == "zhipu-glm-coding-plan" {
|
||||
if row.CatalogSegment != "vendor_top20" || row.MarketRank != 5 {
|
||||
t.Fatalf("智谱榜单字段错误: segment=%q rank=%d", row.CatalogSegment, row.MarketRank)
|
||||
}
|
||||
foundVendorTop20 = true
|
||||
}
|
||||
if row.CatalogCode == "ctyun-coding-plan" {
|
||||
if row.CatalogSegment != "relay_top20plus" || row.MarketRank != 9 {
|
||||
t.Fatalf("天翼云编码套餐榜单字段错误: segment=%q rank=%d", row.CatalogSegment, row.MarketRank)
|
||||
}
|
||||
foundRelayTop20Plus = true
|
||||
}
|
||||
if wantImporterKey, ok := wantImporterKeys[row.CatalogCode]; ok && row.ImporterKey != wantImporterKey {
|
||||
t.Fatalf("%s importerKey 错误: got=%q want=%q", row.CatalogCode, row.ImporterKey, wantImporterKey)
|
||||
}
|
||||
}
|
||||
if !foundVendorTop20 {
|
||||
t.Fatalf("缺少 vendor_top20 覆盖记录")
|
||||
}
|
||||
if !foundRelayTop20Plus {
|
||||
t.Fatalf("缺少 relay_top20plus 覆盖记录")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPlanCatalogImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runPlanCatalogImport(importPlanCatalogConfig{
|
||||
SeedPaths: strings.Join([]string{
|
||||
filepath.Join("..", "seeds", "plan_catalog_inventory_seed.json"),
|
||||
filepath.Join("..", "seeds", "plan_catalog_inventory_seed_cn_vendors_top20.json"),
|
||||
filepath.Join("..", "seeds", "plan_catalog_inventory_seed_cn_relays_top20plus.json"),
|
||||
filepath.Join("..", "seeds", "plan_catalog_inventory_seed_web_research.json"),
|
||||
}, ","),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runPlanCatalogImport 失败: %v", err)
|
||||
}
|
||||
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=plan-catalog-import",
|
||||
"rows=70",
|
||||
"coding_plan:7",
|
||||
"package_plan:1",
|
||||
"pay_as_you_go:51",
|
||||
"token_plan:7",
|
||||
"unknown:4",
|
||||
"confirmed:70",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
88
scripts/import_ppio_pricing.go
Normal file
88
scripts/import_ppio_pricing.go
Normal file
@@ -0,0 +1,88 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ppioPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultPPIOPricingURL, "PPIO Model API 官方价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "PPIO 价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := ppioPricingImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runPPIOPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_ppio_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runPPIOPricingImport(cfg ppioPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parsePPIOPricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=ppio-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "ppio-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=ppio-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
55
scripts/import_ppio_pricing_test.go
Normal file
55
scripts/import_ppio_pricing_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParsePPIOPricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "ppio_pricing_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parsePPIOPricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parsePPIOPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 5 {
|
||||
t.Fatalf("期望 5 条 PPIO 价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].InputPrice != 2 {
|
||||
t.Fatalf("deepseek-v3.1 输入价错误: %v", records[0].InputPrice)
|
||||
}
|
||||
if records[1].OutputPrice != 16 {
|
||||
t.Fatalf("deepseek-r1 输出价错误: %v", records[1].OutputPrice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPPIOPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runPPIOPricingImport(ppioPricingImportConfig{
|
||||
URL: defaultPPIOPricingURL,
|
||||
Fixture: filepath.Join("testdata", "ppio_pricing_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runPPIOPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=ppio-pricing-import",
|
||||
"models=5",
|
||||
"operator=PPIO Model API",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
96
scripts/import_siliconflow_pricing.go
Normal file
96
scripts/import_siliconflow_pricing.go
Normal file
@@ -0,0 +1,96 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
type siliconFlowPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultSiliconFlowPricingURL, "SiliconFlow 官方价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "SiliconFlow 价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := siliconFlowPricingImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runSiliconFlowPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_siliconflow_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runSiliconFlowPricingImport(cfg siliconFlowPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil && cfg.Fixture == "" {
|
||||
raw, err = fetchSubscriptionPage(cfg.URL, filepath.Join("scripts", "testdata", "siliconflow_pricing_sample.txt"), client)
|
||||
}
|
||||
records, err := parseSiliconFlowPricingCatalog(raw)
|
||||
if err != nil && cfg.Fixture == "" {
|
||||
raw, err = fetchSubscriptionPage(cfg.URL, filepath.Join("scripts", "testdata", "siliconflow_pricing_sample.txt"), client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err = parseSiliconFlowPricingCatalog(raw)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=siliconflow-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "siliconflow-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=siliconflow-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
55
scripts/import_siliconflow_pricing_test.go
Normal file
55
scripts/import_siliconflow_pricing_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseSiliconFlowPricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "siliconflow_pricing_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseSiliconFlowPricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseSiliconFlowPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 5 {
|
||||
t.Fatalf("期望 5 条硅基流动价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].ProviderName != "Qwen" {
|
||||
t.Fatalf("Qwen provider 识别错误: %q", records[0].ProviderName)
|
||||
}
|
||||
if !records[4].IsFree {
|
||||
t.Fatalf("免费模型应标记为 free_tier")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSiliconFlowPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runSiliconFlowPricingImport(siliconFlowPricingImportConfig{
|
||||
URL: defaultSiliconFlowPricingURL,
|
||||
Fixture: filepath.Join("testdata", "siliconflow_pricing_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runSiliconFlowPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=siliconflow-pricing-import",
|
||||
"models=5",
|
||||
"operator=SiliconCloud",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
88
scripts/import_ucloud_pricing.go
Normal file
88
scripts/import_ucloud_pricing.go
Normal file
@@ -0,0 +1,88 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ucloudPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultUCloudPricingURL, "UCloud UModelVerse 官方价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "UCloud 价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := ucloudPricingImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runUCloudPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_ucloud_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runUCloudPricingImport(cfg ucloudPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseUCloudPricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=ucloud-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "ucloud-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=ucloud-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
55
scripts/import_ucloud_pricing_test.go
Normal file
55
scripts/import_ucloud_pricing_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseUCloudPricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "ucloud_pricing_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseUCloudPricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseUCloudPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 5 {
|
||||
t.Fatalf("期望 5 条 UCloud 价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].InputPrice != 0.1 {
|
||||
t.Fatalf("gpt-4o-mini 输入价换算错误: %v", records[0].InputPrice)
|
||||
}
|
||||
if records[2].OutputPrice != 16 {
|
||||
t.Fatalf("DeepSeek-R1 输出价错误: %v", records[2].OutputPrice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUCloudPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runUCloudPricingImport(ucloudPricingImportConfig{
|
||||
URL: defaultUCloudPricingURL,
|
||||
Fixture: filepath.Join("testdata", "ucloud_pricing_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runUCloudPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=ucloud-pricing-import",
|
||||
"models=5",
|
||||
"operator=UModelVerse",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
88
scripts/import_youdao_pricing.go
Normal file
88
scripts/import_youdao_pricing.go
Normal file
@@ -0,0 +1,88 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type youdaoPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultYoudaoPricingURL, "有道智云 MaaS 官方价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "有道智云 MaaS 价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := youdaoPricingImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runYoudaoPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_youdao_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runYoudaoPricingImport(cfg youdaoPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseYoudaoPricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=youdao-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "youdao-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=youdao-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
58
scripts/import_youdao_pricing_test.go
Normal file
58
scripts/import_youdao_pricing_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseYoudaoPricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "youdao_pricing_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseYoudaoPricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseYoudaoPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 5 {
|
||||
t.Fatalf("期望 5 条有道价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].ModelID != "youdao-deepseek-v4-flash" {
|
||||
t.Fatalf("首条 modelID 错误: %q", records[0].ModelID)
|
||||
}
|
||||
if records[2].ProviderName != "Moonshot AI" {
|
||||
t.Fatalf("Kimi provider 归一化错误: %q", records[2].ProviderName)
|
||||
}
|
||||
if records[4].ContextLength != 128000 {
|
||||
t.Fatalf("GLM-5 上下文长度错误: %d", records[4].ContextLength)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunYoudaoPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runYoudaoPricingImport(youdaoPricingImportConfig{
|
||||
URL: defaultYoudaoPricingURL,
|
||||
Fixture: filepath.Join("testdata", "youdao_pricing_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runYoudaoPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=youdao-pricing-import",
|
||||
"models=5",
|
||||
"operator=Youdao Zhiyun MaaS",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
100
scripts/import_zhipu_coding_plan.go
Normal file
100
scripts/import_zhipu_coding_plan.go
Normal file
@@ -0,0 +1,100 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type zhipuCodingPlanImportConfig struct {
|
||||
OverviewURL string
|
||||
PromotionURL string
|
||||
OverviewFixture string
|
||||
PromotionFixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var overviewURL string
|
||||
var promotionURL string
|
||||
var overviewFixture string
|
||||
var promotionFixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&overviewURL, "overview-url", defaultZhipuCodingPlanOverviewURL, "智谱 Coding Plan 概览 URL")
|
||||
flag.StringVar(&promotionURL, "promotion-url", defaultZhipuCodingPlanPromotionURL, "智谱 Coding Plan 活动页 URL")
|
||||
flag.StringVar(&overviewFixture, "overview-fixture", "", "智谱 Coding Plan 概览样例文件")
|
||||
flag.StringVar(&promotionFixture, "promotion-fixture", "", "智谱 Coding Plan 活动页样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := zhipuCodingPlanImportConfig{
|
||||
OverviewURL: overviewURL,
|
||||
PromotionURL: promotionURL,
|
||||
OverviewFixture: overviewFixture,
|
||||
PromotionFixture: promotionFixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runZhipuCodingPlanImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_zhipu_coding_plan: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runZhipuCodingPlanImport(cfg zhipuCodingPlanImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
overviewRaw, err := fetchSubscriptionPage(cfg.OverviewURL, cfg.OverviewFixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promotionRaw, err := fetchSubscriptionPage(cfg.PromotionURL, cfg.PromotionFixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
records, err := parseZhipuCodingPlanCatalog(overviewRaw, promotionRaw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=zhipu-coding-plan-import plans=%d provider=%s operator=%s dry_run=true\n", len(records), records[0].ProviderName, records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertSubscriptionImportRecords(db, records); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM subscription_plan`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count subscription_plan: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=zhipu-coding-plan-import plans=%d provider=%s operator=%s table_rows=%d dry_run=false\n", len(records), records[0].ProviderName, records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
66
scripts/import_zhipu_coding_plan_test.go
Normal file
66
scripts/import_zhipu_coding_plan_test.go
Normal file
@@ -0,0 +1,66 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseZhipuCodingPlanBuildsPromoEntry(t *testing.T) {
|
||||
overviewRaw, err := os.ReadFile(filepath.Join("testdata", "zhipu_coding_plan_overview_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 overview fixture 失败: %v", err)
|
||||
}
|
||||
promoRaw, err := os.ReadFile(filepath.Join("testdata", "zhipu_coding_plan_promotion_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 promotion fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
plans, err := parseZhipuCodingPlanCatalog(string(overviewRaw), string(promoRaw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseZhipuCodingPlanCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(plans) != 1 {
|
||||
t.Fatalf("期望 1 条智谱公开活动价记录,实际 %d", len(plans))
|
||||
}
|
||||
if plans[0].PlanCode != "zhipu-coding-plan-promo-floor" {
|
||||
t.Fatalf("planCode 错误: %q", plans[0].PlanCode)
|
||||
}
|
||||
if plans[0].ListPrice != 20 {
|
||||
t.Fatalf("活动价错误: %v", plans[0].ListPrice)
|
||||
}
|
||||
if !strings.Contains(plans[0].Notes, "Lite/Pro/Max") {
|
||||
t.Fatalf("备注缺少套餐分档说明: %q", plans[0].Notes)
|
||||
}
|
||||
if !strings.Contains(plans[0].Notes, "首单 9 折") {
|
||||
t.Fatalf("备注缺少折扣说明: %q", plans[0].Notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunZhipuCodingPlanImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runZhipuCodingPlanImport(zhipuCodingPlanImportConfig{
|
||||
OverviewFixture: filepath.Join("testdata", "zhipu_coding_plan_overview_sample.txt"),
|
||||
PromotionFixture: filepath.Join("testdata", "zhipu_coding_plan_promotion_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runZhipuCodingPlanImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=zhipu-coding-plan-import",
|
||||
"plans=1",
|
||||
"provider=Zhipu AI",
|
||||
"operator=Zhipu",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
188
scripts/minimax_subscription_lib.go
Normal file
188
scripts/minimax_subscription_lib.go
Normal file
@@ -0,0 +1,188 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultMinimaxTokenPlanURL = "https://platform.minimax.io/docs/guides/pricing-token-plan"
|
||||
|
||||
type minimaxPlanSpec struct {
|
||||
billingCycle string
|
||||
priceUnit string
|
||||
blockPattern string
|
||||
modelScope []string
|
||||
tiers []string
|
||||
planCodes []string
|
||||
planNames []string
|
||||
quotaUnit string
|
||||
}
|
||||
|
||||
func parseMinimaxTokenPlans(raw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(raw)
|
||||
normalized := normalizeMinimaxTokenPlanText(raw)
|
||||
|
||||
specs := []minimaxPlanSpec{
|
||||
{
|
||||
billingCycle: "monthly",
|
||||
priceUnit: "USD/month",
|
||||
blockPattern: `Monthly.*?Starter\s+Plus\s+Max\s+Price\s+\$([\d,]+)\s*/month\s+\$([\d,]+)\s*/month\s+\$([\d,]+)\s*/month\s+M2\.7\s+([\d,]+)\s+requests/5hrs\s+([\d,]+)\s+requests/5hrs\s+([\d,]+)\s+requests/5hrs`,
|
||||
modelScope: []string{"MiniMax-M2.7"},
|
||||
tiers: []string{"Starter", "Plus", "Max"},
|
||||
planCodes: []string{"minimax-token-plan-starter", "minimax-token-plan-plus", "minimax-token-plan-max"},
|
||||
planNames: []string{"MiniMax Token Plan Starter", "MiniMax Token Plan Plus", "MiniMax Token Plan Max"},
|
||||
quotaUnit: "requests/5hrs",
|
||||
},
|
||||
{
|
||||
billingCycle: "monthly",
|
||||
priceUnit: "USD/month",
|
||||
blockPattern: `Monthly.*?Highspeed Plans\s+Plus-Highspeed\s+Max-Highspeed\s+Ultra-Highspeed\s+Price\s+\$([\d,]+)\s*/month\s+\$([\d,]+)\s*/month\s+\$([\d,]+)\s*/month\s+M2\.7-highspeed\s+([\d,]+)\s+requests/5hrs\s+([\d,]+)\s+requests/5hrs\s+([\d,]+)\s+requests/5hrs`,
|
||||
modelScope: []string{"MiniMax-M2.7-Highspeed"},
|
||||
tiers: []string{"Plus-Highspeed", "Max-Highspeed", "Ultra-Highspeed"},
|
||||
planCodes: []string{"minimax-token-plan-plus-highspeed", "minimax-token-plan-max-highspeed", "minimax-token-plan-ultra-highspeed"},
|
||||
planNames: []string{"MiniMax Token Plan Plus-Highspeed", "MiniMax Token Plan Max-Highspeed", "MiniMax Token Plan Ultra-Highspeed"},
|
||||
quotaUnit: "requests/5hrs",
|
||||
},
|
||||
{
|
||||
billingCycle: "yearly",
|
||||
priceUnit: "USD/year",
|
||||
blockPattern: `Yearly.*?Starter\s+Plus\s+Max\s+Price\s+\$([\d,]+)\s*/year\s+\$([\d,]+)\s*/year\s+\$([\d,]+)\s*/year\s+M2\.7\s+([\d,]+)\s+requests/5hrs\s+([\d,]+)\s+requests/5hrs\s+([\d,]+)\s+requests/5hrs`,
|
||||
modelScope: []string{"MiniMax-M2.7"},
|
||||
tiers: []string{"Starter-Yearly", "Plus-Yearly", "Max-Yearly"},
|
||||
planCodes: []string{"minimax-token-plan-starter-yearly", "minimax-token-plan-plus-yearly", "minimax-token-plan-max-yearly"},
|
||||
planNames: []string{"MiniMax Token Plan Starter Yearly", "MiniMax Token Plan Plus Yearly", "MiniMax Token Plan Max Yearly"},
|
||||
quotaUnit: "requests/5hrs",
|
||||
},
|
||||
{
|
||||
billingCycle: "yearly",
|
||||
priceUnit: "USD/year",
|
||||
blockPattern: `Yearly.*?Highspeed Plans\s+Plus-Highspeed\s+Max-Highspeed\s+Ultra-Highspeed\s+Price\s+\$([\d,]+)\s*/year\s+\$([\d,]+)\s*/year\s+\$([\d,]+)\s*/year\s+M2\.7-highspeed\s+([\d,]+)\s+requests/5hrs\s+([\d,]+)\s+requests/5hrs\s+([\d,]+)\s+requests/5hrs`,
|
||||
modelScope: []string{"MiniMax-M2.7-Highspeed"},
|
||||
tiers: []string{"Plus-Highspeed-Yearly", "Max-Highspeed-Yearly", "Ultra-Highspeed-Yearly"},
|
||||
planCodes: []string{"minimax-token-plan-plus-highspeed-yearly", "minimax-token-plan-max-highspeed-yearly", "minimax-token-plan-ultra-highspeed-yearly"},
|
||||
planNames: []string{"MiniMax Token Plan Plus-Highspeed Yearly", "MiniMax Token Plan Max-Highspeed Yearly", "MiniMax Token Plan Ultra-Highspeed Yearly"},
|
||||
quotaUnit: "requests/5hrs",
|
||||
},
|
||||
}
|
||||
|
||||
var records []subscriptionImportRecord
|
||||
for _, spec := range specs {
|
||||
parsed, err := parseMinimaxPlanBlock(normalized, spec, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, parsed...)
|
||||
}
|
||||
|
||||
standardNotes := extractMinimaxNotes(normalized, []string{
|
||||
"Speech 2.8",
|
||||
"image-01",
|
||||
"Hailuo-2.3-Fast",
|
||||
"Hailuo-2.3",
|
||||
"Music-2.6",
|
||||
})
|
||||
for i := range records {
|
||||
records[i].PublishedAtKnown = known
|
||||
if strings.Contains(records[i].PlanCode, "highspeed") {
|
||||
records[i].Notes = joinNonEmptyNotes(records[i].Notes, "高速版覆盖 MiniMax-M2.7-Highspeed。")
|
||||
continue
|
||||
}
|
||||
records[i].Notes = joinNonEmptyNotes(records[i].Notes, standardNotes)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseMinimaxPlanBlock(raw string, spec minimaxPlanSpec, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
match := regexp.MustCompile(spec.blockPattern).FindStringSubmatch(raw)
|
||||
if len(match) != 7 {
|
||||
return nil, fmt.Errorf("unexpected minimax %s block", spec.billingCycle)
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, 3)
|
||||
for i := range spec.tiers {
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "MiniMax",
|
||||
ProviderNameCn: "MiniMax",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://platform.minimax.io",
|
||||
OperatorName: "MiniMax",
|
||||
OperatorNameCn: "MiniMax",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://platform.minimax.io/docs/guides/pricing-overview",
|
||||
OperatorType: "official",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: spec.planCodes[i],
|
||||
PlanName: spec.planNames[i],
|
||||
Tier: spec.tiers[i],
|
||||
BillingCycle: spec.billingCycle,
|
||||
Currency: "USD",
|
||||
ListPrice: mustParseSubscriptionPrice(match[i+1]),
|
||||
PriceUnit: spec.priceUnit,
|
||||
QuotaValue: mustParseSubscriptionInt64(match[i+4]),
|
||||
QuotaUnit: spec.quotaUnit,
|
||||
PlanScope: "Token Plan",
|
||||
ModelScope: append([]string(nil), spec.modelScope...),
|
||||
SourceURL: defaultMinimaxTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
PublishedAtKnown: true,
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func normalizeMinimaxTokenPlanText(raw string) string {
|
||||
replacer := strings.NewReplacer(
|
||||
"High-Speed", "Highspeed",
|
||||
"High Speed", "Highspeed",
|
||||
"Max-High-Speed", "Max-Highspeed",
|
||||
"Ultra-High-Speed", "Ultra-Highspeed",
|
||||
"Plus `Cost-Effective`", "Plus",
|
||||
"Max `Extra Large`", "Max",
|
||||
"Starter `Save $20`", "Starter",
|
||||
"Plus `Save $40`", "Plus",
|
||||
"Max `Save $100`", "Max",
|
||||
"Plus-Highspeed `Save $80`", "Plus-Highspeed",
|
||||
"Max-Highspeed `Save $160`", "Max-Highspeed",
|
||||
"Ultra-Highspeed `Save $300`", "Ultra-Highspeed",
|
||||
"Subscribe Now Standard Plans:", "",
|
||||
"Subscribe Now", "",
|
||||
"Standard Plans:", "",
|
||||
"Highspeed Plans:", "Highspeed Plans",
|
||||
)
|
||||
normalized := replacer.Replace(raw)
|
||||
normalized = strings.ReplaceAll(normalized, "\r\n", "\n")
|
||||
normalized = strings.ReplaceAll(normalized, "\r", "\n")
|
||||
normalized = strings.ReplaceAll(normalized, "|", " ")
|
||||
normalized = strings.ReplaceAll(normalized, "---", " ")
|
||||
normalized = regexp.MustCompile(`\s+`).ReplaceAllString(normalized, " ")
|
||||
return strings.TrimSpace(normalized)
|
||||
}
|
||||
|
||||
func extractMinimaxNotes(raw string, markers []string) string {
|
||||
var hits []string
|
||||
for _, marker := range markers {
|
||||
if strings.Contains(raw, marker) {
|
||||
hits = append(hits, marker)
|
||||
}
|
||||
}
|
||||
if len(hits) == 0 {
|
||||
return ""
|
||||
}
|
||||
return "附带配额包含 " + strings.Join(hits, " / ") + "。"
|
||||
}
|
||||
|
||||
func joinNonEmptyNotes(parts ...string) string {
|
||||
filtered := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, part)
|
||||
}
|
||||
return strings.Join(filtered, " ")
|
||||
}
|
||||
508
scripts/official_pricing_import_common.go
Normal file
508
scripts/official_pricing_import_common.go
Normal file
@@ -0,0 +1,508 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const officialPricingFetchMaxAttempts = 3
|
||||
|
||||
type officialPricingFetchOptions struct {
|
||||
AcceptLanguage string
|
||||
}
|
||||
|
||||
type officialPricingRecord struct {
|
||||
ModelID string
|
||||
ModelName string
|
||||
ProviderName string
|
||||
ProviderNameCn string
|
||||
ProviderCountry string
|
||||
ProviderWebsite string
|
||||
OperatorName string
|
||||
OperatorNameCn string
|
||||
OperatorCountry string
|
||||
OperatorWebsite string
|
||||
OperatorType string
|
||||
Region string
|
||||
Currency string
|
||||
InputPrice float64
|
||||
OutputPrice float64
|
||||
ContextLength int
|
||||
IsFree bool
|
||||
SourceURL string
|
||||
ModelSourceURL string
|
||||
ReleaseDate string
|
||||
DateConfidence string
|
||||
DateSourceKind string
|
||||
Modality string
|
||||
}
|
||||
|
||||
func upsertOfficialPricingRecords(db *sql.DB, records []officialPricingRecord, batchID string) error {
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if len(records) == 0 {
|
||||
return fmt.Errorf("official pricing records are empty")
|
||||
}
|
||||
if strings.TrimSpace(batchID) == "" {
|
||||
batchID = fmt.Sprintf("official-pricing-%s", time.Now().Format("20060102-150405"))
|
||||
}
|
||||
|
||||
for _, record := range records {
|
||||
providerID, err := ensureOfficialPricingProvider(db, record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
operatorID, err := ensureOfficialPricingOperator(db, record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modelID, err := ensureOfficialPricingModel(db, record, providerID, batchID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sourceType := officialPricingSourceType(record.OperatorType, record.IsFree)
|
||||
freeQuota := ""
|
||||
freeLimitations := "[]"
|
||||
rateLimit := "{}"
|
||||
if record.IsFree {
|
||||
freeQuota = "See source_url for provider free-tier details"
|
||||
freeLimitations = `["See source_url for current quota and policy"]`
|
||||
}
|
||||
|
||||
_, err = db.Exec(
|
||||
`INSERT INTO region_pricing (
|
||||
model_id, operator_id, region, currency,
|
||||
input_price_per_mtok, output_price_per_mtok,
|
||||
is_free, effective_date, source_url, source_type,
|
||||
free_quota, free_limitations, rate_limit
|
||||
) VALUES (
|
||||
$1, $2, $3, $4,
|
||||
$5, $6, $7, CURRENT_DATE, $8, $9,
|
||||
$10, $11, $12
|
||||
)
|
||||
ON CONFLICT (model_id, operator_id, region, currency, effective_date)
|
||||
DO UPDATE SET
|
||||
input_price_per_mtok = EXCLUDED.input_price_per_mtok,
|
||||
output_price_per_mtok = EXCLUDED.output_price_per_mtok,
|
||||
is_free = EXCLUDED.is_free,
|
||||
source_url = EXCLUDED.source_url,
|
||||
source_type = EXCLUDED.source_type,
|
||||
free_quota = EXCLUDED.free_quota,
|
||||
free_limitations = EXCLUDED.free_limitations,
|
||||
rate_limit = EXCLUDED.rate_limit,
|
||||
updated_at = CURRENT_TIMESTAMP`,
|
||||
modelID, operatorID, record.Region, record.Currency,
|
||||
record.InputPrice, record.OutputPrice, record.IsFree, record.SourceURL, sourceType,
|
||||
nullIfBlank(freeQuota), freeLimitations, rateLimit,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert region_pricing %s: %w", record.ModelID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureOfficialPricingProvider(db *sql.DB, record officialPricingRecord) (int64, error) {
|
||||
var providerID int64
|
||||
err := db.QueryRow(`SELECT id FROM model_provider WHERE name = $1`, record.ProviderName).Scan(&providerID)
|
||||
if err == nil {
|
||||
_, updateErr := db.Exec(
|
||||
`UPDATE model_provider
|
||||
SET name_cn = COALESCE(name_cn, $2),
|
||||
website = COALESCE(NULLIF(website, ''), $3),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = $1`,
|
||||
providerID, nullIfBlank(record.ProviderNameCn), nullIfBlank(record.ProviderWebsite),
|
||||
)
|
||||
return providerID, updateErr
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = db.QueryRow(
|
||||
`INSERT INTO model_provider (name, name_cn, country, website, status)
|
||||
VALUES ($1, $2, $3, $4, 'active')
|
||||
RETURNING id`,
|
||||
record.ProviderName, nullIfBlank(record.ProviderNameCn), record.ProviderCountry, nullIfBlank(record.ProviderWebsite),
|
||||
).Scan(&providerID)
|
||||
return providerID, err
|
||||
}
|
||||
|
||||
func ensureOfficialPricingOperator(db *sql.DB, record officialPricingRecord) (int64, error) {
|
||||
var operatorID int64
|
||||
err := db.QueryRow(`SELECT id FROM operator WHERE name = $1`, record.OperatorName).Scan(&operatorID)
|
||||
if err == nil {
|
||||
_, updateErr := db.Exec(
|
||||
`UPDATE operator
|
||||
SET name_cn = COALESCE(name_cn, $2),
|
||||
website = COALESCE(NULLIF(website, ''), $3),
|
||||
type = COALESCE(NULLIF(type, ''), $4),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = $1`,
|
||||
operatorID, nullIfBlank(record.OperatorNameCn), nullIfBlank(record.OperatorWebsite), nullIfBlank(record.OperatorType),
|
||||
)
|
||||
return operatorID, updateErr
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = db.QueryRow(
|
||||
`INSERT INTO operator (name, name_cn, country, website, description, status, type)
|
||||
VALUES ($1, $2, $3, $4, $5, 'active', $6)
|
||||
RETURNING id`,
|
||||
record.OperatorName, nullIfBlank(record.OperatorNameCn), record.OperatorCountry, nullIfBlank(record.OperatorWebsite),
|
||||
fmt.Sprintf("%s official pricing import", record.OperatorName), record.OperatorType,
|
||||
).Scan(&operatorID)
|
||||
return operatorID, err
|
||||
}
|
||||
|
||||
func ensureOfficialPricingModel(db *sql.DB, record officialPricingRecord, providerID int64, batchID string) (int64, error) {
|
||||
var modelID int64
|
||||
err := db.QueryRow(`SELECT id FROM models WHERE external_id = $1`, record.ModelID).Scan(&modelID)
|
||||
if err == sql.ErrNoRows {
|
||||
err = db.QueryRow(
|
||||
`INSERT INTO models (
|
||||
external_id, name, provider_id, modality, context_length,
|
||||
status, source, batch_id, source_url, release_date,
|
||||
date_confidence, date_source_kind
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5,
|
||||
'active', $6, $7, $8, $9,
|
||||
$10, $11
|
||||
) RETURNING id`,
|
||||
record.ModelID, record.ModelName, providerID, fallbackModality(record.Modality), nullIfZeroIntCommon(record.ContextLength),
|
||||
record.OperatorName, batchID, firstNonEmptyText(record.ModelSourceURL, record.SourceURL), releaseDateValueCommon(record.ReleaseDate),
|
||||
fallbackDateConfidence(record.DateConfidence), fallbackDateSourceKind(record.DateSourceKind),
|
||||
).Scan(&modelID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return modelID, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
_, err = db.Exec(
|
||||
`UPDATE models
|
||||
SET name = $2,
|
||||
provider_id = $3,
|
||||
modality = COALESCE($4, modality),
|
||||
context_length = COALESCE($5, context_length),
|
||||
source = $6,
|
||||
batch_id = $7,
|
||||
source_url = COALESCE(NULLIF(source_url, ''), $8),
|
||||
release_date = COALESCE(release_date, $9),
|
||||
date_confidence = COALESCE(NULLIF(date_confidence, ''), $10),
|
||||
date_source_kind = COALESCE(NULLIF(date_source_kind, ''), $11),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = $1`,
|
||||
modelID, record.ModelName, providerID, nullIfBlank(fallbackModality(record.Modality)), nullIfZeroIntCommon(record.ContextLength),
|
||||
record.OperatorName, batchID, firstNonEmptyText(record.ModelSourceURL, record.SourceURL), releaseDateValueCommon(record.ReleaseDate),
|
||||
fallbackDateConfidence(record.DateConfidence), fallbackDateSourceKind(record.DateSourceKind),
|
||||
)
|
||||
return modelID, err
|
||||
}
|
||||
|
||||
func officialPricingSourceType(operatorType string, isFree bool) string {
|
||||
if isFree {
|
||||
return "free_tier"
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(operatorType)) {
|
||||
case "official":
|
||||
return "official"
|
||||
default:
|
||||
return "reseller"
|
||||
}
|
||||
}
|
||||
|
||||
func releaseDateValueCommon(raw string) any {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return nil
|
||||
}
|
||||
parsed, err := time.Parse("2006-01-02", raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func fallbackDateConfidence(raw string) string {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func fallbackDateSourceKind(raw string) string {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return "official_product_page"
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func fallbackModality(raw string) string {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
return "text"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func fetchRawPricingPage(url string, fixture string, client *http.Client) (string, error) {
|
||||
return fetchRawPricingPageWithOptions(url, fixture, client, officialPricingFetchOptions{
|
||||
AcceptLanguage: "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
})
|
||||
}
|
||||
|
||||
func fetchRawPricingPageWithOptions(url string, fixture string, client *http.Client, opts officialPricingFetchOptions) (string, error) {
|
||||
if fixture != "" {
|
||||
data, err := os.ReadFile(fixture)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read fixture %s: %w", fixture, err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 20 * time.Second}
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= officialPricingFetchMaxAttempts; attempt++ {
|
||||
body, retryable, err := fetchRawPricingPageOnce(url, client, opts)
|
||||
if err == nil {
|
||||
return body, nil
|
||||
}
|
||||
lastErr = err
|
||||
if !retryable || attempt == officialPricingFetchMaxAttempts {
|
||||
return "", err
|
||||
}
|
||||
time.Sleep(time.Duration(attempt) * 200 * time.Millisecond)
|
||||
}
|
||||
return "", lastErr
|
||||
}
|
||||
|
||||
func fetchRawPricingPageOnce(url string, client *http.Client, opts officialPricingFetchOptions) (string, bool, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/json,text/plain;q=0.9,*/*;q=0.8")
|
||||
if strings.TrimSpace(opts.AcceptLanguage) != "" {
|
||||
req.Header.Set("Accept-Language", opts.AcceptLanguage)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", isRetriablePricingFetchError(err), fmt.Errorf("fetch %s: %w", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
retryable := resp.StatusCode == http.StatusTooManyRequests ||
|
||||
resp.StatusCode == http.StatusBadGateway ||
|
||||
resp.StatusCode == http.StatusServiceUnavailable ||
|
||||
resp.StatusCode == http.StatusGatewayTimeout
|
||||
return "", retryable, fmt.Errorf("fetch %s: unexpected status %d", url, resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", isRetriablePricingFetchError(err), fmt.Errorf("read %s: %w", url, err)
|
||||
}
|
||||
return string(body), false, nil
|
||||
}
|
||||
|
||||
func isRetriablePricingFetchError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(err.Error())
|
||||
for _, marker := range []string{
|
||||
"eof",
|
||||
"timeout",
|
||||
"temporarily unavailable",
|
||||
"transport closed",
|
||||
"connection reset",
|
||||
"connection refused",
|
||||
"tls handshake timeout",
|
||||
"i/o timeout",
|
||||
"too many requests",
|
||||
"no such host",
|
||||
} {
|
||||
if strings.Contains(lower, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func cleanHTMLText(raw string) string {
|
||||
tagPattern := regexp.MustCompile(`(?is)<[^>]+>`)
|
||||
spacePattern := regexp.MustCompile(`[ \t]+`)
|
||||
text := html.UnescapeString(raw)
|
||||
text = strings.ReplaceAll(text, "\r\n", "\n")
|
||||
text = strings.ReplaceAll(text, "\r", "\n")
|
||||
text = strings.ReplaceAll(text, "\u00a0", " ")
|
||||
text = tagPattern.ReplaceAllString(text, " ")
|
||||
text = spacePattern.ReplaceAllString(text, " ")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
func firstDollarPrice(raw string) (float64, bool) {
|
||||
pattern := regexp.MustCompile(`\$ ?([0-9]+(?:\.[0-9]+)?)`)
|
||||
match := pattern.FindStringSubmatch(raw)
|
||||
if len(match) != 2 {
|
||||
return 0, false
|
||||
}
|
||||
return mustParseSubscriptionPrice(match[1]), true
|
||||
}
|
||||
|
||||
func normalizeExternalID(parts ...string) string {
|
||||
joined := strings.ToLower(strings.Join(parts, "-"))
|
||||
replacer := regexp.MustCompile(`[^a-z0-9]+`)
|
||||
normalized := replacer.ReplaceAllString(joined, "-")
|
||||
normalized = strings.Trim(normalized, "-")
|
||||
normalized = regexp.MustCompile(`-+`).ReplaceAllString(normalized, "-")
|
||||
return normalized
|
||||
}
|
||||
|
||||
func parseContextLengthCommon(raw string) int {
|
||||
cleaned := strings.TrimSpace(strings.ToUpper(strings.ReplaceAll(raw, ",", "")))
|
||||
if cleaned == "" {
|
||||
return 0
|
||||
}
|
||||
switch {
|
||||
case strings.HasSuffix(cleaned, "M"):
|
||||
return int(parseDecimalMultiplier(strings.TrimSuffix(cleaned, "M"), 1000000))
|
||||
case strings.HasSuffix(cleaned, "K"):
|
||||
return int(parseDecimalMultiplier(strings.TrimSuffix(cleaned, "K"), 1000))
|
||||
default:
|
||||
return int(mustParseSubscriptionInt64(cleaned))
|
||||
}
|
||||
}
|
||||
|
||||
func detectModality(modelName string) string {
|
||||
lower := strings.ToLower(modelName)
|
||||
switch {
|
||||
case strings.Contains(lower, "coder"), strings.Contains(lower, "code"):
|
||||
return "code"
|
||||
case strings.Contains(lower, "vision"), strings.Contains(lower, "vl"), strings.Contains(lower, "omni"), strings.Contains(lower, "multi"), strings.Contains(lower, "live"):
|
||||
return "multimodal"
|
||||
default:
|
||||
return "text"
|
||||
}
|
||||
}
|
||||
|
||||
func providerMetadata(providerName string) (string, string, string) {
|
||||
switch providerName {
|
||||
case "Alibaba", "Qwen":
|
||||
return "阿里云", "CN", "https://tongyi.aliyun.com"
|
||||
case "Amazon":
|
||||
return "亚马逊", "US", "https://aws.amazon.com"
|
||||
case "Anthropic":
|
||||
return "Anthropic", "US", "https://www.anthropic.com"
|
||||
case "Baidu":
|
||||
return "百度", "CN", "https://cloud.baidu.com"
|
||||
case "Cloudflare":
|
||||
return "Cloudflare", "US", "https://www.cloudflare.com"
|
||||
case "Cohere":
|
||||
return "Cohere", "CA", "https://cohere.com"
|
||||
case "DeepSeek":
|
||||
return "深度求索", "CN", "https://www.deepseek.com"
|
||||
case "Google":
|
||||
return "谷歌", "US", "https://ai.google.dev"
|
||||
case "Meta":
|
||||
return "Meta", "US", "https://about.meta.com"
|
||||
case "MiniMax":
|
||||
return "MiniMax", "CN", "https://www.minimax.io"
|
||||
case "Mistral AI":
|
||||
return "Mistral AI", "FR", "https://mistral.ai"
|
||||
case "Moonshot AI":
|
||||
return "月之暗面", "CN", "https://www.moonshot.cn"
|
||||
case "NVIDIA":
|
||||
return "NVIDIA", "US", "https://build.nvidia.com"
|
||||
case "OpenAI":
|
||||
return "OpenAI", "US", "https://openai.com"
|
||||
case "Perplexity":
|
||||
return "Perplexity", "US", "https://www.perplexity.ai"
|
||||
case "xAI":
|
||||
return "xAI", "US", "https://x.ai"
|
||||
case "Zhipu AI":
|
||||
return "智谱", "CN", "https://open.bigmodel.cn"
|
||||
default:
|
||||
return "", "unknown", ""
|
||||
}
|
||||
}
|
||||
|
||||
func providerFromModelPath(modelName string) string {
|
||||
lower := strings.ToLower(modelName)
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "amazon/"):
|
||||
return "Amazon"
|
||||
case strings.HasPrefix(lower, "anthropic/"):
|
||||
return "Anthropic"
|
||||
case strings.HasPrefix(lower, "cohere/"):
|
||||
return "Cohere"
|
||||
case strings.HasPrefix(lower, "qwen/"):
|
||||
return "Qwen"
|
||||
case strings.HasPrefix(lower, "deepseek"), strings.HasPrefix(lower, "deepseek-ai/"):
|
||||
return "DeepSeek"
|
||||
case strings.HasPrefix(lower, "google/"), strings.HasPrefix(lower, "gemini/"):
|
||||
return "Google"
|
||||
case strings.HasPrefix(lower, "meta/"):
|
||||
return "Meta"
|
||||
case strings.HasPrefix(lower, "mistral/"), strings.HasPrefix(lower, "mistralai/"):
|
||||
return "Mistral AI"
|
||||
case strings.HasPrefix(lower, "moonshotai/"):
|
||||
return "Moonshot AI"
|
||||
case strings.HasPrefix(lower, "minimaxai/"):
|
||||
return "MiniMax"
|
||||
case strings.HasPrefix(lower, "nvidia/"):
|
||||
return "NVIDIA"
|
||||
case strings.HasPrefix(lower, "perplexity/"):
|
||||
return "Perplexity"
|
||||
case strings.HasPrefix(lower, "zai-org/"), strings.HasPrefix(lower, "glm/"):
|
||||
return "Zhipu AI"
|
||||
case strings.HasPrefix(lower, "openai/"):
|
||||
return "OpenAI"
|
||||
case strings.HasPrefix(lower, "xai/"):
|
||||
return "xAI"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func dedupeOfficialPricingRecords(records []officialPricingRecord) []officialPricingRecord {
|
||||
seen := make(map[string]officialPricingRecord)
|
||||
order := make([]string, 0, len(records))
|
||||
for _, record := range records {
|
||||
key := strings.Join([]string{
|
||||
record.OperatorName,
|
||||
record.ModelID,
|
||||
record.Region,
|
||||
record.Currency,
|
||||
}, "|")
|
||||
if _, exists := seen[key]; !exists {
|
||||
order = append(order, key)
|
||||
}
|
||||
seen[key] = record
|
||||
}
|
||||
result := make([]officialPricingRecord, 0, len(order))
|
||||
for _, key := range order {
|
||||
result = append(result, seen[key])
|
||||
}
|
||||
return result
|
||||
}
|
||||
49
scripts/official_pricing_import_common_test.go
Normal file
49
scripts/official_pricing_import_common_test.go
Normal file
@@ -0,0 +1,49 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFetchRawPricingPageRetriesTransientStatus(t *testing.T) {
|
||||
var attempts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
current := atomic.AddInt32(&attempts, 1)
|
||||
if current == 1 {
|
||||
http.Error(w, "temporary", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := &http.Client{Timeout: 2 * time.Second}
|
||||
body, err := fetchRawPricingPage(server.URL, "", client)
|
||||
if err != nil {
|
||||
t.Fatalf("fetchRawPricingPage returned error: %v", err)
|
||||
}
|
||||
if body != "ok" {
|
||||
t.Fatalf("body = %q, want ok", body)
|
||||
}
|
||||
if got := atomic.LoadInt32(&attempts); got != 2 {
|
||||
t.Fatalf("attempts = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetriablePricingFetchErrorRecognizesEOF(t *testing.T) {
|
||||
if !isRetriablePricingFetchError(errString("unexpected EOF")) {
|
||||
t.Fatalf("expected EOF to be retriable")
|
||||
}
|
||||
if isRetriablePricingFetchError(errString("bad request")) {
|
||||
t.Fatalf("expected bad request to be non-retriable")
|
||||
}
|
||||
}
|
||||
|
||||
type errString string
|
||||
|
||||
func (e errString) Error() string { return string(e) }
|
||||
76
scripts/platform360_pricing_lib.go
Normal file
76
scripts/platform360_pricing_lib.go
Normal file
@@ -0,0 +1,76 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const default360PricingURL = "https://ai.360.com/open/models"
|
||||
|
||||
var platform360CardPattern = regexp.MustCompile(`(?s)([A-Za-z0-9._/-]+)\n([^\n]+)\n.*?(?:输入价格|Input Price)\s*:\s*¥([\d.]+)\s*/\s*1M tokens.*?(?:输出价格|Output Price)\s*:\s*¥([\d.]+)\s*/\s*1M tokens.*?(?:上下文|Context)\s*:\s*([\d,]+)`)
|
||||
|
||||
func parse360PricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
matches := platform360CardPattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("unexpected 360 pricing content")
|
||||
}
|
||||
|
||||
records := make([]officialPricingRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
modelName := strings.TrimSpace(match[1])
|
||||
providerName := normalize360Provider(match[2], modelName)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("360", modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "360 Open Platform",
|
||||
OperatorNameCn: "360 智脑开放平台",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://ai.360.com",
|
||||
OperatorType: "relay",
|
||||
Region: "CN",
|
||||
Currency: "CNY",
|
||||
InputPrice: mustParseSubscriptionPrice(match[3]),
|
||||
OutputPrice: mustParseSubscriptionPrice(match[4]),
|
||||
ContextLength: parseContextLengthCommon(match[5]),
|
||||
SourceURL: default360PricingURL,
|
||||
ModelSourceURL: default360PricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_product_page",
|
||||
Modality: detectModality(modelName),
|
||||
}
|
||||
record.IsFree = record.InputPrice == 0 && record.OutputPrice == 0
|
||||
records = append(records, record)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func normalize360Provider(raw string, modelName string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case "deepseek", "深度求索":
|
||||
return "DeepSeek"
|
||||
case "moonshot ai", "月之暗面":
|
||||
return "Moonshot AI"
|
||||
case "qwen", "阿里巴巴", "通义千问":
|
||||
return "Qwen"
|
||||
case "zhipu", "智谱":
|
||||
return "Zhipu AI"
|
||||
case "字节跳动":
|
||||
return "ByteDance"
|
||||
case "360智脑":
|
||||
return "360"
|
||||
default:
|
||||
providerByPath := providerFromModelPath(modelName)
|
||||
if providerByPath != "unknown" {
|
||||
return providerByPath
|
||||
}
|
||||
return strings.TrimSpace(raw)
|
||||
}
|
||||
}
|
||||
64
scripts/ppio_pricing_lib.go
Normal file
64
scripts/ppio_pricing_lib.go
Normal file
@@ -0,0 +1,64 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultPPIOPricingURL = "https://resource.ppio.com/pricing?type=enterprise"
|
||||
|
||||
var ppioBlockPattern = regexp.MustCompile(`(?s)([a-z0-9._/-]+)\n([\d,]+)\n(.*?)在线体验`)
|
||||
var ppioPricePattern = regexp.MustCompile(`(?s)¥\s*([\d.]+)\s*/\s*Mt`)
|
||||
|
||||
func parsePPIOPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
matches := ppioBlockPattern.FindAllStringSubmatch(raw, -1)
|
||||
records := make([]officialPricingRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
modelLine := strings.TrimSpace(match[1])
|
||||
contextLength := parseContextLengthCommon(match[2])
|
||||
section := match[3]
|
||||
if strings.Contains(section, "阶梯计费") {
|
||||
continue
|
||||
}
|
||||
priceMatches := ppioPricePattern.FindAllStringSubmatch(section, -1)
|
||||
if len(priceMatches) < 2 {
|
||||
continue
|
||||
}
|
||||
inputPrice := mustParseSubscriptionPrice(priceMatches[len(priceMatches)-2][1])
|
||||
outputPrice := mustParseSubscriptionPrice(priceMatches[len(priceMatches)-1][1])
|
||||
providerName := providerFromModelPath(modelLine)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("ppio", modelLine),
|
||||
ModelName: modelLine,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "PPIO Model API",
|
||||
OperatorNameCn: "PPIO 模型 API",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://ppinfra.com",
|
||||
OperatorType: "relay",
|
||||
Region: "CN",
|
||||
Currency: "CNY",
|
||||
InputPrice: inputPrice,
|
||||
OutputPrice: outputPrice,
|
||||
ContextLength: contextLength,
|
||||
SourceURL: defaultPPIOPricingURL,
|
||||
ModelSourceURL: defaultPPIOPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_product_page",
|
||||
Modality: detectModality(modelLine),
|
||||
}
|
||||
record.IsFree = record.InputPrice == 0 && record.OutputPrice == 0
|
||||
records = append(records, record)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("unexpected ppio pricing content")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
61
scripts/siliconflow_pricing_lib.go
Normal file
61
scripts/siliconflow_pricing_lib.go
Normal file
@@ -0,0 +1,61 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultSiliconFlowPricingURL = "https://siliconflow.cn/pricing"
|
||||
|
||||
var siliconFlowCardPattern = regexp.MustCompile(`(?s)([A-Za-z0-9._/-]+)\n输入 \(元 / M tokens\)\n输出 \(元 / M tokens\)\n(免费|[\d.]+)\n(免费|[\d.]+)`)
|
||||
|
||||
func parseSiliconFlowPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
matches := siliconFlowCardPattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("unexpected siliconflow pricing content")
|
||||
}
|
||||
|
||||
records := make([]officialPricingRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
modelName := strings.TrimSpace(match[1])
|
||||
providerName := providerFromModelPath(modelName)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
inputPrice := parseSiliconFlowPrice(match[2])
|
||||
outputPrice := parseSiliconFlowPrice(match[3])
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("siliconflow", modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "SiliconCloud",
|
||||
OperatorNameCn: "SiliconCloud",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://siliconflow.cn",
|
||||
OperatorType: "relay",
|
||||
Region: "CN",
|
||||
Currency: "CNY",
|
||||
InputPrice: inputPrice,
|
||||
OutputPrice: outputPrice,
|
||||
SourceURL: defaultSiliconFlowPricingURL,
|
||||
ModelSourceURL: defaultSiliconFlowPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_product_page",
|
||||
Modality: detectModality(modelName),
|
||||
}
|
||||
record.IsFree = record.InputPrice == 0 && record.OutputPrice == 0
|
||||
records = append(records, record)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseSiliconFlowPrice(raw string) float64 {
|
||||
if strings.TrimSpace(raw) == "免费" {
|
||||
return 0
|
||||
}
|
||||
return mustParseSubscriptionPrice(raw)
|
||||
}
|
||||
628
scripts/subscription_import_common.go
Normal file
628
scripts/subscription_import_common.go
Normal file
@@ -0,0 +1,628 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
const subscriptionFetchMaxAttempts = 3
|
||||
|
||||
type subscriptionImportRecord struct {
|
||||
ProviderName string
|
||||
ProviderNameCn string
|
||||
ProviderCountry string
|
||||
ProviderWebsite string
|
||||
OperatorName string
|
||||
OperatorNameCn string
|
||||
OperatorCountry string
|
||||
OperatorWebsite string
|
||||
OperatorType string
|
||||
PlanFamily string
|
||||
PlanCode string
|
||||
PlanName string
|
||||
Tier string
|
||||
BillingCycle string
|
||||
Currency string
|
||||
ListPrice float64
|
||||
PriceUnit string
|
||||
QuotaValue int64
|
||||
QuotaUnit string
|
||||
ContextWindow int
|
||||
PlanScope string
|
||||
ModelScope []string
|
||||
SourceURL string
|
||||
PublishedAt string
|
||||
EffectiveDate string
|
||||
Notes string
|
||||
PublishedAtKnown bool
|
||||
}
|
||||
|
||||
func loadSubscriptionImportEnv() {
|
||||
for _, path := range []string{".env.local", ".env"} {
|
||||
loadSubscriptionEnvFile(path)
|
||||
}
|
||||
}
|
||||
|
||||
func loadSubscriptionEnvFile(path string) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
value = strings.Trim(strings.TrimSpace(value), `"'`)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := os.LookupEnv(key); exists {
|
||||
continue
|
||||
}
|
||||
_ = os.Setenv(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
func subscriptionImportDB() (*sql.DB, error) {
|
||||
dsn := os.Getenv("DATABASE_URL")
|
||||
if dsn == "" {
|
||||
dsn = "postgres://long@/llm_intelligence?host=/var/run/postgresql"
|
||||
}
|
||||
return sql.Open("postgres", dsn)
|
||||
}
|
||||
|
||||
func fetchSubscriptionPage(url string, fixture string, client *http.Client) (string, error) {
|
||||
if fixture != "" {
|
||||
data, err := os.ReadFile(fixture)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read fixture %s: %w", fixture, err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= subscriptionFetchMaxAttempts; attempt++ {
|
||||
body, retryable, err := fetchSubscriptionPageOnce(url, client)
|
||||
if err == nil {
|
||||
return body, nil
|
||||
}
|
||||
lastErr = err
|
||||
if !retryable || attempt == subscriptionFetchMaxAttempts {
|
||||
return "", err
|
||||
}
|
||||
time.Sleep(time.Duration(attempt) * 200 * time.Millisecond)
|
||||
}
|
||||
return "", lastErr
|
||||
}
|
||||
|
||||
func fetchSubscriptionPageOnce(url string, client *http.Client) (string, bool, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", isRetriableSubscriptionFetchError(err), fmt.Errorf("fetch %s: %w", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
retryable := resp.StatusCode == http.StatusForbidden ||
|
||||
resp.StatusCode == http.StatusTooManyRequests ||
|
||||
resp.StatusCode == http.StatusBadGateway ||
|
||||
resp.StatusCode == http.StatusServiceUnavailable ||
|
||||
resp.StatusCode == http.StatusGatewayTimeout
|
||||
return "", retryable, fmt.Errorf("fetch %s: unexpected status %d", url, resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", isRetriableSubscriptionFetchError(err), fmt.Errorf("read %s: %w", url, err)
|
||||
}
|
||||
return normalizeSubscriptionPage(string(body)), false, nil
|
||||
}
|
||||
|
||||
func isRetriableSubscriptionFetchError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(err.Error())
|
||||
for _, marker := range []string{
|
||||
"eof",
|
||||
"timeout",
|
||||
"temporarily unavailable",
|
||||
"transport closed",
|
||||
"connection reset",
|
||||
"connection refused",
|
||||
"tls handshake timeout",
|
||||
"i/o timeout",
|
||||
"too many requests",
|
||||
"no such host",
|
||||
"forbidden",
|
||||
"status 403",
|
||||
} {
|
||||
if strings.Contains(lower, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeSubscriptionPage(raw string) string {
|
||||
text := raw
|
||||
scriptPattern := regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)
|
||||
stylePattern := regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)
|
||||
tagPattern := regexp.MustCompile(`(?is)<[^>]+>`)
|
||||
spacePattern := regexp.MustCompile(`[ \t]+`)
|
||||
|
||||
text = scriptPattern.ReplaceAllString(text, "\n")
|
||||
text = stylePattern.ReplaceAllString(text, "\n")
|
||||
text = tagPattern.ReplaceAllString(text, "\n")
|
||||
text = html.UnescapeString(text)
|
||||
text = strings.ReplaceAll(text, "\r\n", "\n")
|
||||
text = strings.ReplaceAll(text, "\r", "\n")
|
||||
|
||||
lines := strings.Split(text, "\n")
|
||||
cleaned := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
line = spacePattern.ReplaceAllString(line, " ")
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
cleaned = append(cleaned, line)
|
||||
}
|
||||
return strings.Join(cleaned, "\n")
|
||||
}
|
||||
|
||||
func publishedAtFromText(raw string) (string, bool) {
|
||||
patterns := []*regexp.Regexp{
|
||||
regexp.MustCompile(`最近更新时间[::]\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})`),
|
||||
regexp.MustCompile(`更新时间[::]?\s*(\d{4}-\d{2}-\d{2})`),
|
||||
}
|
||||
for _, pattern := range patterns {
|
||||
matches := pattern.FindStringSubmatch(raw)
|
||||
if len(matches) != 2 {
|
||||
continue
|
||||
}
|
||||
if len(matches[1]) == len("2006-01-02") {
|
||||
return matches[1] + " 00:00:00", true
|
||||
}
|
||||
return matches[1], true
|
||||
}
|
||||
return time.Now().Format("2006-01-02 15:04:05"), false
|
||||
}
|
||||
|
||||
func effectiveDateFromPublishedAt(publishedAt string) string {
|
||||
if len(publishedAt) >= len("2006-01-02") {
|
||||
return publishedAt[:10]
|
||||
}
|
||||
return time.Now().Format("2006-01-02")
|
||||
}
|
||||
|
||||
func upsertSubscriptionImportRecords(db *sql.DB, records []subscriptionImportRecord) error {
|
||||
type snapshotKey struct {
|
||||
providerID int64
|
||||
planCode string
|
||||
}
|
||||
|
||||
historyKeys := make(map[snapshotKey]struct{})
|
||||
for _, record := range records {
|
||||
providerID, err := ensureSubscriptionProvider(db, record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
operatorID, err := ensureSubscriptionOperator(db, record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !record.PublishedAtKnown {
|
||||
history, err := loadSubscriptionSnapshotHistory(db, providerID, record.PlanCode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := reuseExistingSnapshotDates(&record, history); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
publishedAt, err := time.Parse("2006-01-02 15:04:05", record.PublishedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse published_at for %s: %w", record.PlanCode, err)
|
||||
}
|
||||
effectiveDate, err := time.Parse("2006-01-02", record.EffectiveDate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse effective_date for %s: %w", record.PlanCode, err)
|
||||
}
|
||||
|
||||
modelScopeRaw, err := json.Marshal(record.ModelScope)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal model_scope for %s: %w", record.PlanCode, err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(
|
||||
`INSERT INTO subscription_plan (
|
||||
provider_id, operator_id, plan_family, plan_code, plan_name, tier,
|
||||
billing_cycle, currency, list_price, price_unit, quota_value, quota_unit,
|
||||
context_window, plan_scope, model_scope, source_url, published_at, effective_date, notes
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6,
|
||||
$7, $8, $9, $10, $11, $12,
|
||||
$13, $14, $15, $16, $17, $18, $19
|
||||
)
|
||||
ON CONFLICT (provider_id, plan_code, effective_date)
|
||||
DO UPDATE SET
|
||||
operator_id = EXCLUDED.operator_id,
|
||||
plan_family = EXCLUDED.plan_family,
|
||||
plan_name = EXCLUDED.plan_name,
|
||||
tier = EXCLUDED.tier,
|
||||
billing_cycle = EXCLUDED.billing_cycle,
|
||||
currency = EXCLUDED.currency,
|
||||
list_price = EXCLUDED.list_price,
|
||||
price_unit = EXCLUDED.price_unit,
|
||||
quota_value = EXCLUDED.quota_value,
|
||||
quota_unit = EXCLUDED.quota_unit,
|
||||
context_window = EXCLUDED.context_window,
|
||||
plan_scope = EXCLUDED.plan_scope,
|
||||
model_scope = EXCLUDED.model_scope,
|
||||
source_url = EXCLUDED.source_url,
|
||||
published_at = EXCLUDED.published_at,
|
||||
notes = EXCLUDED.notes,
|
||||
updated_at = CURRENT_TIMESTAMP`,
|
||||
providerID, operatorID, record.PlanFamily, record.PlanCode, record.PlanName, record.Tier,
|
||||
record.BillingCycle, record.Currency, record.ListPrice, record.PriceUnit, nullIfZeroInt64(record.QuotaValue), nullIfBlank(record.QuotaUnit),
|
||||
nullIfZeroIntCommon(record.ContextWindow), nullIfBlank(record.PlanScope), string(modelScopeRaw), record.SourceURL, publishedAt, effectiveDate, nullIfBlank(record.Notes),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert subscription_plan %s: %w", record.PlanCode, err)
|
||||
}
|
||||
historyKeys[snapshotKey{providerID: providerID, planCode: record.PlanCode}] = struct{}{}
|
||||
}
|
||||
for key := range historyKeys {
|
||||
if err := compactSubscriptionSnapshotHistory(db, key.providerID, key.planCode); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type subscriptionSnapshotRow struct {
|
||||
ID int64
|
||||
PlanName string
|
||||
Tier string
|
||||
BillingCycle string
|
||||
Currency string
|
||||
ListPrice float64
|
||||
PriceUnit string
|
||||
QuotaValue int64
|
||||
QuotaUnit string
|
||||
ContextWindow int
|
||||
PlanScope string
|
||||
ModelScope string
|
||||
SourceURL string
|
||||
Notes string
|
||||
PublishedAt time.Time
|
||||
EffectiveDate time.Time
|
||||
}
|
||||
|
||||
func loadSubscriptionSnapshotHistory(db *sql.DB, providerID int64, planCode string) ([]subscriptionSnapshotRow, error) {
|
||||
rows, err := db.Query(
|
||||
`SELECT
|
||||
id,
|
||||
plan_name,
|
||||
tier,
|
||||
billing_cycle,
|
||||
currency,
|
||||
list_price,
|
||||
price_unit,
|
||||
COALESCE(quota_value, 0),
|
||||
COALESCE(quota_unit, ''),
|
||||
COALESCE(context_window, 0),
|
||||
COALESCE(plan_scope, ''),
|
||||
model_scope,
|
||||
source_url,
|
||||
COALESCE(notes, ''),
|
||||
published_at,
|
||||
effective_date
|
||||
FROM subscription_plan
|
||||
WHERE provider_id = $1 AND plan_code = $2
|
||||
ORDER BY effective_date DESC, published_at DESC NULLS LAST, id DESC`,
|
||||
providerID, planCode,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load subscription snapshot history %s: %w", planCode, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
history := make([]subscriptionSnapshotRow, 0)
|
||||
for rows.Next() {
|
||||
row := subscriptionSnapshotRow{}
|
||||
if err := rows.Scan(
|
||||
&row.ID,
|
||||
&row.PlanName,
|
||||
&row.Tier,
|
||||
&row.BillingCycle,
|
||||
&row.Currency,
|
||||
&row.ListPrice,
|
||||
&row.PriceUnit,
|
||||
&row.QuotaValue,
|
||||
&row.QuotaUnit,
|
||||
&row.ContextWindow,
|
||||
&row.PlanScope,
|
||||
&row.ModelScope,
|
||||
&row.SourceURL,
|
||||
&row.Notes,
|
||||
&row.PublishedAt,
|
||||
&row.EffectiveDate,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan subscription snapshot history %s: %w", planCode, err)
|
||||
}
|
||||
history = append(history, row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate subscription snapshot history %s: %w", planCode, err)
|
||||
}
|
||||
return history, nil
|
||||
}
|
||||
|
||||
func reuseExistingSnapshotDates(record *subscriptionImportRecord, history []subscriptionSnapshotRow) (bool, error) {
|
||||
if record == nil || record.PublishedAtKnown || len(history) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
modelScopeRaw, err := json.Marshal(record.ModelScope)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("marshal model_scope for snapshot comparison %s: %w", record.PlanCode, err)
|
||||
}
|
||||
|
||||
for _, existing := range history {
|
||||
if existing.PlanName != record.PlanName ||
|
||||
existing.Tier != record.Tier ||
|
||||
existing.BillingCycle != record.BillingCycle ||
|
||||
existing.Currency != record.Currency ||
|
||||
existing.ListPrice != record.ListPrice ||
|
||||
existing.PriceUnit != record.PriceUnit ||
|
||||
existing.QuotaValue != record.QuotaValue ||
|
||||
existing.QuotaUnit != strings.TrimSpace(record.QuotaUnit) ||
|
||||
existing.ContextWindow != record.ContextWindow ||
|
||||
existing.PlanScope != strings.TrimSpace(record.PlanScope) ||
|
||||
existing.ModelScope != string(modelScopeRaw) ||
|
||||
existing.SourceURL != record.SourceURL ||
|
||||
existing.Notes != strings.TrimSpace(record.Notes) {
|
||||
continue
|
||||
}
|
||||
|
||||
record.PublishedAt = existing.PublishedAt.Format("2006-01-02 15:04:05")
|
||||
record.EffectiveDate = existing.EffectiveDate.Format("2006-01-02")
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func compactSubscriptionSnapshotHistory(db *sql.DB, providerID int64, planCode string) error {
|
||||
history, err := loadSubscriptionSnapshotHistory(db, providerID, planCode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range redundantSnapshotRowIDs(history) {
|
||||
if _, err := db.Exec(`DELETE FROM subscription_plan WHERE id = $1`, id); err != nil {
|
||||
return fmt.Errorf("delete redundant subscription snapshot %d for %s: %w", id, planCode, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func redundantSnapshotRowIDs(history []subscriptionSnapshotRow) []int64 {
|
||||
type signatureKey struct {
|
||||
PlanName string
|
||||
Tier string
|
||||
BillingCycle string
|
||||
Currency string
|
||||
ListPrice float64
|
||||
PriceUnit string
|
||||
QuotaValue int64
|
||||
QuotaUnit string
|
||||
ContextWindow int
|
||||
PlanScope string
|
||||
ModelScope string
|
||||
SourceURL string
|
||||
Notes string
|
||||
}
|
||||
type keptSnapshot struct {
|
||||
ID int64
|
||||
EffectiveDate time.Time
|
||||
PublishedAt time.Time
|
||||
}
|
||||
|
||||
makeKey := func(row subscriptionSnapshotRow) signatureKey {
|
||||
return signatureKey{
|
||||
PlanName: row.PlanName,
|
||||
Tier: row.Tier,
|
||||
BillingCycle: row.BillingCycle,
|
||||
Currency: row.Currency,
|
||||
ListPrice: row.ListPrice,
|
||||
PriceUnit: row.PriceUnit,
|
||||
QuotaValue: row.QuotaValue,
|
||||
QuotaUnit: row.QuotaUnit,
|
||||
ContextWindow: row.ContextWindow,
|
||||
PlanScope: row.PlanScope,
|
||||
ModelScope: row.ModelScope,
|
||||
SourceURL: row.SourceURL,
|
||||
Notes: row.Notes,
|
||||
}
|
||||
}
|
||||
shouldReplace := func(current keptSnapshot, candidate subscriptionSnapshotRow) bool {
|
||||
if candidate.EffectiveDate.Before(current.EffectiveDate) {
|
||||
return true
|
||||
}
|
||||
if candidate.EffectiveDate.Equal(current.EffectiveDate) && candidate.PublishedAt.Before(current.PublishedAt) {
|
||||
return true
|
||||
}
|
||||
return candidate.EffectiveDate.Equal(current.EffectiveDate) && candidate.PublishedAt.Equal(current.PublishedAt) && candidate.ID < current.ID
|
||||
}
|
||||
|
||||
keptBySignature := make(map[signatureKey]keptSnapshot)
|
||||
redundant := make([]int64, 0)
|
||||
for _, row := range history {
|
||||
key := makeKey(row)
|
||||
current, exists := keptBySignature[key]
|
||||
if !exists {
|
||||
keptBySignature[key] = keptSnapshot{ID: row.ID, EffectiveDate: row.EffectiveDate, PublishedAt: row.PublishedAt}
|
||||
continue
|
||||
}
|
||||
if shouldReplace(current, row) {
|
||||
redundant = append(redundant, current.ID)
|
||||
keptBySignature[key] = keptSnapshot{ID: row.ID, EffectiveDate: row.EffectiveDate, PublishedAt: row.PublishedAt}
|
||||
continue
|
||||
}
|
||||
redundant = append(redundant, row.ID)
|
||||
}
|
||||
sort.Slice(redundant, func(i, j int) bool { return redundant[i] < redundant[j] })
|
||||
return redundant
|
||||
}
|
||||
|
||||
func ensureSubscriptionProvider(db *sql.DB, record subscriptionImportRecord) (int64, error) {
|
||||
var providerID int64
|
||||
err := db.QueryRow(`SELECT id FROM model_provider WHERE name = $1`, record.ProviderName).Scan(&providerID)
|
||||
if err == nil {
|
||||
return providerID, nil
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = db.QueryRow(
|
||||
`INSERT INTO model_provider (name, name_cn, country, website, status)
|
||||
VALUES ($1, $2, $3, $4, 'active')
|
||||
RETURNING id`,
|
||||
record.ProviderName, nullIfBlank(record.ProviderNameCn), record.ProviderCountry, nullIfBlank(record.ProviderWebsite),
|
||||
).Scan(&providerID)
|
||||
return providerID, err
|
||||
}
|
||||
|
||||
func ensureSubscriptionOperator(db *sql.DB, record subscriptionImportRecord) (int64, error) {
|
||||
var operatorID int64
|
||||
err := db.QueryRow(`SELECT id FROM operator WHERE name = $1`, record.OperatorName).Scan(&operatorID)
|
||||
if err == nil {
|
||||
return operatorID, nil
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = db.QueryRow(
|
||||
`INSERT INTO operator (name, name_cn, country, website, description, status, type)
|
||||
VALUES ($1, $2, $3, $4, $5, 'active', $6)
|
||||
RETURNING id`,
|
||||
record.OperatorName, nullIfBlank(record.OperatorNameCn), record.OperatorCountry, nullIfBlank(record.OperatorWebsite),
|
||||
fmt.Sprintf("%s subscription import", record.OperatorName), record.OperatorType,
|
||||
).Scan(&operatorID)
|
||||
return operatorID, err
|
||||
}
|
||||
|
||||
func summarizeSubscriptionImport(records []subscriptionImportRecord, getter func(subscriptionImportRecord) string) string {
|
||||
counts := make(map[string]int)
|
||||
keys := make([]string, 0)
|
||||
for _, record := range records {
|
||||
key := getter(record)
|
||||
if _, exists := counts[key]; !exists {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
counts[key]++
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s:%d", key, counts[key]))
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func nullIfBlank(value string) any {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func nullIfZeroInt64(value int64) any {
|
||||
if value == 0 {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func nullIfZeroIntCommon(value int) any {
|
||||
if value == 0 {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func mustParseSubscriptionPrice(raw string) float64 {
|
||||
cleaned := strings.ReplaceAll(raw, ",", "")
|
||||
cleaned = strings.ReplaceAll(cleaned, " ", "")
|
||||
value, _ := strconv.ParseFloat(cleaned, 64)
|
||||
return value
|
||||
}
|
||||
|
||||
func mustParseSubscriptionInt64(raw string) int64 {
|
||||
cleaned := strings.ReplaceAll(raw, ",", "")
|
||||
cleaned = strings.ReplaceAll(cleaned, " ", "")
|
||||
value, _ := strconv.ParseInt(cleaned, 10, 64)
|
||||
return value
|
||||
}
|
||||
|
||||
func firstNonEmptyText(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseDecimalMultiplier(raw string, unit int64) int64 {
|
||||
cleaned := strings.TrimSpace(strings.ReplaceAll(raw, " ", ""))
|
||||
value, _ := strconv.ParseFloat(cleaned, 64)
|
||||
return int64(value * float64(unit))
|
||||
}
|
||||
|
||||
func sliceSection(raw string, start string, end string) string {
|
||||
startIndex := strings.Index(raw, start)
|
||||
if startIndex < 0 {
|
||||
return ""
|
||||
}
|
||||
section := raw[startIndex+len(start):]
|
||||
if end != "" {
|
||||
if endIndex := strings.Index(section, end); endIndex >= 0 {
|
||||
section = section[:endIndex]
|
||||
}
|
||||
}
|
||||
return section
|
||||
}
|
||||
48
scripts/subscription_import_common_test.go
Normal file
48
scripts/subscription_import_common_test.go
Normal file
@@ -0,0 +1,48 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type assertiveError string
|
||||
|
||||
func (e assertiveError) Error() string {
|
||||
return string(e)
|
||||
}
|
||||
|
||||
func TestFetchSubscriptionPageRetriesForbiddenThenSucceeds(t *testing.T) {
|
||||
attempts := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte("blocked"))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("<html><body>套餐价格</body></html>"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := &http.Client{Timeout: 2 * time.Second}
|
||||
body, err := fetchSubscriptionPage(server.URL, "", client)
|
||||
if err != nil {
|
||||
t.Fatalf("fetchSubscriptionPage 返回错误: %v", err)
|
||||
}
|
||||
if body != "套餐价格" {
|
||||
t.Fatalf("返回体归一化错误: %q", body)
|
||||
}
|
||||
if attempts != 2 {
|
||||
t.Fatalf("期望重试 1 次后成功,实际请求 %d 次", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetriableSubscriptionFetchErrorRecognizesForbidden(t *testing.T) {
|
||||
if !isRetriableSubscriptionFetchError(assertiveError("unexpected status 403")) {
|
||||
t.Fatalf("403 应被视作可重试错误")
|
||||
}
|
||||
}
|
||||
21
scripts/testdata/aliyun_coding_plan_sample.txt
vendored
Normal file
21
scripts/testdata/aliyun_coding_plan_sample.txt
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
# Coding Plan概述
|
||||
更新时间:2026-05-14
|
||||
|
||||
## 套餐详情
|
||||
Lite 套餐自 2026 年 3 月 20 日 00:00:00(UTC+08:00)起停止新购。
|
||||
Lite 套餐支持所有套餐模型(含千问、GLM、Kimi、MiniMax),与 Pro 套餐一致。
|
||||
|
||||
Pro 高级套餐
|
||||
支持的模型
|
||||
推荐模型:qwen3.6-plus、kimi-k2.5、glm-5、MiniMax-M2.5
|
||||
更多模型:qwen3.5-plus、qwen3-max-2026-01-23、qwen3-coder-next、qwen3-coder-plus、glm-4.7
|
||||
|
||||
价格
|
||||
¥ 200/月
|
||||
|
||||
用量限制
|
||||
每 5 小时 6,000 次请求
|
||||
每周 45,000 次请求
|
||||
每月 90,000 次请求
|
||||
|
||||
限时优惠:活动已结束,当前价格以下单页为准。
|
||||
31
scripts/testdata/aliyun_token_plan_sample.txt
vendored
Normal file
31
scripts/testdata/aliyun_token_plan_sample.txt
vendored
Normal file
@@ -0,0 +1,31 @@
|
||||
# Token Plan(团队版)概述
|
||||
更新时间:2026-05-14
|
||||
|
||||
## 套餐与定价
|
||||
### Token Plan 团队版
|
||||
提供标准坐席、高级坐席、尊享坐席三个档位,匹配不同使用强度。
|
||||
|
||||
坐席类型
|
||||
价格
|
||||
额度
|
||||
适用场景
|
||||
|
||||
标准坐席
|
||||
¥198/坐席/月
|
||||
25,000 Credits/坐席/月
|
||||
轻度使用 AI 辅助的团队成员
|
||||
|
||||
高级坐席
|
||||
¥698/坐席/月
|
||||
100,000 Credits/坐席/月
|
||||
日常高频使用 AI 编程或办公的团队成员
|
||||
|
||||
尊享坐席
|
||||
¥1,398/坐席/月
|
||||
250,000 Credits/坐席/月
|
||||
重度依赖 AI 的核心开发者或高强度使用者
|
||||
|
||||
### Token Plan 团队版 - 共享用量包
|
||||
Token Plan 团队版 - 共享用量包
|
||||
¥5,000/个
|
||||
625,000 Credits/个
|
||||
69
scripts/testdata/azure_openai_pricing_sample.json
vendored
Normal file
69
scripts/testdata/azure_openai_pricing_sample.json
vendored
Normal file
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"Items": [
|
||||
{
|
||||
"currencyCode": "USD",
|
||||
"retailPrice": 0.0022,
|
||||
"unitPrice": 0.0022,
|
||||
"location": "US East",
|
||||
"meterName": "gpt 4.1 Inp regnl Tokens",
|
||||
"productName": "Azure OpenAI",
|
||||
"skuName": "gpt 4.1 Inp regnl",
|
||||
"serviceName": "Foundry Models",
|
||||
"unitOfMeasure": "1K",
|
||||
"type": "Consumption",
|
||||
"armSkuName": "gpt 4.1 Inp regnl"
|
||||
},
|
||||
{
|
||||
"currencyCode": "USD",
|
||||
"retailPrice": 0.0088,
|
||||
"unitPrice": 0.0088,
|
||||
"location": "US East",
|
||||
"meterName": "gpt 4.1 Outp regnl Tokens",
|
||||
"productName": "Azure OpenAI",
|
||||
"skuName": "gpt 4.1 Outp regnl",
|
||||
"serviceName": "Foundry Models",
|
||||
"unitOfMeasure": "1K",
|
||||
"type": "Consumption",
|
||||
"armSkuName": "gpt 4.1 Outp regnl"
|
||||
},
|
||||
{
|
||||
"currencyCode": "USD",
|
||||
"retailPrice": 1.25,
|
||||
"unitPrice": 1.25,
|
||||
"location": "US West",
|
||||
"meterName": "GPT 5 inp Glbl 1M Tokens",
|
||||
"productName": "Azure OpenAI GPT5",
|
||||
"skuName": "GPT 5 inp Glbl",
|
||||
"serviceName": "Foundry Models",
|
||||
"unitOfMeasure": "1M",
|
||||
"type": "Consumption",
|
||||
"armSkuName": "GPT 5 inp Glbl"
|
||||
},
|
||||
{
|
||||
"currencyCode": "USD",
|
||||
"retailPrice": 10,
|
||||
"unitPrice": 10,
|
||||
"location": "US West",
|
||||
"meterName": "GPT 5 outpt Glbl 1M Tokens",
|
||||
"productName": "Azure OpenAI GPT5",
|
||||
"skuName": "GPT 5 outpt Glbl",
|
||||
"serviceName": "Foundry Models",
|
||||
"unitOfMeasure": "1M",
|
||||
"type": "Consumption",
|
||||
"armSkuName": "GPT 5 outpt Glbl"
|
||||
},
|
||||
{
|
||||
"currencyCode": "USD",
|
||||
"retailPrice": 0.625,
|
||||
"unitPrice": 0.625,
|
||||
"location": "US West",
|
||||
"meterName": "GPT 5.1 Batch inp Gl 1M Tokens",
|
||||
"productName": "Azure OpenAI GPT5",
|
||||
"skuName": "GPT 5.1 Batch inp Gl",
|
||||
"serviceName": "Foundry Models",
|
||||
"unitOfMeasure": "1M",
|
||||
"type": "Consumption",
|
||||
"armSkuName": "GPT 5.1 Batch inp Gl"
|
||||
}
|
||||
]
|
||||
}
|
||||
12
scripts/testdata/baidu_coding_plan_sample.txt
vendored
Normal file
12
scripts/testdata/baidu_coding_plan_sample.txt
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
# Coding Plan
|
||||
更新时间:2026-05-08
|
||||
|
||||
## 套餐详情
|
||||
### 套餐价格与限额
|
||||
套餐类型 价格 用量限制
|
||||
Coding Plan Lite ¥ 40 / 月 每 5 小时:最多约 1,200 次请求
|
||||
每周:最多约 9,000 次请求
|
||||
每订阅月:最多约 18,000 次请求
|
||||
Coding Plan Pro ¥ 200 / 月 每 5 小时:最多约 6,000 次请求
|
||||
每周:最多约 45,000 次请求
|
||||
每订阅月:最多约 90,000 次请求
|
||||
10
scripts/testdata/baidu_token_benefit_pack_sample.txt
vendored
Normal file
10
scripts/testdata/baidu_token_benefit_pack_sample.txt
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
# Token 福利包
|
||||
更新时间:2026-05-08
|
||||
|
||||
## 套餐价格
|
||||
积分额度 有效期 原价 首购优惠价
|
||||
50,000 1个月 ¥50 ¥45
|
||||
100,000 1个月 ¥100 ¥90
|
||||
200,000 1个月 ¥200 ¥170
|
||||
400,000 1个月 ¥400 ¥340
|
||||
800,000 1个月 ¥800 ¥680
|
||||
66
scripts/testdata/bedrock_pricing_sample.html
vendored
Normal file
66
scripts/testdata/bedrock_pricing_sample.html
vendored
Normal file
@@ -0,0 +1,66 @@
|
||||
<h3 id="Model_Pricing" class="lb-txt-none lb-txt-32 lb-h3 lb-title">Model Pricing</h3>
|
||||
<h2 id="Amazon_Nova" class="lb-txt-none lb-h2 lb-title">Amazon Nova</h2>
|
||||
<p><b>Regions: US East (N. Virginia), US East (Ohio), and US West (Oregon)</b></p>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><b>Amazon Nova models</b></td>
|
||||
<td><b>Price per 1M input tokens (text)</b></td>
|
||||
<td><b>Price per 1M input tokens (image)</b></td>
|
||||
<td><b>Price per 1M input tokens (video)</b></td>
|
||||
<td><b>Price per 1M input tokens (audio)</b></td>
|
||||
<td><b>Price per 1M output tokens (text)</b></td>
|
||||
<td><b>Price per 1M output tokens (image)</b></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Amazon Nova 2 Omni (Preview)</td>
|
||||
<td>$0.12</td>
|
||||
<td>$0.24</td>
|
||||
<td>$0.36</td>
|
||||
<td>$0.48</td>
|
||||
<td>$0.96</td>
|
||||
<td>$1.20</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Amazon Nova 2 Lite</td>
|
||||
<td>$0.08</td>
|
||||
<td>$0.10</td>
|
||||
<td>$0.12</td>
|
||||
<td>$0.14</td>
|
||||
<td>$0.40</td>
|
||||
<td>N/A</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h2 id="Qwen" class="lb-txt-none lb-h2 lb-title">Qwen</h2>
|
||||
<p><b>Regions: Europe (Frankfurt) and Asia Pacific (Jakarta)</b></p>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><b>Qwen models</b></td>
|
||||
<td><b>Price per 1M input tokens</b></td>
|
||||
<td><b>Price per 1M output tokens</b></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Qwen3 Coder Next</td>
|
||||
<td>$ 0.60</td>
|
||||
<td>$ 1.44</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p><b>Region: Asia Pacific (Sydney)</b></p>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><b>Qwen models</b></td>
|
||||
<td><b>Price per 1M input tokens</b></td>
|
||||
<td><b>Price per 1M output tokens</b></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Qwen3 Next 80B A3B</td>
|
||||
<td>$ 0.1545</td>
|
||||
<td>$ 1.2360</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h2 id="Pricing_examples" class="lb-txt-none lb-txt-48 lb-h2 lb-title">Pricing examples</h2>
|
||||
3
scripts/testdata/bytedance_coding_plan_notice_sample.txt
vendored
Normal file
3
scripts/testdata/bytedance_coding_plan_notice_sample.txt
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
# 方舟 Coding Plan 服务变更通知
|
||||
每日 10:30 限量开放首购活动,Lite 首月 9.9 元,Pro 首月 49.9 元。
|
||||
续费分别恢复至 40 元/月和 200 元/月。
|
||||
19
scripts/testdata/bytedance_coding_plan_sample.txt
vendored
Normal file
19
scripts/testdata/bytedance_coding_plan_sample.txt
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
# 方舟 Coding Plan
|
||||
最近更新时间: 2026-03-13 10:30:00
|
||||
|
||||
Lite 套餐
|
||||
日常开发、中等强度任务,适合大多数开发者
|
||||
9.9 元
|
||||
40 元/月
|
||||
- 每 5 小时约 1,200 次请求
|
||||
- 每周约 9,000 次请求
|
||||
- 每月约 18,000 次请求
|
||||
|
||||
Pro 套餐
|
||||
高频调用、复杂项目开发,适合重度用户
|
||||
49.9 元
|
||||
200 元/月
|
||||
- Lite 套餐 5 倍额度
|
||||
- 每 5 小时约 6,000 次请求
|
||||
- 每周约 45,000 次请求
|
||||
- 每月约 90,000 次请求
|
||||
32
scripts/testdata/ctyun_coding_plan_sample.txt
vendored
Normal file
32
scripts/testdata/ctyun_coding_plan_sample.txt
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
# 编码套餐(Coding Plan)
|
||||
最近更新时间: 2026-04-09 19:13:32
|
||||
|
||||
GLM Lite
|
||||
GLM Pro
|
||||
GLM Max
|
||||
适用场景
|
||||
适合处理轻量的开发任务
|
||||
适合处理复杂项目的开发任务
|
||||
适合处理海量负载的开发任务
|
||||
|
||||
包月价格 49元/月 149元/月 469元/月
|
||||
包年价格 479/年 1439元/年 4509元/年
|
||||
|
||||
支持模型
|
||||
GLM-5.1
|
||||
GLM-5-Turbo
|
||||
GLM-4.7
|
||||
GLM-4.6
|
||||
GLM-4.5
|
||||
GLM-4.5-Air
|
||||
|
||||
用量限制
|
||||
每 5 小时最多约80 次prompts
|
||||
每周最多约400次prompts
|
||||
每月最多约1,600次prompts
|
||||
每 5 小时最多约400 次prompts
|
||||
每周最多约2,000 次prompts
|
||||
每月最多约8,000次prompts
|
||||
每 5 小时最多约1,600 次prompts
|
||||
每周最多约8,000 次prompts
|
||||
每月最多约32,000次prompts
|
||||
32
scripts/testdata/ctyun_token_plan_sample.txt
vendored
Normal file
32
scripts/testdata/ctyun_token_plan_sample.txt
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
# 天翼云大模型AI专项
|
||||
|
||||
Token Plan Lite1500万Tokens包
|
||||
面向开发者/中小企业,适用于项目开发迭代,大幅提升编码效率与代码质量。
|
||||
支持模型:GLM5
|
||||
支持工具:OpenClaw、OpenCode、Cursor、Cline、Chatbox、Codebuddy、Trae等
|
||||
39 .90 元/个
|
||||
|
||||
Token Plan Pro7000万Tokens包
|
||||
面向开发者/中小企业,适用于项目开发迭代,大幅提升编码效率与代码质量。
|
||||
支持模型:GLM5
|
||||
159 .90 元/个
|
||||
|
||||
Token Plan Max1.5亿Tokens包
|
||||
面向开发者/中小企业,适用于项目开发迭代,大幅提升编码效率与代码质量。
|
||||
支持模型:GLM5
|
||||
299 .90 元/个
|
||||
|
||||
Token Plan 轻享包1000万Tokens包
|
||||
适用于个人/家庭 API 及业务调用,有效解决按需单价高、预算难控等问题。
|
||||
支持模型:Deepseek v3.2
|
||||
9 .90 元/个
|
||||
|
||||
Token Plan 畅享包4000万Tokens包
|
||||
适用于个人/家庭 API 及业务调用,有效解决按需单价高、预算难控等问题。
|
||||
支持模型:Deepseek v3.2
|
||||
29 .90 元/个
|
||||
|
||||
Token Plan 尊享包8000万Tokens包
|
||||
适用于个人/家庭 API 及业务调用,有效解决按需单价高、预算难控等问题。
|
||||
支持模型:Deepseek v3.2
|
||||
49 .90 元/个
|
||||
5
scripts/testdata/cucloud_catalog_sample.txt
vendored
Normal file
5
scripts/testdata/cucloud_catalog_sample.txt
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
# 联通云智算专区
|
||||
AI算力平台AICP
|
||||
实现开发、训练、推理、模型服务部署全流程闭环。
|
||||
AI应用开发平台
|
||||
提供一站式可视化开发、调试和发布智能体应用能力。
|
||||
39
scripts/testdata/huawei_package_plan_sample.txt
vendored
Normal file
39
scripts/testdata/huawei_package_plan_sample.txt
vendored
Normal file
@@ -0,0 +1,39 @@
|
||||
# MaaS 文本生成模型
|
||||
更新时间:2026-05-14
|
||||
按套餐包/资源包计费
|
||||
|
||||
DeepSeek-V3.1
|
||||
100万
|
||||
1个月
|
||||
5.6
|
||||
购买DeepSeek-V3.1模型的套餐包,可抵扣DeepSeek-V3.1模型的Token用量。
|
||||
1000万
|
||||
1个月
|
||||
56
|
||||
购买DeepSeek-V3.1模型的套餐包,可抵扣DeepSeek-V3.1模型的Token用量。
|
||||
1亿
|
||||
3个月
|
||||
558
|
||||
购买DeepSeek-V3.1模型的套餐包,可抵扣DeepSeek-V3.1模型的Token用量。
|
||||
10亿
|
||||
3个月
|
||||
5598
|
||||
购买DeepSeek-V3.1模型的套餐包,可抵扣DeepSeek-V3.1模型的Token用量。
|
||||
|
||||
DeepSeek-V3.2
|
||||
100万
|
||||
1个月
|
||||
2.2
|
||||
购买DeepSeek-V3.2模型的套餐包,可抵扣DeepSeek-V3.2模型的Token用量。
|
||||
1000万
|
||||
1个月
|
||||
22
|
||||
购买DeepSeek-V3.2模型的套餐包,可抵扣DeepSeek-V3.2模型的Token用量。
|
||||
1亿
|
||||
3个月
|
||||
219
|
||||
购买DeepSeek-V3.2模型的套餐包,可抵扣DeepSeek-V3.2模型的Token用量。
|
||||
10亿
|
||||
3个月
|
||||
2199
|
||||
购买DeepSeek-V3.2模型的套餐包,可抵扣DeepSeek-V3.2模型的Token用量。
|
||||
48
scripts/testdata/minimax_token_plan_sample.txt
vendored
Normal file
48
scripts/testdata/minimax_token_plan_sample.txt
vendored
Normal file
@@ -0,0 +1,48 @@
|
||||
更新时间: 2026-05-15
|
||||
Monthly Token Plans
|
||||
Standard Plans:
|
||||
Starter Plus Max
|
||||
Price
|
||||
$10 /month
|
||||
$20 /month
|
||||
$50 /month
|
||||
M2.7
|
||||
1,500 requests/5hrs
|
||||
4,500 requests/5hrs
|
||||
15,000 requests/5hrs
|
||||
Highspeed Plans:
|
||||
Plus-Highspeed Max-Highspeed Ultra-Highspeed
|
||||
Price
|
||||
$40 /month
|
||||
$80 /month
|
||||
$150 /month
|
||||
M2.7-highspeed
|
||||
4,500 requests/5hrs
|
||||
15,000 requests/5hrs
|
||||
30,000 requests/5hrs
|
||||
Speech 2.8 4000 chars/day 11000 chars/day
|
||||
image-01 50 images/day 120 images/day
|
||||
Hailuo 2.3-Fast 2 videos/day
|
||||
Hailuo 2.3 2 videos/day
|
||||
Music-2.6 100 songs/day
|
||||
Yearly Token Plans
|
||||
Standard Plans:
|
||||
Starter Plus Max
|
||||
Price
|
||||
$100 /year
|
||||
$200 /year
|
||||
$500 /year
|
||||
M2.7
|
||||
1,500 requests/5hrs
|
||||
4,500 requests/5hrs
|
||||
15,000 requests/5hrs
|
||||
Highspeed Plans:
|
||||
Plus-Highspeed Max-Highspeed Ultra-Highspeed
|
||||
Price
|
||||
$400 /year
|
||||
$800 /year
|
||||
$1,500 /year
|
||||
M2.7-highspeed
|
||||
4,500 requests/5hrs
|
||||
15,000 requests/5hrs
|
||||
30,000 requests/5hrs
|
||||
4
scripts/testdata/mobile_cloud_catalog_sample.txt
vendored
Normal file
4
scripts/testdata/mobile_cloud_catalog_sample.txt
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
# 移动云市场
|
||||
AI应用专区
|
||||
数据大模型
|
||||
覆盖智能客服、办公提效、数据分析等场景能力。
|
||||
20
scripts/testdata/platform360_pricing_sample.txt
vendored
Normal file
20
scripts/testdata/platform360_pricing_sample.txt
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
deepseek/deepseek-v4-flash
|
||||
DeepSeek
|
||||
Input Price:¥0.5 / 1M tokens
|
||||
Output Price:¥2 / 1M tokens
|
||||
Context:1,000,000
|
||||
deepseek/deepseek-v4-pro
|
||||
DeepSeek
|
||||
Input Price:¥3 / 1M tokens
|
||||
Output Price:¥6 / 1M tokens
|
||||
Context:1,000,000
|
||||
moonshotai/kimi-k2.6
|
||||
Moonshot AI
|
||||
Input Price:¥4 / 1M tokens
|
||||
Output Price:¥16 / 1M tokens
|
||||
Context:262,144
|
||||
qwen/qwen3.6-plus
|
||||
Qwen
|
||||
Input Price:¥0.8 / 1M tokens
|
||||
Output Price:¥3.2 / 1M tokens
|
||||
Context:256,000
|
||||
60
scripts/testdata/ppio_pricing_sample.txt
vendored
Normal file
60
scripts/testdata/ppio_pricing_sample.txt
vendored
Normal file
@@ -0,0 +1,60 @@
|
||||
deepseek/deepseek-v3.1
|
||||
131,072
|
||||
¥
|
||||
4
|
||||
/
|
||||
Mt
|
||||
·Cached(r)
|
||||
¥
|
||||
2
|
||||
/
|
||||
Mt
|
||||
¥
|
||||
12
|
||||
/
|
||||
Mt
|
||||
在线体验
|
||||
deepseek/deepseek-r1-0528
|
||||
131,072
|
||||
¥
|
||||
4
|
||||
/
|
||||
Mt
|
||||
¥
|
||||
16
|
||||
/
|
||||
Mt
|
||||
在线体验
|
||||
moonshotai/kimi-k2.6
|
||||
262,144
|
||||
¥
|
||||
4
|
||||
/
|
||||
Mt
|
||||
¥
|
||||
16
|
||||
/
|
||||
Mt
|
||||
在线体验
|
||||
qwen/qwen3-coder
|
||||
131,072
|
||||
¥
|
||||
2
|
||||
/
|
||||
Mt
|
||||
¥
|
||||
8
|
||||
/
|
||||
Mt
|
||||
在线体验
|
||||
zai-org/glm-4.5-air
|
||||
128,000
|
||||
¥
|
||||
0.8
|
||||
/
|
||||
Mt
|
||||
¥
|
||||
3.2
|
||||
/
|
||||
Mt
|
||||
在线体验
|
||||
25
scripts/testdata/siliconflow_pricing_sample.txt
vendored
Normal file
25
scripts/testdata/siliconflow_pricing_sample.txt
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
Qwen/Qwen3-32B
|
||||
输入 (元 / M tokens)
|
||||
输出 (元 / M tokens)
|
||||
0.40
|
||||
1.80
|
||||
deepseek-ai/DeepSeek-V3.1-Terminus
|
||||
输入 (元 / M tokens)
|
||||
输出 (元 / M tokens)
|
||||
4.00
|
||||
12.00
|
||||
moonshotai/Kimi-K2-Instruct
|
||||
输入 (元 / M tokens)
|
||||
输出 (元 / M tokens)
|
||||
4.00
|
||||
16.00
|
||||
MiniMaxAI/MiniMax-M2.5
|
||||
输入 (元 / M tokens)
|
||||
输出 (元 / M tokens)
|
||||
1.00
|
||||
8.00
|
||||
zai-org/GLM-4.5-Air
|
||||
输入 (元 / M tokens)
|
||||
输出 (元 / M tokens)
|
||||
免费
|
||||
免费
|
||||
5
scripts/testdata/ucloud_pricing_sample.txt
vendored
Normal file
5
scripts/testdata/ucloud_pricing_sample.txt
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
openai/gpt-4o-mini Input 0.0001 CNY/1K tokens, Output 0.0004 CNY/1K tokens
|
||||
moonshotai/kimi-k2.6 Input 4 CNY/million tokens, Output 16 CNY/million tokens
|
||||
deepseek-ai/DeepSeek-R1 Input 4 CNY/million tokens, Output 16 CNY/million tokens
|
||||
deepseek-ai/DeepSeek-V3 Input 2 CNY/million tokens, Output 8 CNY/million tokens
|
||||
qwen/Qwen3-32B Input 0.4 CNY/million tokens, Output 1.8 CNY/million tokens
|
||||
36
scripts/testdata/youdao_pricing_sample.txt
vendored
Normal file
36
scripts/testdata/youdao_pricing_sample.txt
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
DeepSeek V4 Flash
|
||||
DeepSeek
|
||||
上下文长度 1M
|
||||
输入:¥0.5
|
||||
输出:¥2
|
||||
查看详情
|
||||
DeepSeek V4 Pro
|
||||
DeepSeek
|
||||
上下文长度 128K
|
||||
输入:¥2
|
||||
输出:¥8
|
||||
查看详情
|
||||
Kimi K2.6
|
||||
Kimi
|
||||
上下文长度 262K
|
||||
输入:¥4
|
||||
输出:¥16
|
||||
查看详情
|
||||
MiniMax M2.5
|
||||
Minimax
|
||||
上下文长度 1M
|
||||
输入:¥1
|
||||
输出:¥8
|
||||
查看详情
|
||||
GLM-5
|
||||
Zhipu
|
||||
上下文长度 128K
|
||||
输入:¥1
|
||||
输出:¥3
|
||||
查看详情
|
||||
Qwen-3.6-Plus
|
||||
Qwen
|
||||
上下文长度 256K
|
||||
输入:¥0.8
|
||||
输出:¥3.2
|
||||
即将上线
|
||||
6
scripts/testdata/zhipu_coding_plan_overview_sample.txt
vendored
Normal file
6
scripts/testdata/zhipu_coding_plan_overview_sample.txt
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
# 套餐概览
|
||||
GLM Coding Plan 提供 Lite、Pro 和 Max 三个套餐等级,可以按需灵活选择。
|
||||
Lite、Pro 和 Max 套餐额度共享,支持平台中所有上线模型,包含 GLM-4.5-Air、GLM-4.5V、GLM-4.6V、GLM-4.7。
|
||||
每 5 小时最多约 80 / 400 / 1,600 次 prompts。
|
||||
每周最多约 400 / 2,000 / 8,000 次 prompts。
|
||||
Lite 套餐每月包含 100 次 MCP 调用,Pro 套餐每月包含 1,000 次 MCP 调用,Max 套餐每月包含 4,000 次 MCP 调用。
|
||||
3
scripts/testdata/zhipu_coding_plan_promotion_sample.txt
vendored
Normal file
3
scripts/testdata/zhipu_coding_plan_promotion_sample.txt
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
# 限时活动
|
||||
GLM Coding Plan 编码畅享计划,低至20元/月,适用于 Claude Code、Cline、Continue、Open Code 等平台。
|
||||
首单 9 折优惠,新老用户均可参与。
|
||||
58
scripts/ucloud_pricing_lib.go
Normal file
58
scripts/ucloud_pricing_lib.go
Normal file
@@ -0,0 +1,58 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultUCloudPricingURL = "https://www.ucloud-global.com/en/docs/modelverse/price"
|
||||
|
||||
var ucloudPricePattern = regexp.MustCompile(`([A-Za-z0-9._/-]+)\s+Input\s+([\d.]+)\s+CNY/(?:million|1K)\s+tokens,\s+Output\s+([\d.]+)\s+CNY/(?:million|1K)\s+tokens`)
|
||||
|
||||
func parseUCloudPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
matches := ucloudPricePattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("unexpected ucloud pricing content")
|
||||
}
|
||||
|
||||
records := make([]officialPricingRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
modelName := strings.TrimSpace(match[1])
|
||||
providerName := providerFromModelPath(modelName)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
inputPrice := mustParseSubscriptionPrice(match[2])
|
||||
outputPrice := mustParseSubscriptionPrice(match[3])
|
||||
if strings.Contains(strings.ToLower(match[0]), "/1k") {
|
||||
inputPrice *= 1000
|
||||
outputPrice *= 1000
|
||||
}
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("ucloud", modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "UModelVerse",
|
||||
OperatorNameCn: "UModelVerse",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ucloud.cn",
|
||||
OperatorType: "cloud",
|
||||
Region: "CN",
|
||||
Currency: "CNY",
|
||||
InputPrice: inputPrice,
|
||||
OutputPrice: outputPrice,
|
||||
SourceURL: defaultUCloudPricingURL,
|
||||
ModelSourceURL: defaultUCloudPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_product_page",
|
||||
Modality: detectModality(modelName),
|
||||
}
|
||||
record.IsFree = record.InputPrice == 0 && record.OutputPrice == 0
|
||||
records = append(records, record)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
76
scripts/youdao_pricing_lib.go
Normal file
76
scripts/youdao_pricing_lib.go
Normal file
@@ -0,0 +1,76 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultYoudaoPricingURL = "https://ai.youdao.com/new/model-service"
|
||||
|
||||
var youdaoCardPattern = regexp.MustCompile(`(?s)([A-Za-z0-9.+\- ]+)\n([A-Za-z][A-Za-z0-9 ]+)\n.*?上下文长度\s*([0-9A-Za-z., ]+)\n.*?输入[::]?\s*¥?([\d.]+)\n.*?输出[::]?\s*¥?([\d.]+)\n.*?(查看详情|即将上线)`)
|
||||
|
||||
func parseYoudaoPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
matches := youdaoCardPattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("unexpected youdao pricing content")
|
||||
}
|
||||
|
||||
records := make([]officialPricingRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
modelName := strings.TrimSpace(match[1])
|
||||
providerName := normalizeYoudaoProvider(match[2])
|
||||
if strings.TrimSpace(match[6]) != "查看详情" {
|
||||
continue
|
||||
}
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("youdao", modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Youdao Zhiyun MaaS",
|
||||
OperatorNameCn: "有道智云 MaaS",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://sf.163.com",
|
||||
OperatorType: "relay",
|
||||
Region: "CN",
|
||||
Currency: "CNY",
|
||||
InputPrice: mustParseSubscriptionPrice(match[4]),
|
||||
OutputPrice: mustParseSubscriptionPrice(match[5]),
|
||||
ContextLength: parseContextLengthCommon(match[3]),
|
||||
SourceURL: defaultYoudaoPricingURL,
|
||||
ModelSourceURL: defaultYoudaoPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_product_page",
|
||||
Modality: detectModality(modelName),
|
||||
}
|
||||
record.IsFree = record.InputPrice == 0 && record.OutputPrice == 0
|
||||
records = append(records, record)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no active youdao pricing cards found")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func normalizeYoudaoProvider(raw string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case "deepseek":
|
||||
return "DeepSeek"
|
||||
case "qwen":
|
||||
return "Qwen"
|
||||
case "kimi":
|
||||
return "Moonshot AI"
|
||||
case "minimax":
|
||||
return "MiniMax"
|
||||
case "zhipu":
|
||||
return "Zhipu AI"
|
||||
default:
|
||||
return strings.TrimSpace(raw)
|
||||
}
|
||||
}
|
||||
81
scripts/zhipu_coding_plan_lib.go
Normal file
81
scripts/zhipu_coding_plan_lib.go
Normal file
@@ -0,0 +1,81 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultZhipuCodingPlanOverviewURL = "https://docs.bigmodel.cn/cn/coding-plan/overview"
|
||||
defaultZhipuCodingPlanPromotionURL = "https://docs.bigmodel.cn/cn/update/promotion"
|
||||
)
|
||||
|
||||
func parseZhipuCodingPlanCatalog(overviewRaw string, promotionRaw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(firstNonEmptyText(overviewRaw, promotionRaw))
|
||||
pricePattern := regexp.MustCompile(`低至\s*([\d.]+)\s*元/月`)
|
||||
priceMatch := pricePattern.FindStringSubmatch(promotionRaw)
|
||||
if len(priceMatch) != 2 {
|
||||
return nil, fmt.Errorf("zhipu promo floor price not found")
|
||||
}
|
||||
|
||||
modelScope := []string{}
|
||||
modelPattern := regexp.MustCompile(`包含\s+([^\n]+)`)
|
||||
modelMatch := modelPattern.FindStringSubmatch(overviewRaw)
|
||||
if len(modelMatch) == 2 {
|
||||
for _, item := range strings.Split(modelMatch[1], "、") {
|
||||
item = strings.TrimSpace(strings.TrimSuffix(item, "。"))
|
||||
if item != "" {
|
||||
modelScope = append(modelScope, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
notes := []string{
|
||||
"GLM Coding Plan 提供 Lite/Pro/Max 三档能力。",
|
||||
}
|
||||
if strings.Contains(overviewRaw, "每 5 小时最多约 80 / 400 / 1,600 次 prompts") {
|
||||
notes = append(notes, "5 小时额度:Lite 80 / Pro 400 / Max 1600 prompts。")
|
||||
}
|
||||
if strings.Contains(overviewRaw, "每周最多约 400 / 2,000 / 8,000 次 prompts") {
|
||||
notes = append(notes, "每周额度:Lite 400 / Pro 2000 / Max 8000 prompts。")
|
||||
}
|
||||
if strings.Contains(overviewRaw, "Lite 套餐每月包含 100 次 MCP 调用") {
|
||||
notes = append(notes, "MCP 月额度:Lite 100 / Pro 1000 / Max 4000。")
|
||||
}
|
||||
if strings.Contains(promotionRaw, "首单 9 折") {
|
||||
notes = append(notes, "首单 9 折优惠,新老用户均可参与。")
|
||||
}
|
||||
notes = append(notes, "公开文档仅披露活动底价,分档定价仍需控制台核验。")
|
||||
|
||||
return []subscriptionImportRecord{{
|
||||
ProviderName: "Zhipu AI",
|
||||
ProviderNameCn: "智谱 AI",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://open.bigmodel.cn",
|
||||
OperatorName: "Zhipu",
|
||||
OperatorNameCn: "智谱开放平台",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://docs.bigmodel.cn/",
|
||||
OperatorType: "official",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "zhipu-coding-plan-promo-floor",
|
||||
PlanName: "GLM Coding Plan 限时活动价(低至)",
|
||||
Tier: "PromoFloor",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[1]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: 0,
|
||||
QuotaUnit: "",
|
||||
PlanScope: "GLM Coding Plan",
|
||||
ModelScope: modelScope,
|
||||
SourceURL: defaultZhipuCodingPlanPromotionURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: strings.Join(notes, ";"),
|
||||
PublishedAtKnown: known,
|
||||
}}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user