This commit is contained in:
61
scripts/fetch_tencent_catalog.go
Normal file
61
scripts/fetch_tencent_catalog.go
Normal file
@@ -0,0 +1,61 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var rawURL string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
var fixturePath string
|
||||
|
||||
flag.StringVar(&rawURL, "url", defaultTencentCatalogURL, "腾讯云公开目录 URL")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅抓取并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", int(defaultTencentCatalogTimeout/time.Second), "请求超时(秒)")
|
||||
flag.StringVar(&fixturePath, "fixture", "", "本地 HTML/Text 样例文件,优先用于离线 dry-run")
|
||||
flag.Parse()
|
||||
|
||||
cfg := fetchTencentCatalogConfig{
|
||||
URL: rawURL,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
Fixture: fixturePath,
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
if err := runTencentCatalog(cfg, client, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fetch_tencent_catalog: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runTencentCatalog(cfg fetchTencentCatalogConfig, client *http.Client, out io.Writer) error {
|
||||
raw, err := fetchTencentCatalogContent(cfg, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
catalog, err := parseTencentCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(
|
||||
out,
|
||||
"source=tencent-public-catalog updated_at=%s plans=%d models=%d series=%s dry_run=%t\n",
|
||||
catalog.UpdatedAt,
|
||||
len(catalog.Plans),
|
||||
len(catalog.Models),
|
||||
formatSeriesSummary(catalog.Plans),
|
||||
cfg.DryRun,
|
||||
)
|
||||
return err
|
||||
}
|
||||
Reference in New Issue
Block a user