From a32a1067d3964e8ff8995767a2df46510ed77969 Mon Sep 17 00:00:00 2001 From: spiritlhl <103393591+spiritLHLS@users.noreply.github.com> Date: Tue, 21 Jul 2026 00:42:30 +0800 Subject: [PATCH] feat: refresh ASN prefixes with bounded Go loader --- .github/workflows/build.yml | 33 +++--- bk/backtrace.go | 1 + bk/ipv6_asn.go | 13 ++- bk/prefix_registry.go | 194 +++++++++++++++++++++++++++++++ bk/prefix_registry_test.go | 38 ++++++ bk/utils.go | 6 +- cmd/main.go | 104 ++++++++++++++--- cmd/main_test.go | 65 +++++++++++ cmd/update-prefixes/main.go | 186 +++++++++++++++++++++++++++++ cmd/update-prefixes/main_test.go | 32 +++++ model/model.go | 2 +- 11 files changed, 632 insertions(+), 42 deletions(-) create mode 100644 bk/prefix_registry.go create mode 100644 bk/prefix_registry_test.go create mode 100644 cmd/main_test.go create mode 100644 cmd/update-prefixes/main.go create mode 100644 cmd/update-prefixes/main_test.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 48e1796..4a2e591 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,11 +3,18 @@ name: 创建IPv6检测的前缀 on: workflow_dispatch: schedule: - - cron: '0 0 * * 0' + - cron: '0 0 * * *' + +concurrency: + group: backtrace-ipv6-prefix-sync + cancel-in-progress: false env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" +permissions: + contents: write + jobs: fetch-ipv6-prefixes: runs-on: ubuntu-latest @@ -17,26 +24,14 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} + - name: 设置 Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + - name: 获取并处理多个ASN的IPv6前缀 run: | - mkdir -p bk/prefix/ - for asn in AS4809 AS4134 AS9929 AS4837 AS58807 AS9808 AS58453 AS23764; do - echo "处理 $asn..." - curl -s -A "Mozilla/5.0" \ - "https://bgp.he.net/$asn" > "${asn}.html" - grep -oE '[0-9a-f:]+::/[0-9]+' "${asn}.html" | sort -u > tmp_prefixes.txt - { - while read prefix; do - ip_part=$(echo "$prefix" | cut -d/ -f1) - prefix_len=$(echo "$prefix" | cut -d/ -f2) - keep_segments=$((prefix_len / 16)) - segments=$(echo "$ip_part" | tr ':' '\n' | grep -v '^$') - kept=$(echo "$segments" | head -n "$keep_segments" | tr '\n' ':' | sed 's/:$//') - echo "$kept" - done < tmp_prefixes.txt - } | sort -u > "bk/prefix/${asn,,}.txt" - rm -f "${asn}.html" tmp_prefixes.txt - done + go run ./cmd/update-prefixes -output-dir bk/prefix - name: 提交更新到仓库 run: | diff --git a/bk/backtrace.go b/bk/backtrace.go index 79543e2..d337030 100644 --- a/bk/backtrace.go +++ b/bk/backtrace.go @@ -16,6 +16,7 @@ func safeTraceCall(fn func()) { } func BackTrace(enableIpv6 bool) string { + StartASNPrefixRefresh() if model.CachedIcmpData == "" || model.ParsedIcmpTargets == nil || time.Since(model.CachedIcmpDataFetchTime) > time.Hour { model.CachedIcmpData = getData(model.IcmpTargets) model.CachedIcmpDataFetchTime = time.Now() diff --git a/bk/ipv6_asn.go b/bk/ipv6_asn.go index 27f931b..26af69c 100644 --- a/bk/ipv6_asn.go +++ b/bk/ipv6_asn.go @@ -1,8 +1,9 @@ package backtrace import ( - "strings" _ "embed" + "net/netip" + "strings" ) //go:embed prefix/as4809.txt @@ -44,12 +45,20 @@ var asnPrefixes = map[string][]string{ // 判断 IPv6 地址是否匹配 ASN 中的某个前缀 func ipv6Asn(ip string) string { ip = strings.ToLower(ip) - for asn, prefixes := range asnPrefixes { + for asn, prefixes := range currentASNPrefixes() { for _, prefix := range prefixes { prefix = strings.TrimSpace(prefix) if prefix == "" { continue } + if strings.Contains(prefix, "/") { + address, addressErr := netip.ParseAddr(ip) + network, networkErr := netip.ParsePrefix(prefix) + if addressErr == nil && networkErr == nil && network.Contains(address) { + return asn + } + continue + } if strings.HasPrefix(ip, prefix) { return asn } diff --git a/bk/prefix_registry.go b/bk/prefix_registry.go new file mode 100644 index 0000000..49d9d7d --- /dev/null +++ b/bk/prefix_registry.go @@ -0,0 +1,194 @@ +package backtrace + +import ( + "context" + _ "embed" + "errors" + "fmt" + "io" + "net/http" + "net/netip" + "net/url" + "regexp" + "sort" + "strings" + "sync" + "sync/atomic" + "time" +) + +const ( + ASNPrefixRegistryRawBaseURL = "https://raw.githubusercontent.com/oneclickvirt/backtrace/main/bk/prefix" + ASNPrefixRegistryCDNBaseURL = "https://cdn.spiritlhl.net/" + ASNPrefixRegistryRawBaseURL +) + +var knownPrefixASNs = []string{"AS23764", "AS4134", "AS4809", "AS4837", "AS58453", "AS58807", "AS9808", "AS9929"} +var prefixFragmentPattern = regexp.MustCompile(`^[0-9a-fA-F:]+$`) + +type ASNPrefixRegistrySource struct { + Name string + Base string +} + +type ASNPrefixRegistryLoadResult struct { + Prefixes map[string][]string + Source string + Fallback bool +} + +var activeASNPrefixes atomic.Value // stores map[string][]string; maps are immutable after publication +var refreshASNPrefixesOnce sync.Once + +func init() { + activeASNPrefixes.Store(cloneASNPrefixes(asnPrefixes)) +} + +func DefaultASNPrefixRegistrySources() []ASNPrefixRegistrySource { + return []ASNPrefixRegistrySource{ + {Name: "cdn", Base: ASNPrefixRegistryCDNBaseURL}, + {Name: "raw", Base: ASNPrefixRegistryRawBaseURL}, + } +} + +func LoadASNPrefixRegistry(ctx context.Context, client *http.Client, sources []ASNPrefixRegistrySource) (ASNPrefixRegistryLoadResult, error) { + if ctx == nil { + ctx = context.Background() + } + if client == nil { + client = &http.Client{Timeout: 6 * time.Second} + } + var lastErr error + for index, source := range sources { + prefixes, err := fetchASNPrefixSource(ctx, client, source.Base) + if err != nil { + lastErr = fmt.Errorf("load %s ASN prefixes: %w", source.Name, err) + continue + } + return ASNPrefixRegistryLoadResult{Prefixes: prefixes, Source: source.Name, Fallback: index > 0}, nil + } + embedded := cloneASNPrefixes(asnPrefixes) + if len(embedded) > 0 { + return ASNPrefixRegistryLoadResult{Prefixes: embedded, Source: "embedded", Fallback: true}, nil + } + if lastErr == nil { + lastErr = errors.New("no ASN prefix registry sources configured") + } + return ASNPrefixRegistryLoadResult{}, lastErr +} + +func RefreshASNPrefixRegistry(ctx context.Context, client *http.Client, sources []ASNPrefixRegistrySource) (ASNPrefixRegistryLoadResult, error) { + loaded, err := LoadASNPrefixRegistry(ctx, client, sources) + if err != nil { + return ASNPrefixRegistryLoadResult{}, err + } + activeASNPrefixes.Store(cloneASNPrefixes(loaded.Prefixes)) + return loaded, nil +} + +// StartASNPrefixRefresh performs one bounded background refresh. It never +// delays legacy tracing and the embedded map remains active on any error. +func StartASNPrefixRefresh() { + refreshASNPrefixesOnce.Do(func() { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _, _ = RefreshASNPrefixRegistry(ctx, nil, DefaultASNPrefixRegistrySources()) + }() + }) +} + +func fetchASNPrefixSource(ctx context.Context, client *http.Client, base string) (map[string][]string, error) { + parsed, err := url.Parse(strings.TrimRight(base, "/")) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" { + return nil, errors.New("invalid ASN prefix source") + } + type prefixResult struct { + asn string + values []string + err error + } + results := make(chan prefixResult, len(knownPrefixASNs)) + for _, asn := range knownPrefixASNs { + go func(asn string) { + endpoint := strings.TrimRight(base, "/") + "/" + strings.ToLower(asn) + ".txt" + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + results <- prefixResult{asn: asn, err: err} + return + } + request.Header.Set("Accept", "text/plain") + request.Header.Set("User-Agent", "oneclickvirt-backtrace/asn-prefix-registry-v1") + response, err := client.Do(request) + if err != nil { + results <- prefixResult{asn: asn, err: err} + return + } + data, readErr := io.ReadAll(io.LimitReader(response.Body, 4<<20)) + response.Body.Close() + if readErr != nil { + results <- prefixResult{asn: asn, err: readErr} + return + } + if response.StatusCode != http.StatusOK { + results <- prefixResult{asn: asn, err: fmt.Errorf("HTTP %d", response.StatusCode)} + return + } + values, parseErr := parseASNPrefixLines(data) + results <- prefixResult{asn: asn, values: values, err: parseErr} + }(asn) + } + result := make(map[string][]string, len(knownPrefixASNs)) + for range knownPrefixASNs { + loaded := <-results + if loaded.err != nil { + return nil, fmt.Errorf("%s: %w", loaded.asn, loaded.err) + } + result[loaded.asn] = loaded.values + } + return result, nil +} + +func parseASNPrefixLines(data []byte) ([]string, error) { + seen := make(map[string]struct{}) + values := make([]string, 0) + for _, raw := range strings.Split(string(data), "\n") { + value := strings.TrimSpace(raw) + if value == "" { + continue + } + if strings.Contains(value, "/") { + prefix, err := netip.ParsePrefix(value) + if err != nil || !prefix.Addr().Is6() { + return nil, fmt.Errorf("invalid IPv6 prefix %q", value) + } + } else if !prefixFragmentPattern.MatchString(value) { + return nil, fmt.Errorf("invalid prefix fragment %q", value) + } + value = strings.ToLower(value) + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + values = append(values, value) + } + if len(values) == 0 { + return nil, errors.New("prefix file is empty") + } + sort.Strings(values) + return values, nil +} + +func cloneASNPrefixes(input map[string][]string) map[string][]string { + result := make(map[string][]string, len(input)) + for asn, values := range input { + result[asn] = append([]string(nil), values...) + } + return result +} + +func currentASNPrefixes() map[string][]string { + if value := activeASNPrefixes.Load(); value != nil { + return value.(map[string][]string) + } + return asnPrefixes +} diff --git a/bk/prefix_registry_test.go b/bk/prefix_registry_test.go new file mode 100644 index 0000000..02593a6 --- /dev/null +++ b/bk/prefix_registry_test.go @@ -0,0 +1,38 @@ +package backtrace + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +func TestParseASNPrefixLinesNormalizesAndDeduplicates(t *testing.T) { + values, err := parseASNPrefixLines([]byte("2409:8000\n2409:8000\n2401:db8::/32\n")) + if err != nil || len(values) != 2 || values[0] != "2401:db8::/32" { + t.Fatalf("unexpected prefixes: %#v, %v", values, err) + } +} + +func TestLoadASNPrefixRegistryUsesRawAfterCDNFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/bad/as23764.txt" { + http.Error(writer, "bad", http.StatusBadGateway) + return + } + _, _ = writer.Write([]byte("2409:8000\n")) + })) + defer server.Close() + sources := []ASNPrefixRegistrySource{{Name: "cdn", Base: server.URL + "/bad"}, {Name: "raw", Base: server.URL + "/good"}} + loaded, err := LoadASNPrefixRegistry(context.Background(), server.Client(), sources) + if err != nil || loaded.Source != "raw" || !loaded.Fallback || len(loaded.Prefixes) != len(knownPrefixASNs) { + t.Fatalf("unexpected load: %+v, %v", loaded, err) + } +} + +func TestLoadASNPrefixRegistryEmbeddedFallback(t *testing.T) { + loaded, err := LoadASNPrefixRegistry(context.Background(), nil, nil) + if err != nil || loaded.Source != "embedded" || !loaded.Fallback || len(loaded.Prefixes) == 0 { + t.Fatalf("unexpected embedded load: %+v, %v", loaded, err) + } +} diff --git a/bk/utils.go b/bk/utils.go index 3ce97c1..8fb4838 100644 --- a/bk/utils.go +++ b/bk/utils.go @@ -80,11 +80,11 @@ func getData(endpoint string) string { InitLogger() defer Logger.Sync() } - + // 先测试CDN可用性 testUrl := "https://raw.githubusercontent.com/spiritLHLS/ecs/main/back/test" cdnUrl := checkCdn(testUrl) - + // 如果有可用的CDN，使用CDN获取数据 if cdnUrl != "" { url := cdnUrl + endpoint @@ -106,7 +106,7 @@ func getData(endpoint string) string { Logger.Info(fmt.Sprintf("CDN request failed: %v", err)) } } - + // CDN不可用，尝试直连 if model.EnableLoger { Logger.Info(fmt.Sprintf("Trying direct connection: %s", endpoint)) diff --git a/cmd/main.go b/cmd/main.go index 97bacd7..8c143a7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,9 +1,11 @@ package main import ( + "context" "encoding/json" "flag" "fmt" + "io" "net/http" "os" "runtime" @@ -32,6 +34,32 @@ type ConcurrentResults struct { // backtraceError error } +type cliOptions struct { + showVersion bool + showIPInfo bool + help bool + ipv6 bool + jsonOutput bool + deep bool + specifiedIP string + timeout time.Duration +} + +func newBacktraceFlagSet(options *cliOptions) *flag.FlagSet { + set := flag.NewFlagSet("backtrace", flag.ContinueOnError) + set.BoolVar(&options.help, "h", false, "Show help information") + set.BoolVar(&options.showVersion, "v", false, "Show version") + set.BoolVar(&options.showIPInfo, "s", true, "Disabe show ip info") + set.BoolVar(&model.EnableLoger, "log", false, "Enable logging") + set.BoolVar(&options.ipv6, "ipv6", false, "Enable ipv6 testing") + set.StringVar(&options.specifiedIP, "ip", "", "Specify IP address for bgptools") + set.BoolVar(&options.jsonOutput, "json", false, "Output structured RDAP/BGP report as JSON") + set.BoolVar(&options.jsonOutput, "structured", false, "Alias for -json") + set.BoolVar(&options.deep, "deep", false, "Fetch geofeed and enable WHOIS fallback") + set.DurationVar(&options.timeout, "timeout", 15*time.Second, "Structured report timeout") + return set +} + func safeGo(wg *sync.WaitGroup, fn func()) { go func() { defer wg.Done() @@ -50,28 +78,44 @@ func main() { resp.Body.Close() } }() - fmt.Println(Green("Repo:"), Yellow("https://github.com/oneclickvirt/backtrace")) - var showVersion, showIpInfo, help, ipv6 bool - var specifiedIP string - backtraceFlag := flag.NewFlagSet("backtrace", flag.ContinueOnError) - backtraceFlag.BoolVar(&help, "h", false, "Show help information") - backtraceFlag.BoolVar(&showVersion, "v", false, "Show version") - backtraceFlag.BoolVar(&showIpInfo, "s", true, "Disabe show ip info") - backtraceFlag.BoolVar(&model.EnableLoger, "log", false, "Enable logging") - backtraceFlag.BoolVar(&ipv6, "ipv6", false, "Enable ipv6 testing") - backtraceFlag.StringVar(&specifiedIP, "ip", "", "Specify IP address for bgptools") - backtraceFlag.Parse(os.Args[1:]) - if help { + var options cliOptions + backtraceFlag := newBacktraceFlagSet(&options) + if err := backtraceFlag.Parse(os.Args[1:]); err != nil { + os.Exit(2) + } + if !options.jsonOutput { + fmt.Println(Green("Repo:"), Yellow("https://github.com/oneclickvirt/backtrace")) + } + if options.help { fmt.Printf("Usage: %s [options]\n", os.Args[0]) backtraceFlag.PrintDefaults() return } - if showVersion { + if options.showVersion { fmt.Println(model.BackTraceVersion) return } + if err := validateStructuredOptions(options); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + if options.jsonOutput { + config := bgptools.IPBGPReportConfig{ + Timeout: options.timeout, + FetchGeofeed: options.deep, + EnableWHOISFallback: options.deep, + ResolveASN: func(ctx context.Context, ip string) (string, error) { + return bgptools.ResolveOriginASNWithConfig(ctx, ip, bgptools.OriginASNConfig{Timeout: options.timeout}) + }, + } + if err := writeStructuredReport(context.Background(), os.Stdout, options.specifiedIP, config, bgptools.QueryIPBGPReport); err != nil { + fmt.Fprintf(os.Stderr, "structured report failed: %v\n", err) + os.Exit(1) + } + return + } info := IpInfo{} - if showIpInfo { + if options.showIPInfo { rsp, err := http.Get("http://ipinfo.io") if err != nil { fmt.Printf("get ip info err %v \n", err.Error()) @@ -98,7 +142,7 @@ func main() { var useIPv6 bool switch preCheck.StackType { case "DualStack": - useIPv6 = ipv6 + useIPv6 = options.ipv6 case "IPv4": useIPv6 = false case "IPv6": @@ -114,8 +158,8 @@ func main() { results := ConcurrentResults{} var wg sync.WaitGroup var targetIP string - if specifiedIP != "" { - targetIP = specifiedIP + if options.specifiedIP != "" { + targetIP = options.specifiedIP } else if info.Ip != "" { targetIP = info.Ip } @@ -154,3 +198,29 @@ func main() { fmt.Scanln() } } + +func validateStructuredOptions(options cliOptions) error { + if options.deep && !options.jsonOutput { + return fmt.Errorf("-deep requires -json or -structured") + } + if !options.jsonOutput { + return nil + } + if options.specifiedIP == "" { + return fmt.Errorf("structured report requires -ip") + } + if options.timeout <= 0 { + return fmt.Errorf("structured report timeout must be positive") + } + return nil +} + +func writeStructuredReport(ctx context.Context, output io.Writer, ip string, config bgptools.IPBGPReportConfig, query func(context.Context, string, bgptools.IPBGPReportConfig) (*bgptools.IPBGPReport, error)) error { + report, err := query(ctx, ip, config) + if err != nil { + return err + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/main_test.go b/cmd/main_test.go new file mode 100644 index 0000000..8d27684 --- /dev/null +++ b/cmd/main_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "testing" + "time" + + "github.com/oneclickvirt/backtrace/bgptools" +) + +func TestBacktraceStructuredFlagParsing(t *testing.T) { + var options cliOptions + set := newBacktraceFlagSet(&options) + if err := set.Parse([]string{"-structured", "-deep", "-ip", "192.0.2.1", "-timeout", "900ms"}); err != nil { + t.Fatal(err) + } + if !options.jsonOutput || !options.deep || options.specifiedIP != "192.0.2.1" || options.timeout != 900*time.Millisecond { + t.Fatalf("unexpected options: %+v", options) + } +} + +func TestBacktraceDefaultsPreserveLegacyMode(t *testing.T) { + var options cliOptions + if err := newBacktraceFlagSet(&options).Parse(nil); err != nil { + t.Fatal(err) + } + if !options.showIPInfo || options.jsonOutput || options.deep || options.ipv6 || options.specifiedIP != "" || options.timeout != 15*time.Second { + t.Fatalf("legacy defaults changed: %+v", options) + } +} + +func TestValidateStructuredOptionsRequiresExplicitModeAndTarget(t *testing.T) { + tests := []cliOptions{ + {deep: true, timeout: time.Second}, + {jsonOutput: true, timeout: time.Second}, + {jsonOutput: true, specifiedIP: "192.0.2.1"}, + } + for _, options := range tests { + if err := validateStructuredOptions(options); err == nil { + t.Fatalf("expected invalid structured options: %+v", options) + } + } + if err := validateStructuredOptions(cliOptions{jsonOutput: true, deep: true, specifiedIP: "192.0.2.1", timeout: time.Second}); err != nil { + t.Fatalf("valid structured options rejected: %v", err) + } +} + +func TestWriteStructuredReportKeepsStdoutJSONOnly(t *testing.T) { + var output bytes.Buffer + err := writeStructuredReport(context.Background(), &output, "192.0.2.1", bgptools.IPBGPReportConfig{Timeout: time.Second}, func(_ context.Context, ip string, _ bgptools.IPBGPReportConfig) (*bgptools.IPBGPReport, error) { + return &bgptools.IPBGPReport{IP: ip, Status: bgptools.ReportAvailable, Sources: []bgptools.IPBGPSourceStatus{}}, nil + }) + if err != nil { + t.Fatal(err) + } + var report bgptools.IPBGPReport + if err := json.Unmarshal(output.Bytes(), &report); err != nil { + t.Fatalf("stdout is not JSON: %v (%q)", err, output.String()) + } + if report.IP != "192.0.2.1" { + t.Fatalf("unexpected report: %+v", report) + } +} diff --git a/cmd/update-prefixes/main.go b/cmd/update-prefixes/main.go new file mode 100644 index 0000000..048d292 --- /dev/null +++ b/cmd/update-prefixes/main.go @@ -0,0 +1,186 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/netip" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +var knownASNs = []string{"AS23764", "AS4134", "AS4809", "AS4837", "AS58453", "AS58807", "AS9808", "AS9929"} +var errPrefixCountDrop = errors.New("prefix count dropped beyond protection threshold") + +type updateConfig struct { + SourceBase string + OutputDir string + Timeout time.Duration +} + +func main() { + config := updateConfig{} + flag.StringVar(&config.SourceBase, "source-base", "https://stat.ripe.net/data/announced-prefixes/data.json", "upstream announced-prefixes API") + flag.StringVar(&config.OutputDir, "output-dir", "bk/prefix", "prefix snapshot directory") + flag.DurationVar(&config.Timeout, "timeout", 30*time.Second, "upstream request timeout") + flag.Parse() + if err := updatePrefixes(context.Background(), http.DefaultClient, config); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func updatePrefixes(ctx context.Context, client *http.Client, config updateConfig) error { + if ctx == nil { + ctx = context.Background() + } + if client == nil { + client = http.DefaultClient + } + if config.SourceBase == "" || config.OutputDir == "" || config.Timeout <= 0 { + return errors.New("source-base, output-dir, and timeout must be valid") + } + valuesByASN := make(map[string][]string, len(knownASNs)) + for _, asn := range knownASNs { + requestCtx, cancel := context.WithTimeout(ctx, config.Timeout) + endpoint, err := url.Parse(config.SourceBase) + if err != nil { + cancel() + return fmt.Errorf("parse source URL: %w", err) + } + query := endpoint.Query() + query.Set("resource", asn) + endpoint.RawQuery = query.Encode() + request, err := http.NewRequestWithContext(requestCtx, http.MethodGet, endpoint.String(), nil) + if err != nil { + cancel() + return err + } + request.Header.Set("User-Agent", "oneclickvirt-backtrace-prefix-sync/1") + response, err := client.Do(request) + if err != nil { + cancel() + return fmt.Errorf("fetch %s: %w", asn, err) + } + body, readErr := io.ReadAll(io.LimitReader(response.Body, 8<<20)) + response.Body.Close() + cancel() + if readErr != nil { + return readErr + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("fetch %s: HTTP %d", asn, response.StatusCode) + } + values, err := extractIPv6Prefixes(body) + if err != nil { + return fmt.Errorf("parse %s: %w", asn, err) + } + valuesByASN[asn] = values + } + for _, asn := range knownASNs { + if err := writePrefixFile(filepath.Join(config.OutputDir, strings.ToLower(asn)+".txt"), valuesByASN[asn]); err != nil { + if errors.Is(err, errPrefixCountDrop) { + fmt.Fprintf(os.Stderr, "skip %s: %v\n", asn, err) + continue + } + return err + } + } + return nil +} + +func extractIPv6Prefixes(data []byte) ([]string, error) { + var response struct { + Status string `json:"status"` + Data struct { + Resource string `json:"resource"` + Prefixes []struct { + Prefix string `json:"prefix"` + } `json:"prefixes"` + } `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil { + return nil, fmt.Errorf("decode announced-prefixes response: %w", err) + } + if response.Status != "ok" || response.Data.Resource == "" || response.Data.Prefixes == nil { + return nil, errors.New("announced-prefixes response is missing required fields") + } + seen := make(map[string]struct{}) + for _, record := range response.Data.Prefixes { + prefix, err := netip.ParsePrefix(strings.TrimSpace(record.Prefix)) + if err != nil || !prefix.Addr().Is6() || prefix.Bits() < 16 || prefix.Bits() > 128 { + continue + } + seen[prefix.String()] = struct{}{} + } + values := make([]string, 0, len(seen)) + for value := range seen { + values = append(values, value) + } + sort.Strings(values) + if len(values) == 0 { + return nil, errors.New("no IPv6 prefixes found") + } + return values, nil +} + +func writePrefixFile(output string, values []string) error { + if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil { + return err + } + candidate := []byte(strings.Join(values, "\n") + "\n") + current, readErr := os.ReadFile(output) + if readErr == nil { + currentCount := countNonemptyLines(current) + if currentCount > 0 && len(values)*100 < currentCount*65 { + return fmt.Errorf("%w: prefix count dropped from %d to %d for %s", errPrefixCountDrop, currentCount, len(values), filepath.Base(output)) + } + if bytes.Equal(current, candidate) { + return nil + } + } + if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { + return readErr + } + temporary, err := os.CreateTemp(filepath.Dir(output), ".prefix-*") + if err != nil { + return err + } + temporaryName := temporary.Name() + defer os.Remove(temporaryName) + if err := temporary.Chmod(0o644); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(candidate); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + return os.Rename(temporaryName, output) +} + +func countNonemptyLines(data []byte) int { + count := 0 + for _, line := range strings.Split(string(data), "\n") { + if strings.TrimSpace(line) != "" { + count++ + } + } + return count +} diff --git a/cmd/update-prefixes/main_test.go b/cmd/update-prefixes/main_test.go new file mode 100644 index 0000000..42cdb92 --- /dev/null +++ b/cmd/update-prefixes/main_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestExtractIPv6Prefixes(t *testing.T) { + values, err := extractIPv6Prefixes([]byte(`{"status":"ok","data":{"resource":"64500","prefixes":[{"prefix":"2409:8000::/32"},{"prefix":"2409:8000::/32"},{"prefix":"192.0.2.0/24"}]}}`)) + if err != nil || len(values) != 1 || values[0] != "2409:8000::/32" { + t.Fatalf("unexpected prefixes: %#v, %v", values, err) + } +} + +func TestExtractIPv6PrefixesRejectsSchemaDrift(t *testing.T) { + if _, err := extractIPv6Prefixes([]byte(`{"status":"ok","data":{"resource":"64500"}}`)); err == nil { + t.Fatal("missing prefixes unexpectedly accepted") + } +} + +func TestWritePrefixFileProtectsLargeCountDrop(t *testing.T) { + path := filepath.Join(t.TempDir(), "as64500.txt") + if err := os.WriteFile(path, []byte("2001:db8:1::/48\n2001:db8:2::/48\n2001:db8:3::/48\n2001:db8:4::/48\n"), 0o644); err != nil { + t.Fatal(err) + } + err := writePrefixFile(path, []string{"2001:db8:1::/48"}) + if !errors.Is(err, errPrefixCountDrop) { + t.Fatalf("count drop error = %v", err) + } +} diff --git a/model/model.go b/model/model.go index 6b6d323..fd38ccd 100644 --- a/model/model.go +++ b/model/model.go @@ -2,7 +2,7 @@ package model import "time" -const BackTraceVersion = "v0.0.11" +const BackTraceVersion = "v0.0.12" var EnableLoger = false