diff --git a/.github/workflows/sync-asn-metadata.yml b/.github/workflows/sync-asn-metadata.yml new file mode 100644 index 0000000..db42368 --- /dev/null +++ b/.github/workflows/sync-asn-metadata.yml @@ -0,0 +1,39 @@ +name: Sync ASN metadata + +on: + schedule: + - cron: "29 3 * * *" + workflow_dispatch: + +concurrency: + group: backtrace-asn-metadata-sync + cancel-in-progress: false + +permissions: + contents: write + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + - name: Update snapshot + run: go run ./cmd/update-asn-metadata + - name: Validate updater and runtime loader + run: go test -race ./cmd/update-asn-metadata ./bgptools + - name: Vet updater and runtime loader + run: go vet ./cmd/update-asn-metadata ./bgptools + - name: Commit semantic changes + run: | + if git diff --quiet -- bgptools/data/bgp-asn-map.json; then + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add bgptools/data/bgp-asn-map.json + git commit -m "chore(data): update ASN metadata" + git push diff --git a/bgptools/asn_metadata.go b/bgptools/asn_metadata.go new file mode 100644 index 0000000..9e68083 --- /dev/null +++ b/bgptools/asn_metadata.go @@ -0,0 +1,167 @@ +package bgptools + +import ( + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "sort" + "strconv" + "strings" + "time" +) + +const ( + ASNMetadataSchema = "backtrace.asn-metadata/v1" + ASNMetadataMinimum = 50 +) + +var asnMetadataSnapshotURLs = []string{ + "https://cdn.spiritlhl.net/https://raw.githubusercontent.com/oneclickvirt/backtrace/main/bgptools/data/bgp-asn-map.json", + "https://raw.githubusercontent.com/oneclickvirt/backtrace/main/bgptools/data/bgp-asn-map.json", +} + +//go:embed data/bgp-asn-map.json +var embeddedASNMetadata []byte + +type ASNMetadata struct { + ASN uint32 `json:"asn"` + Name string `json:"name"` +} + +type ASNMetadataSource struct { + Schema string `json:"schema"` + Count int `json:"count"` + GeneratedAt time.Time `json:"generated_at,omitempty"` + Source string `json:"source"` + Fallback bool `json:"fallback"` +} + +type asnMetadataDocument struct { + Schema string `json:"schema"` + GeneratedAt time.Time `json:"generated_at"` + Entries []ASNMetadata `json:"entries"` +} + +// LoadASNMetadata prefers the component's validated remote snapshot and +// falls back to the compile-time snapshot. Upstream registry URLs and parsing +// rules are intentionally not exposed to callers. +func LoadASNMetadata(ctx context.Context, client *http.Client) ([]ASNMetadata, ASNMetadataSource, error) { + return loadASNMetadata(ctx, client, asnMetadataSnapshotURLs, embeddedASNMetadata, ASNMetadataMinimum) +} + +// EmbeddedASNMetadata returns the validated compile-time snapshot without +// performing network access. +func EmbeddedASNMetadata() ([]ASNMetadata, ASNMetadataSource, error) { + entries, generatedAt, err := parseASNMetadataDocument(embeddedASNMetadata, ASNMetadataMinimum) + if err != nil { + return nil, ASNMetadataSource{}, err + } + return entries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(entries), GeneratedAt: generatedAt, Source: "embedded", Fallback: true}, nil +} + +// LookupASNMetadata resolves one ASN from the current component registry. +func LookupASNMetadata(ctx context.Context, client *http.Client, asn string) (ASNMetadata, ASNMetadataSource, bool, error) { + normalized, err := normalizeASN(asn) + if err != nil { + return ASNMetadata{}, ASNMetadataSource{}, false, err + } + number, _ := strconv.ParseUint(normalized, 10, 32) + entries, source, err := LoadASNMetadata(ctx, client) + if err != nil { + return ASNMetadata{}, ASNMetadataSource{}, false, err + } + index := sort.Search(len(entries), func(index int) bool { return entries[index].ASN >= uint32(number) }) + if index >= len(entries) || entries[index].ASN != uint32(number) { + return ASNMetadata{}, source, false, nil + } + return entries[index], source, true, nil +} + +func loadASNMetadata(ctx context.Context, client *http.Client, urls []string, embedded []byte, minimum int) ([]ASNMetadata, ASNMetadataSource, error) { + if ctx == nil { + ctx = context.Background() + } + embeddedEntries, embeddedAt, err := parseASNMetadataDocument(embedded, minimum) + if err != nil { + return nil, ASNMetadataSource{}, fmt.Errorf("invalid embedded ASN metadata: %w", err) + } + if client == nil { + client = &http.Client{Timeout: 6 * time.Second} + } + minimumRemote := max(minimum, len(embeddedEntries)*65/100) + for _, snapshotURL := range urls { + data, fetchErr := fetchASNMetadata(ctx, client, snapshotURL) + if fetchErr != nil { + continue + } + entries, generatedAt, parseErr := parseASNMetadataDocument(data, minimumRemote) + if parseErr == nil { + return entries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(entries), GeneratedAt: generatedAt, Source: "remote"}, nil + } + } + return embeddedEntries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(embeddedEntries), GeneratedAt: embeddedAt, Source: "embedded", Fallback: true}, nil +} + +func fetchASNMetadata(ctx context.Context, client *http.Client, snapshotURL string) ([]byte, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, snapshotURL, nil) + if err != nil { + return nil, err + } + request.Header.Set("Accept", "application/json") + request.Header.Set("User-Agent", "oneclickvirt-backtrace-asn-metadata/1") + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d", response.StatusCode) + } + const maximumSize = 4 << 20 + data, err := io.ReadAll(io.LimitReader(response.Body, maximumSize+1)) + if err != nil { + return nil, err + } + if len(data) > maximumSize { + return nil, fmt.Errorf("ASN metadata exceeds %d bytes", maximumSize) + } + return data, nil +} + +func parseASNMetadataDocument(data []byte, minimum int) ([]ASNMetadata, time.Time, error) { + var document asnMetadataDocument + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&document); err != nil { + return nil, time.Time{}, err + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return nil, time.Time{}, errors.New("ASN metadata contains trailing JSON") + } + if document.Schema != ASNMetadataSchema || document.GeneratedAt.IsZero() { + return nil, time.Time{}, errors.New("ASN metadata schema or generated_at is invalid") + } + seen := make(map[uint32]struct{}, len(document.Entries)) + entries := make([]ASNMetadata, 0, len(document.Entries)) + for _, entry := range document.Entries { + entry.Name = strings.TrimSpace(entry.Name) + if entry.ASN == 0 || entry.Name == "" { + return nil, time.Time{}, errors.New("ASN metadata contains an invalid entry") + } + if _, exists := seen[entry.ASN]; exists { + return nil, time.Time{}, fmt.Errorf("ASN metadata contains duplicate AS%d", entry.ASN) + } + seen[entry.ASN] = struct{}{} + entries = append(entries, entry) + } + if len(entries) < minimum { + return nil, time.Time{}, fmt.Errorf("ASN metadata count %d is below minimum %d", len(entries), minimum) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].ASN < entries[j].ASN }) + return entries, document.GeneratedAt, nil +} diff --git a/bgptools/asn_metadata_test.go b/bgptools/asn_metadata_test.go new file mode 100644 index 0000000..c7b207b --- /dev/null +++ b/bgptools/asn_metadata_test.go @@ -0,0 +1,41 @@ +package bgptools + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +func TestLoadASNMetadataPrefersValidatedRemote(t *testing.T) { + payload := `{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":2,"name":"Two"},{"asn":1,"name":"One"}]}` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(payload)) })) + defer server.Close() + entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL}, []byte(payload), 1) + if err != nil { + t.Fatal(err) + } + if source.Source != "remote" || source.Fallback || source.Count != 2 || entries[0].ASN != 1 { + t.Fatalf("unexpected result: %#v %#v", entries, source) + } +} + +func TestLoadASNMetadataFallsBackOnSchemaFailure(t *testing.T) { + embedded := `{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"}]}` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"schema":"wrong"}`)) })) + defer server.Close() + entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL}, []byte(embedded), 1) + if err != nil { + t.Fatal(err) + } + if source.Source != "embedded" || !source.Fallback || len(entries) != 1 { + t.Fatalf("unexpected fallback: %#v %#v", entries, source) + } +} + +func TestParseASNMetadataRejectsDuplicateAndDrop(t *testing.T) { + payload := []byte(`{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"},{"asn":1,"name":"Again"}]}`) + if _, _, err := parseASNMetadataDocument(payload, 1); err == nil { + t.Fatal("duplicate ASN unexpectedly accepted") + } +} diff --git a/bgptools/data/bgp-asn-map.json b/bgptools/data/bgp-asn-map.json new file mode 100644 index 0000000..805bb70 --- /dev/null +++ b/bgptools/data/bgp-asn-map.json @@ -0,0 +1,342 @@ +{ + "schema": "backtrace.asn-metadata/v1", + "generated_at": "2026-07-21T15:33:49.459211Z", + "entries": [ + { + "asn": 174, + "name": "Cogent" + }, + { + "asn": 701, + "name": "Verizon" + }, + { + "asn": 702, + "name": "Verizon" + }, + { + "asn": 1239, + "name": "Sprint" + }, + { + "asn": 1273, + "name": "Vodafone" + }, + { + "asn": 1299, + "name": "Arelion" + }, + { + "asn": 1729, + "name": "Telia" + }, + { + "asn": 2497, + "name": "IIJ" + }, + { + "asn": 2516, + "name": "KDDI" + }, + { + "asn": 2914, + "name": "NTT" + }, + { + "asn": 3209, + "name": "Vodafone" + }, + { + "asn": 3257, + "name": "GTT" + }, + { + "asn": 3258, + "name": "xTom" + }, + { + "asn": 3301, + "name": "Telia" + }, + { + "asn": 3308, + "name": "Telia" + }, + { + "asn": 3320, + "name": "DTAG" + }, + { + "asn": 3356, + "name": "Lumen" + }, + { + "asn": 3462, + "name": "Hinet" + }, + { + "asn": 3491, + "name": "PCCW" + }, + { + "asn": 3786, + "name": "LG" + }, + { + "asn": 4445, + "name": "Vodafone" + }, + { + "asn": 4609, + "name": "CTM" + }, + { + "asn": 4637, + "name": "Telstra" + }, + { + "asn": 4657, + "name": "StarHub" + }, + { + "asn": 4766, + "name": "KT" + }, + { + "asn": 4788, + "name": "TMNet" + }, + { + "asn": 5378, + "name": "Vodafone" + }, + { + "asn": 5511, + "name": "Orange" + }, + { + "asn": 6233, + "name": "xTom" + }, + { + "asn": 6453, + "name": "TATA" + }, + { + "asn": 6461, + "name": "Zayo" + }, + { + "asn": 6762, + "name": "Sparkle" + }, + { + "asn": 6830, + "name": "Liberty" + }, + { + "asn": 6939, + "name": "HE" + }, + { + "asn": 7018, + "name": "AT\u0026T" + }, + { + "asn": 7473, + "name": "SingTel" + }, + { + "asn": 7922, + "name": "Comcast" + }, + { + "asn": 7979, + "name": "Servers" + }, + { + "asn": 8075, + "name": "MS" + }, + { + "asn": 8888, + "name": "xTom" + }, + { + "asn": 9002, + "name": "RETN" + }, + { + "asn": 9269, + "name": "HKBN" + }, + { + "asn": 9304, + "name": "HGC" + }, + { + "asn": 9505, + "name": "CHT" + }, + { + "asn": 9644, + "name": "SK" + }, + { + "asn": 9886, + "name": "CTCSCI" + }, + { + "asn": 9924, + "name": "TFN" + }, + { + "asn": 12389, + "name": "RT_RU" + }, + { + "asn": 12876, + "name": "Scaleway" + }, + { + "asn": 12956, + "name": "Telxius" + }, + { + "asn": 13335, + "name": "CF" + }, + { + "asn": 15169, + "name": "Google" + }, + { + "asn": 15802, + "name": "DU" + }, + { + "asn": 16276, + "name": "OVH" + }, + { + "asn": 16509, + "name": "Amazon" + }, + { + "asn": 17676, + "name": "Softbank" + }, + { + "asn": 18403, + "name": "FPT" + }, + { + "asn": 20473, + "name": "Vultr" + }, + { + "asn": 20485, + "name": "TTK_RU" + }, + { + "asn": 23961, + "name": "Misaka" + }, + { + "asn": 24940, + "name": "Hetzner" + }, + { + "asn": 25820, + "name": "IT7" + }, + { + "asn": 30844, + "name": "Liquid" + }, + { + "asn": 31898, + "name": "Oracle" + }, + { + "asn": 37700, + "name": "JINX" + }, + { + "asn": 38731, + "name": "CHT" + }, + { + "asn": 45102, + "name": "Alibaba" + }, + { + "asn": 45899, + "name": "VNPT" + }, + { + "asn": 49304, + "name": "SAKURA" + }, + { + "asn": 51847, + "name": "Nearoute" + }, + { + "asn": 58652, + "name": "Level3" + }, + { + "asn": 60068, + "name": "Datacamp" + }, + { + "asn": 63150, + "name": "Bage" + }, + { + "asn": 132203, + "name": "Tencent" + }, + { + "asn": 133613, + "name": "MTel" + }, + { + "asn": 136907, + "name": "Huawei" + }, + { + "asn": 137409, + "name": "GSL" + }, + { + "asn": 140096, + "name": "JINX" + }, + { + "asn": 152672, + "name": "Aiyun" + }, + { + "asn": 202662, + "name": "Hytron" + }, + { + "asn": 209242, + "name": "CF" + }, + { + "asn": 211392, + "name": "Dream" + }, + { + "asn": 214996, + "name": "netcup" + }, + { + "asn": 216382, + "name": "Layer" + } + ] +} diff --git a/cmd/update-asn-metadata/main.go b/cmd/update-asn-metadata/main.go new file mode 100644 index 0000000..c95206a --- /dev/null +++ b/cmd/update-asn-metadata/main.go @@ -0,0 +1,151 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/oneclickvirt/backtrace/bgptools" +) + +const defaultSource = "https://raw.githubusercontent.com/xykt/NetQuality/main/ref/AS_Mapping.txt" + +type document struct { + Schema string `json:"schema"` + GeneratedAt time.Time `json:"generated_at"` + Entries []bgptools.ASNMetadata `json:"entries"` +} + +func main() { + source := flag.String("source", defaultSource, "ASN name registry URL") + output := flag.String("output", "bgptools/data/bgp-asn-map.json", "snapshot output path") + minimum := flag.Int("min-count", bgptools.ASNMetadataMinimum, "minimum accepted ASN count") + timeout := flag.Duration("timeout", 30*time.Second, "fetch timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + entries, err := fetchEntries(ctx, http.DefaultClient, *source) + if err != nil { + fatal(err) + } + if err := updateSnapshot(*output, entries, *minimum); err != nil { + fatal(err) + } +} + +func fetchEntries(ctx context.Context, client *http.Client, source string) ([]bgptools.ASNMetadata, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil) + if err != nil { + return nil, err + } + request.Header.Set("User-Agent", "oneclickvirt-backtrace-asn-sync/1") + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("ASN registry returned HTTP %d", response.StatusCode) + } + return parseEntries(io.LimitReader(response.Body, 4<<20)) +} + +func parseEntries(reader io.Reader) ([]bgptools.ASNMetadata, error) { + seen := make(map[uint32]string) + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 2 || !strings.HasPrefix(strings.ToUpper(fields[0]), "AS") { + continue + } + number, err := strconv.ParseUint(strings.TrimPrefix(strings.ToUpper(fields[0]), "AS"), 10, 32) + if err != nil || number == 0 { + continue + } + if _, exists := seen[uint32(number)]; !exists { + seen[uint32(number)] = strings.Join(fields[1:], " ") + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + entries := make([]bgptools.ASNMetadata, 0, len(seen)) + for asn, name := range seen { + entries = append(entries, bgptools.ASNMetadata{ASN: asn, Name: name}) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].ASN < entries[j].ASN }) + if len(entries) == 0 { + return nil, fmt.Errorf("ASN registry contains no valid entries") + } + return entries, nil +} + +func updateSnapshot(path string, entries []bgptools.ASNMetadata, minimum int) error { + if len(entries) < minimum { + return fmt.Errorf("ASN count %d is below minimum %d", len(entries), minimum) + } + currentData, err := os.ReadFile(path) + if err != nil { + return err + } + var current document + if err := json.Unmarshal(currentData, ¤t); err != nil || current.Schema != bgptools.ASNMetadataSchema { + return fmt.Errorf("current ASN metadata schema is invalid") + } + if len(current.Entries) >= minimum && len(entries)*100 < len(current.Entries)*65 { + return fmt.Errorf("ASN count dropped from %d to %d", len(current.Entries), len(entries)) + } + if sameEntries(current.Entries, entries) && !current.GeneratedAt.IsZero() { + return nil + } + next, err := json.MarshalIndent(document{Schema: bgptools.ASNMetadataSchema, GeneratedAt: time.Now().UTC(), Entries: entries}, "", " ") + if err != nil { + return err + } + next = append(next, '\n') + temporary, err := os.CreateTemp(filepath.Dir(path), ".asn-metadata-*.json") + if err != nil { + return err + } + name := temporary.Name() + defer os.Remove(name) + if _, err := temporary.Write(next); err != nil { + temporary.Close() + return err + } + if err := temporary.Chmod(0o644); err != nil { + temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + return os.Rename(name, path) +} + +func sameEntries(left, right []bgptools.ASNMetadata) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/cmd/update-asn-metadata/main_test.go b/cmd/update-asn-metadata/main_test.go new file mode 100644 index 0000000..9fed245 --- /dev/null +++ b/cmd/update-asn-metadata/main_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/oneclickvirt/backtrace/bgptools" +) + +func TestParseEntriesDeduplicatesAndSorts(t *testing.T) { + entries, err := parseEntries(strings.NewReader("AS3356 Lumen\nAS174 Cogent\nAS3356 Level3\ninvalid\n")) + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 || entries[0] != (bgptools.ASNMetadata{ASN: 174, Name: "Cogent"}) || entries[1].Name != "Lumen" { + t.Fatalf("unexpected entries: %#v", entries) + } +} + +func TestUpdateSnapshotRejectsCountDrop(t *testing.T) { + path := filepath.Join(t.TempDir(), "asn.json") + current := `{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"},{"asn":2,"name":"Two"}]}` + if err := os.WriteFile(path, []byte(current), 0o644); err != nil { + t.Fatal(err) + } + if err := updateSnapshot(path, []bgptools.ASNMetadata{{ASN: 1, Name: "One"}}, 1); err == nil { + t.Fatal("count drop unexpectedly accepted") + } +} diff --git a/model/model.go b/model/model.go index fd38ccd..cdf3472 100644 --- a/model/model.go +++ b/model/model.go @@ -2,7 +2,7 @@ package model import "time" -const BackTraceVersion = "v0.0.12" +const BackTraceVersion = "v0.0.14" var EnableLoger = false