diff --git a/.github/workflows/sync-asn-metadata.yml b/.github/workflows/sync-asn-metadata.yml index db42368..15fd670 100644 --- a/.github/workflows/sync-asn-metadata.yml +++ b/.github/workflows/sync-asn-metadata.yml @@ -29,11 +29,11 @@ jobs: run: go vet ./cmd/update-asn-metadata ./bgptools - name: Commit semantic changes run: | - if git diff --quiet -- bgptools/data/bgp-asn-map.json; then + if git diff --quiet -- bgptools/data/bgp-asn-map.json bgptools/data/bgp-asn-map.manifest.json; then exit 0 fi git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add bgptools/data/bgp-asn-map.json + git add bgptools/data/bgp-asn-map.json bgptools/data/bgp-asn-map.manifest.json git commit -m "chore(data): update ASN metadata" git push diff --git a/bgptools/asn_metadata.go b/bgptools/asn_metadata.go index 9e68083..0a26073 100644 --- a/bgptools/asn_metadata.go +++ b/bgptools/asn_metadata.go @@ -2,7 +2,9 @@ package bgptools import ( "context" + "crypto/sha256" _ "embed" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -15,8 +17,9 @@ import ( ) const ( - ASNMetadataSchema = "backtrace.asn-metadata/v1" - ASNMetadataMinimum = 50 + ASNMetadataSchema = "backtrace.asn-metadata/v1" + ASNMetadataManifestSchema = "backtrace.asn-metadata-manifest/v1" + ASNMetadataMinimum = 50 ) var asnMetadataSnapshotURLs = []string{ @@ -27,6 +30,9 @@ var asnMetadataSnapshotURLs = []string{ //go:embed data/bgp-asn-map.json var embeddedASNMetadata []byte +//go:embed data/bgp-asn-map.manifest.json +var embeddedASNMetadataManifest []byte + type ASNMetadata struct { ASN uint32 `json:"asn"` Name string `json:"name"` @@ -46,11 +52,19 @@ type asnMetadataDocument struct { Entries []ASNMetadata `json:"entries"` } +type asnMetadataManifest struct { + Schema string `json:"schema"` + File string `json:"file"` + Count int `json:"count"` + SHA256 string `json:"sha256"` + GeneratedAt time.Time `json:"generated_at"` +} + // LoadASNMetadata prefers the component's validated remote snapshot and // falls back to the compile-time snapshot. Upstream registry URLs and parsing // rules are intentionally not exposed to callers. func LoadASNMetadata(ctx context.Context, client *http.Client) ([]ASNMetadata, ASNMetadataSource, error) { - return loadASNMetadata(ctx, client, asnMetadataSnapshotURLs, embeddedASNMetadata, ASNMetadataMinimum) + return loadASNMetadata(ctx, client, asnMetadataSnapshotURLs, embeddedASNMetadata, embeddedASNMetadataManifest, ASNMetadataMinimum) } // EmbeddedASNMetadata returns the validated compile-time snapshot without @@ -60,6 +74,9 @@ func EmbeddedASNMetadata() ([]ASNMetadata, ASNMetadataSource, error) { if err != nil { return nil, ASNMetadataSource{}, err } + if err := validateASNMetadataManifest(embeddedASNMetadataManifest, embeddedASNMetadata, len(entries), generatedAt); err != nil { + return nil, ASNMetadataSource{}, fmt.Errorf("validate embedded ASN metadata manifest: %w", err) + } return entries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(entries), GeneratedAt: generatedAt, Source: "embedded", Fallback: true}, nil } @@ -81,7 +98,7 @@ func LookupASNMetadata(ctx context.Context, client *http.Client, asn string) (AS return entries[index], source, true, nil } -func loadASNMetadata(ctx context.Context, client *http.Client, urls []string, embedded []byte, minimum int) ([]ASNMetadata, ASNMetadataSource, error) { +func loadASNMetadata(ctx context.Context, client *http.Client, urls []string, embedded, embeddedManifest []byte, minimum int) ([]ASNMetadata, ASNMetadataSource, error) { if ctx == nil { ctx = context.Background() } @@ -89,23 +106,65 @@ func loadASNMetadata(ctx context.Context, client *http.Client, urls []string, em if err != nil { return nil, ASNMetadataSource{}, fmt.Errorf("invalid embedded ASN metadata: %w", err) } + if err := validateASNMetadataManifest(embeddedManifest, embedded, len(embeddedEntries), embeddedAt); err != nil { + return nil, ASNMetadataSource{}, fmt.Errorf("invalid embedded ASN metadata manifest: %w", err) + } if client == nil { client = &http.Client{Timeout: 6 * time.Second} } minimumRemote := max(minimum, len(embeddedEntries)*65/100) - for _, snapshotURL := range urls { + for index, snapshotURL := range urls { + manifest, fetchErr := fetchASNMetadata(ctx, client, asnMetadataManifestURL(snapshotURL)) + if fetchErr != nil { + continue + } data, fetchErr := fetchASNMetadata(ctx, client, snapshotURL) if fetchErr != nil { continue } entries, generatedAt, parseErr := parseASNMetadataDocument(data, minimumRemote) - if parseErr == nil { - return entries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(entries), GeneratedAt: generatedAt, Source: "remote"}, nil + if parseErr == nil && validateASNMetadataManifest(manifest, data, len(entries), generatedAt) == nil { + return entries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(entries), GeneratedAt: generatedAt, Source: metadataRemoteSource(index), Fallback: index > 0}, nil } } return embeddedEntries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(embeddedEntries), GeneratedAt: embeddedAt, Source: "embedded", Fallback: true}, nil } +func metadataRemoteSource(index int) string { + if index == 0 { + return "cdn" + } + if index == 1 { + return "raw" + } + return "remote" +} + +func asnMetadataManifestURL(snapshotURL string) string { + return strings.TrimSuffix(snapshotURL, ".json") + ".manifest.json" +} + +func validateASNMetadataManifest(data, snapshot []byte, count int, generatedAt time.Time) error { + var manifest asnMetadataManifest + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&manifest); err != nil { + return fmt.Errorf("decode manifest: %w", err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return errors.New("manifest contains trailing JSON") + } + if manifest.Schema != ASNMetadataManifestSchema || manifest.File != "bgp-asn-map.json" || manifest.Count != count || manifest.GeneratedAt.IsZero() || !manifest.GeneratedAt.Equal(generatedAt) { + return errors.New("manifest schema, file, count, or generated_at is invalid") + } + hash := sha256.Sum256(snapshot) + if !strings.EqualFold(manifest.SHA256, hex.EncodeToString(hash[:])) { + return errors.New("manifest SHA-256 does not match snapshot") + } + return nil +} + func fetchASNMetadata(ctx context.Context, client *http.Client, snapshotURL string) ([]byte, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, snapshotURL, nil) if err != nil { diff --git a/bgptools/asn_metadata_test.go b/bgptools/asn_metadata_test.go index c7b207b..eefbfcc 100644 --- a/bgptools/asn_metadata_test.go +++ b/bgptools/asn_metadata_test.go @@ -2,20 +2,35 @@ package bgptools import ( "context" + "crypto/sha256" + "encoding/hex" + "fmt" "net/http" "net/http/httptest" + "strings" "testing" + "time" ) func TestLoadASNMetadataPrefersValidatedRemote(t *testing.T) { payload := `{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":2,"name":"Two"},{"asn":1,"name":"One"}]}` - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(payload)) })) + manifest := manifestFor([]byte(payload), "bgp-asn-map.json", 2) + if err := validateASNMetadataManifest(manifest, []byte(payload), 2, time.Date(2026, 7, 20, 0, 0, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if strings.HasSuffix(request.URL.Path, ".manifest.json") { + _, _ = w.Write(manifest) + return + } + _, _ = w.Write([]byte(payload)) + })) defer server.Close() - entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL}, []byte(payload), 1) + entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL + "/bgp-asn-map.json"}, []byte(payload), manifest, 1) if err != nil { t.Fatal(err) } - if source.Source != "remote" || source.Fallback || source.Count != 2 || entries[0].ASN != 1 { + if source.Source != "cdn" || source.Fallback || source.Count != 2 || entries[0].ASN != 1 { t.Fatalf("unexpected result: %#v %#v", entries, source) } } @@ -24,7 +39,8 @@ func TestLoadASNMetadataFallsBackOnSchemaFailure(t *testing.T) { embedded := `{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"}]}` server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"schema":"wrong"}`)) })) defer server.Close() - entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL}, []byte(embedded), 1) + manifest := manifestFor([]byte(embedded), "bgp-asn-map.json", 1) + entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL + "/bgp-asn-map.json"}, []byte(embedded), manifest, 1) if err != nil { t.Fatal(err) } @@ -33,6 +49,32 @@ func TestLoadASNMetadataFallsBackOnSchemaFailure(t *testing.T) { } } +func TestLoadASNMetadataFallsBackFromCDNToRaw(t *testing.T) { + payload := []byte(`{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"}]}`) + validManifest := manifestFor(payload, "bgp-asn-map.json", 1) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/cdn/bgp-asn-map.manifest.json" { + _, _ = writer.Write([]byte(`{"schema":"backtrace.asn-metadata-manifest/v1","file":"bgp-asn-map.json","count":1,"sha256":"bad","generated_at":"2026-07-20T00:00:00Z"}`)) + return + } + if request.URL.Path == "/raw/bgp-asn-map.manifest.json" { + _, _ = writer.Write(validManifest) + return + } + _, _ = writer.Write(payload) + })) + defer server.Close() + entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL + "/cdn/bgp-asn-map.json", server.URL + "/raw/bgp-asn-map.json"}, payload, validManifest, 1) + if err != nil || len(entries) != 1 || source.Source != "raw" || !source.Fallback { + t.Fatalf("unexpected raw fallback: entries=%#v source=%#v err=%v", entries, source, err) + } +} + +func manifestFor(snapshot []byte, file string, count int) []byte { + hash := sha256.Sum256(snapshot) + return []byte(fmt.Sprintf(`{"schema":"backtrace.asn-metadata-manifest/v1","file":"%s","count":%d,"sha256":"%s","generated_at":"2026-07-20T00:00:00Z"}`, file, count, hex.EncodeToString(hash[:]))) +} + func TestParseASNMetadataRejectsDuplicateAndDrop(t *testing.T) { payload := []byte(`{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"},{"asn":1,"name":"Again"}]}`) if _, _, err := parseASNMetadataDocument(payload, 1); err == nil { diff --git a/bgptools/data/bgp-asn-map.manifest.json b/bgptools/data/bgp-asn-map.manifest.json new file mode 100644 index 0000000..000929e --- /dev/null +++ b/bgptools/data/bgp-asn-map.manifest.json @@ -0,0 +1,7 @@ +{ + "schema": "backtrace.asn-metadata-manifest/v1", + "file": "bgp-asn-map.json", + "count": 84, + "sha256": "f681dd78d55c27d4615d2e705de9bde24d111e8897d37a489b91e263f27f1a66", + "generated_at": "2026-07-21T15:33:49.459211Z" +} diff --git a/cmd/update-asn-metadata/main.go b/cmd/update-asn-metadata/main.go index c95206a..d519e7d 100644 --- a/cmd/update-asn-metadata/main.go +++ b/cmd/update-asn-metadata/main.go @@ -3,6 +3,8 @@ package main import ( "bufio" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "flag" "fmt" @@ -26,6 +28,14 @@ type document struct { Entries []bgptools.ASNMetadata `json:"entries"` } +type manifest struct { + Schema string `json:"schema"` + File string `json:"file"` + Count int `json:"count"` + SHA256 string `json:"sha256"` + GeneratedAt time.Time `json:"generated_at"` +} + func main() { source := flag.String("source", defaultSource, "ASN name registry URL") output := flag.String("output", "bgptools/data/bgp-asn-map.json", "snapshot output path") @@ -106,7 +116,7 @@ func updateSnapshot(path string, entries []bgptools.ASNMetadata, minimum int) er return fmt.Errorf("ASN count dropped from %d to %d", len(current.Entries), len(entries)) } if sameEntries(current.Entries, entries) && !current.GeneratedAt.IsZero() { - return nil + return writeManifest(path, currentData, len(current.Entries)) } next, err := json.MarshalIndent(document{Schema: bgptools.ASNMetadataSchema, GeneratedAt: time.Now().UTC(), Entries: entries}, "", " ") if err != nil { @@ -130,6 +140,52 @@ func updateSnapshot(path string, entries []bgptools.ASNMetadata, minimum int) er if err := temporary.Close(); err != nil { return err } + if err := os.Rename(name, path); err != nil { + return err + } + return writeManifest(path, next, len(entries)) +} + +func writeManifest(snapshotPath string, snapshot []byte, count int) error { + var snapshotDocument document + if err := json.Unmarshal(snapshot, &snapshotDocument); err != nil { + return err + } + hash := sha256.Sum256(snapshot) + value := manifest{ + Schema: bgptools.ASNMetadataManifestSchema, File: filepath.Base(snapshotPath), Count: count, + SHA256: hex.EncodeToString(hash[:]), GeneratedAt: snapshotDocument.GeneratedAt, + } + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + manifestPath := strings.TrimSuffix(snapshotPath, ".json") + ".manifest.json" + if current, readErr := os.ReadFile(manifestPath); readErr == nil && string(current) == string(data) { + return nil + } + return writeAtomic(manifestPath, data) +} + +func writeAtomic(path string, data []byte) error { + temporary, err := os.CreateTemp(filepath.Dir(path), ".asn-metadata-manifest-*.json") + if err != nil { + return err + } + name := temporary.Name() + defer os.Remove(name) + if _, err := temporary.Write(data); err != nil { + temporary.Close() + return err + } + if err := temporary.Chmod(0o644); err != nil { + temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } return os.Rename(name, path) } diff --git a/cmd/update-asn-metadata/main_test.go b/cmd/update-asn-metadata/main_test.go index 9fed245..7209d79 100644 --- a/cmd/update-asn-metadata/main_test.go +++ b/cmd/update-asn-metadata/main_test.go @@ -1,6 +1,9 @@ package main import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" "os" "path/filepath" "strings" @@ -19,6 +22,34 @@ func TestParseEntriesDeduplicatesAndSorts(t *testing.T) { } } +func TestUpdateSnapshotWritesMatchingManifestWithoutRewritingSnapshot(t *testing.T) { + path := filepath.Join(t.TempDir(), "asn.json") + current := []byte(`{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"},{"asn":2,"name":"Two"}]}`) + if err := os.WriteFile(path, current, 0o644); err != nil { + t.Fatal(err) + } + entries := []bgptools.ASNMetadata{{ASN: 1, Name: "One"}, {ASN: 2, Name: "Two"}} + if err := updateSnapshot(path, entries, 1); err != nil { + t.Fatal(err) + } + after, _ := os.ReadFile(path) + if string(after) != string(current) { + t.Fatal("semantic no-op rewrote snapshot") + } + data, err := os.ReadFile(filepath.Join(filepath.Dir(path), "asn.manifest.json")) + if err != nil { + t.Fatal(err) + } + var value manifest + if err := json.Unmarshal(data, &value); err != nil { + t.Fatal(err) + } + hash := sha256.Sum256(current) + if value.Schema != bgptools.ASNMetadataManifestSchema || value.File != "asn.json" || value.Count != 2 || value.SHA256 != hex.EncodeToString(hash[:]) { + t.Fatalf("manifest mismatch: %#v", value) + } +} + func TestUpdateSnapshotRejectsCountDrop(t *testing.T) { path := filepath.Join(t.TempDir(), "asn.json") current := `{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"},{"asn":2,"name":"Two"}]}` diff --git a/model/model.go b/model/model.go index cdf3472..232005b 100644 --- a/model/model.go +++ b/model/model.go @@ -2,7 +2,7 @@ package model import "time" -const BackTraceVersion = "v0.0.14" +const BackTraceVersion = "v0.0.15" var EnableLoger = false