mirror of
https://github.com/oneclickvirt/backtrace.git
synced 2026-07-23 11:20:13 +08:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a5f09cf874 | ||
|
|
881d397bbc | ||
|
|
f0ba5fa410 | ||
|
|
46e20f680e | ||
|
|
765774cb06 |
16
.github/workflows/sync-asn-metadata.yml
vendored
16
.github/workflows/sync-asn-metadata.yml
vendored
@ -4,6 +4,18 @@ on:
|
||||
schedule:
|
||||
- cron: "29 3 * * *"
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- ".github/workflows/sync-asn-metadata.yml"
|
||||
- "cmd/update-asn-metadata/**"
|
||||
- "bgptools/data/**"
|
||||
- "bgptools/asn_metadata.go"
|
||||
- "bgptools/asn_metadata_test.go"
|
||||
- "model/model.go"
|
||||
- "go.mod"
|
||||
- "go.sum"
|
||||
|
||||
concurrency:
|
||||
group: backtrace-asn-metadata-sync
|
||||
@ -29,11 +41,11 @@ jobs:
|
||||
run: go vet ./cmd/update-asn-metadata ./bgptools
|
||||
- name: Commit semantic changes
|
||||
run: |
|
||||
if git diff --quiet -- bgptools/data/bgp-asn-map.json; then
|
||||
if git diff --quiet -- bgptools/data/bgp-asn-map.json bgptools/data/bgp-asn-map.manifest.json; then
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add bgptools/data/bgp-asn-map.json
|
||||
git add bgptools/data/bgp-asn-map.json bgptools/data/bgp-asn-map.manifest.json
|
||||
git commit -m "chore(data): update ASN metadata"
|
||||
git push
|
||||
|
||||
@ -2,7 +2,9 @@ package bgptools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
_ "embed"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -16,6 +18,7 @@ import (
|
||||
|
||||
const (
|
||||
ASNMetadataSchema = "backtrace.asn-metadata/v1"
|
||||
ASNMetadataManifestSchema = "backtrace.asn-metadata-manifest/v1"
|
||||
ASNMetadataMinimum = 50
|
||||
)
|
||||
|
||||
@ -27,6 +30,9 @@ var asnMetadataSnapshotURLs = []string{
|
||||
//go:embed data/bgp-asn-map.json
|
||||
var embeddedASNMetadata []byte
|
||||
|
||||
//go:embed data/bgp-asn-map.manifest.json
|
||||
var embeddedASNMetadataManifest []byte
|
||||
|
||||
type ASNMetadata struct {
|
||||
ASN uint32 `json:"asn"`
|
||||
Name string `json:"name"`
|
||||
@ -46,11 +52,19 @@ type asnMetadataDocument struct {
|
||||
Entries []ASNMetadata `json:"entries"`
|
||||
}
|
||||
|
||||
type asnMetadataManifest struct {
|
||||
Schema string `json:"schema"`
|
||||
File string `json:"file"`
|
||||
Count int `json:"count"`
|
||||
SHA256 string `json:"sha256"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
}
|
||||
|
||||
// LoadASNMetadata prefers the component's validated remote snapshot and
|
||||
// falls back to the compile-time snapshot. Upstream registry URLs and parsing
|
||||
// rules are intentionally not exposed to callers.
|
||||
func LoadASNMetadata(ctx context.Context, client *http.Client) ([]ASNMetadata, ASNMetadataSource, error) {
|
||||
return loadASNMetadata(ctx, client, asnMetadataSnapshotURLs, embeddedASNMetadata, ASNMetadataMinimum)
|
||||
return loadASNMetadata(ctx, client, asnMetadataSnapshotURLs, embeddedASNMetadata, embeddedASNMetadataManifest, ASNMetadataMinimum)
|
||||
}
|
||||
|
||||
// EmbeddedASNMetadata returns the validated compile-time snapshot without
|
||||
@ -60,6 +74,9 @@ func EmbeddedASNMetadata() ([]ASNMetadata, ASNMetadataSource, error) {
|
||||
if err != nil {
|
||||
return nil, ASNMetadataSource{}, err
|
||||
}
|
||||
if err := validateASNMetadataManifest(embeddedASNMetadataManifest, embeddedASNMetadata, len(entries), generatedAt); err != nil {
|
||||
return nil, ASNMetadataSource{}, fmt.Errorf("validate embedded ASN metadata manifest: %w", err)
|
||||
}
|
||||
return entries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(entries), GeneratedAt: generatedAt, Source: "embedded", Fallback: true}, nil
|
||||
}
|
||||
|
||||
@ -81,7 +98,7 @@ func LookupASNMetadata(ctx context.Context, client *http.Client, asn string) (AS
|
||||
return entries[index], source, true, nil
|
||||
}
|
||||
|
||||
func loadASNMetadata(ctx context.Context, client *http.Client, urls []string, embedded []byte, minimum int) ([]ASNMetadata, ASNMetadataSource, error) {
|
||||
func loadASNMetadata(ctx context.Context, client *http.Client, urls []string, embedded, embeddedManifest []byte, minimum int) ([]ASNMetadata, ASNMetadataSource, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
@ -89,33 +106,78 @@ func loadASNMetadata(ctx context.Context, client *http.Client, urls []string, em
|
||||
if err != nil {
|
||||
return nil, ASNMetadataSource{}, fmt.Errorf("invalid embedded ASN metadata: %w", err)
|
||||
}
|
||||
if err := validateASNMetadataManifest(embeddedManifest, embedded, len(embeddedEntries), embeddedAt); err != nil {
|
||||
return nil, ASNMetadataSource{}, fmt.Errorf("invalid embedded ASN metadata manifest: %w", err)
|
||||
}
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 6 * time.Second}
|
||||
}
|
||||
minimumRemote := max(minimum, len(embeddedEntries)*65/100)
|
||||
for _, snapshotURL := range urls {
|
||||
for index, snapshotURL := range urls {
|
||||
manifest, fetchErr := fetchASNMetadata(ctx, client, asnMetadataManifestURL(snapshotURL))
|
||||
if fetchErr != nil {
|
||||
continue
|
||||
}
|
||||
data, fetchErr := fetchASNMetadata(ctx, client, snapshotURL)
|
||||
if fetchErr != nil {
|
||||
continue
|
||||
}
|
||||
entries, generatedAt, parseErr := parseASNMetadataDocument(data, minimumRemote)
|
||||
if parseErr == nil {
|
||||
return entries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(entries), GeneratedAt: generatedAt, Source: "remote"}, nil
|
||||
if parseErr == nil && validateASNMetadataManifest(manifest, data, len(entries), generatedAt) == nil {
|
||||
return entries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(entries), GeneratedAt: generatedAt, Source: metadataRemoteSource(index), Fallback: index > 0}, nil
|
||||
}
|
||||
}
|
||||
return embeddedEntries, ASNMetadataSource{Schema: ASNMetadataSchema, Count: len(embeddedEntries), GeneratedAt: embeddedAt, Source: "embedded", Fallback: true}, nil
|
||||
}
|
||||
|
||||
func metadataRemoteSource(index int) string {
|
||||
if index == 0 {
|
||||
return "cdn"
|
||||
}
|
||||
if index == 1 {
|
||||
return "raw"
|
||||
}
|
||||
return "remote"
|
||||
}
|
||||
|
||||
func asnMetadataManifestURL(snapshotURL string) string {
|
||||
return strings.TrimSuffix(snapshotURL, ".json") + ".manifest.json"
|
||||
}
|
||||
|
||||
func validateASNMetadataManifest(data, snapshot []byte, count int, generatedAt time.Time) error {
|
||||
var manifest asnMetadataManifest
|
||||
decoder := json.NewDecoder(strings.NewReader(string(data)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&manifest); err != nil {
|
||||
return fmt.Errorf("decode manifest: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return errors.New("manifest contains trailing JSON")
|
||||
}
|
||||
if manifest.Schema != ASNMetadataManifestSchema || manifest.File != "bgp-asn-map.json" || manifest.Count != count || manifest.GeneratedAt.IsZero() || !manifest.GeneratedAt.Equal(generatedAt) {
|
||||
return errors.New("manifest schema, file, count, or generated_at is invalid")
|
||||
}
|
||||
hash := sha256.Sum256(snapshot)
|
||||
if !strings.EqualFold(manifest.SHA256, hex.EncodeToString(hash[:])) {
|
||||
return errors.New("manifest SHA-256 does not match snapshot")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchASNMetadata(ctx context.Context, client *http.Client, snapshotURL string) ([]byte, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, snapshotURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.New("create ASN metadata request failed")
|
||||
}
|
||||
request.Header.Set("Accept", "application/json")
|
||||
request.Header.Set("User-Agent", "oneclickvirt-backtrace-asn-metadata/1")
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return nil, ctxErr
|
||||
}
|
||||
return nil, errors.New("ASN metadata request failed")
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
@ -124,7 +186,7 @@ func fetchASNMetadata(ctx context.Context, client *http.Client, snapshotURL stri
|
||||
const maximumSize = 4 << 20
|
||||
data, err := io.ReadAll(io.LimitReader(response.Body, maximumSize+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.New("ASN metadata response read failed")
|
||||
}
|
||||
if len(data) > maximumSize {
|
||||
return nil, fmt.Errorf("ASN metadata exceeds %d bytes", maximumSize)
|
||||
|
||||
@ -2,20 +2,35 @@ package bgptools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadASNMetadataPrefersValidatedRemote(t *testing.T) {
|
||||
payload := `{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":2,"name":"Two"},{"asn":1,"name":"One"}]}`
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(payload)) }))
|
||||
manifest := manifestFor([]byte(payload), "bgp-asn-map.json", 2)
|
||||
if err := validateASNMetadataManifest(manifest, []byte(payload), 2, time.Date(2026, 7, 20, 0, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||
if strings.HasSuffix(request.URL.Path, ".manifest.json") {
|
||||
_, _ = w.Write(manifest)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(payload))
|
||||
}))
|
||||
defer server.Close()
|
||||
entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL}, []byte(payload), 1)
|
||||
entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL + "/bgp-asn-map.json"}, []byte(payload), manifest, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if source.Source != "remote" || source.Fallback || source.Count != 2 || entries[0].ASN != 1 {
|
||||
if source.Source != "cdn" || source.Fallback || source.Count != 2 || entries[0].ASN != 1 {
|
||||
t.Fatalf("unexpected result: %#v %#v", entries, source)
|
||||
}
|
||||
}
|
||||
@ -24,7 +39,8 @@ func TestLoadASNMetadataFallsBackOnSchemaFailure(t *testing.T) {
|
||||
embedded := `{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"}]}`
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"schema":"wrong"}`)) }))
|
||||
defer server.Close()
|
||||
entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL}, []byte(embedded), 1)
|
||||
manifest := manifestFor([]byte(embedded), "bgp-asn-map.json", 1)
|
||||
entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL + "/bgp-asn-map.json"}, []byte(embedded), manifest, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -33,6 +49,32 @@ func TestLoadASNMetadataFallsBackOnSchemaFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadASNMetadataFallsBackFromCDNToRaw(t *testing.T) {
|
||||
payload := []byte(`{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"}]}`)
|
||||
validManifest := manifestFor(payload, "bgp-asn-map.json", 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path == "/cdn/bgp-asn-map.manifest.json" {
|
||||
_, _ = writer.Write([]byte(`{"schema":"backtrace.asn-metadata-manifest/v1","file":"bgp-asn-map.json","count":1,"sha256":"bad","generated_at":"2026-07-20T00:00:00Z"}`))
|
||||
return
|
||||
}
|
||||
if request.URL.Path == "/raw/bgp-asn-map.manifest.json" {
|
||||
_, _ = writer.Write(validManifest)
|
||||
return
|
||||
}
|
||||
_, _ = writer.Write(payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
entries, source, err := loadASNMetadata(context.Background(), server.Client(), []string{server.URL + "/cdn/bgp-asn-map.json", server.URL + "/raw/bgp-asn-map.json"}, payload, validManifest, 1)
|
||||
if err != nil || len(entries) != 1 || source.Source != "raw" || !source.Fallback {
|
||||
t.Fatalf("unexpected raw fallback: entries=%#v source=%#v err=%v", entries, source, err)
|
||||
}
|
||||
}
|
||||
|
||||
func manifestFor(snapshot []byte, file string, count int) []byte {
|
||||
hash := sha256.Sum256(snapshot)
|
||||
return []byte(fmt.Sprintf(`{"schema":"backtrace.asn-metadata-manifest/v1","file":"%s","count":%d,"sha256":"%s","generated_at":"2026-07-20T00:00:00Z"}`, file, count, hex.EncodeToString(hash[:])))
|
||||
}
|
||||
|
||||
func TestParseASNMetadataRejectsDuplicateAndDrop(t *testing.T) {
|
||||
payload := []byte(`{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"},{"asn":1,"name":"Again"}]}`)
|
||||
if _, _, err := parseASNMetadataDocument(payload, 1); err == nil {
|
||||
|
||||
7
bgptools/data/bgp-asn-map.manifest.json
Normal file
7
bgptools/data/bgp-asn-map.manifest.json
Normal file
@ -0,0 +1,7 @@
|
||||
{
|
||||
"schema": "backtrace.asn-metadata-manifest/v1",
|
||||
"file": "bgp-asn-map.json",
|
||||
"count": 84,
|
||||
"sha256": "f681dd78d55c27d4615d2e705de9bde24d111e8897d37a489b91e263f27f1a66",
|
||||
"generated_at": "2026-07-21T15:33:49.459211Z"
|
||||
}
|
||||
@ -53,19 +53,22 @@ func ResolveOriginASNWithConfig(parent context.Context, ip string, config Origin
|
||||
}
|
||||
requestURL, err := addQuery(config.BaseURL, "resource", parsed.String())
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", errors.New("invalid origin ASN source")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, config.Timeout)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", errors.New("create origin ASN request failed")
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("User-Agent", "oneclickvirt-backtrace-origin-asn/1")
|
||||
response, err := config.Client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return "", ctxErr
|
||||
}
|
||||
return "", errors.New("origin ASN request failed")
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode == http.StatusTooManyRequests {
|
||||
@ -76,7 +79,7 @@ func ResolveOriginASNWithConfig(parent context.Context, ip string, config Origin
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, config.MaxResponseSize+1))
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", errors.New("origin ASN response read failed")
|
||||
}
|
||||
if int64(len(body)) > config.MaxResponseSize {
|
||||
return "", fmt.Errorf("origin ASN response exceeds %d bytes", config.MaxResponseSize)
|
||||
|
||||
97
bgptools/privacy_test.go
Normal file
97
bgptools/privacy_test.go
Normal file
@ -0,0 +1,97 @@
|
||||
package bgptools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type privacyRoundTripper func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (fn privacyRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
return fn(request)
|
||||
}
|
||||
|
||||
func TestStructuredReportErrorsDoNotExposeRemoteURLs(t *testing.T) {
|
||||
var rdap *httptest.Server
|
||||
rdap = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = fmt.Fprintf(w, `{"handle":"TEST","links":[{"rel":"geofeed","href":"https://private.example/geofeed?token=secret"}]}`)
|
||||
}))
|
||||
defer rdap.Close()
|
||||
failing := &http.Client{Transport: privacyRoundTripper(func(request *http.Request) (*http.Response, error) {
|
||||
return nil, errors.New("dial " + request.URL.String())
|
||||
})}
|
||||
report, err := QueryIPBGPReport(context.Background(), "192.0.2.1", IPBGPReportConfig{
|
||||
RDAPClient: rdap.Client(),
|
||||
RDAPBaseURL: rdap.URL,
|
||||
FetchGeofeed: true,
|
||||
GeofeedClient: failing,
|
||||
ResolveASN: func(context.Context, string) (string, error) {
|
||||
return "", errors.New("resolver https://private.example/asn?key=secret")
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, source := range report.Sources {
|
||||
for _, field := range []string{source.Source, source.Error} {
|
||||
for _, forbidden := range []string{"private.example", "token=", "key=", "secret"} {
|
||||
if strings.Contains(field, forbidden) {
|
||||
t.Fatalf("structured source leaked %q: %+v", forbidden, source)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(report.Geofeeds) != 1 || report.Geofeeds[0].Error != "request_failed" {
|
||||
t.Fatalf("geofeed error was not stabilized: %+v", report.Geofeeds)
|
||||
}
|
||||
encoded, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"token=", "key=", "secret", "userinfo@"} {
|
||||
if strings.Contains(string(encoded), forbidden) {
|
||||
t.Fatalf("structured report leaked %q: %s", forbidden, encoded)
|
||||
}
|
||||
}
|
||||
if report.RDAP == nil || len(report.RDAP.GeofeedURLs) != 1 || report.RDAP.GeofeedURLs[0] != "https://private.example/geofeed" || report.Geofeeds[0].URL != "https://private.example/geofeed" {
|
||||
t.Fatalf("geofeed URL was not sanitized consistently: %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteFetchErrorsDoNotExposeSourceURL(t *testing.T) {
|
||||
client := &http.Client{Transport: privacyRoundTripper(func(request *http.Request) (*http.Response, error) {
|
||||
return nil, errors.New("dial " + request.URL.String())
|
||||
})}
|
||||
tests := []struct {
|
||||
name string
|
||||
run func() error
|
||||
}{
|
||||
{name: "ASN metadata", run: func() error {
|
||||
_, err := fetchASNMetadata(context.Background(), client, "https://private.example/asn?token=secret")
|
||||
return err
|
||||
}},
|
||||
{name: "RDAP", run: func() error {
|
||||
_, err := QueryRDAP(context.Background(), "192.0.2.1", client, "https://private.example/rdap?token=secret")
|
||||
return err
|
||||
}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
err := test.run()
|
||||
if err == nil {
|
||||
t.Fatal("expected request failure")
|
||||
}
|
||||
for _, forbidden := range []string{"private.example", "token=", "secret"} {
|
||||
if strings.Contains(err.Error(), forbidden) {
|
||||
t.Fatalf("error leaked %q: %v", forbidden, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -60,13 +60,16 @@ func QueryRDAP(ctx context.Context, ip string, client *http.Client, baseURL stri
|
||||
requestURL := strings.TrimRight(baseURL, "/") + "/" + parsed.String()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
return RDAPRecord{}, err
|
||||
return RDAPRecord{}, errors.New("create RDAP request failed")
|
||||
}
|
||||
req.Header.Set("Accept", "application/rdap+json, application/json")
|
||||
req.Header.Set("User-Agent", "oneclickvirt-backtrace-rdap/1")
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
return RDAPRecord{}, err
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return RDAPRecord{}, ctxErr
|
||||
}
|
||||
return RDAPRecord{}, errors.New("RDAP request failed")
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode == http.StatusTooManyRequests {
|
||||
@ -77,7 +80,7 @@ func QueryRDAP(ctx context.Context, ip string, client *http.Client, baseURL stri
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(response.Body, (4<<20)+1))
|
||||
if err != nil {
|
||||
return RDAPRecord{}, err
|
||||
return RDAPRecord{}, errors.New("RDAP response read failed")
|
||||
}
|
||||
if len(data) > 4<<20 {
|
||||
return RDAPRecord{}, ErrRDAPResponseTooLarge
|
||||
@ -125,7 +128,7 @@ func parseRDAPRecord(ip, source string, data []byte) (RDAPRecord, error) {
|
||||
record := RDAPRecord{
|
||||
IP: ip, Handle: payload.Handle, Name: payload.Name, Type: payload.Type,
|
||||
Country: payload.Country, StartAddress: payload.StartAddress, EndAddress: payload.EndAddress,
|
||||
ParentHandle: payload.ParentHandle, Port43: payload.Port43, Status: payload.Status, Source: source,
|
||||
ParentHandle: payload.ParentHandle, Port43: payload.Port43, Status: payload.Status, Source: "rdap",
|
||||
}
|
||||
for _, event := range payload.Events {
|
||||
switch strings.ToLower(event.Action) {
|
||||
|
||||
@ -131,7 +131,7 @@ func QueryASNRelationships(ctx context.Context, asn string, cfg RelationshipConf
|
||||
report := &ASNRelationshipReport{TargetASN: normalized}
|
||||
ripeURL, err := addQuery(cfg.RIPEstatURL, "resource", normalized)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.New("invalid relationship source configuration")
|
||||
}
|
||||
ripeRaw, ripeStatus, ripeErr := fetchRelationshipJSON(ctx, cfg, "ripestat", ripeURL)
|
||||
report.SourceStatuses = append(report.SourceStatuses, sourceStatus("ripestat", ripeStatus, ripeErr))
|
||||
@ -153,7 +153,7 @@ func QueryASNRelationships(ctx context.Context, asn string, cfg RelationshipConf
|
||||
|
||||
peeringURL, err := addQuery(cfg.PeeringDBURL, "asn", normalized)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.New("invalid relationship source configuration")
|
||||
}
|
||||
peeringRaw, peeringStatus, peeringErr := fetchRelationshipJSON(ctx, cfg, "peeringdb", peeringURL)
|
||||
report.SourceStatuses = append(report.SourceStatuses, sourceStatus("peeringdb", peeringStatus, peeringErr))
|
||||
@ -255,7 +255,16 @@ func sourceStatus(source string, status RelationshipStatus, err error) Relations
|
||||
Timeout: status == RelationshipTimeout,
|
||||
}
|
||||
if err != nil {
|
||||
item.Error = err.Error()
|
||||
switch status {
|
||||
case RelationshipRateLimited:
|
||||
item.Error = "rate_limited"
|
||||
case RelationshipTimeout:
|
||||
item.Error = "timeout"
|
||||
case RelationshipMissingFields:
|
||||
item.Error = "missing_fields"
|
||||
default:
|
||||
item.Error = "request_failed"
|
||||
}
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
@ -217,6 +217,12 @@ func QueryIPBGPReport(parent context.Context, ip string, cfg IPBGPReportConfig)
|
||||
geofeedURLs = append(geofeedURLs, report.WHOIS.GeofeedURLs...)
|
||||
}
|
||||
geofeedURLs = uniqueSortedStrings(geofeedURLs)
|
||||
if report.RDAP != nil {
|
||||
report.RDAP.GeofeedURLs = sanitizeReportURLs(report.RDAP.GeofeedURLs)
|
||||
}
|
||||
if report.WHOIS != nil {
|
||||
report.WHOIS.GeofeedURLs = sanitizeReportURLs(report.WHOIS.GeofeedURLs)
|
||||
}
|
||||
for _, geofeedURL := range geofeedURLs {
|
||||
if !cfg.FetchGeofeed {
|
||||
report.Geofeeds = append(report.Geofeeds, GeofeedResult{URL: geofeedURL, Status: ReportUnsupported, Error: "fetch disabled"})
|
||||
@ -224,7 +230,7 @@ func QueryIPBGPReport(parent context.Context, ip string, cfg IPBGPReportConfig)
|
||||
}
|
||||
result := fetchGeofeed(ctx, geofeedURL, cfg)
|
||||
report.Geofeeds = append(report.Geofeeds, result)
|
||||
report.Sources = append(report.Sources, IPBGPSourceStatus{Source: "geofeed:" + geofeedURL, Status: result.Status, Error: result.Error})
|
||||
report.Sources = append(report.Sources, IPBGPSourceStatus{Source: "geofeed", Status: result.Status, Error: result.Error})
|
||||
}
|
||||
|
||||
asn := strings.TrimSpace(cfg.ASN)
|
||||
@ -286,7 +292,7 @@ func inferRIR(record RDAPRecord, available bool) RIRInfo {
|
||||
}
|
||||
|
||||
func fetchGeofeed(parent context.Context, rawURL string, cfg IPBGPReportConfig) GeofeedResult {
|
||||
result := GeofeedResult{URL: rawURL}
|
||||
result := GeofeedResult{URL: sanitizeReportURL(rawURL)}
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
result.Status = ReportUnsupported
|
||||
@ -310,7 +316,7 @@ func fetchGeofeed(parent context.Context, rawURL string, cfg IPBGPReportConfig)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
result.Status = ReportError
|
||||
result.Error = err.Error()
|
||||
result.Error = stableReportError(result.Status, err)
|
||||
return result
|
||||
}
|
||||
req.Header.Set("Accept", "text/csv, text/plain, */*")
|
||||
@ -318,7 +324,7 @@ func fetchGeofeed(parent context.Context, rawURL string, cfg IPBGPReportConfig)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
result.Status = reportStatusForError(err)
|
||||
result.Error = err.Error()
|
||||
result.Error = stableReportError(result.Status, err)
|
||||
return result
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
@ -341,7 +347,7 @@ func fetchGeofeed(parent context.Context, rawURL string, cfg IPBGPReportConfig)
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
|
||||
if err != nil {
|
||||
result.Status = reportStatusForError(err)
|
||||
result.Error = err.Error()
|
||||
result.Error = stableReportError(result.Status, err)
|
||||
return result
|
||||
}
|
||||
if int64(len(body)) > maxBytes {
|
||||
@ -517,11 +523,51 @@ func rirNameFromText(value string) string {
|
||||
func sourceStatusForError(source string, status ReportStatus, err error) IPBGPSourceStatus {
|
||||
item := IPBGPSourceStatus{Source: source, Status: status}
|
||||
if err != nil {
|
||||
item.Error = err.Error()
|
||||
item.Error = stableReportError(status, err)
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func stableReportError(status ReportStatus, err error) string {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return "canceled"
|
||||
}
|
||||
switch status {
|
||||
case ReportRateLimited:
|
||||
return "rate_limited"
|
||||
case ReportTimeout:
|
||||
return "timeout"
|
||||
case ReportUnsupported:
|
||||
return "unsupported"
|
||||
case ReportMissingFields:
|
||||
return "missing_fields"
|
||||
default:
|
||||
return "request_failed"
|
||||
}
|
||||
}
|
||||
|
||||
func sanitizeReportURL(raw string) string {
|
||||
parsed, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||||
return ""
|
||||
}
|
||||
parsed.User = nil
|
||||
parsed.RawQuery = ""
|
||||
parsed.ForceQuery = false
|
||||
parsed.Fragment = ""
|
||||
return parsed.String()
|
||||
}
|
||||
|
||||
func sanitizeReportURLs(values []string) []string {
|
||||
result := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
if sanitized := sanitizeReportURL(value); sanitized != "" {
|
||||
result = append(result, sanitized)
|
||||
}
|
||||
}
|
||||
return uniqueSortedStrings(result)
|
||||
}
|
||||
|
||||
func reportStatusForError(err error) ReportStatus {
|
||||
if err == nil {
|
||||
return ReportAvailable
|
||||
|
||||
@ -210,6 +210,7 @@
|
||||
240e:4b:8000::/33
|
||||
240e:4b::/33
|
||||
240e:4c:4006::/48
|
||||
240e:4c:5000::/48
|
||||
240e:4c:8000::/33
|
||||
240e:4c::/33
|
||||
240e:4d:50ff::/48
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema": "backtrace.asn-prefixes/v1",
|
||||
"file": "as4134.txt",
|
||||
"count": 628,
|
||||
"sha256": "9e032d30479db6c5bd167681048922765c23d1a35b01ed9e6968b2099b8b916f",
|
||||
"generated_at": "2026-07-20T19:12:53Z"
|
||||
"count": 629,
|
||||
"sha256": "00941f76c022566166b926670ee9358084c53465cbaaf75a3da7b73d997d1947",
|
||||
"generated_at": "2026-07-23T02:56:31Z"
|
||||
}
|
||||
|
||||
@ -6639,7 +6639,6 @@
|
||||
2409:8c7c::/32
|
||||
2409:8c7e:9301::/48
|
||||
2409:8c7e::/32
|
||||
2409:8c85:1000::/48
|
||||
2409:8c85:1001::/48
|
||||
2409:8c85:1::/48
|
||||
2409:8c85:200::/48
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema": "backtrace.asn-prefixes/v1",
|
||||
"file": "as9808.txt",
|
||||
"count": 7171,
|
||||
"sha256": "c362e447b515ca510b10fd37ba50189c3551a35dede88399190bc869b6fbe66b",
|
||||
"generated_at": "2026-07-20T19:12:54Z"
|
||||
"count": 7170,
|
||||
"sha256": "7532e5bba93ed6b7d5cf010b22d9f5c87606392673fd9877ce28417f2baadd5d",
|
||||
"generated_at": "2026-07-23T02:56:31Z"
|
||||
}
|
||||
|
||||
12
cmd/main.go
12
cmd/main.go
@ -96,7 +96,7 @@ func main() {
|
||||
return
|
||||
}
|
||||
if err := validateStructuredOptions(options); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
fmt.Fprintln(os.Stderr, sanitizeErrorText(err.Error()))
|
||||
os.Exit(2)
|
||||
}
|
||||
if options.jsonOutput {
|
||||
@ -109,7 +109,7 @@ func main() {
|
||||
},
|
||||
}
|
||||
if err := writeStructuredReport(context.Background(), os.Stdout, options.specifiedIP, config, bgptools.QueryIPBGPReport); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "structured report failed: %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "structured report failed: %s\n", sanitizeErrorText(err.Error()))
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
@ -118,12 +118,12 @@ func main() {
|
||||
if options.showIPInfo {
|
||||
rsp, err := http.Get("http://ipinfo.io")
|
||||
if err != nil {
|
||||
fmt.Printf("get ip info err %v \n", err.Error())
|
||||
fmt.Printf("get ip info err %s \n", sanitizeErrorText(err.Error()))
|
||||
} else {
|
||||
defer rsp.Body.Close()
|
||||
err = json.NewDecoder(rsp.Body).Decode(&info)
|
||||
if err != nil {
|
||||
fmt.Printf("json decode err %v \n", err.Error())
|
||||
fmt.Printf("json decode err %s \n", sanitizeErrorText(err.Error()))
|
||||
} else {
|
||||
fmt.Println(Green("国家: ") + White(info.Country) + Green(" 城市: ") + White(info.City) +
|
||||
Green(" 服务商: ") + Blue(info.Org))
|
||||
@ -186,10 +186,10 @@ func main() {
|
||||
})
|
||||
wg.Wait()
|
||||
if results.bgpResult != "" {
|
||||
fmt.Print(results.bgpResult)
|
||||
fmt.Print(indentLegacyOutput(results.bgpResult))
|
||||
}
|
||||
if results.backtraceResult != "" {
|
||||
fmt.Printf("%s\n", results.backtraceResult)
|
||||
fmt.Printf("%s\n", indentLegacyOutput(results.backtraceResult))
|
||||
}
|
||||
fmt.Println(Yellow("准确线路自行查看详细路由,本测试结果仅作参考"))
|
||||
fmt.Println(Yellow("同一目标地址多个线路时,检测可能已越过汇聚层,除第一个线路外,后续信息可能无效"))
|
||||
|
||||
27
cmd/output.go
Normal file
27
cmd/output.go
Normal file
@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var privateErrorPattern = regexp.MustCompile("(?i)(https?://)[^\\s]+")
|
||||
var privateTokenPattern = regexp.MustCompile("(?i)(^|[?&\\s])(token|api[_-]?key|secret|authorization|password|key)=([^&\\s]+)")
|
||||
var privatePathPattern = regexp.MustCompile("(?:[A-Za-z]:\\\\|/)[^\\s]+")
|
||||
|
||||
func indentLegacyOutput(value string) string {
|
||||
lines := strings.Split(value, "\n")
|
||||
for index, line := range lines {
|
||||
if line != "" && line[0] != ' ' && line[0] != '\t' {
|
||||
lines[index] = " " + line
|
||||
}
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// sanitizeErrorText keeps remote-source and credential details out of user-visible errors.
|
||||
func sanitizeErrorText(value string) string {
|
||||
value = privateErrorPattern.ReplaceAllString(value, "<remote>")
|
||||
value = privateTokenPattern.ReplaceAllString(value, "$1$2=<redacted>")
|
||||
return privatePathPattern.ReplaceAllString(value, "<path>")
|
||||
}
|
||||
24
cmd/output_test.go
Normal file
24
cmd/output_test.go
Normal file
@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIndentLegacyOutputLeavesOneLeadingCell(t *testing.T) {
|
||||
got := indentLegacyOutput("alpha\n beta\n\tgamma\n")
|
||||
want := " alpha\n beta\n\tgamma\n"
|
||||
if got != want {
|
||||
t.Fatalf("indentLegacyOutput() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeErrorTextRedactsRemoteSecretsAndPaths(t *testing.T) {
|
||||
input := "load https://private.example/repo/data?token=secret from /private/cache failed: api_key=hidden"
|
||||
got := sanitizeErrorText(input)
|
||||
for _, forbidden := range []string{"private.example", "secret", "hidden", "/private/cache"} {
|
||||
if strings.Contains(got, forbidden) {
|
||||
t.Fatalf("sanitizeErrorText() leaked %q in %q", forbidden, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -3,6 +3,8 @@ package main
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
@ -26,6 +28,14 @@ type document struct {
|
||||
Entries []bgptools.ASNMetadata `json:"entries"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Schema string `json:"schema"`
|
||||
File string `json:"file"`
|
||||
Count int `json:"count"`
|
||||
SHA256 string `json:"sha256"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
source := flag.String("source", defaultSource, "ASN name registry URL")
|
||||
output := flag.String("output", "bgptools/data/bgp-asn-map.json", "snapshot output path")
|
||||
@ -106,7 +116,7 @@ func updateSnapshot(path string, entries []bgptools.ASNMetadata, minimum int) er
|
||||
return fmt.Errorf("ASN count dropped from %d to %d", len(current.Entries), len(entries))
|
||||
}
|
||||
if sameEntries(current.Entries, entries) && !current.GeneratedAt.IsZero() {
|
||||
return nil
|
||||
return writeManifest(path, currentData, len(current.Entries))
|
||||
}
|
||||
next, err := json.MarshalIndent(document{Schema: bgptools.ASNMetadataSchema, GeneratedAt: time.Now().UTC(), Entries: entries}, "", " ")
|
||||
if err != nil {
|
||||
@ -130,6 +140,52 @@ func updateSnapshot(path string, entries []bgptools.ASNMetadata, minimum int) er
|
||||
if err := temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(name, path); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeManifest(path, next, len(entries))
|
||||
}
|
||||
|
||||
func writeManifest(snapshotPath string, snapshot []byte, count int) error {
|
||||
var snapshotDocument document
|
||||
if err := json.Unmarshal(snapshot, &snapshotDocument); err != nil {
|
||||
return err
|
||||
}
|
||||
hash := sha256.Sum256(snapshot)
|
||||
value := manifest{
|
||||
Schema: bgptools.ASNMetadataManifestSchema, File: filepath.Base(snapshotPath), Count: count,
|
||||
SHA256: hex.EncodeToString(hash[:]), GeneratedAt: snapshotDocument.GeneratedAt,
|
||||
}
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
manifestPath := strings.TrimSuffix(snapshotPath, ".json") + ".manifest.json"
|
||||
if current, readErr := os.ReadFile(manifestPath); readErr == nil && string(current) == string(data) {
|
||||
return nil
|
||||
}
|
||||
return writeAtomic(manifestPath, data)
|
||||
}
|
||||
|
||||
func writeAtomic(path string, data []byte) error {
|
||||
temporary, err := os.CreateTemp(filepath.Dir(path), ".asn-metadata-manifest-*.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := temporary.Name()
|
||||
defer os.Remove(name)
|
||||
if _, err := temporary.Write(data); err != nil {
|
||||
temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Chmod(0o644); err != nil {
|
||||
temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(name, path)
|
||||
}
|
||||
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@ -19,6 +22,34 @@ func TestParseEntriesDeduplicatesAndSorts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateSnapshotWritesMatchingManifestWithoutRewritingSnapshot(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "asn.json")
|
||||
current := []byte(`{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"},{"asn":2,"name":"Two"}]}`)
|
||||
if err := os.WriteFile(path, current, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries := []bgptools.ASNMetadata{{ASN: 1, Name: "One"}, {ASN: 2, Name: "Two"}}
|
||||
if err := updateSnapshot(path, entries, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := os.ReadFile(path)
|
||||
if string(after) != string(current) {
|
||||
t.Fatal("semantic no-op rewrote snapshot")
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(filepath.Dir(path), "asn.manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value manifest
|
||||
if err := json.Unmarshal(data, &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash := sha256.Sum256(current)
|
||||
if value.Schema != bgptools.ASNMetadataManifestSchema || value.File != "asn.json" || value.Count != 2 || value.SHA256 != hex.EncodeToString(hash[:]) {
|
||||
t.Fatalf("manifest mismatch: %#v", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateSnapshotRejectsCountDrop(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "asn.json")
|
||||
current := `{"schema":"backtrace.asn-metadata/v1","generated_at":"2026-07-20T00:00:00Z","entries":[{"asn":1,"name":"One"},{"asn":2,"name":"Two"}]}`
|
||||
|
||||
@ -2,7 +2,7 @@ package model
|
||||
|
||||
import "time"
|
||||
|
||||
const BackTraceVersion = "v0.0.14"
|
||||
const BackTraceVersion = "v0.0.17"
|
||||
|
||||
var EnableLoger = false
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user