From 8706bab893ad436030258b566c557d67a389ce38 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 6 Jul 2021 16:37:05 -0700 Subject: [PATCH 1/9] base limits on reservations issued --- v2/relay/constraints.go | 187 +++++++++++++++++++++++++++++++++++ v2/relay/constraints_test.go | 152 ++++++++++++++++++++++++++++ v2/relay/ipcs.go | 110 --------------------- v2/relay/relay.go | 22 ++--- v2/relay/resources.go | 8 +- v2/test/ipcs_test.go | 69 ------------- 6 files changed, 352 insertions(+), 196 deletions(-) create mode 100644 v2/relay/constraints.go create mode 100644 v2/relay/constraints_test.go delete mode 100644 v2/relay/ipcs.go delete mode 100644 v2/test/ipcs_test.go diff --git a/v2/relay/constraints.go b/v2/relay/constraints.go new file mode 100644 index 0000000..7c40d29 --- /dev/null +++ b/v2/relay/constraints.go @@ -0,0 +1,187 @@ +package relay + +import ( + crand "crypto/rand" + "encoding/binary" + "errors" + "math/rand" + "sync" + "time" + + asnutil "github.com/libp2p/go-libp2p-asn-util" + "github.com/libp2p/go-libp2p-core/peer" + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" +) + +var cleanupInterval = 2 * time.Minute +var validity = 30 * time.Minute + +var ( + errTooManyReservations = errors.New("too many reservations") + errTooManyReservationsForPeer = errors.New("too many reservations for peer") + errTooManyReservationsForIP = errors.New("too many peers for IP address") + errTooManyReservationsForASN = errors.New("too many peers for ASN") +) + +// constraints implements various reservation constraints +type constraints struct { + rc *Resources + + closed bool + closing, cleanupRunning chan struct{} + + mutex sync.Mutex + rand rand.Rand + total map[uint64]time.Time + peers map[peer.ID]map[uint64]time.Time + ips map[string]map[uint64]time.Time + asns map[string]map[uint64]time.Time +} + +// NewConstraints creates a new constraints object. +// The methods are *not* thread-safe; an external lock must be held if synchronization +// is required. +func NewConstraints(rc *Resources) *constraints { + b := make([]byte, 8) + crand.Read(b) + random := rand.New(rand.NewSource(int64(binary.BigEndian.Uint64(b)))) + + c := &constraints{ + rc: rc, + closing: make(chan struct{}), + cleanupRunning: make(chan struct{}), + rand: *random, + total: make(map[uint64]time.Time), + peers: make(map[peer.ID]map[uint64]time.Time), + ips: make(map[string]map[uint64]time.Time), + asns: make(map[string]map[uint64]time.Time), + } + go c.cleanup() + return c +} + +// AddReservation adds a reservation for a given peer with a given multiaddr. +// If adding this reservation violates IP constraints, an error is returned. +func (c *constraints) AddReservation(p peer.ID, a ma.Multiaddr) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + if len(c.total) >= c.rc.MaxReservations { + return errTooManyReservations + } + + ip, err := manet.ToIP(a) + if err != nil { + return errors.New("no IP address associated with peer") + } + + peerReservations := c.peers[p] + if len(peerReservations) >= c.rc.MaxReservationsPerPeer { + return errTooManyReservationsForPeer + } + + ipStr := ip.String() + ipReservations := c.ips[ipStr] + if len(ipReservations) >= c.rc.MaxReservationsPerIP { + return errTooManyReservationsForIP + } + + var ansReservations map[uint64]time.Time + var asn string + if ip.To4() == nil { + asn, _ = asnutil.Store.AsnForIPv6(ip) + if asn != "" { + ansReservations = c.asns[asn] + if len(ansReservations) >= c.rc.MaxReservationsPerASN { + return errTooManyReservationsForASN + } + } + } + + now := time.Now() + id := c.rand.Uint64() + + c.total[id] = now + + if peerReservations == nil { + peerReservations = make(map[uint64]time.Time) + c.peers[p] = peerReservations + } + peerReservations[id] = now + + if ipReservations == nil { + ipReservations = make(map[uint64]time.Time) + c.ips[ipStr] = ipReservations + } + ipReservations[id] = now + + if asn != "" { + if ansReservations == nil { + ansReservations = make(map[uint64]time.Time) + c.asns[asn] = ansReservations + } + ansReservations[id] = now + } + + return nil +} + +func (c *constraints) cleanup() { + defer close(c.cleanupRunning) + closeChan := c.closing + ticker := time.NewTicker(cleanupInterval) + defer ticker.Stop() + for { + select { + case <-closeChan: + return + case now := <-ticker.C: + c.mutex.Lock() + for id, t := range c.total { + if t.Add(validity).Before(now) { + delete(c.total, id) + } + } + for p, values := range c.peers { + for id, t := range values { + if t.Add(validity).Before(now) { + delete(values, id) + } + } + if len(values) == 0 { + delete(c.peers, p) + } + } + for ip, values := range c.ips { + for id, t := range values { + if t.Add(validity).Before(now) { + delete(values, id) + } + } + if len(values) == 0 { + delete(c.ips, ip) + } + } + for asn, values := range c.asns { + for id, t := range values { + if t.Add(validity).Before(now) { + delete(values, id) + } + } + if len(values) == 0 { + delete(c.asns, asn) + } + } + c.mutex.Unlock() + } + } +} + +func (c *constraints) Close() { + if !c.closed { + close(c.closing) + c.closed = true + <-c.cleanupRunning + } +} diff --git a/v2/relay/constraints_test.go b/v2/relay/constraints_test.go new file mode 100644 index 0000000..47ddaa3 --- /dev/null +++ b/v2/relay/constraints_test.go @@ -0,0 +1,152 @@ +package relay + +import ( + "crypto/rand" + "fmt" + "math" + "net" + "testing" + "time" + + "github.com/libp2p/go-libp2p-core/test" + ma "github.com/multiformats/go-multiaddr" +) + +func randomIPv4Addr(t *testing.T) ma.Multiaddr { + t.Helper() + b := make([]byte, 4) + rand.Read(b) + addr, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/1234", net.IP(b))) + if err != nil { + t.Fatal(err) + } + return addr +} + +func TestConstraints(t *testing.T) { + infResources := func() *Resources { + return &Resources{ + MaxReservations: math.MaxInt32, + MaxReservationsPerPeer: math.MaxInt32, + MaxReservationsPerIP: math.MaxInt32, + MaxReservationsPerASN: math.MaxInt32, + } + } + const limit = 7 + + t.Run("total reservations", func(t *testing.T) { + res := infResources() + res.MaxReservations = limit + c := NewConstraints(res) + defer c.Close() + for i := 0; i < limit; i++ { + if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { + t.Fatal(err) + } + } + if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != errTooManyReservations { + t.Fatalf("expected to run into total reservation limit, got %v", err) + } + }) + + t.Run("reservations per peer", func(t *testing.T) { + p := test.RandPeerIDFatal(t) + res := infResources() + res.MaxReservationsPerPeer = limit + c := NewConstraints(res) + defer c.Close() + for i := 0; i < limit; i++ { + if err := c.AddReservation(p, randomIPv4Addr(t)); err != nil { + t.Fatal(err) + } + } + if err := c.AddReservation(p, randomIPv4Addr(t)); err != errTooManyReservationsForPeer { + t.Fatalf("expected to run into total reservation limit, got %v", err) + } + if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { + t.Fatalf("expected reservation for different peer to be possible, got %v", err) + } + }) + + t.Run("reservations per IP", func(t *testing.T) { + ip := randomIPv4Addr(t) + res := infResources() + res.MaxReservationsPerIP = limit + c := NewConstraints(res) + defer c.Close() + for i := 0; i < limit; i++ { + if err := c.AddReservation(test.RandPeerIDFatal(t), ip); err != nil { + t.Fatal(err) + } + } + if err := c.AddReservation(test.RandPeerIDFatal(t), ip); err != errTooManyReservationsForIP { + t.Fatalf("expected to run into total reservation limit, got %v", err) + } + if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { + t.Fatalf("expected reservation for different IP to be possible, got %v", err) + } + }) + + t.Run("reservations per ASN", func(t *testing.T) { + getAddr := func(t *testing.T, ip net.IP) ma.Multiaddr { + t.Helper() + addr, err := ma.NewMultiaddr(fmt.Sprintf("/ip6/%s/tcp/1234", ip)) + if err != nil { + t.Fatal(err) + } + return addr + } + + res := infResources() + res.MaxReservationsPerASN = limit + c := NewConstraints(res) + defer c.Close() + const ipv6Prefix = "2a03:2880:f003:c07:face:b00c::" + for i := 0; i < limit; i++ { + addr := getAddr(t, net.ParseIP(fmt.Sprintf("%s%d", ipv6Prefix, i+1))) + if err := c.AddReservation(test.RandPeerIDFatal(t), addr); err != nil { + t.Fatal(err) + } + } + if err := c.AddReservation(test.RandPeerIDFatal(t), getAddr(t, net.ParseIP(fmt.Sprintf("%s%d", ipv6Prefix, 42)))); err != errTooManyReservationsForASN { + t.Fatalf("expected to run into total reservation limit, got %v", err) + } + if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { + t.Fatalf("expected reservation for different IP to be possible, got %v", err) + } + }) +} + +func TestConstraintsCleanup(t *testing.T) { + origValidity := validity + origCleanupInterval := cleanupInterval + defer func() { + validity = origValidity + cleanupInterval = origCleanupInterval + }() + validity = 500 * time.Millisecond + cleanupInterval = validity / 10 + + const limit = 7 + res := &Resources{ + MaxReservations: limit, + MaxReservationsPerPeer: math.MaxInt32, + MaxReservationsPerIP: math.MaxInt32, + MaxReservationsPerASN: math.MaxInt32, + } + c := NewConstraints(res) + defer c.Close() + for i := 0; i < limit; i++ { + if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { + t.Fatal(err) + } + } + if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != errTooManyReservations { + t.Fatalf("expected to run into total reservation limit, got %v", err) + } + + time.Sleep(validity + 2*cleanupInterval) + if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { + t.Fatalf("expected old reservations to have been garbage collected, %v", err) + } +} diff --git a/v2/relay/ipcs.go b/v2/relay/ipcs.go deleted file mode 100644 index 167e08e..0000000 --- a/v2/relay/ipcs.go +++ /dev/null @@ -1,110 +0,0 @@ -package relay - -import ( - "errors" - "net" - - "github.com/libp2p/go-libp2p-core/peer" - - asnutil "github.com/libp2p/go-libp2p-asn-util" - ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" -) - -var ( - ErrNoIP = errors.New("no IP address associated with peer") - ErrTooManyPeersInIP = errors.New("too many peers in IP address") - ErrTooManyPeersInASN = errors.New("too many peers in ASN") -) - -// IPConstraints implements reservation constraints per IP -type IPConstraints struct { - iplimit, asnlimit int - - peers map[peer.ID]net.IP - ips map[string]map[peer.ID]struct{} - asns map[string]map[peer.ID]struct{} -} - -// NewIPConstraints creates a new IPConstraints object. -// The methods are *not* thread-safe; an external lock must be held if synchronization -// is required. -func NewIPConstraints(rc Resources) *IPConstraints { - return &IPConstraints{ - iplimit: rc.MaxReservationsPerIP, - asnlimit: rc.MaxReservationsPerASN, - - peers: make(map[peer.ID]net.IP), - ips: make(map[string]map[peer.ID]struct{}), - asns: make(map[string]map[peer.ID]struct{}), - } -} - -// AddReservation adds a reservation for a given peer with a given multiaddr. -// If adding this reservation violates IP constraints, an error is returned. -func (ipcs *IPConstraints) AddReservation(p peer.ID, a ma.Multiaddr) error { - ip, err := manet.ToIP(a) - if err != nil { - return ErrNoIP - } - - ips := ip.String() - peersInIP := ipcs.ips[ips] - if len(peersInIP) >= ipcs.iplimit { - return ErrTooManyPeersInIP - } - - var peersInAsn map[peer.ID]struct{} - asn, _ := asnutil.Store.AsnForIPv6(ip) - peersInAsn = ipcs.asns[asn] - if len(peersInAsn) >= ipcs.asnlimit { - return ErrTooManyPeersInASN - } - - ipcs.peers[p] = ip - - if peersInIP == nil { - peersInIP = make(map[peer.ID]struct{}) - ipcs.ips[ips] = peersInIP - } - peersInIP[p] = struct{}{} - - if asn != "" { - if peersInAsn == nil { - peersInAsn = make(map[peer.ID]struct{}) - ipcs.asns[asn] = peersInAsn - } - peersInAsn[p] = struct{}{} - } - - return nil -} - -// RemoveReservation removes a peer from the constraints. -func (ipcs *IPConstraints) RemoveReservation(p peer.ID) { - ip, ok := ipcs.peers[p] - if !ok { - return - } - - ips := ip.String() - asn, _ := asnutil.Store.AsnForIPv6(ip) - - delete(ipcs.peers, p) - - peersInIP, ok := ipcs.ips[ips] - if ok { - delete(peersInIP, p) - if len(peersInIP) == 0 { - delete(ipcs.ips, ips) - } - } - - peersInAsn, ok := ipcs.asns[asn] - if ok { - delete(peersInAsn, p) - if len(peersInAsn) == 0 { - delete(ipcs.asns, asn) - } - } -} diff --git a/v2/relay/relay.go b/v2/relay/relay.go index 8ef4e49..e73afb1 100644 --- a/v2/relay/relay.go +++ b/v2/relay/relay.go @@ -41,10 +41,10 @@ type Relay struct { ctx context.Context cancel func() - host host.Host - rc Resources - acl ACLFilter - ipcs *IPConstraints + host host.Host + rc Resources + acl ACLFilter + constraints *constraints mx sync.Mutex rsvp map[peer.ID]time.Time @@ -74,7 +74,7 @@ func New(h host.Host, opts ...Option) (*Relay, error) { } } - r.ipcs = NewIPConstraints(r.rc) + r.constraints = NewConstraints(&r.rc) r.selfAddr = ma.StringCast(fmt.Sprintf("/p2p/%s", h.ID())) h.SetStreamHandler(proto.ProtoIDv2Hop, r.handleStream) @@ -95,6 +95,7 @@ func (r *Relay) Close() error { for p := range r.rsvp { r.host.ConnManager().UntagPeer(p, "relay-reservation") } + r.constraints.Close() r.mx.Unlock() } return nil @@ -153,14 +154,7 @@ func (r *Relay) handleReserve(s network.Stream) { _, exists := r.rsvp[p] if !exists { - active := len(r.rsvp) - if active >= r.rc.MaxReservations { - r.mx.Unlock() - log.Debugf("refusing relay reservation for %s; too many reservations", p) - r.handleError(s, pbv2.Status_RESERVATION_REFUSED) - return - } - if err := r.ipcs.AddReservation(p, a); err != nil { + if err := r.constraints.AddReservation(p, a); err != nil { r.mx.Unlock() log.Debugf("refusing relay reservation for %s; IP constraint violation: %s", p, err) r.handleError(s, pbv2.Status_RESERVATION_REFUSED) @@ -493,7 +487,6 @@ func (r *Relay) gc() { for p, expire := range r.rsvp { if expire.Before(now) { delete(r.rsvp, p) - r.ipcs.RemoveReservation(p) r.host.ConnManager().UntagPeer(p, "relay-reservation") } } @@ -515,5 +508,4 @@ func (r *Relay) disconnected(n network.Network, c network.Conn) { defer r.mx.Unlock() delete(r.rsvp, p) - r.ipcs.RemoveReservation(p) } diff --git a/v2/relay/resources.go b/v2/relay/resources.go index 345b527..4026a00 100644 --- a/v2/relay/resources.go +++ b/v2/relay/resources.go @@ -20,6 +20,9 @@ type Resources struct { // BufferSize is the size of the relayed connection buffers; defaults to 2048. BufferSize int + // MaxReservationsPerPeer is the maximum number of reservations originating from the same + // peer; default is 8. + MaxReservationsPerPeer int // MaxReservationsPerIP is the maximum number of reservations originating from the same // IP address; default is 4. MaxReservationsPerIP int @@ -48,8 +51,9 @@ func DefaultResources() Resources { MaxCircuits: 16, BufferSize: 2048, - MaxReservationsPerIP: 4, - MaxReservationsPerASN: 32, + MaxReservationsPerPeer: 8, + MaxReservationsPerIP: 4, + MaxReservationsPerASN: 32, } } diff --git a/v2/test/ipcs_test.go b/v2/test/ipcs_test.go deleted file mode 100644 index 09a1a9b..0000000 --- a/v2/test/ipcs_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package test - -import ( - "fmt" - "net" - "testing" - - "github.com/libp2p/go-libp2p-circuit/v2/relay" - - "github.com/libp2p/go-libp2p-core/peer" - - ma "github.com/multiformats/go-multiaddr" -) - -func TestIPConstraints(t *testing.T) { - ipcs := relay.NewIPConstraints(relay.Resources{ - MaxReservationsPerIP: 1, - MaxReservationsPerASN: 2, - }) - - peerA := peer.ID("A") - peerB := peer.ID("B") - peerC := peer.ID("C") - peerD := peer.ID("D") - peerE := peer.ID("E") - - ipA := net.ParseIP("1.2.3.4") - ipB := ipA - ipC := net.ParseIP("2001:200::1") - ipD := net.ParseIP("2001:200::2") - ipE := net.ParseIP("2001:200::3") - - err := ipcs.AddReservation(peerA, ma.StringCast(fmt.Sprintf("/ip4/%s/tcp/1234", ipA))) - if err != nil { - t.Fatal(err) - } - - err = ipcs.AddReservation(peerB, ma.StringCast(fmt.Sprintf("/ip4/%s/tcp/1234", ipB))) - if err != relay.ErrTooManyPeersInIP { - t.Fatalf("unexpected error: %s", err) - } - - ipcs.RemoveReservation(peerA) - err = ipcs.AddReservation(peerB, ma.StringCast(fmt.Sprintf("/ip4/%s/tcp/1234", ipB))) - if err != nil { - t.Fatal(err) - } - - err = ipcs.AddReservation(peerC, ma.StringCast(fmt.Sprintf("/ip6/%s/tcp/1234", ipC))) - if err != nil { - t.Fatal(err) - } - - err = ipcs.AddReservation(peerD, ma.StringCast(fmt.Sprintf("/ip6/%s/tcp/1234", ipD))) - if err != nil { - t.Fatal(err) - } - - err = ipcs.AddReservation(peerE, ma.StringCast(fmt.Sprintf("/ip6/%s/tcp/1234", ipE))) - if err != relay.ErrTooManyPeersInASN { - t.Fatalf("unexpected error: %s", err) - } - - ipcs.RemoveReservation(peerD) - err = ipcs.AddReservation(peerE, ma.StringCast(fmt.Sprintf("/ip6/%s/tcp/1234", ipE))) - if err != nil { - t.Fatal(err) - } -} From 168a60012149df90fc6a4e2b6303d9aaeadda1f9 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 25 Jul 2021 11:42:20 +0200 Subject: [PATCH 2/9] switch default reservation limits per peer and per IP --- v2/relay/resources.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/relay/resources.go b/v2/relay/resources.go index 4026a00..0ae0169 100644 --- a/v2/relay/resources.go +++ b/v2/relay/resources.go @@ -21,10 +21,10 @@ type Resources struct { BufferSize int // MaxReservationsPerPeer is the maximum number of reservations originating from the same - // peer; default is 8. + // peer; default is 4. MaxReservationsPerPeer int // MaxReservationsPerIP is the maximum number of reservations originating from the same - // IP address; default is 4. + // IP address; default is 8. MaxReservationsPerIP int // MaxReservationsPerASN is the maximum number of reservations origination from the same // ASN; default is 32 @@ -51,8 +51,8 @@ func DefaultResources() Resources { MaxCircuits: 16, BufferSize: 2048, - MaxReservationsPerPeer: 8, - MaxReservationsPerIP: 4, + MaxReservationsPerPeer: 4, + MaxReservationsPerIP: 8, MaxReservationsPerASN: 32, } } From 2dbf65a9702a681216da414763c98fbb28914420 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 29 Jul 2021 14:05:09 +0200 Subject: [PATCH 3/9] don't export the constructor for relay.constraints --- v2/relay/constraints.go | 4 ++-- v2/relay/constraints_test.go | 10 +++++----- v2/relay/relay.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/v2/relay/constraints.go b/v2/relay/constraints.go index 7c40d29..db0845d 100644 --- a/v2/relay/constraints.go +++ b/v2/relay/constraints.go @@ -39,10 +39,10 @@ type constraints struct { asns map[string]map[uint64]time.Time } -// NewConstraints creates a new constraints object. +// newConstraints creates a new constraints object. // The methods are *not* thread-safe; an external lock must be held if synchronization // is required. -func NewConstraints(rc *Resources) *constraints { +func newConstraints(rc *Resources) *constraints { b := make([]byte, 8) crand.Read(b) random := rand.New(rand.NewSource(int64(binary.BigEndian.Uint64(b)))) diff --git a/v2/relay/constraints_test.go b/v2/relay/constraints_test.go index 47ddaa3..5f8b361 100644 --- a/v2/relay/constraints_test.go +++ b/v2/relay/constraints_test.go @@ -37,7 +37,7 @@ func TestConstraints(t *testing.T) { t.Run("total reservations", func(t *testing.T) { res := infResources() res.MaxReservations = limit - c := NewConstraints(res) + c := newConstraints(res) defer c.Close() for i := 0; i < limit; i++ { if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { @@ -53,7 +53,7 @@ func TestConstraints(t *testing.T) { p := test.RandPeerIDFatal(t) res := infResources() res.MaxReservationsPerPeer = limit - c := NewConstraints(res) + c := newConstraints(res) defer c.Close() for i := 0; i < limit; i++ { if err := c.AddReservation(p, randomIPv4Addr(t)); err != nil { @@ -72,7 +72,7 @@ func TestConstraints(t *testing.T) { ip := randomIPv4Addr(t) res := infResources() res.MaxReservationsPerIP = limit - c := NewConstraints(res) + c := newConstraints(res) defer c.Close() for i := 0; i < limit; i++ { if err := c.AddReservation(test.RandPeerIDFatal(t), ip); err != nil { @@ -99,7 +99,7 @@ func TestConstraints(t *testing.T) { res := infResources() res.MaxReservationsPerASN = limit - c := NewConstraints(res) + c := newConstraints(res) defer c.Close() const ipv6Prefix = "2a03:2880:f003:c07:face:b00c::" for i := 0; i < limit; i++ { @@ -134,7 +134,7 @@ func TestConstraintsCleanup(t *testing.T) { MaxReservationsPerIP: math.MaxInt32, MaxReservationsPerASN: math.MaxInt32, } - c := NewConstraints(res) + c := newConstraints(res) defer c.Close() for i := 0; i < limit; i++ { if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { diff --git a/v2/relay/relay.go b/v2/relay/relay.go index e73afb1..cfa77b1 100644 --- a/v2/relay/relay.go +++ b/v2/relay/relay.go @@ -74,7 +74,7 @@ func New(h host.Host, opts ...Option) (*Relay, error) { } } - r.constraints = NewConstraints(&r.rc) + r.constraints = newConstraints(&r.rc) r.selfAddr = ma.StringCast(fmt.Sprintf("/p2p/%s", h.ID())) h.SetStreamHandler(proto.ProtoIDv2Hop, r.handleStream) From 88d9aab70a443e5b120b6ffa79d245a83fb82ac9 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 29 Jul 2021 14:06:35 +0200 Subject: [PATCH 4/9] panic when reading from crypto/rand fails --- v2/relay/constraints.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v2/relay/constraints.go b/v2/relay/constraints.go index db0845d..4bb6e84 100644 --- a/v2/relay/constraints.go +++ b/v2/relay/constraints.go @@ -44,7 +44,9 @@ type constraints struct { // is required. func newConstraints(rc *Resources) *constraints { b := make([]byte, 8) - crand.Read(b) + if _, err := crand.Read(b); err != nil { + panic("failed to read from crypto/rand") + } random := rand.New(rand.NewSource(int64(binary.BigEndian.Uint64(b)))) c := &constraints{ From e1f7b8682863b0ace683cd03257179774c3e0e87 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 29 Jul 2021 14:07:46 +0200 Subject: [PATCH 5/9] optimize IP-based reservation lookup --- v2/relay/constraints.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/v2/relay/constraints.go b/v2/relay/constraints.go index 4bb6e84..f18b1e3 100644 --- a/v2/relay/constraints.go +++ b/v2/relay/constraints.go @@ -83,8 +83,7 @@ func (c *constraints) AddReservation(p peer.ID, a ma.Multiaddr) error { return errTooManyReservationsForPeer } - ipStr := ip.String() - ipReservations := c.ips[ipStr] + ipReservations := c.ips[ip.String()] if len(ipReservations) >= c.rc.MaxReservationsPerIP { return errTooManyReservationsForIP } @@ -114,7 +113,7 @@ func (c *constraints) AddReservation(p peer.ID, a ma.Multiaddr) error { if ipReservations == nil { ipReservations = make(map[uint64]time.Time) - c.ips[ipStr] = ipReservations + c.ips[ip.String()] = ipReservations } ipReservations[id] = now From 6fe81423cfd164e8b4b344d9686771abaae165b3 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 4 Aug 2021 17:41:27 +0200 Subject: [PATCH 6/9] use lists instead of maps to save reservations --- v2/relay/constraints.go | 144 +++++++++++++---------------------- v2/relay/constraints_test.go | 14 +--- v2/relay/relay.go | 1 - 3 files changed, 56 insertions(+), 103 deletions(-) diff --git a/v2/relay/constraints.go b/v2/relay/constraints.go index f18b1e3..f3c4956 100644 --- a/v2/relay/constraints.go +++ b/v2/relay/constraints.go @@ -1,6 +1,7 @@ package relay import ( + "container/list" crand "crypto/rand" "encoding/binary" "errors" @@ -14,7 +15,6 @@ import ( manet "github.com/multiformats/go-multiaddr/net" ) -var cleanupInterval = 2 * time.Minute var validity = 30 * time.Minute var ( @@ -24,19 +24,20 @@ var ( errTooManyReservationsForASN = errors.New("too many peers for ASN") ) +type listEntry struct { + t time.Time +} + // constraints implements various reservation constraints type constraints struct { rc *Resources - closed bool - closing, cleanupRunning chan struct{} - mutex sync.Mutex rand rand.Rand - total map[uint64]time.Time - peers map[peer.ID]map[uint64]time.Time - ips map[string]map[uint64]time.Time - asns map[string]map[uint64]time.Time + total *list.List + peers map[peer.ID]*list.List + ips map[string]*list.List + asns map[string]*list.List } // newConstraints creates a new constraints object. @@ -49,18 +50,14 @@ func newConstraints(rc *Resources) *constraints { } random := rand.New(rand.NewSource(int64(binary.BigEndian.Uint64(b)))) - c := &constraints{ - rc: rc, - closing: make(chan struct{}), - cleanupRunning: make(chan struct{}), - rand: *random, - total: make(map[uint64]time.Time), - peers: make(map[peer.ID]map[uint64]time.Time), - ips: make(map[string]map[uint64]time.Time), - asns: make(map[string]map[uint64]time.Time), + return &constraints{ + rc: rc, + rand: *random, + total: list.New(), + peers: make(map[peer.ID]*list.List), + ips: make(map[string]*list.List), + asns: make(map[string]*list.List), } - go c.cleanup() - return c } // AddReservation adds a reservation for a given peer with a given multiaddr. @@ -69,7 +66,10 @@ func (c *constraints) AddReservation(p peer.ID, a ma.Multiaddr) error { c.mutex.Lock() defer c.mutex.Unlock() - if len(c.total) >= c.rc.MaxReservations { + now := time.Now() + c.cleanup(now) + + if c.total.Len() >= c.rc.MaxReservations { return errTooManyReservations } @@ -78,111 +78,75 @@ func (c *constraints) AddReservation(p peer.ID, a ma.Multiaddr) error { return errors.New("no IP address associated with peer") } - peerReservations := c.peers[p] - if len(peerReservations) >= c.rc.MaxReservationsPerPeer { + peerReservations, ok := c.peers[p] + if ok && peerReservations.Len() >= c.rc.MaxReservationsPerPeer { return errTooManyReservationsForPeer } - ipReservations := c.ips[ip.String()] - if len(ipReservations) >= c.rc.MaxReservationsPerIP { + ipReservations, ok := c.ips[ip.String()] + if ok && ipReservations.Len() >= c.rc.MaxReservationsPerIP { return errTooManyReservationsForIP } - var ansReservations map[uint64]time.Time + var asnReservations *list.List var asn string if ip.To4() == nil { asn, _ = asnutil.Store.AsnForIPv6(ip) if asn != "" { - ansReservations = c.asns[asn] - if len(ansReservations) >= c.rc.MaxReservationsPerASN { + var ok bool + asnReservations, ok = c.asns[asn] + if ok && asnReservations.Len() >= c.rc.MaxReservationsPerASN { return errTooManyReservationsForASN } } } - now := time.Now() - id := c.rand.Uint64() - - c.total[id] = now + c.total.PushBack(listEntry{t: now}) if peerReservations == nil { - peerReservations = make(map[uint64]time.Time) + peerReservations = list.New() c.peers[p] = peerReservations } - peerReservations[id] = now + peerReservations.PushBack(listEntry{t: now}) if ipReservations == nil { - ipReservations = make(map[uint64]time.Time) + ipReservations = list.New() c.ips[ip.String()] = ipReservations } - ipReservations[id] = now + ipReservations.PushBack(listEntry{t: now}) if asn != "" { - if ansReservations == nil { - ansReservations = make(map[uint64]time.Time) - c.asns[asn] = ansReservations + if asnReservations == nil { + asnReservations = list.New() + c.asns[asn] = asnReservations } - ansReservations[id] = now + asnReservations.PushBack(listEntry{t: now}) } return nil } -func (c *constraints) cleanup() { - defer close(c.cleanupRunning) - closeChan := c.closing - ticker := time.NewTicker(cleanupInterval) - defer ticker.Stop() - for { - select { - case <-closeChan: +func (c *constraints) cleanupList(l *list.List, now time.Time) { + for el := l.Front(); el != nil; { + entry := el.Value.(listEntry) + if entry.t.Add(validity).After(now) { return - case now := <-ticker.C: - c.mutex.Lock() - for id, t := range c.total { - if t.Add(validity).Before(now) { - delete(c.total, id) - } - } - for p, values := range c.peers { - for id, t := range values { - if t.Add(validity).Before(now) { - delete(values, id) - } - } - if len(values) == 0 { - delete(c.peers, p) - } - } - for ip, values := range c.ips { - for id, t := range values { - if t.Add(validity).Before(now) { - delete(values, id) - } - } - if len(values) == 0 { - delete(c.ips, ip) - } - } - for asn, values := range c.asns { - for id, t := range values { - if t.Add(validity).Before(now) { - delete(values, id) - } - } - if len(values) == 0 { - delete(c.asns, asn) - } - } - c.mutex.Unlock() } + nextEl := el.Next() + l.Remove(el) + el = nextEl } } -func (c *constraints) Close() { - if !c.closed { - close(c.closing) - c.closed = true - <-c.cleanupRunning +func (c *constraints) cleanup(now time.Time) { + c.cleanupList(c.total, now) + for _, peerReservations := range c.peers { + c.cleanupList(peerReservations, now) + } + for _, ipReservations := range c.ips { + c.cleanupList(ipReservations, now) + } + for _, asnReservations := range c.asns { + c.cleanupList(asnReservations, now) } } diff --git a/v2/relay/constraints_test.go b/v2/relay/constraints_test.go index 5f8b361..0eaaa68 100644 --- a/v2/relay/constraints_test.go +++ b/v2/relay/constraints_test.go @@ -38,7 +38,6 @@ func TestConstraints(t *testing.T) { res := infResources() res.MaxReservations = limit c := newConstraints(res) - defer c.Close() for i := 0; i < limit; i++ { if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { t.Fatal(err) @@ -54,7 +53,6 @@ func TestConstraints(t *testing.T) { res := infResources() res.MaxReservationsPerPeer = limit c := newConstraints(res) - defer c.Close() for i := 0; i < limit; i++ { if err := c.AddReservation(p, randomIPv4Addr(t)); err != nil { t.Fatal(err) @@ -73,7 +71,6 @@ func TestConstraints(t *testing.T) { res := infResources() res.MaxReservationsPerIP = limit c := newConstraints(res) - defer c.Close() for i := 0; i < limit; i++ { if err := c.AddReservation(test.RandPeerIDFatal(t), ip); err != nil { t.Fatal(err) @@ -100,7 +97,6 @@ func TestConstraints(t *testing.T) { res := infResources() res.MaxReservationsPerASN = limit c := newConstraints(res) - defer c.Close() const ipv6Prefix = "2a03:2880:f003:c07:face:b00c::" for i := 0; i < limit; i++ { addr := getAddr(t, net.ParseIP(fmt.Sprintf("%s%d", ipv6Prefix, i+1))) @@ -119,13 +115,8 @@ func TestConstraints(t *testing.T) { func TestConstraintsCleanup(t *testing.T) { origValidity := validity - origCleanupInterval := cleanupInterval - defer func() { - validity = origValidity - cleanupInterval = origCleanupInterval - }() + defer func() { validity = origValidity }() validity = 500 * time.Millisecond - cleanupInterval = validity / 10 const limit = 7 res := &Resources{ @@ -135,7 +126,6 @@ func TestConstraintsCleanup(t *testing.T) { MaxReservationsPerASN: math.MaxInt32, } c := newConstraints(res) - defer c.Close() for i := 0; i < limit; i++ { if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { t.Fatal(err) @@ -145,7 +135,7 @@ func TestConstraintsCleanup(t *testing.T) { t.Fatalf("expected to run into total reservation limit, got %v", err) } - time.Sleep(validity + 2*cleanupInterval) + time.Sleep(validity + time.Millisecond) if err := c.AddReservation(test.RandPeerIDFatal(t), randomIPv4Addr(t)); err != nil { t.Fatalf("expected old reservations to have been garbage collected, %v", err) } diff --git a/v2/relay/relay.go b/v2/relay/relay.go index cfa77b1..efba742 100644 --- a/v2/relay/relay.go +++ b/v2/relay/relay.go @@ -95,7 +95,6 @@ func (r *Relay) Close() error { for p := range r.rsvp { r.host.ConnManager().UntagPeer(p, "relay-reservation") } - r.constraints.Close() r.mx.Unlock() } return nil From eff0cbe500206839b0ec0df8ca85645ff0601afc Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 4 Aug 2021 17:44:02 +0200 Subject: [PATCH 7/9] save expiry timestamp in reservations --- v2/relay/constraints.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/v2/relay/constraints.go b/v2/relay/constraints.go index f3c4956..bf6450f 100644 --- a/v2/relay/constraints.go +++ b/v2/relay/constraints.go @@ -101,26 +101,27 @@ func (c *constraints) AddReservation(p peer.ID, a ma.Multiaddr) error { } } - c.total.PushBack(listEntry{t: now}) + expiry := now.Add(validity) + c.total.PushBack(listEntry{t: expiry}) if peerReservations == nil { peerReservations = list.New() c.peers[p] = peerReservations } - peerReservations.PushBack(listEntry{t: now}) + peerReservations.PushBack(listEntry{t: expiry}) if ipReservations == nil { ipReservations = list.New() c.ips[ip.String()] = ipReservations } - ipReservations.PushBack(listEntry{t: now}) + ipReservations.PushBack(listEntry{t: expiry}) if asn != "" { if asnReservations == nil { asnReservations = list.New() c.asns[asn] = asnReservations } - asnReservations.PushBack(listEntry{t: now}) + asnReservations.PushBack(listEntry{t: expiry}) } return nil @@ -129,7 +130,7 @@ func (c *constraints) AddReservation(p peer.ID, a ma.Multiaddr) error { func (c *constraints) cleanupList(l *list.List, now time.Time) { for el := l.Front(); el != nil; { entry := el.Value.(listEntry) - if entry.t.Add(validity).After(now) { + if entry.t.After(now) { return } nextEl := el.Next() From 54c4a93b6a67f8d322f45c892f5adb557072cc45 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 4 Aug 2021 18:54:09 +0200 Subject: [PATCH 8/9] use slices instead of linked lists for reservations --- v2/relay/constraints.go | 90 +++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 54 deletions(-) diff --git a/v2/relay/constraints.go b/v2/relay/constraints.go index bf6450f..8083432 100644 --- a/v2/relay/constraints.go +++ b/v2/relay/constraints.go @@ -1,7 +1,6 @@ package relay import ( - "container/list" crand "crypto/rand" "encoding/binary" "errors" @@ -24,20 +23,16 @@ var ( errTooManyReservationsForASN = errors.New("too many peers for ASN") ) -type listEntry struct { - t time.Time -} - // constraints implements various reservation constraints type constraints struct { rc *Resources mutex sync.Mutex rand rand.Rand - total *list.List - peers map[peer.ID]*list.List - ips map[string]*list.List - asns map[string]*list.List + total []time.Time + peers map[peer.ID][]time.Time + ips map[string][]time.Time + asns map[string][]time.Time } // newConstraints creates a new constraints object. @@ -53,10 +48,9 @@ func newConstraints(rc *Resources) *constraints { return &constraints{ rc: rc, rand: *random, - total: list.New(), - peers: make(map[peer.ID]*list.List), - ips: make(map[string]*list.List), - asns: make(map[string]*list.List), + peers: make(map[peer.ID][]time.Time), + ips: make(map[string][]time.Time), + asns: make(map[string][]time.Time), } } @@ -69,7 +63,7 @@ func (c *constraints) AddReservation(p peer.ID, a ma.Multiaddr) error { now := time.Now() c.cleanup(now) - if c.total.Len() >= c.rc.MaxReservations { + if len(c.total) >= c.rc.MaxReservations { return errTooManyReservations } @@ -78,76 +72,64 @@ func (c *constraints) AddReservation(p peer.ID, a ma.Multiaddr) error { return errors.New("no IP address associated with peer") } - peerReservations, ok := c.peers[p] - if ok && peerReservations.Len() >= c.rc.MaxReservationsPerPeer { + peerReservations := c.peers[p] + if len(peerReservations) >= c.rc.MaxReservationsPerPeer { return errTooManyReservationsForPeer } - ipReservations, ok := c.ips[ip.String()] - if ok && ipReservations.Len() >= c.rc.MaxReservationsPerIP { + ipReservations := c.ips[ip.String()] + if len(ipReservations) >= c.rc.MaxReservationsPerIP { return errTooManyReservationsForIP } - var asnReservations *list.List + var asnReservations []time.Time var asn string if ip.To4() == nil { asn, _ = asnutil.Store.AsnForIPv6(ip) if asn != "" { - var ok bool - asnReservations, ok = c.asns[asn] - if ok && asnReservations.Len() >= c.rc.MaxReservationsPerASN { + asnReservations = c.asns[asn] + if len(asnReservations) >= c.rc.MaxReservationsPerASN { return errTooManyReservationsForASN } } } expiry := now.Add(validity) - c.total.PushBack(listEntry{t: expiry}) + c.total = append(c.total, expiry) - if peerReservations == nil { - peerReservations = list.New() - c.peers[p] = peerReservations - } - peerReservations.PushBack(listEntry{t: expiry}) + peerReservations = append(peerReservations, expiry) + c.peers[p] = peerReservations - if ipReservations == nil { - ipReservations = list.New() - c.ips[ip.String()] = ipReservations - } - ipReservations.PushBack(listEntry{t: expiry}) + ipReservations = append(ipReservations, expiry) + c.ips[ip.String()] = ipReservations if asn != "" { - if asnReservations == nil { - asnReservations = list.New() - c.asns[asn] = asnReservations - } - asnReservations.PushBack(listEntry{t: expiry}) + asnReservations = append(asnReservations, expiry) + c.asns[asn] = asnReservations } - return nil } -func (c *constraints) cleanupList(l *list.List, now time.Time) { - for el := l.Front(); el != nil; { - entry := el.Value.(listEntry) - if entry.t.After(now) { - return +func (c *constraints) cleanupList(l []time.Time, now time.Time) []time.Time { + var index int + for i, t := range l { + if t.After(now) { + break } - nextEl := el.Next() - l.Remove(el) - el = nextEl + index = i + 1 } + return l[index:] } func (c *constraints) cleanup(now time.Time) { - c.cleanupList(c.total, now) - for _, peerReservations := range c.peers { - c.cleanupList(peerReservations, now) + c.total = c.cleanupList(c.total, now) + for k, peerReservations := range c.peers { + c.peers[k] = c.cleanupList(peerReservations, now) } - for _, ipReservations := range c.ips { - c.cleanupList(ipReservations, now) + for k, ipReservations := range c.ips { + c.ips[k] = c.cleanupList(ipReservations, now) } - for _, asnReservations := range c.asns { - c.cleanupList(asnReservations, now) + for k, asnReservations := range c.asns { + c.asns[k] = c.cleanupList(asnReservations, now) } } From 0ebfb5e264a11f7923e8062bef3926fdf0875b1d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 5 Aug 2021 19:10:58 +0200 Subject: [PATCH 9/9] remove unused rand in constraints --- v2/relay/constraints.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/v2/relay/constraints.go b/v2/relay/constraints.go index 8083432..1b0087e 100644 --- a/v2/relay/constraints.go +++ b/v2/relay/constraints.go @@ -1,10 +1,7 @@ package relay import ( - crand "crypto/rand" - "encoding/binary" "errors" - "math/rand" "sync" "time" @@ -28,7 +25,6 @@ type constraints struct { rc *Resources mutex sync.Mutex - rand rand.Rand total []time.Time peers map[peer.ID][]time.Time ips map[string][]time.Time @@ -39,15 +35,8 @@ type constraints struct { // The methods are *not* thread-safe; an external lock must be held if synchronization // is required. func newConstraints(rc *Resources) *constraints { - b := make([]byte, 8) - if _, err := crand.Read(b); err != nil { - panic("failed to read from crypto/rand") - } - random := rand.New(rand.NewSource(int64(binary.BigEndian.Uint64(b)))) - return &constraints{ rc: rc, - rand: *random, peers: make(map[peer.ID][]time.Time), ips: make(map[string][]time.Time), asns: make(map[string][]time.Time),